hive git commit: HIVE-19127: Concurrency fixes in QueryResultsCache (Jason Dere, reviewed by Deepak Jaiswal)
Repository: hive
Updated Branches:
refs/heads/master 76b696c26 -> a1034102d
HIVE-19127: Concurrency fixes in QueryResultsCache (Jason Dere, reviewed by
Deepak Jaiswal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a1034102
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a1034102
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a1034102
Branch: refs/heads/master
Commit: a1034102d3580922f6c8f9d186272280d6917802
Parents: 76b696c
Author: Jason Dere
Authored: Mon Apr 9 16:48:23 2018 -0700
Committer: Jason Dere
Committed: Mon Apr 9 16:48:23 2018 -0700
--
.../ql/cache/results/QueryResultsCache.java | 112 +++
1 file changed, 68 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/a1034102/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
index ac5ae57..b1a3646 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
@@ -502,32 +502,39 @@ public final class QueryResultsCache {
return false;
}
- if (requiresMove) {
-// Move the query results to the query cache directory.
-cachedResultsPath = moveResultsToCacheDirectory(queryResultsPath);
-dataDirMoved = true;
- }
- LOG.info("Moved query results from {} to {} (size {}) for query '{}'",
- queryResultsPath, cachedResultsPath, resultSize, queryText);
-
- // Create a new FetchWork to reference the new cache location.
- FetchWork fetchWorkForCache =
- new FetchWork(cachedResultsPath, fetchWork.getTblDesc(),
fetchWork.getLimit());
- fetchWorkForCache.setCachedResult(true);
- cacheEntry.fetchWork = fetchWorkForCache;
- cacheEntry.cachedResultsPath = cachedResultsPath;
- cacheEntry.size = resultSize;
- this.cacheSize += resultSize;
- cacheEntry.createTime = System.currentTimeMillis();
-
- cacheEntry.setStatus(CacheEntryStatus.VALID);
- // Mark this entry as being in use. Caller will need to release later.
- cacheEntry.addReader();
-
- scheduleEntryInvalidation(cacheEntry);
-
- // Notify any queries waiting on this cacheEntry to become valid.
+ // Synchronize on the cache entry so that no one else can invalidate
this entry
+ // while we are in the process of setting it to valid.
synchronized (cacheEntry) {
+if (cacheEntry.getStatus() == CacheEntryStatus.INVALID) {
+ // Entry either expired, or was invalidated due to table updates
+ return false;
+}
+
+if (requiresMove) {
+ // Move the query results to the query cache directory.
+ cachedResultsPath = moveResultsToCacheDirectory(queryResultsPath);
+ dataDirMoved = true;
+}
+LOG.info("Moved query results from {} to {} (size {}) for query '{}'",
+queryResultsPath, cachedResultsPath, resultSize, queryText);
+
+// Create a new FetchWork to reference the new cache location.
+FetchWork fetchWorkForCache =
+new FetchWork(cachedResultsPath, fetchWork.getTblDesc(),
fetchWork.getLimit());
+fetchWorkForCache.setCachedResult(true);
+cacheEntry.fetchWork = fetchWorkForCache;
+cacheEntry.cachedResultsPath = cachedResultsPath;
+cacheEntry.size = resultSize;
+this.cacheSize += resultSize;
+cacheEntry.createTime = System.currentTimeMillis();
+
+cacheEntry.setStatus(CacheEntryStatus.VALID);
+// Mark this entry as being in use. Caller will need to release later.
+cacheEntry.addReader();
+
+scheduleEntryInvalidation(cacheEntry);
+
+// Notify any queries waiting on this cacheEntry to become valid.
cacheEntry.notifyAll();
}
@@ -564,7 +571,11 @@ public final class QueryResultsCache {
try {
writeLock.lock();
LOG.info("Clearing the results cache");
- for (CacheEntry entry : lru.keySet().toArray(EMPTY_CACHEENTRY_ARRAY)) {
+ CacheEntry[] allEntries = null;
+ synchronized (lru) {
+allEntries = lru.keySet().toArray(EMPTY_CACHEENTRY_ARRAY);
+ }
+ for (CacheEntry entry : allEntries) {
try {
removeEntry(entry);
} catch (Exception err) {
@@ -611,10 +622,15 @@ public final class QueryResultsCache {
public void removeEntry(CacheEntry entry) {
entry.invalidate();
-removeFromLookup(entry);
-lru.remove(entry);
-// Should the cache si
hive git commit: HIVE-19127: Concurrency fixes in QueryResultsCache (Jason Dere, reviewed by Deepak Jaiswal)
Repository: hive
Updated Branches:
refs/heads/branch-3 1cd74b451 -> 3f56b44fb
HIVE-19127: Concurrency fixes in QueryResultsCache (Jason Dere, reviewed by
Deepak Jaiswal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3f56b44f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3f56b44f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3f56b44f
Branch: refs/heads/branch-3
Commit: 3f56b44fb9dcd427ae1a93e9739da84a33eb6aed
Parents: 1cd74b4
Author: Jason Dere
Authored: Mon Apr 9 16:48:23 2018 -0700
Committer: Jason Dere
Committed: Mon Apr 9 16:49:17 2018 -0700
--
.../ql/cache/results/QueryResultsCache.java | 112 +++
1 file changed, 68 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/3f56b44f/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
index ac5ae57..b1a3646 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/cache/results/QueryResultsCache.java
@@ -502,32 +502,39 @@ public final class QueryResultsCache {
return false;
}
- if (requiresMove) {
-// Move the query results to the query cache directory.
-cachedResultsPath = moveResultsToCacheDirectory(queryResultsPath);
-dataDirMoved = true;
- }
- LOG.info("Moved query results from {} to {} (size {}) for query '{}'",
- queryResultsPath, cachedResultsPath, resultSize, queryText);
-
- // Create a new FetchWork to reference the new cache location.
- FetchWork fetchWorkForCache =
- new FetchWork(cachedResultsPath, fetchWork.getTblDesc(),
fetchWork.getLimit());
- fetchWorkForCache.setCachedResult(true);
- cacheEntry.fetchWork = fetchWorkForCache;
- cacheEntry.cachedResultsPath = cachedResultsPath;
- cacheEntry.size = resultSize;
- this.cacheSize += resultSize;
- cacheEntry.createTime = System.currentTimeMillis();
-
- cacheEntry.setStatus(CacheEntryStatus.VALID);
- // Mark this entry as being in use. Caller will need to release later.
- cacheEntry.addReader();
-
- scheduleEntryInvalidation(cacheEntry);
-
- // Notify any queries waiting on this cacheEntry to become valid.
+ // Synchronize on the cache entry so that no one else can invalidate
this entry
+ // while we are in the process of setting it to valid.
synchronized (cacheEntry) {
+if (cacheEntry.getStatus() == CacheEntryStatus.INVALID) {
+ // Entry either expired, or was invalidated due to table updates
+ return false;
+}
+
+if (requiresMove) {
+ // Move the query results to the query cache directory.
+ cachedResultsPath = moveResultsToCacheDirectory(queryResultsPath);
+ dataDirMoved = true;
+}
+LOG.info("Moved query results from {} to {} (size {}) for query '{}'",
+queryResultsPath, cachedResultsPath, resultSize, queryText);
+
+// Create a new FetchWork to reference the new cache location.
+FetchWork fetchWorkForCache =
+new FetchWork(cachedResultsPath, fetchWork.getTblDesc(),
fetchWork.getLimit());
+fetchWorkForCache.setCachedResult(true);
+cacheEntry.fetchWork = fetchWorkForCache;
+cacheEntry.cachedResultsPath = cachedResultsPath;
+cacheEntry.size = resultSize;
+this.cacheSize += resultSize;
+cacheEntry.createTime = System.currentTimeMillis();
+
+cacheEntry.setStatus(CacheEntryStatus.VALID);
+// Mark this entry as being in use. Caller will need to release later.
+cacheEntry.addReader();
+
+scheduleEntryInvalidation(cacheEntry);
+
+// Notify any queries waiting on this cacheEntry to become valid.
cacheEntry.notifyAll();
}
@@ -564,7 +571,11 @@ public final class QueryResultsCache {
try {
writeLock.lock();
LOG.info("Clearing the results cache");
- for (CacheEntry entry : lru.keySet().toArray(EMPTY_CACHEENTRY_ARRAY)) {
+ CacheEntry[] allEntries = null;
+ synchronized (lru) {
+allEntries = lru.keySet().toArray(EMPTY_CACHEENTRY_ARRAY);
+ }
+ for (CacheEntry entry : allEntries) {
try {
removeEntry(entry);
} catch (Exception err) {
@@ -611,10 +622,15 @@ public final class QueryResultsCache {
public void removeEntry(CacheEntry entry) {
entry.invalidate();
-removeFromLookup(entry);
-lru.remove(entry);
-// Should the cach
hive git commit: HIVE-19143 : Update golden files for negative tests
Repository: hive Updated Branches: refs/heads/master a1034102d -> 55fb0a196 HIVE-19143 : Update golden files for negative tests Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/55fb0a19 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/55fb0a19 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/55fb0a19 Branch: refs/heads/master Commit: 55fb0a196d870ff4452137bf2d34c7e415b77b76 Parents: a103410 Author: Ashutosh Chauhan Authored: Mon Apr 9 17:50:32 2018 -0700 Committer: Ashutosh Chauhan Committed: Mon Apr 9 17:52:22 2018 -0700 -- .../results/clientnegative/authorization_caseinsensitivity.q.out | 2 +- ql/src/test/results/clientnegative/authorization_fail_1.q.out | 2 +- .../results/clientnegative/authorization_grant_table_dup.q.out | 2 +- ql/src/test/results/clientnegative/authorization_role_case.q.out | 2 +- .../clientnegative/authorization_role_grant_nosuchrole.q.out | 2 +- .../clientnegative/authorization_table_grant_nosuchrole.q.out | 2 +- ql/src/test/results/clientnegative/subquery_subquery_chain.q.out | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/55fb0a19/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out -- diff --git a/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out b/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out index 1dbc3e2..8bc747e 100644 --- a/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out +++ b/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out @@ -55,4 +55,4 @@ public testrole PREHOOK: query: create role TESTRoLE PREHOOK: type: CREATEROLE -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Role testrole already exists. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Error create role: Got exception: org.apache.hadoop.hive.metastore.api.InvalidObjectException Role testrole already exists. http://git-wip-us.apache.org/repos/asf/hive/blob/55fb0a19/ql/src/test/results/clientnegative/authorization_fail_1.q.out -- diff --git a/ql/src/test/results/clientnegative/authorization_fail_1.q.out b/ql/src/test/results/clientnegative/authorization_fail_1.q.out index 5c78f2a..fc52cb3 100644 --- a/ql/src/test/results/clientnegative/authorization_fail_1.q.out +++ b/ql/src/test/results/clientnegative/authorization_fail_1.q.out @@ -15,4 +15,4 @@ POSTHOOK: Output: default@authorization_fail_1 PREHOOK: query: grant Create on table authorization_fail_1 to user hive_test_user PREHOOK: type: GRANT_PRIVILEGE PREHOOK: Output: default@authorization_fail_1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: InvalidObjectException(message:CREATE is already granted on table [default,authorization_fail_1] by hive_test_user) +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:Got exception: org.apache.hadoop.hive.metastore.api.InvalidObjectException CREATE is already granted on table [default,authorization_fail_1] by hive_test_user) http://git-wip-us.apache.org/repos/asf/hive/blob/55fb0a19/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out -- diff --git a/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out b/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out index 9449474..795dc83 100644 --- a/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out +++ b/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out @@ -22,4 +22,4 @@ default tauth_gdup user1 USER UPDATE true-1 user1 PREHOOK: query: GRANT INSERT ON tauth_gdup TO USER user1 PREHOOK: type: GRANT_PRIVILEGE PREHOOK: Output: default@tauth_gdup -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Error granting privileges: InvalidObjectException(message:INSERT is already granted on table [default,tauth_gdup] by user1) +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Error granting privileges: Got exception: org.apache.hadoop.hive.metastore.api.InvalidObjectException INSERT is already granted on table [default,tauth_gdup] by user1 http://git-wip-us.apache.org/repos/asf/hive/blob/55fb0a19/ql/src/test/results/clientnegative/authoriz
hive git commit: HIVE-19143 : Update golden files for negative tests
Repository: hive Updated Branches: refs/heads/branch-3 3f56b44fb -> 946f619e7 HIVE-19143 : Update golden files for negative tests Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/946f619e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/946f619e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/946f619e Branch: refs/heads/branch-3 Commit: 946f619e78fbe9e4ab32d77eacc672135157253c Parents: 3f56b44 Author: Ashutosh Chauhan Authored: Mon Apr 9 17:50:32 2018 -0700 Committer: Ashutosh Chauhan Committed: Mon Apr 9 17:50:32 2018 -0700 -- .../results/clientnegative/authorization_caseinsensitivity.q.out | 2 +- ql/src/test/results/clientnegative/authorization_fail_1.q.out | 2 +- .../results/clientnegative/authorization_grant_table_dup.q.out | 2 +- ql/src/test/results/clientnegative/authorization_role_case.q.out | 2 +- .../clientnegative/authorization_role_grant_nosuchrole.q.out | 2 +- .../clientnegative/authorization_table_grant_nosuchrole.q.out | 2 +- ql/src/test/results/clientnegative/subquery_subquery_chain.q.out | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/946f619e/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out -- diff --git a/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out b/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out index 1dbc3e2..8bc747e 100644 --- a/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out +++ b/ql/src/test/results/clientnegative/authorization_caseinsensitivity.q.out @@ -55,4 +55,4 @@ public testrole PREHOOK: query: create role TESTRoLE PREHOOK: type: CREATEROLE -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Role testrole already exists. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Error create role: Got exception: org.apache.hadoop.hive.metastore.api.InvalidObjectException Role testrole already exists. http://git-wip-us.apache.org/repos/asf/hive/blob/946f619e/ql/src/test/results/clientnegative/authorization_fail_1.q.out -- diff --git a/ql/src/test/results/clientnegative/authorization_fail_1.q.out b/ql/src/test/results/clientnegative/authorization_fail_1.q.out index 5c78f2a..fc52cb3 100644 --- a/ql/src/test/results/clientnegative/authorization_fail_1.q.out +++ b/ql/src/test/results/clientnegative/authorization_fail_1.q.out @@ -15,4 +15,4 @@ POSTHOOK: Output: default@authorization_fail_1 PREHOOK: query: grant Create on table authorization_fail_1 to user hive_test_user PREHOOK: type: GRANT_PRIVILEGE PREHOOK: Output: default@authorization_fail_1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: InvalidObjectException(message:CREATE is already granted on table [default,authorization_fail_1] by hive_test_user) +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: MetaException(message:Got exception: org.apache.hadoop.hive.metastore.api.InvalidObjectException CREATE is already granted on table [default,authorization_fail_1] by hive_test_user) http://git-wip-us.apache.org/repos/asf/hive/blob/946f619e/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out -- diff --git a/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out b/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out index 9449474..795dc83 100644 --- a/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out +++ b/ql/src/test/results/clientnegative/authorization_grant_table_dup.q.out @@ -22,4 +22,4 @@ default tauth_gdup user1 USER UPDATE true-1 user1 PREHOOK: query: GRANT INSERT ON tauth_gdup TO USER user1 PREHOOK: type: GRANT_PRIVILEGE PREHOOK: Output: default@tauth_gdup -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Error granting privileges: InvalidObjectException(message:INSERT is already granted on table [default,tauth_gdup] by user1) +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Error granting privileges: Got exception: org.apache.hadoop.hive.metastore.api.InvalidObjectException INSERT is already granted on table [default,tauth_gdup] by user1 http://git-wip-us.apache.org/repos/asf/hive/blob/946f619e/ql/src/test/results/clientnegative/auth
hive git commit: HIVE-18857: Store default value text instead of default value expression in metastore(Vineet Garg, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 55fb0a196 -> 2e92451a6
HIVE-18857: Store default value text instead of default value expression in
metastore(Vineet Garg, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2e92451a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2e92451a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2e92451a
Branch: refs/heads/master
Commit: 2e92451a61557248e47453e7917d81c428379bfa
Parents: 55fb0a1
Author: Vineet Garg
Authored: Mon Apr 9 19:31:14 2018 -0700
Committer: Vineet Garg
Committed: Mon Apr 9 19:31:14 2018 -0700
--
.../hadoop/hive/ql/parse/BaseSemanticAnalyzer.java | 15 +--
.../hadoop/hive/ql/parse/DDLSemanticAnalyzer.java| 3 ++-
2 files changed, 11 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/2e92451a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index 5301b2a..d940cdd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -765,10 +765,11 @@ public abstract class BaseSemanticAnalyzer {
}
protected static void processDefaultConstraints(String catName, String
databaseName, String tableName,
- ASTNode child, List columnNames, List
defaultConstraints, final ASTNode typeChild)
+ ASTNode child, List columnNames, List
defaultConstraints, final ASTNode typeChild,
+ final TokenRewriteStream
tokenRewriteStream)
throws SemanticException {
List defaultInfos = new ArrayList();
-generateConstraintInfos(child, columnNames, defaultInfos, typeChild, null);
+generateConstraintInfos(child, columnNames, defaultInfos, typeChild,
tokenRewriteStream);
constraintInfosToDefaultConstraints(catName, databaseName, tableName,
defaultInfos, defaultConstraints);
}
@@ -930,7 +931,8 @@ public abstract class BaseSemanticAnalyzer {
* @return retrieve the default value and return it as string
* @throws SemanticException
*/
- private static String getDefaultValue(ASTNode defaultValueAST, ASTNode
typeChild) throws SemanticException{
+ private static String getDefaultValue(ASTNode defaultValueAST, ASTNode
typeChild,
+final TokenRewriteStream tokenStream)
throws SemanticException{
// first create expression from defaultValueAST
TypeCheckCtx typeCheckCtx = new TypeCheckCtx(null);
ExprNodeDesc defaultValExpr = TypeCheckProcFactory
@@ -942,7 +944,8 @@ public abstract class BaseSemanticAnalyzer {
}
//get default value to be be stored in metastore
-String defaultValueText = defaultValExpr.getExprString();
+String defaultValueText =
tokenStream.toOriginalString(defaultValueAST.getTokenStartIndex(),
+
defaultValueAST.getTokenStopIndex());
final int DEFAULT_MAX_LEN = 255;
if(defaultValueText.length() > DEFAULT_MAX_LEN) {
throw new SemanticException(
@@ -1026,7 +1029,7 @@ public abstract class BaseSemanticAnalyzer {
rely = false;
} else if( child.getToken().getType() == HiveParser.TOK_DEFAULT_VALUE){
// try to get default value only if this is DEFAULT constraint
-checkOrDefaultValue = getDefaultValue(grandChild, typeChildForDefault);
+checkOrDefaultValue = getDefaultValue(grandChild, typeChildForDefault,
tokenRewriteStream);
}
else if(child.getToken().getType() == HiveParser.TOK_CHECK_CONSTRAINT) {
checkOrDefaultValue = getCheckExpression(grandChild,
tokenRewriteStream);
@@ -1259,7 +1262,7 @@ public abstract class BaseSemanticAnalyzer {
break;
case HiveParser.TOK_DEFAULT_VALUE:
processDefaultConstraints(catName, qualifiedTabName[0],
qualifiedTabName[1], constraintChild,
-ImmutableList.of(col.getName()), defaultConstraints,
typeChild);
+ImmutableList.of(col.getName()), defaultConstraints,
typeChild, tokenRewriteStream);
break;
case HiveParser.TOK_NOT_NULL:
processNotNullConstraints(catName, qualifiedTabName[0],
qualifiedTabName[1], constraintChild,
http://git-wip-us.apache.org/repos/asf/hive/blob/2e92451a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
--
diff --git
a/ql/src/ja
hive git commit: HIVE-18857: Store default value text instead of default value expression in metastore(Vineet Garg, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 946f619e7 -> d1a935816
HIVE-18857: Store default value text instead of default value expression in
metastore(Vineet Garg, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d1a93581
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d1a93581
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d1a93581
Branch: refs/heads/branch-3
Commit: d1a9358162f32ef489541cc72aef059e13349497
Parents: 946f619
Author: Vineet Garg
Authored: Mon Apr 9 19:32:07 2018 -0700
Committer: Vineet Garg
Committed: Mon Apr 9 19:32:07 2018 -0700
--
.../hadoop/hive/ql/parse/BaseSemanticAnalyzer.java | 15 +--
.../hadoop/hive/ql/parse/DDLSemanticAnalyzer.java| 3 ++-
2 files changed, 11 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/d1a93581/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index 5301b2a..d940cdd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -765,10 +765,11 @@ public abstract class BaseSemanticAnalyzer {
}
protected static void processDefaultConstraints(String catName, String
databaseName, String tableName,
- ASTNode child, List columnNames, List
defaultConstraints, final ASTNode typeChild)
+ ASTNode child, List columnNames, List
defaultConstraints, final ASTNode typeChild,
+ final TokenRewriteStream
tokenRewriteStream)
throws SemanticException {
List defaultInfos = new ArrayList();
-generateConstraintInfos(child, columnNames, defaultInfos, typeChild, null);
+generateConstraintInfos(child, columnNames, defaultInfos, typeChild,
tokenRewriteStream);
constraintInfosToDefaultConstraints(catName, databaseName, tableName,
defaultInfos, defaultConstraints);
}
@@ -930,7 +931,8 @@ public abstract class BaseSemanticAnalyzer {
* @return retrieve the default value and return it as string
* @throws SemanticException
*/
- private static String getDefaultValue(ASTNode defaultValueAST, ASTNode
typeChild) throws SemanticException{
+ private static String getDefaultValue(ASTNode defaultValueAST, ASTNode
typeChild,
+final TokenRewriteStream tokenStream)
throws SemanticException{
// first create expression from defaultValueAST
TypeCheckCtx typeCheckCtx = new TypeCheckCtx(null);
ExprNodeDesc defaultValExpr = TypeCheckProcFactory
@@ -942,7 +944,8 @@ public abstract class BaseSemanticAnalyzer {
}
//get default value to be be stored in metastore
-String defaultValueText = defaultValExpr.getExprString();
+String defaultValueText =
tokenStream.toOriginalString(defaultValueAST.getTokenStartIndex(),
+
defaultValueAST.getTokenStopIndex());
final int DEFAULT_MAX_LEN = 255;
if(defaultValueText.length() > DEFAULT_MAX_LEN) {
throw new SemanticException(
@@ -1026,7 +1029,7 @@ public abstract class BaseSemanticAnalyzer {
rely = false;
} else if( child.getToken().getType() == HiveParser.TOK_DEFAULT_VALUE){
// try to get default value only if this is DEFAULT constraint
-checkOrDefaultValue = getDefaultValue(grandChild, typeChildForDefault);
+checkOrDefaultValue = getDefaultValue(grandChild, typeChildForDefault,
tokenRewriteStream);
}
else if(child.getToken().getType() == HiveParser.TOK_CHECK_CONSTRAINT) {
checkOrDefaultValue = getCheckExpression(grandChild,
tokenRewriteStream);
@@ -1259,7 +1262,7 @@ public abstract class BaseSemanticAnalyzer {
break;
case HiveParser.TOK_DEFAULT_VALUE:
processDefaultConstraints(catName, qualifiedTabName[0],
qualifiedTabName[1], constraintChild,
-ImmutableList.of(col.getName()), defaultConstraints,
typeChild);
+ImmutableList.of(col.getName()), defaultConstraints,
typeChild, tokenRewriteStream);
break;
case HiveParser.TOK_NOT_NULL:
processNotNullConstraints(catName, qualifiedTabName[0],
qualifiedTabName[1], constraintChild,
http://git-wip-us.apache.org/repos/asf/hive/blob/d1a93581/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
--
diff --git
a/ql/sr
hive git commit: HIVE-19144 : TestSparkCliDriver:subquery_scalar - golden file needs to be udpated (Vineet Garg via Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/master 2e92451a6 -> 65abf418a HIVE-19144 : TestSparkCliDriver:subquery_scalar - golden file needs to be udpated (Vineet Garg via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/65abf418 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/65abf418 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/65abf418 Branch: refs/heads/master Commit: 65abf418ab6504b640dc60c1454e7642157e5667 Parents: 2e92451 Author: Vineet Garg Authored: Mon Apr 9 21:26:42 2018 -0700 Committer: Ashutosh Chauhan Committed: Mon Apr 9 21:26:42 2018 -0700 -- .../clientpositive/spark/subquery_scalar.q.out | 222 +-- 1 file changed, 107 insertions(+), 115 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/65abf418/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out -- diff --git a/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out b/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out index 7488f2e..d044da9 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_scalar.q.out @@ -119,14 +119,14 @@ STAGE PLANS: outputColumnNames: p_size Statistics: Num rows: 1 Data size: 32560 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: avg(p_size) + aggregations: sum(p_size), count(p_size) mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: -Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: struct) +Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: bigint), _col1 (type: bigint) Reducer 2 Reduce Operator Tree: Join Operator @@ -136,17 +136,17 @@ STAGE PLANS: 0 1 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 -Statistics: Num rows: 26 Data size: 5149 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 26 Data size: 3589 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (UDFToDouble(_col5) > _col9) (type: boolean) - Statistics: Num rows: 8 Data size: 1584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1104 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 -Statistics: Num rows: 8 Data size: 1584 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 8 Data size: 1104 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 8 Data size: 1584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1104 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -154,14 +154,18 @@ STAGE PLANS: Reducer 4 Reduce Operator Tree: Group By Operator -aggregations: avg(VALUE._col0) +aggregations: sum(VALUE._col0), count(VALUE._col1) mode: mergepartial -outputColumnNames: _col0 -Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: NONE -Reduce Output Operator - sort order: - Statistics: N
hive git commit: HIVE-19145 : Stabilize statsoptimizer.q test
Repository: hive Updated Branches: refs/heads/master 65abf418a -> 91a0cb8fa HIVE-19145 : Stabilize statsoptimizer.q test Signed-off-by: Ashutosh Chauhan Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/91a0cb8f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/91a0cb8f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/91a0cb8f Branch: refs/heads/master Commit: 91a0cb8fa34d8fcc2f1086209e6f6986e54eb95e Parents: 65abf41 Author: Ashutosh Chauhan Authored: Mon Apr 9 21:28:08 2018 -0700 Committer: Ashutosh Chauhan Committed: Mon Apr 9 21:28:08 2018 -0700 -- .../test/queries/clientpositive/statsoptimizer.q | 8 .../results/clientpositive/statsoptimizer.q.out | 18 +- 2 files changed, 13 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/91a0cb8f/ql/src/test/queries/clientpositive/statsoptimizer.q -- diff --git a/ql/src/test/queries/clientpositive/statsoptimizer.q b/ql/src/test/queries/clientpositive/statsoptimizer.q index 428b741..a3dbe4e 100644 --- a/ql/src/test/queries/clientpositive/statsoptimizer.q +++ b/ql/src/test/queries/clientpositive/statsoptimizer.q @@ -2,13 +2,13 @@ set hive.cbo.enable=false; set hive.compute.query.using.stats=true; EXPLAIN -SELECT to_date(current_date()) as GROUP_BY_FIELD, count (*) as src_cnt +SELECT round(year(to_date(current_date())),-3) as GROUP_BY_FIELD, count (*) as src_cnt from src WHERE 1=1 -group by to_date(current_date()); +group by round(year(to_date(current_date())),-3); -SELECT to_date(current_date()) as GROUP_BY_FIELD, count (*) as src_cnt +SELECT round(year(to_date(current_date())),-3) as GROUP_BY_FIELD, count (*) as src_cnt from src WHERE 1=1 -group by to_date(current_date()); +group by round(year(to_date(current_date())),-3); http://git-wip-us.apache.org/repos/asf/hive/blob/91a0cb8f/ql/src/test/results/clientpositive/statsoptimizer.q.out -- diff --git a/ql/src/test/results/clientpositive/statsoptimizer.q.out b/ql/src/test/results/clientpositive/statsoptimizer.q.out index 1f97de3..35330d9 100644 --- a/ql/src/test/results/clientpositive/statsoptimizer.q.out +++ b/ql/src/test/results/clientpositive/statsoptimizer.q.out @@ -1,14 +1,14 @@ PREHOOK: query: EXPLAIN -SELECT to_date(current_date()) as GROUP_BY_FIELD, count (*) as src_cnt +SELECT round(year(to_date(current_date())),-3) as GROUP_BY_FIELD, count (*) as src_cnt from src WHERE 1=1 -group by to_date(current_date()) +group by round(year(to_date(current_date())),-3) PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT to_date(current_date()) as GROUP_BY_FIELD, count (*) as src_cnt +SELECT round(year(to_date(current_date())),-3) as GROUP_BY_FIELD, count (*) as src_cnt from src WHERE 1=1 -group by to_date(current_date()) +group by round(year(to_date(current_date())),-3) POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage @@ -20,18 +20,18 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: SELECT to_date(current_date()) as GROUP_BY_FIELD, count (*) as src_cnt +PREHOOK: query: SELECT round(year(to_date(current_date())),-3) as GROUP_BY_FIELD, count (*) as src_cnt from src WHERE 1=1 -group by to_date(current_date()) +group by round(year(to_date(current_date())),-3) PREHOOK: type: QUERY PREHOOK: Input: default@src A masked pattern was here -POSTHOOK: query: SELECT to_date(current_date()) as GROUP_BY_FIELD, count (*) as src_cnt +POSTHOOK: query: SELECT round(year(to_date(current_date())),-3) as GROUP_BY_FIELD, count (*) as src_cnt from src WHERE 1=1 -group by to_date(current_date()) +group by round(year(to_date(current_date())),-3) POSTHOOK: type: QUERY POSTHOOK: Input: default@src A masked pattern was here -2018-04-04 500 +2000 500
hive git commit: HIVE-19146 : Delete dangling q.out
Repository: hive
Updated Branches:
refs/heads/master 91a0cb8fa -> dcd9b5941
HIVE-19146 : Delete dangling q.out
Signed-off-by: Ashutosh Chauhan
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dcd9b594
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dcd9b594
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dcd9b594
Branch: refs/heads/master
Commit: dcd9b5941c53d5219eb35f9f682ba2da69288291
Parents: 91a0cb8
Author: Ashutosh Chauhan
Authored: Mon Apr 9 21:30:34 2018 -0700
Committer: Ashutosh Chauhan
Committed: Mon Apr 9 21:30:34 2018 -0700
--
.../materialized_view_create.q.out | 288 ---
1 file changed, 288 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/dcd9b594/ql/src/test/results/clientpositive/materialized_view_create.q.out
--
diff --git a/ql/src/test/results/clientpositive/materialized_view_create.q.out
b/ql/src/test/results/clientpositive/materialized_view_create.q.out
deleted file mode 100644
index 8952304..000
--- a/ql/src/test/results/clientpositive/materialized_view_create.q.out
+++ /dev/null
@@ -1,288 +0,0 @@
-PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c
decimal(10,2))
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@cmv_basetable
-POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c
decimal(10,2))
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@cmv_basetable
-PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2,
'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@cmv_basetable
-POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2,
'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@cmv_basetable
-POSTHOOK: Lineage: cmv_basetable.a SCRIPT []
-POSTHOOK: Lineage: cmv_basetable.b SCRIPT []
-POSTHOOK: Lineage: cmv_basetable.c SCRIPT []
-PREHOOK: query: create materialized view cmv_mat_view as select a, b, c from
cmv_basetable
-PREHOOK: type: CREATE_MATERIALIZED_VIEW
-PREHOOK: Input: default@cmv_basetable
-PREHOOK: Output: database:default
-PREHOOK: Output: default@cmv_mat_view
-POSTHOOK: query: create materialized view cmv_mat_view as select a, b, c from
cmv_basetable
-POSTHOOK: type: CREATE_MATERIALIZED_VIEW
-POSTHOOK: Input: default@cmv_basetable
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@cmv_mat_view
-PREHOOK: query: desc formatted cmv_mat_view
-PREHOOK: type: DESCTABLE
-PREHOOK: Input: default@cmv_mat_view
-POSTHOOK: query: desc formatted cmv_mat_view
-POSTHOOK: type: DESCTABLE
-POSTHOOK: Input: default@cmv_mat_view
-# col_name data_type comment
-a int
-b varchar(256)
-c decimal(10,2)
-
-# Detailed Table Information
-Database: default
- A masked pattern was here
-Retention: 0
- A masked pattern was here
-Table Type:MATERIALIZED_VIEW
-Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
- numFiles1
- numRows 5
- rawDataSize 1025
- totalSize 497
- A masked pattern was here
-
-# Storage Information
-SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-Compressed:No
-Num Buckets: -1
-Bucket Columns:[]
-Sort Columns: []
-
-# View Information
-View Original Text:select a, b, c from cmv_basetable
-View Expanded Text:select `cmv_basetable`.`a`, `cmv_basetable`.`b`,
`cmv_basetable`.`c` from `default`.`cmv_basetable`
-View Rewrite Enabled: No
-PREHOOK: query: select * from cmv_mat_view
-PREHOOK: type: QUERY
-PREHOOK: Input: default@cmv_mat_view
- A masked pattern was here
-POSTHOOK: query: sel
hive git commit: HIVE-18783: ALTER TABLE post-commit listener does not include the transactional listener responses (Sergio Pena, reviewed by Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/master a263f0831 -> b290468c0
HIVE-18783: ALTER TABLE post-commit listener does not include the transactional
listener responses (Sergio Pena, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b290468c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b290468c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b290468c
Branch: refs/heads/master
Commit: b290468c0ffe53daa76fad1e2a063d4596ea2ece
Parents: a263f08
Author: Sergio Pena
Authored: Mon Apr 9 10:04:43 2018 -0500
Committer: Sergio Pena
Committed: Mon Apr 9 10:04:43 2018 -0500
--
.../listener/TestDbNotificationListener.java| 1 +
.../hadoop/hive/metastore/HiveAlterHandler.java | 59 ++--
.../hadoop/hive/metastore/HiveMetaStore.java| 30 ++
.../hadoop/hive/metastore/IHMSHandler.java | 6 ++
4 files changed, 65 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/b290468c/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
--
diff --git
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 823312b..70c6a94 100644
---
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -472,6 +472,7 @@ public class TestDbNotificationListener {
assertEquals(TableType.MANAGED_TABLE.toString(),
alterTableMessage.getTableType());
// Verify the eventID was passed to the non-transactional listener
+MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ALTER_TABLE,
firstEventId + 2);
MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE,
firstEventId + 1);
// When hive.metastore.transactional.event.listeners is set,
http://git-wip-us.apache.org/repos/asf/hive/blob/b290468c/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index ed1b8c5..60bed98 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -55,6 +55,7 @@ import
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -122,15 +123,24 @@ public class HiveAlterHandler implements AlterHandler {
boolean success = false;
boolean dataWasMoved = false;
-Table oldt;
+boolean isPartitionedTable = false;
+
+Table oldt = null;
+
List transactionalListeners = null;
+List listeners = null;
+Map txnAlterTableEventResponses = Collections.emptyMap();
+Map txnDropTableEventResponses = Collections.emptyMap();
+Map txnCreateTableEventResponses = Collections.emptyMap();
+Map txnAddPartitionEventResponses = Collections.emptyMap();
+
if (handler != null) {
transactionalListeners = handler.getTransactionalListeners();
+ listeners = handler.getListeners();
}
try {
boolean rename = false;
- boolean isPartitionedTable = false;
List parts;
// Switching tables between catalogs is not allowed.
@@ -337,23 +347,23 @@ public class HiveAlterHandler implements AlterHandler {
if (transactionalListeners != null && !transactionalListeners.isEmpty())
{
if (oldt.getDbName().equalsIgnoreCase(newt.getDbName())) {
- MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
+ txnAlterTableEventResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventMessage.EventType.ALTER_TABLE,
new AlterTableEvent(oldt, newt, false, true, handler),
environmentContext);
} else {
- MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
+ txnDropTableEventResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventMessage.EventType.DROP_TABLE,
new DropTableEvent(oldt
hive git commit: HIVE-18859 : Incorrect handling of thrift metastore exceptions (Ganesha Shreedhara via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master b290468c0 -> 959e77257
HIVE-18859 : Incorrect handling of thrift metastore exceptions (Ganesha
Shreedhara via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/959e7725
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/959e7725
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/959e7725
Branch: refs/heads/master
Commit: 959e77257a006b36769cffd9efb01dd16b139474
Parents: b290468
Author: Ganesha Shreedhara
Authored: Mon Mar 12 03:06:00 2018 -0700
Committer: Ashutosh Chauhan
Committed: Mon Apr 9 08:06:58 2018 -0700
--
.../AbstractTestAuthorizationApiAuthorizer.java | 19 +++--
.../hadoop/hive/metastore/HiveMetaStore.java| 30
2 files changed, 40 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/959e7725/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
index abd5e32..69692d0 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java
@@ -35,6 +35,7 @@ import
org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import
org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly;
import
org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener;
+import org.apache.thrift.TException;
import org.junit.Test;
/**
@@ -91,15 +92,27 @@ public abstract class
AbstractTestAuthorizationApiAuthorizer {
// authorization checks passed.
String exStackString = ExceptionUtils.getStackTrace(e);
assertTrue("Verifying this exception came after authorization check",
-
exStackString.contains("org.apache.hadoop.hive.metastore.ObjectStore"));
+
exStackString.contains("org.apache.hadoop.hive.metastore.ObjectStore"));
// If its not an exception caused by auth check, ignore it
}
assertFalse("Authz Exception should have been thrown in remote mode",
isRemoteMetastoreMode);
System.err.println("No auth exception thrown");
} catch (MetaException e) {
System.err.println("Caught exception");
- caughtEx = true;
-
assertTrue(e.getMessage().contains(MetaStoreAuthzAPIAuthorizerEmbedOnly.errMsg));
+ String exStackString = ExceptionUtils.getStackTrace(e);
+ // Check if MetaException has one of InvalidObjectException or
NoSuchObjectExcetion or any exception thrown from ObjectStore , which means
that the
+ // authorization checks passed.
+
if(exStackString.contains("org.apache.hadoop.hive.metastore.api.NoSuchObjectException")
||
+
exStackString.contains("org.apache.hadoop.hive.metastore.api.InvalidObjectException"))
{
+assertFalse("No Authz exception thrown in embedded mode",
isRemoteMetastoreMode);
+ } else {
+caughtEx = true;
+
assertTrue(e.getMessage().contains(MetaStoreAuthzAPIAuthorizerEmbedOnly.errMsg));
+ }
+} catch (TException e) {
+ String exStackString = ExceptionUtils.getStackTrace(e);
+ assertTrue("Verifying this exception came after authorization check",
+
exStackString.contains("org.apache.hadoop.hive.metastore.ObjectStore"));
}
if (!isRemoteMetastoreMode) {
assertFalse("No exception should be thrown in embedded mode", caughtEx);
http://git-wip-us.apache.org/repos/asf/hive/blob/959e7725/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 102e5b4..a2fe7d7 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -5981,8 +5981,11 @@ public class HiveMetaStore extends ThriftHiveMetastore {
ret = ms.grantRole(role, principalName, principalType, grantor,
grantorType, grantOption);
} catch (MetaException e) {
thro
hive git commit: HIVE-18783: ALTER TABLE post-commit listener does not include the transactional listener responses (Sergio Pena, reviewed by Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/branch-2 941f037c2 -> 7cea4d0da
HIVE-18783: ALTER TABLE post-commit listener does not include the transactional
listener responses (Sergio Pena, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7cea4d0d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7cea4d0d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7cea4d0d
Branch: refs/heads/branch-2
Commit: 7cea4d0da85b6f4a23fefb1ebb43c442b989c58d
Parents: 941f037
Author: Sergio Pena
Authored: Mon Apr 9 10:26:10 2018 -0500
Committer: Sergio Pena
Committed: Mon Apr 9 10:26:10 2018 -0500
--
.../hcatalog/listener/TestDbNotificationListener.java | 1 +
.../apache/hadoop/hive/metastore/HiveAlterHandler.java | 13 -
.../apache/hadoop/hive/metastore/HiveMetaStore.java| 10 --
3 files changed, 17 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/7cea4d0d/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
--
diff --git
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 5a40780..8e9eb3d 100644
---
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -447,6 +447,7 @@ public class TestDbNotificationListener {
assertEquals(TableType.MANAGED_TABLE.toString(),
alterTableMessage.getTableType());
// Verify the eventID was passed to the non-transactional listener
+MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ALTER_TABLE,
firstEventId + 2);
MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE,
firstEventId + 1);
// When hive.metastore.transactional.event.listeners is set,
http://git-wip-us.apache.org/repos/asf/hive/blob/7cea4d0d/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 3e7c59b..83c68a2 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -54,6 +54,7 @@ import org.apache.hive.common.util.HiveStringUtils;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -126,8 +127,12 @@ public class HiveAlterHandler implements AlterHandler {
boolean dataWasMoved = false;
Table oldt = null;
List transactionalListeners = null;
+List listeners = null;
+Map txnAlterTableEventResponses = Collections.emptyMap();
+
if (handler != null) {
transactionalListeners = handler.getTransactionalListeners();
+ listeners = handler.getListeners();
}
try {
@@ -309,7 +314,7 @@ public class HiveAlterHandler implements AlterHandler {
}
if (transactionalListeners != null && !transactionalListeners.isEmpty())
{
-MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
+txnAlterTableEventResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventMessage.EventType.ALTER_TABLE,
new AlterTableEvent(oldt, newt,
true, handler),
environmentContext);
@@ -349,6 +354,12 @@ public class HiveAlterHandler implements AlterHandler {
}
}
}
+
+ if (!listeners.isEmpty()) {
+MetaStoreListenerNotifier.notifyEvent(listeners,
EventMessage.EventType.ALTER_TABLE,
+new AlterTableEvent(oldt, newt, success, handler),
+environmentContext, txnAlterTableEventResponses, msdb);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7cea4d0d/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index d32e89c..aa233dd 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStor
hive git commit: HIVE-18783: ALTER TABLE post-commit listener does not include the transactional listener responses (Sergio Pena, reviewed by Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/branch-2.3 d3fa8e0c5 -> f78128a71
HIVE-18783: ALTER TABLE post-commit listener does not include the transactional
listener responses (Sergio Pena, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f78128a7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f78128a7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f78128a7
Branch: refs/heads/branch-2.3
Commit: f78128a719772c1157ce6f0b9bce52a0cac0893c
Parents: d3fa8e0
Author: Sergio Pena
Authored: Mon Apr 9 10:26:10 2018 -0500
Committer: Sergio Pena
Committed: Mon Apr 9 11:05:50 2018 -0500
--
.../hcatalog/listener/TestDbNotificationListener.java | 1 +
.../hadoop/hive/metastore/HiveAlterHandler.java | 14 +-
.../apache/hadoop/hive/metastore/HiveMetaStore.java | 10 --
3 files changed, 18 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/f78128a7/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
--
diff --git
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index 976c3c5..786c4c2 100644
---
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -444,6 +444,7 @@ public class TestDbNotificationListener {
assertEquals(table, alterTableMessage.getTableObjAfter());
// Verify the eventID was passed to the non-transactional listener
+MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ALTER_TABLE,
firstEventId + 2);
MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE,
firstEventId + 1);
// When hive.metastore.transactional.event.listeners is set,
http://git-wip-us.apache.org/repos/asf/hive/blob/f78128a7/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 15f2597..7730a24 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -54,8 +54,10 @@ import org.apache.hive.common.util.HiveStringUtils;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
/**
* Hive specific implementation of alter
@@ -115,8 +117,12 @@ public class HiveAlterHandler implements AlterHandler {
boolean rename = false;
Table oldt = null;
List transactionalListeners = null;
+List listeners = null;
+Map txnAlterTableEventResponses = Collections.emptyMap();
+
if (handler != null) {
transactionalListeners = handler.getTransactionalListeners();
+ listeners = handler.getListeners();
}
try {
@@ -270,7 +276,7 @@ public class HiveAlterHandler implements AlterHandler {
alterTableUpdateTableColumnStats(msdb, oldt, newt);
if (transactionalListeners != null && !transactionalListeners.isEmpty())
{
-MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
+txnAlterTableEventResponses =
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventMessage.EventType.ALTER_TABLE,
new AlterTableEvent(oldt, newt,
true, handler),
environmentContext);
@@ -305,6 +311,12 @@ public class HiveAlterHandler implements AlterHandler {
}
}
}
+
+ if (!listeners.isEmpty()) {
+MetaStoreListenerNotifier.notifyEvent(listeners,
EventMessage.EventType.ALTER_TABLE,
+new AlterTableEvent(oldt, newt, success, handler),
+environmentContext, txnAlterTableEventResponses, msdb);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f78128a7/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index befd15e..d799acf 100644
---
hive git commit: HIVE-19129: Support DEFAULT keyword with MERGE(Vineet Garg, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 959e77257 -> e7c15d234
HIVE-19129: Support DEFAULT keyword with MERGE(Vineet Garg, reviewed by
Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e7c15d23
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e7c15d23
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e7c15d23
Branch: refs/heads/master
Commit: e7c15d2348602b6890aff24a088c2b04fb46af8a
Parents: 959e772
Author: Vineet Garg
Authored: Mon Apr 9 10:51:46 2018 -0700
Committer: Vineet Garg
Committed: Mon Apr 9 10:51:46 2018 -0700
--
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 26 +-
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 1 +
.../insert_into_default_keyword.q | 49 ++
.../llap/insert_into_default_keyword.q.out | 787 +++
4 files changed, 861 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/e7c15d23/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index ff0a2e6..3b74aba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -665,13 +665,13 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
/**
* This method creates a list of default constraints which corresponds to
- * given schema (taretSchema) or target table's column schema (if
targetSchema is null)
+ * given schema (targetSchema) or target table's column schema (if
targetSchema is null)
* @param tbl
* @param targetSchema
* @return List of default constraints (including NULL if there is no
default)
* @throws SemanticException
*/
- private List getDefaultConstraints(Table tbl, List
targetSchema) throws SemanticException{
+ private static List getDefaultConstraints(Table tbl, List
targetSchema) throws SemanticException{
Map colNameToDefaultVal = null;
try {
DefaultConstraint dc =
Hive.get().getEnabledDefaultConstraints(tbl.getDbName(), tbl.getTableName());
@@ -718,6 +718,28 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer
{
return newNode;
}
+ public static String replaceDefaultKeywordForMerge(String valueClause,Table
targetTable)
+ throws SemanticException {
+List defaultConstraints = null;
+String[] values = valueClause.trim().split(",");
+StringBuilder newValueClause = new StringBuilder();
+for (int i = 0; i < values.length; i++) {
+ if (values[i].trim().toLowerCase().equals("`default`")) {
+if (defaultConstraints == null) {
+ defaultConstraints = getDefaultConstraints(targetTable, null);
+}
+newValueClause.append(defaultConstraints.get(i));
+ }
+ else {
+newValueClause.append(values[i]);
+ }
+ if(i != values.length-1) {
+newValueClause.append(",");
+ }
+}
+return newValueClause.toString();
+ }
+
/**
* This method replaces ASTNode corresponding to DEFAULT keyword with either
DEFAULT constraint
* expression if exists or NULL otherwise
http://git-wip-us.apache.org/repos/asf/hive/blob/e7c15d23/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
index a660747..0effd92 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
@@ -1101,6 +1101,7 @@ public class UpdateDeleteSemanticAnalyzer extends
SemanticAnalyzer {
List partCols = targetTable.getPartCols();
String valuesClause =
getMatchedText((ASTNode)getWhenClauseOperation(whenNotMatchedClause).getChild(0));
valuesClause = valuesClause.substring(1, valuesClause.length() -
1);//strip '(' and ')'
+valuesClause =
SemanticAnalyzer.replaceDefaultKeywordForMerge(valuesClause, targetTable);
rewrittenQueryStr.append("INSERT INTO
").append(getFullTableNameForSQL(target));
addPartitionColsToInsert(partCols, rewrittenQueryStr);
http://git-wip-us.apache.org/repos/asf/hive/blob/e7c15d23/ql/src/test/queries/clientpositive/insert_into_default_keyword.q
--
diff --git a/ql/src/test/queries/clientpositive/insert_into_default_keyword.q
b/ql/src/test/queries/c
[2/3] hive git commit: HIVE-19074: Vectorization: Add llap vectorization_div0.q.out Q output file (missing changes) (Matt McCline, reviewed by Teddy Choi)
http://git-wip-us.apache.org/repos/asf/hive/blob/a21302f1/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out b/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out new file mode 100644 index 000..e00dcd9 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out @@ -0,0 +1,870 @@ +PREHOOK: query: explain vectorization expression +select cint, cint / 0 as cint_div, ctinyint, ctinyint / 0 as ctinyint_div, cbigint, cbigint / 0 as cbigint_div, cdouble, cdouble / 0.0 as cdouble_div +from alltypesorc order by cint, ctinyint, cbigint, cdouble limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization expression +select cint, cint / 0 as cint_div, ctinyint, ctinyint / 0 as ctinyint_div, cbigint, cbigint / 0 as cbigint_div, cdouble, cdouble / 0.0 as cdouble_div +from alltypesorc order by cint, ctinyint, cbigint, cdouble limit 100 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Tez + A masked pattern was here + Edges: +Reducer 2 <- Map 1 (SIMPLE_EDGE) + A masked pattern was here + Vertices: +Map 1 +Map Operator Tree: +TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Select Operator +expressions: cint (type: int), (UDFToDouble(cint) / 0.0D) (type: double), ctinyint (type: tinyint), (UDFToDouble(ctinyint) / 0.0D) (type: double), cbigint (type: bigint), (UDFToDouble(cbigint) / 0.0D) (type: double), cdouble (type: double), (cdouble / 0.0D) (type: double) +outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 +Select Vectorization: +className: VectorSelectOperator +native: true +projectedOutputColumnNums: [2, 14, 0, 15, 3, 16, 5, 13] +selectExpressions: DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 2:int) -> 13:double) -> 14:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 0:tinyint) -> 13:double) -> 15:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 3:bigint) -> 13:double) -> 16:double, DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 13:double +Statistics: Num rows: 12288 Data size: 613400 Basic stats: COMPLETE Column stats: COMPLETE +Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: tinyint), _col4 (type: bigint), _col6 (type: double) + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 12288 Data size: 613400 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: double), _col3 (type: double), _col5 (type: double), _col7 (type: double) +Execution mode: vectorized, llap +LLAP IO: all inputs +Map Vectorization: +enabled: true +enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true +inputFormatFeatureSupport: [] +featureSupportInUse: [] +inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +allNative: true +usesVectorUDFAdaptor: false +vectorized: true +Reducer 2 +Execution mode: vectorized, llap +Reduce Vectorization: +enabled: true +enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true +allNative: false +usesVectorUDFAdaptor: false +vectorized: true +Reduce Operator Tree: + Select Operator +expressions: KEY.reducesinkkey0 (type: int), VALUE._
[3/3] hive git commit: HIVE-19074: Vectorization: Add llap vectorization_div0.q.out Q output file (missing changes) (Matt McCline, reviewed by Teddy Choi)
HIVE-19074: Vectorization: Add llap vectorization_div0.q.out Q output file (missing changes) (Matt McCline, reviewed by Teddy Choi) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a21302f1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a21302f1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a21302f1 Branch: refs/heads/master Commit: a21302f10777047c2a09c0b74de648b4e3b15f2d Parents: e7c15d2 Author: Matt McCline Authored: Mon Apr 9 13:22:42 2018 -0500 Committer: Matt McCline Committed: Mon Apr 9 13:36:02 2018 -0500 -- .../resources/testconfiguration.properties.orig | 1685 -- .../llap/vectorization_div0.q.out | 870 + .../clientpositive/tez/vectorization_div0.q.out | 870 - 3 files changed, 870 insertions(+), 2555 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/a21302f1/itests/src/test/resources/testconfiguration.properties.orig -- diff --git a/itests/src/test/resources/testconfiguration.properties.orig b/itests/src/test/resources/testconfiguration.properties.orig deleted file mode 100644 index a42ae80..000 --- a/itests/src/test/resources/testconfiguration.properties.orig +++ /dev/null @@ -1,1685 +0,0 @@ -# Note: the *.shared groups also run on TestCliDriver - -# NOTE: files should be listed in alphabetical order -minimr.query.files=infer_bucket_sort_map_operators.q,\ - infer_bucket_sort_dyn_part.q,\ - infer_bucket_sort_merge.q,\ - infer_bucket_sort_reducers_power_two.q,\ - infer_bucket_sort_num_buckets.q,\ - root_dir_external_table.q,\ - parallel_orderby.q,\ - bucket_num_reducers.q,\ - udf_using.q,\ - index_bitmap3.q,\ - index_bitmap_auto.q,\ - scriptfile1.q,\ - bucket_num_reducers2.q,\ - bucket_num_reducers_acid.q,\ - bucket_num_reducers_acid2.q,\ - scriptfile1_win.q - -# These tests are disabled for minimr -# ql_rewrite_gbtoidx.q,\ -# ql_rewrite_gbtoidx_cbo_1.q,\ -# ql_rewrite_gbtoidx_cbo_2.q,\ -# smb_mapjoin_8.q,\ - - -# Tests that are not enabled for CLI Driver -disabled.query.files=ql_rewrite_gbtoidx.q,\ - ql_rewrite_gbtoidx_cbo_1.q,\ - cbo_rp_subq_in.q,\ - cbo_rp_subq_not_in.q,\ - cbo_rp_subq_exists.q,\ - orc_llap.q,\ - ql_rewrite_gbtoidx_cbo_2.q,\ - rcfile_merge1.q,\ - stats_filemetadata.q,\ - cbo_rp_insert.q,\ - cbo_rp_lineage2.q - -# NOTE: Add tests to minitez only if it is very -# specific to tez and cannot be added to minillap. -minitez.query.files.shared=delete_orig_table.q,\ - orc_merge12.q,\ - orc_vectorization_ppd.q,\ - update_orig_table.q,\ - vector_join_part_col_char.q,\ - vector_non_string_partition.q,\ - vectorization_div0.q,\ - vectorization_limit.q - -# NOTE: Add tests to minitez only if it is very -# specific to tez and cannot be added to minillap. -minitez.query.files=acid_vectorization_original_tez.q,\ - explainuser_3.q,\ - explainanalyze_1.q,\ - explainanalyze_3.q,\ - explainanalyze_4.q,\ - explainanalyze_5.q,\ - hybridgrace_hashjoin_1.q,\ - hybridgrace_hashjoin_2.q,\ - multi_count_distinct.q,\ - tez-tag.q,\ - tez_union_with_udf.q,\ - vectorization_div0.q - - -minillap.shared.query.files=insert_into1.q,\ - insert_into2.q,\ - llapdecider.q,\ - mapreduce1.q,\ - mapreduce2.q,\ - mm_all.q,\ - mm_cttas.q,\ - orc_merge1.q,\ - orc_merge10.q,\ - orc_merge2.q,\ - orc_merge3.q,\ - orc_merge4.q,\ - orc_merge_diff_fs.q,\ - parallel_colstats.q,\ - parquet_types_vectorization.q,\ - parquet_complex_types_vectorization.q,\ - parquet_map_type_vectorization.q,\ - parquet_struct_type_vectorization.q,\ - orc_struct_type_vectorization.q,\ - union_type_chk.q,\ - cte_2.q,\ - cte_4.q,\ - llap_nullscan.q,\ - dynamic_partition_pruning_2.q,\ - tez_union_dynamic_partition.q - -minillaplocal.shared.query.files=alter_merge_2_orc.q,\ - alter_merge_orc.q,\ - alter_merge_stats_orc.q,\ - authorization_view_8.q,\ - auto_join0.q,\ - auto_join1.q,\ - auto_join21.q,\ - auto_join29.q,\ - auto_join30.q,\ - auto_join_filters.q,\ - auto_join_nulls.q,\ - auto_sortmerge_join_1.q,\ - auto_sortmerge_join_10.q,\ - auto_sortmerge_join_11.q,\ - auto_sortmerge_join_12.q,\ - auto_sortmerge_join_13.q,\ - auto_sortmerge_join_14.q,\ - auto_sortmerge_join_15.q,\ - auto_sortmerge_join_16.q,\ - auto_sortmerge_join_2.q,\ - auto_sortmerge_join_3.q,\ - auto_sortmerge_join_4.q,\ - auto_sortmerge_join_5.q,\ - auto_sortmerge_join_6.q,\ - auto_sortmerge_join_7.q,\ - auto_sortmerge_join_8.q,\ - auto_sortmerge_join_9.q,\ - autoColumnStats_1.q,\ - autoColumnStats_10.q,\ - autoColumnStats_2.q,\ - bucket2.q,\ - bucket3.q,\ - bucket4.q,\ - bucket_map_join_tez1.q,\ - bucket_map_join_tez2.q,\ - cbo_gby.q,\ - cbo_gby_empty.q,\ - cbo_join.q,\ - cbo_limit.q,\ - cbo_semijoin.q,\ - cbo_s
[1/3] hive git commit: HIVE-19074: Vectorization: Add llap vectorization_div0.q.out Q output file (missing changes) (Matt McCline, reviewed by Teddy Choi)
Repository: hive Updated Branches: refs/heads/master e7c15d234 -> a21302f10 http://git-wip-us.apache.org/repos/asf/hive/blob/a21302f1/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out -- diff --git a/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out b/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out deleted file mode 100644 index e00dcd9..000 --- a/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out +++ /dev/null @@ -1,870 +0,0 @@ -PREHOOK: query: explain vectorization expression -select cint, cint / 0 as cint_div, ctinyint, ctinyint / 0 as ctinyint_div, cbigint, cbigint / 0 as cbigint_div, cdouble, cdouble / 0.0 as cdouble_div -from alltypesorc order by cint, ctinyint, cbigint, cdouble limit 100 -PREHOOK: type: QUERY -POSTHOOK: query: explain vectorization expression -select cint, cint / 0 as cint_div, ctinyint, ctinyint / 0 as ctinyint_div, cbigint, cbigint / 0 as cbigint_div, cdouble, cdouble / 0.0 as cdouble_div -from alltypesorc order by cint, ctinyint, cbigint, cdouble limit 100 -POSTHOOK: type: QUERY -Explain -PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 -Tez - A masked pattern was here - Edges: -Reducer 2 <- Map 1 (SIMPLE_EDGE) - A masked pattern was here - Vertices: -Map 1 -Map Operator Tree: -TableScan - alias: alltypesorc - Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE - TableScan Vectorization: - native: true - Select Operator -expressions: cint (type: int), (UDFToDouble(cint) / 0.0D) (type: double), ctinyint (type: tinyint), (UDFToDouble(ctinyint) / 0.0D) (type: double), cbigint (type: bigint), (UDFToDouble(cbigint) / 0.0D) (type: double), cdouble (type: double), (cdouble / 0.0D) (type: double) -outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 -Select Vectorization: -className: VectorSelectOperator -native: true -projectedOutputColumnNums: [2, 14, 0, 15, 3, 16, 5, 13] -selectExpressions: DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 2:int) -> 13:double) -> 14:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 0:tinyint) -> 13:double) -> 15:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 3:bigint) -> 13:double) -> 16:double, DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 13:double -Statistics: Num rows: 12288 Data size: 613400 Basic stats: COMPLETE Column stats: COMPLETE -Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: tinyint), _col4 (type: bigint), _col6 (type: double) - sort order: - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 12288 Data size: 613400 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 - value expressions: _col1 (type: double), _col3 (type: double), _col5 (type: double), _col7 (type: double) -Execution mode: vectorized, llap -LLAP IO: all inputs -Map Vectorization: -enabled: true -enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true -inputFormatFeatureSupport: [] -featureSupportInUse: [] -inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat -allNative: true -usesVectorUDFAdaptor: false -vectorized: true -Reducer 2 -Execution mode: vectorized, llap -Reduce Vectorization: -enabled: true -enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true -allNative: false -usesVectorUDFAdaptor: false -vectorized: true -Reduce Operator Tree: - Selec
hive git commit: HIVE-18781 broke WarehouseInstance (Alan Gates, reviewed by Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master a21302f10 -> 7fa2ba8c7
HIVE-18781 broke WarehouseInstance (Alan Gates, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7fa2ba8c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7fa2ba8c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7fa2ba8c
Branch: refs/heads/master
Commit: 7fa2ba8c77b77d176c6d6180e6444b97aebf1503
Parents: a21302f
Author: Alan Gates
Authored: Mon Apr 9 11:49:55 2018 -0700
Committer: Alan Gates
Committed: Mon Apr 9 11:49:55 2018 -0700
--
.../java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java | 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/7fa2ba8c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index fe4660c..7c8020d 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -123,6 +123,9 @@ public class WarehouseInstance implements Closeable {
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
+if
(!hiveConf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER).equals("org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"))
{
+ hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname,
"false");
+}
System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
hive git commit: HIVE-19119 Fix the TestAppendPartitions tests which are failing in the pre-commit runs (Marta Kuczora via Alan Gates)
Repository: hive
Updated Branches:
refs/heads/master 7fa2ba8c7 -> d589ee755
HIVE-19119 Fix the TestAppendPartitions tests which are failing in the
pre-commit runs (Marta Kuczora via Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d589ee75
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d589ee75
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d589ee75
Branch: refs/heads/master
Commit: d589ee7550abb52864b71acb94e1efab601ea67e
Parents: 7fa2ba8
Author: Alan Gates
Authored: Mon Apr 9 12:03:02 2018 -0700
Committer: Alan Gates
Committed: Mon Apr 9 12:03:02 2018 -0700
--
.../apache/hadoop/hive/metastore/HiveMetaStore.java | 2 +-
.../hive/metastore/client/TestAppendPartitions.java | 14 +-
2 files changed, 6 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/d589ee75/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index a2fe7d7..c81b8fa 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -3094,7 +3094,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
final String tableName, final List part_vals, final
EnvironmentContext envContext)
throws InvalidObjectException, AlreadyExistsException, MetaException {
if (part_vals == null || part_vals.isEmpty()) {
-throw new MetaException("The partition values must not be null.");
+throw new MetaException("The partition values must not be null or
empty.");
}
String[] parsedDbName = parseDbName(dbName, conf);
startPartitionFunction("append_partition", parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tableName, part_vals);
http://git-wip-us.apache.org/repos/asf/hive/blob/d589ee75/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java
--
diff --git
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java
index 75b26f2..37ca40c 100644
---
a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java
+++
b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java
@@ -226,22 +226,18 @@ public class TestAppendPartitions extends
MetaStoreClientTest {
client.appendPartition(tableWithPartitions.getDbName(), null,
partitionValues);
}
- @Test(expected = InvalidObjectException.class)
+ @Test(expected = MetaException.class)
public void testAppendPartitionEmptyPartValues() throws Exception {
Table table = tableWithPartitions;
client.appendPartition(table.getDbName(), table.getTableName(), new
ArrayList<>());
}
- @Test
+ @Test(expected = MetaException.class)
public void testAppendPartitionNullPartValues() throws Exception {
-try {
- Table table = tableWithPartitions;
- client.appendPartition(table.getDbName(), table.getTableName(),
(List) null);
- Assert.fail("Exception should have been thrown.");
-} catch (TTransportException | InvalidObjectException e) {
- // TODO: NPE should not be thrown
-}
+
+Table table = tableWithPartitions;
+client.appendPartition(table.getDbName(), table.getTableName(),
(List) null);
}
@Test
[hive] Git Push Summary
Repository: hive Updated Branches: refs/heads/branch-3.0.0 [created] d589ee755
[hive] Git Push Summary
Repository: hive Updated Branches: refs/heads/branch-3 [created] d589ee755
[hive] Git Push Summary
Repository: hive Updated Branches: refs/heads/branch-3.0.0 [deleted] d589ee755
hive git commit: HIVE-19134: Update copyright NOTICE and fix rat check failures (Vineet Garg, reviewed by Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/master d589ee755 -> 2d3a41045 HIVE-19134: Update copyright NOTICE and fix rat check failures (Vineet Garg, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d3a4104 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d3a4104 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d3a4104 Branch: refs/heads/master Commit: 2d3a410451626b32b60ce8edaa0d8d00d951c7d0 Parents: d589ee7 Author: Vineet Garg Authored: Mon Apr 9 14:34:29 2018 -0700 Committer: Vineet Garg Committed: Mon Apr 9 14:34:29 2018 -0700 -- NOTICE| 2 +- .../hadoop/hive/ql/TestTxnAddPartition.java | 18 ++ 2 files changed, 19 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/2d3a4104/NOTICE -- diff --git a/NOTICE b/NOTICE index 0235613..d018205 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Hive -Copyright 2008-2016 The Apache Software Foundation +Copyright 2008-2018 The Apache Software Foundation This product includes software developed by The Apache Software Foundation (http://www.apache.org/). http://git-wip-us.apache.org/repos/asf/hive/blob/2d3a4104/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java -- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java index 901b5db..c821365 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.ql; import org.apache.hadoop.fs.FileStatus;
hive git commit: HIVE-19134: Update copyright NOTICE and fix rat check failures(Vineet Garg, reviewed by Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/branch-3 d589ee755 -> 43cb101ea HIVE-19134: Update copyright NOTICE and fix rat check failures(Vineet Garg, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/43cb101e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/43cb101e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/43cb101e Branch: refs/heads/branch-3 Commit: 43cb101eac4dba1405c2e6337a04b0b8c1240e85 Parents: d589ee7 Author: Vineet Garg Authored: Mon Apr 9 14:35:17 2018 -0700 Committer: Vineet Garg Committed: Mon Apr 9 14:35:17 2018 -0700 -- NOTICE| 2 +- .../hadoop/hive/ql/TestTxnAddPartition.java | 18 ++ 2 files changed, 19 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/43cb101e/NOTICE -- diff --git a/NOTICE b/NOTICE index 0235613..d018205 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Hive -Copyright 2008-2016 The Apache Software Foundation +Copyright 2008-2018 The Apache Software Foundation This product includes software developed by The Apache Software Foundation (http://www.apache.org/). http://git-wip-us.apache.org/repos/asf/hive/blob/43cb101e/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java -- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java index 901b5db..c821365 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnAddPartition.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.ql; import org.apache.hadoop.fs.FileStatus;
[04/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query78.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query78.q.out b/ql/src/test/results/clientpositive/perf/spark/query78.q.out index 1467c5f..c2e8577 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query78.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query78.q.out @@ -120,57 +120,41 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 14 (PARTITION-LEVEL SORT, 85) -Reducer 12 <- Reducer 11 (GROUP, 93) -Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 164), Map 15 (PARTITION-LEVEL SORT, 164) -Reducer 17 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 20 (PARTITION-LEVEL SORT, 85) +Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 164), Map 14 (PARTITION-LEVEL SORT, 164) +Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 85), Reducer 11 (PARTITION-LEVEL SORT, 85) +Reducer 13 <- Reducer 12 (GROUP, 93) +Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 85), Reducer 20 (PARTITION-LEVEL SORT, 85) Reducer 18 <- Reducer 17 (GROUP, 93) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 219), Reducer 8 (PARTITION-LEVEL SORT, 219) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 432), Map 8 (PARTITION-LEVEL SORT, 432) Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 177), Map 21 (PARTITION-LEVEL SORT, 177) -Reducer 3 <- Reducer 2 (GROUP, 241) -Reducer 4 <- Reducer 12 (PARTITION-LEVEL SORT, 167), Reducer 3 (PARTITION-LEVEL SORT, 167) -Reducer 5 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 4 (PARTITION-LEVEL SORT, 91) -Reducer 6 <- Reducer 5 (SORT, 1) -Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 432), Map 9 (PARTITION-LEVEL SORT, 432) +Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 219), Reducer 2 (PARTITION-LEVEL SORT, 219) +Reducer 4 <- Reducer 3 (GROUP, 241) +Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 167), Reducer 4 (PARTITION-LEVEL SORT, 167) +Reducer 6 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 5 (PARTITION-LEVEL SORT, 91) +Reducer 7 <- Reducer 6 (SORT, 1) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE + alias: store_sales + Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: ((d_year = 2000) and d_date_sk is not null) (type: boolean) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE +predicate: ss_sold_date_sk is not null (type: boolean) +Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE + expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator -key expressions: _col0 (type: int) -sort order: + -Map-reduce partition columns: _col0 (type: int) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE +key expressions: _col1 (type: int), _col3 (type: int) +sort order: ++ +Map-reduce partition columns: _col1 (type: int), _col3 (type: int) +Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)) Map 10 Map Operator Tree: TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE - Filter Operator -
[08/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query54.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query54.q.out b/ql/src/test/results/clientpositive/perf/spark/query54.q.out index 43132bc..251d7ad 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query54.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query54.q.out @@ -1,7 +1,7 @@ -Warning: Shuffle Join JOIN[111][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product -Warning: Shuffle Join JOIN[107][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Work 'Reducer 14' is a cross product -Warning: Shuffle Join JOIN[114][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 4' is a cross product -Warning: Map Join MAPJOIN[144][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[111][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[104][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Work 'Reducer 14' is a cross product +Warning: Shuffle Join JOIN[114][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 5' is a cross product +Warning: Map Join MAPJOIN[143][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain with my_customers as ( select distinct c_customer_sk @@ -122,11 +122,11 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: -Reducer 29 <- Map 28 (GROUP, 2) -Reducer 30 <- Reducer 29 (GROUP, 1) +Reducer 31 <- Map 30 (GROUP, 2) +Reducer 32 <- Reducer 31 (GROUP, 1) A masked pattern was here Vertices: -Map 28 +Map 30 Map Operator Tree: TableScan alias: date_dim @@ -148,7 +148,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE -Reducer 29 +Reducer 31 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) @@ -166,7 +166,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 30 +Reducer 32 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -211,23 +211,23 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 10 <- Reducer 9 (GROUP, 1) +Reducer 10 <- Map 1 (GROUP, 2) Reducer 12 <- Map 11 (PARTITION-LEVEL SORT, 398), Map 15 (PARTITION-LEVEL SORT, 398) Reducer 13 <- Reducer 12 (PARTITION-LEVEL SORT, 772), Reducer 17 (PARTITION-LEVEL SORT, 772) -Reducer 14 <- Reducer 13 (PARTITION-LEVEL SORT, 1), Reducer 32 (PARTITION-LEVEL SORT, 1) +Reducer 14 <- Reducer 13 (PARTITION-LEVEL SORT, 1), Reducer 29 (PARTITION-LEVEL SORT, 1) Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 654), Reducer 23 (PARTITION-LEVEL SORT, 654) Reducer 2 <- Map 1 (GROUP, 2) -Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 458), Map 24 (PARTITION-LEVEL SORT, 458), Map 25 (PARTITION-LEVEL SORT, 458) -Reducer 21 <- Map 26 (PARTITION-LEVEL SORT, 505), Reducer 20 (PARTITION-LEVEL SORT, 505) +Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 459), Map 24 (PARTITION-LEVEL SORT, 459), Map 25 (PARTITION-LEVEL SORT, 459) +Reducer 21 <- Map 26 (PARTITION-LEVEL SORT, 504), Reducer 20 (PARTITION-LEVEL SORT, 504) Reducer 22 <- Map 27 (PARTITION-LEVEL SORT, 1009), Reducer 21 (PARTITION-LEVEL SORT, 1009) Reducer 23 <- Reducer 22 (GROUP, 610) -Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) -Reducer 32 <- Map 31 (GROUP, 2) -Reducer 4 <- Reducer 14 (PARTITION-LEVEL SORT, 1), Reducer 3 (PARTITION-LEVEL SORT, 1) -Reducer 5 <- Reducer 4 (GROUP, 1009) +Reducer 29 <- Map 28 (GROUP, 2) +Reducer 3 <- Reducer 2 (GROUP, 1) +Reducer 4 <- Reducer 10 (PARTITION-LEVEL SORT, 1), Reducer 3 (PARTITION-LEVEL SORT, 1) +Reducer 5 <- Reducer 14 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1) Reducer 6 <- Reducer 5 (GROUP, 1009) -Reducer 7 <- Reducer 6 (SORT, 1) -Reducer 9 <- Map 1 (GROUP, 2) +Reducer 7 <- Reducer 6 (GROUP, 1009) +Reducer 8 <- Reducer 7 (SORT, 1) A masked pattern was here Vertices: Map 1 @@ -330,11 +330,11 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 287989836 Data size: 38999608952 Basic stat
[11/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query30.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query30.q.out b/ql/src/test/results/clientpositive/perf/spark/query30.q.out index 6385984..399251d 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query30.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query30.q.out @@ -66,72 +66,37 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 10 <- Reducer 16 (PARTITION-LEVEL SORT, 262), Reducer 9 (PARTITION-LEVEL SORT, 262) -Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 11), Map 17 (PARTITION-LEVEL SORT, 11) -Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 329), Reducer 14 (PARTITION-LEVEL SORT, 329) -Reducer 16 <- Reducer 15 (GROUP PARTITION-LEVEL SORT, 349) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 697), Map 5 (PARTITION-LEVEL SORT, 697) -Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 656), Reducer 2 (PARTITION-LEVEL SORT, 656) -Reducer 4 <- Reducer 3 (SORT, 1) -Reducer 7 <- Map 11 (PARTITION-LEVEL SORT, 11), Map 6 (PARTITION-LEVEL SORT, 11) -Reducer 8 <- Map 12 (PARTITION-LEVEL SORT, 329), Reducer 7 (PARTITION-LEVEL SORT, 329) -Reducer 9 <- Reducer 8 (GROUP, 349) +Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 11), Map 14 (PARTITION-LEVEL SORT, 11) +Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 329), Reducer 11 (PARTITION-LEVEL SORT, 329) +Reducer 13 <- Reducer 12 (GROUP PARTITION-LEVEL SORT, 349) +Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 697), Map 18 (PARTITION-LEVEL SORT, 697) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 11), Map 8 (PARTITION-LEVEL SORT, 11) +Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 329), Reducer 2 (PARTITION-LEVEL SORT, 329) +Reducer 4 <- Reducer 3 (GROUP, 349) +Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 262), Reducer 4 (PARTITION-LEVEL SORT, 262) +Reducer 6 <- Reducer 17 (PARTITION-LEVEL SORT, 656), Reducer 5 (PARTITION-LEVEL SORT, 656) +Reducer 7 <- Reducer 6 (SORT, 1) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: customer - Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean) -Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: c_customer_sk (type: int), c_customer_id (type: string), c_current_addr_sk (type: int), c_salutation (type: string), c_first_name (type: string), c_last_name (type: string), c_preferred_cust_flag (type: string), c_birth_day (type: int), c_birth_month (type: int), c_birth_year (type: int), c_birth_country (type: string), c_login (type: string), c_email_address (type: string), c_last_review_date (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col2 (type: int) -sort order: + -Map-reduce partition columns: _col2 (type: int) -Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string) -Map 11 -Map Operator Tree: -TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col0 (type: i
[02/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query85.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query85.q.out
b/ql/src/test/results/clientpositive/perf/spark/query85.q.out
index d60751c..c82dcf7 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query85.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query85.q.out
@@ -175,7 +175,7 @@ STAGE PLANS:
Spark
A masked pattern was here
Vertices:
-Map 13
+Map 15
Map Operator Tree:
TableScan
alias: reason
@@ -212,21 +212,21 @@ STAGE PLANS:
Statistics: Num rows: 4602 Data size: 2696178 Basic
stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col10 (type: int)
- 1 _col0 (type: int)
+ 0 _col0 (type: int)
+ 1 _col2 (type: int)
Local Work:
Map Reduce Local Work
Stage: Stage-1
Spark
Edges:
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 28), Map 9 (PARTITION-LEVEL
SORT, 28)
-Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 98), Reducer 2
(PARTITION-LEVEL SORT, 98)
-Reducer 4 <- Map 12 (PARTITION-LEVEL SORT, 5), Reducer 3
(PARTITION-LEVEL SORT, 5)
-Reducer 5 <- Map 14 (PARTITION-LEVEL SORT, 11), Reducer 4
(PARTITION-LEVEL SORT, 11)
-Reducer 6 <- Map 15 (PARTITION-LEVEL SORT, 7), Reducer 5
(PARTITION-LEVEL SORT, 7)
-Reducer 7 <- Reducer 6 (GROUP, 7)
-Reducer 8 <- Reducer 7 (SORT, 1)
+Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 20), Map 14
(PARTITION-LEVEL SORT, 20)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 90), Map 8 (PARTITION-LEVEL
SORT, 90)
+Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 17), Reducer 2
(PARTITION-LEVEL SORT, 17)
+Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 19), Reducer 3
(PARTITION-LEVEL SORT, 19)
+Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 35), Reducer 4
(PARTITION-LEVEL SORT, 35)
+Reducer 6 <- Reducer 5 (GROUP, 2)
+Reducer 7 <- Reducer 6 (SORT, 1)
A masked pattern was here
Vertices:
Map 1
@@ -242,32 +242,61 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7
Statistics: Num rows: 14398467 Data size: 1325194184
Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
-key expressions: _col0 (type: int), _col5 (type: int)
-sort order: ++
-Map-reduce partition columns: _col0 (type: int), _col5
(type: int)
+key expressions: _col2 (type: int)
+sort order: +
+Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 14398467 Data size: 1325194184
Basic stats: COMPLETE Column stats: NONE
-value expressions: _col1 (type: int), _col2 (type:
int), _col3 (type: int), _col4 (type: int), _col6 (type: decimal(7,2)), _col7
(type: decimal(7,2))
+value expressions: _col0 (type: int), _col1 (type:
int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type:
decimal(7,2)), _col7 (type: decimal(7,2))
Map 10
Map Operator Tree:
TableScan
- alias: customer_address
- Statistics: Num rows: 4000 Data size: 40595195284 Basic
stats: COMPLETE Column stats: NONE
+ alias: cd2
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic
stats: COMPLETE Column stats: NONE
Filter Operator
-predicate: ((ca_country = 'United States') and (ca_state)
IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and ca_address_sk is
not null) (type: boolean)
-Statistics: Num rows: 1000 Data size: 10148798821
Basic stats: COMPLETE Column stats: NONE
+predicate: (((cd_education_status = '4 yr Degree') or
(cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree'))
and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or
(cd_marital_status = 'U')) and cd_demo_sk is not null and cd_education_status
is not null and cd_marital_status is not null) (type: boolean)
+Statistics: Num rows: 1861800 Data size: 717186159 Basic
stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ca_address_sk (type: int), ca_state (type:
string)
- outputColumnNames: _col0, _col1
- Statistics: Num r
[01/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
Repository: hive Updated Branches: refs/heads/master 2d3a41045 -> 328d3f935 http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query92.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query92.q.out b/ql/src/test/results/clientpositive/perf/spark/query92.q.out index e7b8632..70c4c5a 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query92.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query92.q.out @@ -67,7 +67,7 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 5 +Map 7 Map Operator Tree: TableScan alias: date_dim @@ -90,7 +90,7 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 9 +Map 10 Map Operator Tree: TableScan alias: date_dim @@ -112,11 +112,11 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 261), Reducer 8 (PARTITION-LEVEL SORT, 261) -Reducer 3 <- Reducer 2 (GROUP, 1) -Reducer 4 <- Reducer 3 (SORT, 1) -Reducer 7 <- Map 6 (GROUP, 169) -Reducer 8 <- Map 10 (PARTITION-LEVEL SORT, 87), Reducer 7 (PARTITION-LEVEL SORT, 87) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 156), Map 6 (PARTITION-LEVEL SORT, 156) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 270), Reducer 9 (PARTITION-LEVEL SORT, 270) +Reducer 4 <- Reducer 3 (GROUP, 1) +Reducer 5 <- Reducer 4 (SORT, 1) +Reducer 9 <- Map 8 (GROUP, 169) A masked pattern was here Vertices: Map 1 @@ -131,25 +131,13 @@ STAGE PLANS: expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_ext_discount_amt (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE - Map Join Operator -condition map: - Inner Join 0 to 1 -keys: - 0 _col0 (type: int) - 1 _col0 (type: int) -outputColumnNames: _col1, _col2 -input vertices: - 1 Map 5 -Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE -Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: decimal(7,2)) -Local Work: - Map Reduce Local Work -Map 10 + Reduce Output Operator +key expressions: _col1 (type: int) +sort order: + +Map-reduce partition columns: _col1 (type: int) +Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: int), _col2 (type: decimal(7,2)) +Map 6 Map Operator Tree: TableScan alias: item @@ -166,7 +154,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE -Map 6 +Map 8 Map Operator Tree: TableScan alias: web_sales @@ -186,7 +174,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2 input vertices: - 1 Map 9 + 1 Map 10 Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col2), count(_col2) @@ -203,22 +191,50 @@ STAGE PLANS: Local Work: Map Reduce Local Work Reducer 2 +Local Work: + Map Reduce Local Work Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: 0 _col1 (type: int) - 1 _col2 (type: int) -output
[12/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query24.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query24.q.out b/ql/src/test/results/clientpositive/perf/spark/query24.q.out index 1f291c0..13ac1e8 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query24.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query24.q.out @@ -131,10 +131,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: -Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 975), Map 20 (PARTITION-LEVEL SORT, 975) -Reducer 14 <- Map 21 (PARTITION-LEVEL SORT, 486), Reducer 13 (PARTITION-LEVEL SORT, 486) -Reducer 15 <- Map 22 (PARTITION-LEVEL SORT, 564), Reducer 14 (PARTITION-LEVEL SORT, 564) -Reducer 16 <- Map 23 (PARTITION-LEVEL SORT, 899), Reducer 15 (PARTITION-LEVEL SORT, 899) +Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 472), Map 20 (PARTITION-LEVEL SORT, 472) +Reducer 14 <- Map 21 (PARTITION-LEVEL SORT, 1009), Reducer 13 (PARTITION-LEVEL SORT, 1009) +Reducer 15 <- Map 22 (PARTITION-LEVEL SORT, 846), Reducer 14 (PARTITION-LEVEL SORT, 846) +Reducer 16 <- Map 23 (PARTITION-LEVEL SORT, 587), Reducer 15 (PARTITION-LEVEL SORT, 587) Reducer 17 <- Reducer 16 (GROUP, 640) Reducer 18 <- Reducer 17 (GROUP, 1) A masked pattern was here @@ -162,16 +162,33 @@ STAGE PLANS: 1 Map 19 Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) + key expressions: _col0 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col3 (type: int) Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string) + value expressions: _col1 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string) Local Work: Map Reduce Local Work Map 20 Map Operator Tree: TableScan + alias: store_returns + Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean) +Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: sr_item_sk (type: int), sr_ticket_number (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col0 (type: int), _col1 (type: int) +sort order: ++ +Map-reduce partition columns: _col0 (type: int), _col1 (type: int) +Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE +Map 21 +Map Operator Tree: +TableScan alias: customer Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -187,44 +204,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) -Map 21 -Map Operator Tree: -TableScan - alias: item - Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: i_item_sk is not null (type: boolean) -Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: i_item_sk (type: int), i_current_price (type: decimal(7,2)), i_size (type: string),
[13/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
HIVE-19128 : Update golden files for spark perf tests Signed-off-by: Ashutosh Chauhan Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/328d3f93 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/328d3f93 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/328d3f93 Branch: refs/heads/master Commit: 328d3f93596c82ac79544f111200659e28f5f8fc Parents: 2d3a410 Author: Ashutosh Chauhan Authored: Mon Apr 9 14:57:11 2018 -0700 Committer: Ashutosh Chauhan Committed: Mon Apr 9 14:57:11 2018 -0700 -- .../clientpositive/perf/spark/query11.q.out | 48 +- .../clientpositive/perf/spark/query15.q.out | 164 +++ .../clientpositive/perf/spark/query16.q.out | 70 ++- .../clientpositive/perf/spark/query18.q.out | 216 .../clientpositive/perf/spark/query19.q.out | 218 - .../clientpositive/perf/spark/query21.q.out | 114 ++--- .../clientpositive/perf/spark/query24.q.out | 282 +-- .../clientpositive/perf/spark/query25.q.out | 118 ++--- .../clientpositive/perf/spark/query29.q.out | 280 ++- .../clientpositive/perf/spark/query30.q.out | 262 +- .../clientpositive/perf/spark/query32.q.out | 132 ++--- .../clientpositive/perf/spark/query34.q.out | 34 +- .../clientpositive/perf/spark/query35.q.out | 74 +-- .../clientpositive/perf/spark/query37.q.out | 16 +- .../clientpositive/perf/spark/query4.q.out | 214 .../clientpositive/perf/spark/query40.q.out | 116 ++--- .../clientpositive/perf/spark/query44.q.out | 246 +- .../clientpositive/perf/spark/query45.q.out | 204 .../clientpositive/perf/spark/query46.q.out | 104 ++-- .../clientpositive/perf/spark/query47.q.out | 92 ++-- .../clientpositive/perf/spark/query48.q.out | 94 ++-- .../clientpositive/perf/spark/query5.q.out | 38 +- .../clientpositive/perf/spark/query50.q.out | 196 .../clientpositive/perf/spark/query53.q.out | 105 ++-- .../clientpositive/perf/spark/query54.q.out | 277 +-- .../clientpositive/perf/spark/query57.q.out | 92 ++-- .../clientpositive/perf/spark/query58.q.out | 490 +-- .../clientpositive/perf/spark/query6.q.out | 350 ++--- .../clientpositive/perf/spark/query61.q.out | 40 +- .../clientpositive/perf/spark/query63.q.out | 105 ++-- .../clientpositive/perf/spark/query65.q.out | 100 ++-- .../clientpositive/perf/spark/query66.q.out | 20 +- .../clientpositive/perf/spark/query67.q.out | 137 +++--- .../clientpositive/perf/spark/query68.q.out | 104 ++-- .../clientpositive/perf/spark/query72.q.out | 461 - .../clientpositive/perf/spark/query73.q.out | 34 +- .../clientpositive/perf/spark/query75.q.out | 248 +- .../clientpositive/perf/spark/query76.q.out | 130 ++--- .../clientpositive/perf/spark/query77.q.out | 66 +-- .../clientpositive/perf/spark/query78.q.out | 273 ++- .../clientpositive/perf/spark/query79.q.out | 10 +- .../clientpositive/perf/spark/query8.q.out | 174 +++ .../clientpositive/perf/spark/query80.q.out | 417 .../clientpositive/perf/spark/query81.q.out | 270 +- .../clientpositive/perf/spark/query82.q.out | 16 +- .../clientpositive/perf/spark/query83.q.out | 354 +++--- .../clientpositive/perf/spark/query85.q.out | 309 ++-- .../clientpositive/perf/spark/query88.q.out | 272 +- .../clientpositive/perf/spark/query89.q.out | 26 +- .../clientpositive/perf/spark/query90.q.out | 68 +-- .../clientpositive/perf/spark/query91.q.out | 14 +- .../clientpositive/perf/spark/query92.q.out | 106 ++-- .../clientpositive/perf/spark/query94.q.out | 70 ++- .../clientpositive/perf/spark/query95.q.out | 100 ++-- .../clientpositive/perf/spark/query97.q.out | 54 +- .../clientpositive/perf/spark/query99.q.out | 42 +- 56 files changed, 4313 insertions(+), 4353 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query11.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query11.q.out b/ql/src/test/results/clientpositive/perf/spark/query11.q.out index e46aa21..227068c 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query11.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query11.q.out @@ -300,7 +300,7 @@ STAGE PLANS: alias: date_dim Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: ((d_y
[03/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query81.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query81.q.out b/ql/src/test/results/clientpositive/perf/spark/query81.q.out index 5b2d5b3..23998d6 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query81.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query81.q.out @@ -66,72 +66,37 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 10 <- Reducer 16 (PARTITION-LEVEL SORT, 262), Reducer 9 (PARTITION-LEVEL SORT, 262) -Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 25), Map 17 (PARTITION-LEVEL SORT, 25) -Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 344), Reducer 14 (PARTITION-LEVEL SORT, 344) -Reducer 16 <- Reducer 15 (GROUP PARTITION-LEVEL SORT, 349) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 697), Map 5 (PARTITION-LEVEL SORT, 697) -Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 656), Reducer 2 (PARTITION-LEVEL SORT, 656) -Reducer 4 <- Reducer 3 (SORT, 1) -Reducer 7 <- Map 11 (PARTITION-LEVEL SORT, 25), Map 6 (PARTITION-LEVEL SORT, 25) -Reducer 8 <- Map 12 (PARTITION-LEVEL SORT, 344), Reducer 7 (PARTITION-LEVEL SORT, 344) -Reducer 9 <- Reducer 8 (GROUP, 349) +Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 25), Map 14 (PARTITION-LEVEL SORT, 25) +Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 344), Reducer 11 (PARTITION-LEVEL SORT, 344) +Reducer 13 <- Reducer 12 (GROUP PARTITION-LEVEL SORT, 349) +Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 697), Map 18 (PARTITION-LEVEL SORT, 697) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 25), Map 8 (PARTITION-LEVEL SORT, 25) +Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 344), Reducer 2 (PARTITION-LEVEL SORT, 344) +Reducer 4 <- Reducer 3 (GROUP, 349) +Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 262), Reducer 4 (PARTITION-LEVEL SORT, 262) +Reducer 6 <- Reducer 17 (PARTITION-LEVEL SORT, 656), Reducer 5 (PARTITION-LEVEL SORT, 656) +Reducer 7 <- Reducer 6 (SORT, 1) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: customer - Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean) -Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: c_customer_sk (type: int), c_customer_id (type: string), c_current_addr_sk (type: int), c_salutation (type: string), c_first_name (type: string), c_last_name (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col2 (type: int) -sort order: + -Map-reduce partition columns: _col2 (type: int) -Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) -Map 11 -Map Operator Tree: -TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: ((d_year = 1998) and d_date_sk is not null) (type: boolean) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col0 (type: int) -sort order: + -Map-reduce partition columns: _col0 (type: int) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE -Map 12 -Map Operator Tree: -TableScan - alias: customer_address - Statistics: Num rows: 4000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE +
[05/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query75.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query75.q.out b/ql/src/test/results/clientpositive/perf/spark/query75.q.out index 4b76570..f4fca34 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query75.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query75.q.out @@ -144,25 +144,25 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 398), Map 16 (PARTITION-LEVEL SORT, 398) -Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 440), Reducer 13 (PARTITION-LEVEL SORT, 440) +Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 400), Map 16 (PARTITION-LEVEL SORT, 400) +Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 438), Reducer 13 (PARTITION-LEVEL SORT, 438) Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 516), Reducer 14 (PARTITION-LEVEL SORT, 516) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 16 (PARTITION-LEVEL SORT, 306) -Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 154), Map 19 (PARTITION-LEVEL SORT, 154) -Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 171), Reducer 20 (PARTITION-LEVEL SORT, 171) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 308), Map 16 (PARTITION-LEVEL SORT, 308) +Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 156), Map 19 (PARTITION-LEVEL SORT, 156) +Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 169), Reducer 20 (PARTITION-LEVEL SORT, 169) Reducer 22 <- Map 25 (PARTITION-LEVEL SORT, 196), Reducer 21 (PARTITION-LEVEL SORT, 196) -Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 306), Map 32 (PARTITION-LEVEL SORT, 306) -Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 338), Reducer 27 (PARTITION-LEVEL SORT, 338) +Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 308), Map 32 (PARTITION-LEVEL SORT, 308) +Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 336), Reducer 27 (PARTITION-LEVEL SORT, 336) Reducer 29 <- Map 34 (PARTITION-LEVEL SORT, 393), Reducer 28 (PARTITION-LEVEL SORT, 393) -Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 338), Reducer 2 (PARTITION-LEVEL SORT, 338) +Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 336), Reducer 2 (PARTITION-LEVEL SORT, 336) Reducer 30 <- Reducer 29 (GROUP, 934), Reducer 38 (GROUP, 934) Reducer 31 <- Reducer 30 (GROUP PARTITION-LEVEL SORT, 671), Reducer 45 (GROUP PARTITION-LEVEL SORT, 671) -Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 398), Map 35 (PARTITION-LEVEL SORT, 398) -Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 440), Reducer 36 (PARTITION-LEVEL SORT, 440) +Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 400), Map 35 (PARTITION-LEVEL SORT, 400) +Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 438), Reducer 36 (PARTITION-LEVEL SORT, 438) Reducer 38 <- Map 41 (PARTITION-LEVEL SORT, 516), Reducer 37 (PARTITION-LEVEL SORT, 516) Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 393), Reducer 3 (PARTITION-LEVEL SORT, 393) -Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 154), Map 42 (PARTITION-LEVEL SORT, 154) -Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 171), Reducer 43 (PARTITION-LEVEL SORT, 171) +Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 156), Map 42 (PARTITION-LEVEL SORT, 156) +Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 169), Reducer 43 (PARTITION-LEVEL SORT, 169) Reducer 45 <- Map 48 (PARTITION-LEVEL SORT, 196), Reducer 44 (PARTITION-LEVEL SORT, 196) Reducer 5 <- Reducer 15 (GROUP, 934), Reducer 4 (GROUP, 934) Reducer 6 <- Reducer 22 (GROUP PARTITION-LEVEL SORT, 671), Reducer 5 (GROUP PARTITION-LEVEL SORT, 671) @@ -183,29 +183,28 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator -key expressions: _col0 (type: int) +key expressions: _col1 (type: int) sort order: + -Map-reduce partition columns: _col0 (type: int) +Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE -value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)) +value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)) Map 10 Map Operator Tree: TableScan - alias: item - Statistics: Num rows: 462000 Data size: 663560457 Basic stats:
[10/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query40.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query40.q.out b/ql/src/test/results/clientpositive/perf/spark/query40.q.out index f286294..5360385 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query40.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query40.q.out @@ -54,8 +54,7 @@ limit 100 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 - Stage-1 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 Stage-0 depends on stages: Stage-1 STAGE PLANS: @@ -63,44 +62,39 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 9 +Map 8 Map Operator Tree: TableScan - alias: warehouse - Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE + alias: date_dim + Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: w_warehouse_sk is not null (type: boolean) -Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE +predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-03-08 23:00:00.0' AND TIMESTAMP'1998-05-08 00:00:00.0' and d_date_sk is not null) (type: boolean) +Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: w_warehouse_sk (type: int), w_state (type: string) + expressions: d_date_sk (type: int), d_date (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col1 (type: int) + 0 _col0 (type: int) 1 _col0 (type: int) Local Work: Map Reduce Local Work - - Stage: Stage-3 -Spark - A masked pattern was here - Vertices: -Map 7 +Map 9 Map Operator Tree: TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE + alias: warehouse + Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-03-08 23:00:00.0' AND TIMESTAMP'1998-05-08 00:00:00.0' and d_date_sk is not null) (type: boolean) -Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE +predicate: w_warehouse_sk is not null (type: boolean) +Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: d_date_sk (type: int), d_date (type: string) + expressions: w_warehouse_sk (type: int), w_state (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: int) + 0 _col1 (type: int) 1 _col0 (type: int) Local Work: Map Reduce Local Work @@ -109,7 +103,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 329), Map 6 (PARTITION-LEVEL SORT, 329) -Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 370), Reducer 2 (PARTITION-LEVEL SORT, 370) +Reducer 3 <- Map 7 (PARTITION-LEVEL SORT, 336), Reducer 2 (PARTITION-LEVEL SORT, 336) Reducer 4 <- Reducer 3 (GROUP, 447) Reducer 5 <- Reducer 4 (SORT, 1) A masked pattern was here @@ -150,7 +144,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type:
[07/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query58.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query58.q.out b/ql/src/test/results/clientpositive/perf/spark/query58.q.out index f06cbef..eb5dffe 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query58.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query58.q.out @@ -1,6 +1,6 @@ -Warning: Map Join MAPJOIN[183][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[184][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[185][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[180][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[181][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[182][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain with ss_items as (select i_item_id item_id @@ -140,10 +140,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: -Reducer 11 <- Map 10 (GROUP, 1) +Reducer 12 <- Map 11 (GROUP, 1) A masked pattern was here Vertices: -Map 10 +Map 11 Map Operator Tree: TableScan alias: date_dim @@ -162,7 +162,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 11 +Reducer 12 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -184,10 +184,10 @@ STAGE PLANS: Stage: Stage-3 Spark Edges: -Reducer 24 <- Map 23 (GROUP, 1) +Reducer 25 <- Map 24 (GROUP, 1) A masked pattern was here Vertices: -Map 23 +Map 24 Map Operator Tree: TableScan alias: date_dim @@ -206,7 +206,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 24 +Reducer 25 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -228,10 +228,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: -Reducer 37 <- Map 36 (GROUP, 1) +Reducer 38 <- Map 37 (GROUP, 1) A masked pattern was here Vertices: -Map 36 +Map 37 Map Operator Tree: TableScan alias: date_dim @@ -250,7 +250,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 37 +Reducer 38 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -272,76 +272,47 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 2), Map 15 (PARTITION-LEVEL SORT, 2) -Reducer 14 <- Reducer 13 (GROUP, 2) -Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 403), Map 20 (PARTITION-LEVEL SORT, 403) -Reducer 18 <- Reducer 17 (PARTITION-LEVEL SORT, 438), Reducer 22 (PARTITION-LEVEL SORT, 438) -Reducer 19 <- Reducer 18 (GROUP, 481) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 310), Map 7 (PARTITION-LEVEL SORT, 310) -Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 2), Reducer 27 (PARTITION-LEVEL SORT, 2) -Reducer 26 <- Map 25 (PARTITION-LEVEL SORT, 2), Map 28 (PARTITION-LEVEL SORT, 2) -Reducer 27 <- Reducer 26 (GROUP, 2) -Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 336), Reducer 9 (PARTITION-LEVEL SORT, 336) -Reducer 30 <- Map 29 (PARTITION-LEVEL SORT, 159), Map 33 (PARTITION-LEVEL SORT, 159) -Reducer 31 <- Reducer 30 (PARTITION-LEVEL SORT, 169), Reducer 35 (PARTITION-LEVEL SORT, 169) -Reducer 32 <- Reducer 31 (GROUP, 186) -Reducer 35 <- Map 34 (PARTITION-LEVEL SORT, 2), Reducer 40 (PARTITION-LEVEL SORT, 2) -Reducer 39 <- Map 38 (PARTITION-LEVEL SORT, 2), Map 41 (PARTITION-LEVEL SORT, 2) -Reducer 4 <- Reducer 3 (GROUP, 369) -Reducer 40 <- Reducer 39 (GROUP, 2) -Reducer 5 <- Reducer 19 (PARTITION-LEVEL SORT, 518), Reducer 32 (PARTITION-LEVEL SORT, 518), Reducer 4 (PARTITION-LEVEL SORT, 518) -Reducer 6 <- Reducer 5 (SORT, 1) -Reducer 9 <- Map 8 (PARTITION-LEVEL SORT, 2), Reducer 14 (PARTITION-LEVEL SORT, 2) +Reducer 10 <- Reducer
[06/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query65.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query65.q.out b/ql/src/test/results/clientpositive/perf/spark/query65.q.out index 3b3baef..575fc5c 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query65.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query65.q.out @@ -57,7 +57,8 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 - Stage-1 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-4 Stage-0 depends on stages: Stage-1 STAGE PLANS: @@ -65,6 +66,29 @@ STAGE PLANS: Spark A masked pattern was here Vertices: +Map 11 +Map Operator Tree: +TableScan + alias: store + Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: s_store_sk is not null (type: boolean) +Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: s_store_sk (type: int), s_store_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator +keys: + 0 _col0 (type: int) + 1 _col0 (type: int) +Local Work: + Map Reduce Local Work + + Stage: Stage-3 +Spark + A masked pattern was here + Vertices: Map 6 Map Operator Tree: TableScan @@ -84,7 +108,7 @@ STAGE PLANS: Local Work: Map Reduce Local Work - Stage: Stage-3 + Stage: Stage-4 Spark A masked pattern was here Vertices: @@ -111,8 +135,8 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 437) -Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 328), Reducer 2 (PARTITION-LEVEL SORT, 328), Reducer 8 (PARTITION-LEVEL SORT, 328) -Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 166), Reducer 3 (PARTITION-LEVEL SORT, 166) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 328), Reducer 8 (PARTITION-LEVEL SORT, 328) +Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 86), Reducer 3 (PARTITION-LEVEL SORT, 86) Reducer 5 <- Reducer 4 (SORT, 1) Reducer 8 <- Map 7 (GROUP PARTITION-LEVEL SORT, 437) A masked pattern was here @@ -156,24 +180,6 @@ STAGE PLANS: Map 10 Map Operator Tree: TableScan - alias: store - Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: s_store_sk is not null (type: boolean) -Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: s_store_sk (type: int), s_store_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col0 (type: int) -sort order: + -Map-reduce partition columns: _col0 (type: int) -Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE -value expressions: _col1 (type: string) -Map 11 -Map Operator Tree: -TableScan alias: item Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -244,23 +250,23 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 keys: 0 _col0 (type: int) 1 _col0 (type: int) - 2 _col0 (type: int) -outputColumnNames: _col1, _col2, _col4, _col6 -Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE +outputColumnNames: _col0, _col1, _col2, _col4 +Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE Filter Operator predica
[09/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/328d3f93/ql/src/test/results/clientpositive/perf/spark/query47.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query47.q.out b/ql/src/test/results/clientpositive/perf/spark/query47.q.out index 5175f80..bc97fdf 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query47.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query47.q.out @@ -397,47 +397,47 @@ STAGE PLANS: Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col3) -keys: _col5 (type: int), _col6 (type: int), _col8 (type: string), _col9 (type: string), _col11 (type: string), _col12 (type: string) +keys: _col11 (type: string), _col12 (type: string), _col8 (type: string), _col9 (type: string), _col5 (type: int), _col6 (type: int) mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int) sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int) Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE value expressions: _col6 (type: decimal(17,2)) Reducer 14 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) -keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string) +keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._col5 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col0 (type: int) + key expressions: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: int) sort order: + - Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col0 (type: int) + Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: int) Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col6 (type: decimal(17,2)) + value expressions: _col5 (type: int), _col6 (type: decimal(17,2)) Reducer 15 Reduce Operator Tree: Select Operator -expressions: KEY.reducesinkkey4 (type: int), VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), VALUE._col1 (type: decimal(17,2)) +expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey4 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: decimal(17,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition input alias: ptf_0 -output shape: _col0: int, _col1: int, _col2: string, _col3: string, _col4: string
[06/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query65.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query65.q.out b/ql/src/test/results/clientpositive/perf/spark/query65.q.out index 3b3baef..575fc5c 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query65.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query65.q.out @@ -57,7 +57,8 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 - Stage-1 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-4 Stage-0 depends on stages: Stage-1 STAGE PLANS: @@ -65,6 +66,29 @@ STAGE PLANS: Spark A masked pattern was here Vertices: +Map 11 +Map Operator Tree: +TableScan + alias: store + Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: s_store_sk is not null (type: boolean) +Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: s_store_sk (type: int), s_store_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + Spark HashTable Sink Operator +keys: + 0 _col0 (type: int) + 1 _col0 (type: int) +Local Work: + Map Reduce Local Work + + Stage: Stage-3 +Spark + A masked pattern was here + Vertices: Map 6 Map Operator Tree: TableScan @@ -84,7 +108,7 @@ STAGE PLANS: Local Work: Map Reduce Local Work - Stage: Stage-3 + Stage: Stage-4 Spark A masked pattern was here Vertices: @@ -111,8 +135,8 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 437) -Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 328), Reducer 2 (PARTITION-LEVEL SORT, 328), Reducer 8 (PARTITION-LEVEL SORT, 328) -Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 166), Reducer 3 (PARTITION-LEVEL SORT, 166) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 328), Reducer 8 (PARTITION-LEVEL SORT, 328) +Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 86), Reducer 3 (PARTITION-LEVEL SORT, 86) Reducer 5 <- Reducer 4 (SORT, 1) Reducer 8 <- Map 7 (GROUP PARTITION-LEVEL SORT, 437) A masked pattern was here @@ -156,24 +180,6 @@ STAGE PLANS: Map 10 Map Operator Tree: TableScan - alias: store - Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: s_store_sk is not null (type: boolean) -Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: s_store_sk (type: int), s_store_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col0 (type: int) -sort order: + -Map-reduce partition columns: _col0 (type: int) -Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE -value expressions: _col1 (type: string) -Map 11 -Map Operator Tree: -TableScan alias: item Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -244,23 +250,23 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 keys: 0 _col0 (type: int) 1 _col0 (type: int) - 2 _col0 (type: int) -outputColumnNames: _col1, _col2, _col4, _col6 -Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE +outputColumnNames: _col0, _col1, _col2, _col4 +Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE Filter Operator predica
[02/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query85.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query85.q.out
b/ql/src/test/results/clientpositive/perf/spark/query85.q.out
index d60751c..c82dcf7 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query85.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query85.q.out
@@ -175,7 +175,7 @@ STAGE PLANS:
Spark
A masked pattern was here
Vertices:
-Map 13
+Map 15
Map Operator Tree:
TableScan
alias: reason
@@ -212,21 +212,21 @@ STAGE PLANS:
Statistics: Num rows: 4602 Data size: 2696178 Basic
stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col10 (type: int)
- 1 _col0 (type: int)
+ 0 _col0 (type: int)
+ 1 _col2 (type: int)
Local Work:
Map Reduce Local Work
Stage: Stage-1
Spark
Edges:
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 28), Map 9 (PARTITION-LEVEL
SORT, 28)
-Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 98), Reducer 2
(PARTITION-LEVEL SORT, 98)
-Reducer 4 <- Map 12 (PARTITION-LEVEL SORT, 5), Reducer 3
(PARTITION-LEVEL SORT, 5)
-Reducer 5 <- Map 14 (PARTITION-LEVEL SORT, 11), Reducer 4
(PARTITION-LEVEL SORT, 11)
-Reducer 6 <- Map 15 (PARTITION-LEVEL SORT, 7), Reducer 5
(PARTITION-LEVEL SORT, 7)
-Reducer 7 <- Reducer 6 (GROUP, 7)
-Reducer 8 <- Reducer 7 (SORT, 1)
+Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 20), Map 14
(PARTITION-LEVEL SORT, 20)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 90), Map 8 (PARTITION-LEVEL
SORT, 90)
+Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 17), Reducer 2
(PARTITION-LEVEL SORT, 17)
+Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 19), Reducer 3
(PARTITION-LEVEL SORT, 19)
+Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 35), Reducer 4
(PARTITION-LEVEL SORT, 35)
+Reducer 6 <- Reducer 5 (GROUP, 2)
+Reducer 7 <- Reducer 6 (SORT, 1)
A masked pattern was here
Vertices:
Map 1
@@ -242,32 +242,61 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4,
_col5, _col6, _col7
Statistics: Num rows: 14398467 Data size: 1325194184
Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
-key expressions: _col0 (type: int), _col5 (type: int)
-sort order: ++
-Map-reduce partition columns: _col0 (type: int), _col5
(type: int)
+key expressions: _col2 (type: int)
+sort order: +
+Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 14398467 Data size: 1325194184
Basic stats: COMPLETE Column stats: NONE
-value expressions: _col1 (type: int), _col2 (type:
int), _col3 (type: int), _col4 (type: int), _col6 (type: decimal(7,2)), _col7
(type: decimal(7,2))
+value expressions: _col0 (type: int), _col1 (type:
int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type:
decimal(7,2)), _col7 (type: decimal(7,2))
Map 10
Map Operator Tree:
TableScan
- alias: customer_address
- Statistics: Num rows: 4000 Data size: 40595195284 Basic
stats: COMPLETE Column stats: NONE
+ alias: cd2
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic
stats: COMPLETE Column stats: NONE
Filter Operator
-predicate: ((ca_country = 'United States') and (ca_state)
IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and ca_address_sk is
not null) (type: boolean)
-Statistics: Num rows: 1000 Data size: 10148798821
Basic stats: COMPLETE Column stats: NONE
+predicate: (((cd_education_status = '4 yr Degree') or
(cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree'))
and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or
(cd_marital_status = 'U')) and cd_demo_sk is not null and cd_education_status
is not null and cd_marital_status is not null) (type: boolean)
+Statistics: Num rows: 1861800 Data size: 717186159 Basic
stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ca_address_sk (type: int), ca_state (type:
string)
- outputColumnNames: _col0, _col1
- Statistics: Num r
[09/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query47.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query47.q.out b/ql/src/test/results/clientpositive/perf/spark/query47.q.out index 5175f80..bc97fdf 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query47.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query47.q.out @@ -397,47 +397,47 @@ STAGE PLANS: Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col3) -keys: _col5 (type: int), _col6 (type: int), _col8 (type: string), _col9 (type: string), _col11 (type: string), _col12 (type: string) +keys: _col11 (type: string), _col12 (type: string), _col8 (type: string), _col9 (type: string), _col5 (type: int), _col6 (type: int) mode: hash outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int) sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int) Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE value expressions: _col6 (type: decimal(17,2)) Reducer 14 Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) -keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string) +keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._col5 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col0 (type: int) + key expressions: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: int) sort order: + - Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col0 (type: int) + Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: int) Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: int), _col6 (type: decimal(17,2)) + value expressions: _col5 (type: int), _col6 (type: decimal(17,2)) Reducer 15 Reduce Operator Tree: Select Operator -expressions: KEY.reducesinkkey4 (type: int), VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), VALUE._col1 (type: decimal(17,2)) +expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey4 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: decimal(17,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition input alias: ptf_0 -output shape: _col0: int, _col1: int, _col2: string, _col3: string, _col4: string
[04/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query78.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query78.q.out b/ql/src/test/results/clientpositive/perf/spark/query78.q.out index 1467c5f..c2e8577 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query78.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query78.q.out @@ -120,57 +120,41 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 14 (PARTITION-LEVEL SORT, 85) -Reducer 12 <- Reducer 11 (GROUP, 93) -Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 164), Map 15 (PARTITION-LEVEL SORT, 164) -Reducer 17 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 20 (PARTITION-LEVEL SORT, 85) +Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 164), Map 14 (PARTITION-LEVEL SORT, 164) +Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 85), Reducer 11 (PARTITION-LEVEL SORT, 85) +Reducer 13 <- Reducer 12 (GROUP, 93) +Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 85), Reducer 20 (PARTITION-LEVEL SORT, 85) Reducer 18 <- Reducer 17 (GROUP, 93) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 219), Reducer 8 (PARTITION-LEVEL SORT, 219) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 432), Map 8 (PARTITION-LEVEL SORT, 432) Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 177), Map 21 (PARTITION-LEVEL SORT, 177) -Reducer 3 <- Reducer 2 (GROUP, 241) -Reducer 4 <- Reducer 12 (PARTITION-LEVEL SORT, 167), Reducer 3 (PARTITION-LEVEL SORT, 167) -Reducer 5 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 4 (PARTITION-LEVEL SORT, 91) -Reducer 6 <- Reducer 5 (SORT, 1) -Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 432), Map 9 (PARTITION-LEVEL SORT, 432) +Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 219), Reducer 2 (PARTITION-LEVEL SORT, 219) +Reducer 4 <- Reducer 3 (GROUP, 241) +Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 167), Reducer 4 (PARTITION-LEVEL SORT, 167) +Reducer 6 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 5 (PARTITION-LEVEL SORT, 91) +Reducer 7 <- Reducer 6 (SORT, 1) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE + alias: store_sales + Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: ((d_year = 2000) and d_date_sk is not null) (type: boolean) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE +predicate: ss_sold_date_sk is not null (type: boolean) +Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE + expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator -key expressions: _col0 (type: int) -sort order: + -Map-reduce partition columns: _col0 (type: int) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE +key expressions: _col1 (type: int), _col3 (type: int) +sort order: ++ +Map-reduce partition columns: _col1 (type: int), _col3 (type: int) +Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)) Map 10 Map Operator Tree: TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE - Filter Operator -
[05/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query75.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query75.q.out b/ql/src/test/results/clientpositive/perf/spark/query75.q.out index 4b76570..f4fca34 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query75.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query75.q.out @@ -144,25 +144,25 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 398), Map 16 (PARTITION-LEVEL SORT, 398) -Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 440), Reducer 13 (PARTITION-LEVEL SORT, 440) +Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 400), Map 16 (PARTITION-LEVEL SORT, 400) +Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 438), Reducer 13 (PARTITION-LEVEL SORT, 438) Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 516), Reducer 14 (PARTITION-LEVEL SORT, 516) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 16 (PARTITION-LEVEL SORT, 306) -Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 154), Map 19 (PARTITION-LEVEL SORT, 154) -Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 171), Reducer 20 (PARTITION-LEVEL SORT, 171) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 308), Map 16 (PARTITION-LEVEL SORT, 308) +Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 156), Map 19 (PARTITION-LEVEL SORT, 156) +Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 169), Reducer 20 (PARTITION-LEVEL SORT, 169) Reducer 22 <- Map 25 (PARTITION-LEVEL SORT, 196), Reducer 21 (PARTITION-LEVEL SORT, 196) -Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 306), Map 32 (PARTITION-LEVEL SORT, 306) -Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 338), Reducer 27 (PARTITION-LEVEL SORT, 338) +Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 308), Map 32 (PARTITION-LEVEL SORT, 308) +Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 336), Reducer 27 (PARTITION-LEVEL SORT, 336) Reducer 29 <- Map 34 (PARTITION-LEVEL SORT, 393), Reducer 28 (PARTITION-LEVEL SORT, 393) -Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 338), Reducer 2 (PARTITION-LEVEL SORT, 338) +Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 336), Reducer 2 (PARTITION-LEVEL SORT, 336) Reducer 30 <- Reducer 29 (GROUP, 934), Reducer 38 (GROUP, 934) Reducer 31 <- Reducer 30 (GROUP PARTITION-LEVEL SORT, 671), Reducer 45 (GROUP PARTITION-LEVEL SORT, 671) -Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 398), Map 35 (PARTITION-LEVEL SORT, 398) -Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 440), Reducer 36 (PARTITION-LEVEL SORT, 440) +Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 400), Map 35 (PARTITION-LEVEL SORT, 400) +Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 438), Reducer 36 (PARTITION-LEVEL SORT, 438) Reducer 38 <- Map 41 (PARTITION-LEVEL SORT, 516), Reducer 37 (PARTITION-LEVEL SORT, 516) Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 393), Reducer 3 (PARTITION-LEVEL SORT, 393) -Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 154), Map 42 (PARTITION-LEVEL SORT, 154) -Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 171), Reducer 43 (PARTITION-LEVEL SORT, 171) +Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 156), Map 42 (PARTITION-LEVEL SORT, 156) +Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 169), Reducer 43 (PARTITION-LEVEL SORT, 169) Reducer 45 <- Map 48 (PARTITION-LEVEL SORT, 196), Reducer 44 (PARTITION-LEVEL SORT, 196) Reducer 5 <- Reducer 15 (GROUP, 934), Reducer 4 (GROUP, 934) Reducer 6 <- Reducer 22 (GROUP PARTITION-LEVEL SORT, 671), Reducer 5 (GROUP PARTITION-LEVEL SORT, 671) @@ -183,29 +183,28 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator -key expressions: _col0 (type: int) +key expressions: _col1 (type: int) sort order: + -Map-reduce partition columns: _col0 (type: int) +Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE -value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)) +value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)) Map 10 Map Operator Tree: TableScan - alias: item - Statistics: Num rows: 462000 Data size: 663560457 Basic stats:
[10/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query40.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query40.q.out b/ql/src/test/results/clientpositive/perf/spark/query40.q.out index f286294..5360385 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query40.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query40.q.out @@ -54,8 +54,7 @@ limit 100 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-3 depends on stages: Stage-2 - Stage-1 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 Stage-0 depends on stages: Stage-1 STAGE PLANS: @@ -63,44 +62,39 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 9 +Map 8 Map Operator Tree: TableScan - alias: warehouse - Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE + alias: date_dim + Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: w_warehouse_sk is not null (type: boolean) -Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE +predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-03-08 23:00:00.0' AND TIMESTAMP'1998-05-08 00:00:00.0' and d_date_sk is not null) (type: boolean) +Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: w_warehouse_sk (type: int), w_state (type: string) + expressions: d_date_sk (type: int), d_date (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col1 (type: int) + 0 _col0 (type: int) 1 _col0 (type: int) Local Work: Map Reduce Local Work - - Stage: Stage-3 -Spark - A masked pattern was here - Vertices: -Map 7 +Map 9 Map Operator Tree: TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE + alias: warehouse + Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-03-08 23:00:00.0' AND TIMESTAMP'1998-05-08 00:00:00.0' and d_date_sk is not null) (type: boolean) -Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE +predicate: w_warehouse_sk is not null (type: boolean) +Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: d_date_sk (type: int), d_date (type: string) + expressions: w_warehouse_sk (type: int), w_state (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: int) + 0 _col1 (type: int) 1 _col0 (type: int) Local Work: Map Reduce Local Work @@ -109,7 +103,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 329), Map 6 (PARTITION-LEVEL SORT, 329) -Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 370), Reducer 2 (PARTITION-LEVEL SORT, 370) +Reducer 3 <- Map 7 (PARTITION-LEVEL SORT, 336), Reducer 2 (PARTITION-LEVEL SORT, 336) Reducer 4 <- Reducer 3 (GROUP, 447) Reducer 5 <- Reducer 4 (SORT, 1) A masked pattern was here @@ -150,7 +144,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type:
[12/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query24.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query24.q.out b/ql/src/test/results/clientpositive/perf/spark/query24.q.out index 1f291c0..13ac1e8 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query24.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query24.q.out @@ -131,10 +131,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: -Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 975), Map 20 (PARTITION-LEVEL SORT, 975) -Reducer 14 <- Map 21 (PARTITION-LEVEL SORT, 486), Reducer 13 (PARTITION-LEVEL SORT, 486) -Reducer 15 <- Map 22 (PARTITION-LEVEL SORT, 564), Reducer 14 (PARTITION-LEVEL SORT, 564) -Reducer 16 <- Map 23 (PARTITION-LEVEL SORT, 899), Reducer 15 (PARTITION-LEVEL SORT, 899) +Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 472), Map 20 (PARTITION-LEVEL SORT, 472) +Reducer 14 <- Map 21 (PARTITION-LEVEL SORT, 1009), Reducer 13 (PARTITION-LEVEL SORT, 1009) +Reducer 15 <- Map 22 (PARTITION-LEVEL SORT, 846), Reducer 14 (PARTITION-LEVEL SORT, 846) +Reducer 16 <- Map 23 (PARTITION-LEVEL SORT, 587), Reducer 15 (PARTITION-LEVEL SORT, 587) Reducer 17 <- Reducer 16 (GROUP, 640) Reducer 18 <- Reducer 17 (GROUP, 1) A masked pattern was here @@ -162,16 +162,33 @@ STAGE PLANS: 1 Map 19 Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) + key expressions: _col0 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col3 (type: int) Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string) + value expressions: _col1 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string) Local Work: Map Reduce Local Work Map 20 Map Operator Tree: TableScan + alias: store_returns + Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean) +Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: sr_item_sk (type: int), sr_ticket_number (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col0 (type: int), _col1 (type: int) +sort order: ++ +Map-reduce partition columns: _col0 (type: int), _col1 (type: int) +Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE +Map 21 +Map Operator Tree: +TableScan alias: customer Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -187,44 +204,9 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) -Map 21 -Map Operator Tree: -TableScan - alias: item - Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: i_item_sk is not null (type: boolean) -Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: i_item_sk (type: int), i_current_price (type: decimal(7,2)), i_size (type: string),
[07/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query58.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query58.q.out b/ql/src/test/results/clientpositive/perf/spark/query58.q.out index f06cbef..eb5dffe 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query58.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query58.q.out @@ -1,6 +1,6 @@ -Warning: Map Join MAPJOIN[183][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[184][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[185][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[180][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[181][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[182][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain with ss_items as (select i_item_id item_id @@ -140,10 +140,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: -Reducer 11 <- Map 10 (GROUP, 1) +Reducer 12 <- Map 11 (GROUP, 1) A masked pattern was here Vertices: -Map 10 +Map 11 Map Operator Tree: TableScan alias: date_dim @@ -162,7 +162,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 11 +Reducer 12 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -184,10 +184,10 @@ STAGE PLANS: Stage: Stage-3 Spark Edges: -Reducer 24 <- Map 23 (GROUP, 1) +Reducer 25 <- Map 24 (GROUP, 1) A masked pattern was here Vertices: -Map 23 +Map 24 Map Operator Tree: TableScan alias: date_dim @@ -206,7 +206,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 24 +Reducer 25 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -228,10 +228,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: -Reducer 37 <- Map 36 (GROUP, 1) +Reducer 38 <- Map 37 (GROUP, 1) A masked pattern was here Vertices: -Map 36 +Map 37 Map Operator Tree: TableScan alias: date_dim @@ -250,7 +250,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 37 +Reducer 38 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -272,76 +272,47 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 2), Map 15 (PARTITION-LEVEL SORT, 2) -Reducer 14 <- Reducer 13 (GROUP, 2) -Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 403), Map 20 (PARTITION-LEVEL SORT, 403) -Reducer 18 <- Reducer 17 (PARTITION-LEVEL SORT, 438), Reducer 22 (PARTITION-LEVEL SORT, 438) -Reducer 19 <- Reducer 18 (GROUP, 481) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 310), Map 7 (PARTITION-LEVEL SORT, 310) -Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 2), Reducer 27 (PARTITION-LEVEL SORT, 2) -Reducer 26 <- Map 25 (PARTITION-LEVEL SORT, 2), Map 28 (PARTITION-LEVEL SORT, 2) -Reducer 27 <- Reducer 26 (GROUP, 2) -Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 336), Reducer 9 (PARTITION-LEVEL SORT, 336) -Reducer 30 <- Map 29 (PARTITION-LEVEL SORT, 159), Map 33 (PARTITION-LEVEL SORT, 159) -Reducer 31 <- Reducer 30 (PARTITION-LEVEL SORT, 169), Reducer 35 (PARTITION-LEVEL SORT, 169) -Reducer 32 <- Reducer 31 (GROUP, 186) -Reducer 35 <- Map 34 (PARTITION-LEVEL SORT, 2), Reducer 40 (PARTITION-LEVEL SORT, 2) -Reducer 39 <- Map 38 (PARTITION-LEVEL SORT, 2), Map 41 (PARTITION-LEVEL SORT, 2) -Reducer 4 <- Reducer 3 (GROUP, 369) -Reducer 40 <- Reducer 39 (GROUP, 2) -Reducer 5 <- Reducer 19 (PARTITION-LEVEL SORT, 518), Reducer 32 (PARTITION-LEVEL SORT, 518), Reducer 4 (PARTITION-LEVEL SORT, 518) -Reducer 6 <- Reducer 5 (SORT, 1) -Reducer 9 <- Map 8 (PARTITION-LEVEL SORT, 2), Reducer 14 (PARTITION-LEVEL SORT, 2) +Reducer 10 <- Reducer
[13/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
HIVE-19128 : Update golden files for spark perf tests Signed-off-by: Ashutosh Chauhan Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9ce42cba Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9ce42cba Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9ce42cba Branch: refs/heads/branch-3 Commit: 9ce42cba8dfa21855463977f0d2006c77dd04c73 Parents: 43cb101 Author: Ashutosh Chauhan Authored: Mon Apr 9 14:57:11 2018 -0700 Committer: Ashutosh Chauhan Committed: Mon Apr 9 15:07:02 2018 -0700 -- .../clientpositive/perf/spark/query11.q.out | 48 +- .../clientpositive/perf/spark/query15.q.out | 164 +++ .../clientpositive/perf/spark/query16.q.out | 70 ++- .../clientpositive/perf/spark/query18.q.out | 216 .../clientpositive/perf/spark/query19.q.out | 218 - .../clientpositive/perf/spark/query21.q.out | 114 ++--- .../clientpositive/perf/spark/query24.q.out | 282 +-- .../clientpositive/perf/spark/query25.q.out | 118 ++--- .../clientpositive/perf/spark/query29.q.out | 280 ++- .../clientpositive/perf/spark/query30.q.out | 262 +- .../clientpositive/perf/spark/query32.q.out | 132 ++--- .../clientpositive/perf/spark/query34.q.out | 34 +- .../clientpositive/perf/spark/query35.q.out | 74 +-- .../clientpositive/perf/spark/query37.q.out | 16 +- .../clientpositive/perf/spark/query4.q.out | 214 .../clientpositive/perf/spark/query40.q.out | 116 ++--- .../clientpositive/perf/spark/query44.q.out | 246 +- .../clientpositive/perf/spark/query45.q.out | 204 .../clientpositive/perf/spark/query46.q.out | 104 ++-- .../clientpositive/perf/spark/query47.q.out | 92 ++-- .../clientpositive/perf/spark/query48.q.out | 94 ++-- .../clientpositive/perf/spark/query5.q.out | 38 +- .../clientpositive/perf/spark/query50.q.out | 196 .../clientpositive/perf/spark/query53.q.out | 105 ++-- .../clientpositive/perf/spark/query54.q.out | 277 +-- .../clientpositive/perf/spark/query57.q.out | 92 ++-- .../clientpositive/perf/spark/query58.q.out | 490 +-- .../clientpositive/perf/spark/query6.q.out | 350 ++--- .../clientpositive/perf/spark/query61.q.out | 40 +- .../clientpositive/perf/spark/query63.q.out | 105 ++-- .../clientpositive/perf/spark/query65.q.out | 100 ++-- .../clientpositive/perf/spark/query66.q.out | 20 +- .../clientpositive/perf/spark/query67.q.out | 137 +++--- .../clientpositive/perf/spark/query68.q.out | 104 ++-- .../clientpositive/perf/spark/query72.q.out | 461 - .../clientpositive/perf/spark/query73.q.out | 34 +- .../clientpositive/perf/spark/query75.q.out | 248 +- .../clientpositive/perf/spark/query76.q.out | 130 ++--- .../clientpositive/perf/spark/query77.q.out | 66 +-- .../clientpositive/perf/spark/query78.q.out | 273 ++- .../clientpositive/perf/spark/query79.q.out | 10 +- .../clientpositive/perf/spark/query8.q.out | 174 +++ .../clientpositive/perf/spark/query80.q.out | 417 .../clientpositive/perf/spark/query81.q.out | 270 +- .../clientpositive/perf/spark/query82.q.out | 16 +- .../clientpositive/perf/spark/query83.q.out | 354 +++--- .../clientpositive/perf/spark/query85.q.out | 309 ++-- .../clientpositive/perf/spark/query88.q.out | 272 +- .../clientpositive/perf/spark/query89.q.out | 26 +- .../clientpositive/perf/spark/query90.q.out | 68 +-- .../clientpositive/perf/spark/query91.q.out | 14 +- .../clientpositive/perf/spark/query92.q.out | 106 ++-- .../clientpositive/perf/spark/query94.q.out | 70 ++- .../clientpositive/perf/spark/query95.q.out | 100 ++-- .../clientpositive/perf/spark/query97.q.out | 54 +- .../clientpositive/perf/spark/query99.q.out | 42 +- 56 files changed, 4313 insertions(+), 4353 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query11.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query11.q.out b/ql/src/test/results/clientpositive/perf/spark/query11.q.out index e46aa21..227068c 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query11.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query11.q.out @@ -300,7 +300,7 @@ STAGE PLANS: alias: date_dim Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: ((d
[03/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query81.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query81.q.out b/ql/src/test/results/clientpositive/perf/spark/query81.q.out index 5b2d5b3..23998d6 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query81.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query81.q.out @@ -66,72 +66,37 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 10 <- Reducer 16 (PARTITION-LEVEL SORT, 262), Reducer 9 (PARTITION-LEVEL SORT, 262) -Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 25), Map 17 (PARTITION-LEVEL SORT, 25) -Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 344), Reducer 14 (PARTITION-LEVEL SORT, 344) -Reducer 16 <- Reducer 15 (GROUP PARTITION-LEVEL SORT, 349) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 697), Map 5 (PARTITION-LEVEL SORT, 697) -Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 656), Reducer 2 (PARTITION-LEVEL SORT, 656) -Reducer 4 <- Reducer 3 (SORT, 1) -Reducer 7 <- Map 11 (PARTITION-LEVEL SORT, 25), Map 6 (PARTITION-LEVEL SORT, 25) -Reducer 8 <- Map 12 (PARTITION-LEVEL SORT, 344), Reducer 7 (PARTITION-LEVEL SORT, 344) -Reducer 9 <- Reducer 8 (GROUP, 349) +Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 25), Map 14 (PARTITION-LEVEL SORT, 25) +Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 344), Reducer 11 (PARTITION-LEVEL SORT, 344) +Reducer 13 <- Reducer 12 (GROUP PARTITION-LEVEL SORT, 349) +Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 697), Map 18 (PARTITION-LEVEL SORT, 697) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 25), Map 8 (PARTITION-LEVEL SORT, 25) +Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 344), Reducer 2 (PARTITION-LEVEL SORT, 344) +Reducer 4 <- Reducer 3 (GROUP, 349) +Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 262), Reducer 4 (PARTITION-LEVEL SORT, 262) +Reducer 6 <- Reducer 17 (PARTITION-LEVEL SORT, 656), Reducer 5 (PARTITION-LEVEL SORT, 656) +Reducer 7 <- Reducer 6 (SORT, 1) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: customer - Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean) -Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: c_customer_sk (type: int), c_customer_id (type: string), c_current_addr_sk (type: int), c_salutation (type: string), c_first_name (type: string), c_last_name (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col2 (type: int) -sort order: + -Map-reduce partition columns: _col2 (type: int) -Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) -Map 11 -Map Operator Tree: -TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: ((d_year = 1998) and d_date_sk is not null) (type: boolean) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col0 (type: int) -sort order: + -Map-reduce partition columns: _col0 (type: int) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE -Map 12 -Map Operator Tree: -TableScan - alias: customer_address - Statistics: Num rows: 4000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE +
[08/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query54.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query54.q.out b/ql/src/test/results/clientpositive/perf/spark/query54.q.out index 43132bc..251d7ad 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query54.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query54.q.out @@ -1,7 +1,7 @@ -Warning: Shuffle Join JOIN[111][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product -Warning: Shuffle Join JOIN[107][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Work 'Reducer 14' is a cross product -Warning: Shuffle Join JOIN[114][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 4' is a cross product -Warning: Map Join MAPJOIN[144][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Shuffle Join JOIN[111][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product +Warning: Shuffle Join JOIN[104][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Work 'Reducer 14' is a cross product +Warning: Shuffle Join JOIN[114][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 5' is a cross product +Warning: Map Join MAPJOIN[143][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain with my_customers as ( select distinct c_customer_sk @@ -122,11 +122,11 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: -Reducer 29 <- Map 28 (GROUP, 2) -Reducer 30 <- Reducer 29 (GROUP, 1) +Reducer 31 <- Map 30 (GROUP, 2) +Reducer 32 <- Reducer 31 (GROUP, 1) A masked pattern was here Vertices: -Map 28 +Map 30 Map Operator Tree: TableScan alias: date_dim @@ -148,7 +148,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE -Reducer 29 +Reducer 31 Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int) @@ -166,7 +166,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 30 +Reducer 32 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -211,23 +211,23 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 10 <- Reducer 9 (GROUP, 1) +Reducer 10 <- Map 1 (GROUP, 2) Reducer 12 <- Map 11 (PARTITION-LEVEL SORT, 398), Map 15 (PARTITION-LEVEL SORT, 398) Reducer 13 <- Reducer 12 (PARTITION-LEVEL SORT, 772), Reducer 17 (PARTITION-LEVEL SORT, 772) -Reducer 14 <- Reducer 13 (PARTITION-LEVEL SORT, 1), Reducer 32 (PARTITION-LEVEL SORT, 1) +Reducer 14 <- Reducer 13 (PARTITION-LEVEL SORT, 1), Reducer 29 (PARTITION-LEVEL SORT, 1) Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 654), Reducer 23 (PARTITION-LEVEL SORT, 654) Reducer 2 <- Map 1 (GROUP, 2) -Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 458), Map 24 (PARTITION-LEVEL SORT, 458), Map 25 (PARTITION-LEVEL SORT, 458) -Reducer 21 <- Map 26 (PARTITION-LEVEL SORT, 505), Reducer 20 (PARTITION-LEVEL SORT, 505) +Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 459), Map 24 (PARTITION-LEVEL SORT, 459), Map 25 (PARTITION-LEVEL SORT, 459) +Reducer 21 <- Map 26 (PARTITION-LEVEL SORT, 504), Reducer 20 (PARTITION-LEVEL SORT, 504) Reducer 22 <- Map 27 (PARTITION-LEVEL SORT, 1009), Reducer 21 (PARTITION-LEVEL SORT, 1009) Reducer 23 <- Reducer 22 (GROUP, 610) -Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) -Reducer 32 <- Map 31 (GROUP, 2) -Reducer 4 <- Reducer 14 (PARTITION-LEVEL SORT, 1), Reducer 3 (PARTITION-LEVEL SORT, 1) -Reducer 5 <- Reducer 4 (GROUP, 1009) +Reducer 29 <- Map 28 (GROUP, 2) +Reducer 3 <- Reducer 2 (GROUP, 1) +Reducer 4 <- Reducer 10 (PARTITION-LEVEL SORT, 1), Reducer 3 (PARTITION-LEVEL SORT, 1) +Reducer 5 <- Reducer 14 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1) Reducer 6 <- Reducer 5 (GROUP, 1009) -Reducer 7 <- Reducer 6 (SORT, 1) -Reducer 9 <- Map 1 (GROUP, 2) +Reducer 7 <- Reducer 6 (GROUP, 1009) +Reducer 8 <- Reducer 7 (SORT, 1) A masked pattern was here Vertices: Map 1 @@ -330,11 +330,11 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 287989836 Data size: 38999608952 Basic stat
[01/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
Repository: hive Updated Branches: refs/heads/branch-3 43cb101ea -> 9ce42cba8 http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query92.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query92.q.out b/ql/src/test/results/clientpositive/perf/spark/query92.q.out index e7b8632..70c4c5a 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query92.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query92.q.out @@ -67,7 +67,7 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 5 +Map 7 Map Operator Tree: TableScan alias: date_dim @@ -90,7 +90,7 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 9 +Map 10 Map Operator Tree: TableScan alias: date_dim @@ -112,11 +112,11 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 261), Reducer 8 (PARTITION-LEVEL SORT, 261) -Reducer 3 <- Reducer 2 (GROUP, 1) -Reducer 4 <- Reducer 3 (SORT, 1) -Reducer 7 <- Map 6 (GROUP, 169) -Reducer 8 <- Map 10 (PARTITION-LEVEL SORT, 87), Reducer 7 (PARTITION-LEVEL SORT, 87) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 156), Map 6 (PARTITION-LEVEL SORT, 156) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 270), Reducer 9 (PARTITION-LEVEL SORT, 270) +Reducer 4 <- Reducer 3 (GROUP, 1) +Reducer 5 <- Reducer 4 (SORT, 1) +Reducer 9 <- Map 8 (GROUP, 169) A masked pattern was here Vertices: Map 1 @@ -131,25 +131,13 @@ STAGE PLANS: expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_ext_discount_amt (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE - Map Join Operator -condition map: - Inner Join 0 to 1 -keys: - 0 _col0 (type: int) - 1 _col0 (type: int) -outputColumnNames: _col1, _col2 -input vertices: - 1 Map 5 -Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE -Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: decimal(7,2)) -Local Work: - Map Reduce Local Work -Map 10 + Reduce Output Operator +key expressions: _col1 (type: int) +sort order: + +Map-reduce partition columns: _col1 (type: int) +Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: int), _col2 (type: decimal(7,2)) +Map 6 Map Operator Tree: TableScan alias: item @@ -166,7 +154,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE -Map 6 +Map 8 Map Operator Tree: TableScan alias: web_sales @@ -186,7 +174,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2 input vertices: - 1 Map 9 + 1 Map 10 Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col2), count(_col2) @@ -203,22 +191,50 @@ STAGE PLANS: Local Work: Map Reduce Local Work Reducer 2 +Local Work: + Map Reduce Local Work Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: 0 _col1 (type: int) - 1 _col2 (type: int) -outp
[11/13] hive git commit: HIVE-19128 : Update golden files for spark perf tests
http://git-wip-us.apache.org/repos/asf/hive/blob/9ce42cba/ql/src/test/results/clientpositive/perf/spark/query30.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query30.q.out b/ql/src/test/results/clientpositive/perf/spark/query30.q.out index 6385984..399251d 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query30.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query30.q.out @@ -66,72 +66,37 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 10 <- Reducer 16 (PARTITION-LEVEL SORT, 262), Reducer 9 (PARTITION-LEVEL SORT, 262) -Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 11), Map 17 (PARTITION-LEVEL SORT, 11) -Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 329), Reducer 14 (PARTITION-LEVEL SORT, 329) -Reducer 16 <- Reducer 15 (GROUP PARTITION-LEVEL SORT, 349) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 697), Map 5 (PARTITION-LEVEL SORT, 697) -Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 656), Reducer 2 (PARTITION-LEVEL SORT, 656) -Reducer 4 <- Reducer 3 (SORT, 1) -Reducer 7 <- Map 11 (PARTITION-LEVEL SORT, 11), Map 6 (PARTITION-LEVEL SORT, 11) -Reducer 8 <- Map 12 (PARTITION-LEVEL SORT, 329), Reducer 7 (PARTITION-LEVEL SORT, 329) -Reducer 9 <- Reducer 8 (GROUP, 349) +Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 11), Map 14 (PARTITION-LEVEL SORT, 11) +Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 329), Reducer 11 (PARTITION-LEVEL SORT, 329) +Reducer 13 <- Reducer 12 (GROUP PARTITION-LEVEL SORT, 349) +Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 697), Map 18 (PARTITION-LEVEL SORT, 697) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 11), Map 8 (PARTITION-LEVEL SORT, 11) +Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 329), Reducer 2 (PARTITION-LEVEL SORT, 329) +Reducer 4 <- Reducer 3 (GROUP, 349) +Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 262), Reducer 4 (PARTITION-LEVEL SORT, 262) +Reducer 6 <- Reducer 17 (PARTITION-LEVEL SORT, 656), Reducer 5 (PARTITION-LEVEL SORT, 656) +Reducer 7 <- Reducer 6 (SORT, 1) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: customer - Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean) -Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: c_customer_sk (type: int), c_customer_id (type: string), c_current_addr_sk (type: int), c_salutation (type: string), c_first_name (type: string), c_last_name (type: string), c_preferred_cust_flag (type: string), c_birth_day (type: int), c_birth_month (type: int), c_birth_year (type: int), c_birth_country (type: string), c_login (type: string), c_email_address (type: string), c_last_review_date (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col2 (type: int) -sort order: + -Map-reduce partition columns: _col2 (type: int) -Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string) -Map 11 -Map Operator Tree: -TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean) -Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col0 (type: i
hive git commit: Preparing for 3.1.0 development - Updated pom to 3.1.0-SNAPSHOT
Repository: hive Updated Branches: refs/heads/master 328d3f935 -> 109c594a1 Preparing for 3.1.0 development - Updated pom to 3.1.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/109c594a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/109c594a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/109c594a Branch: refs/heads/master Commit: 109c594a1dbe14e27ecb5c6f6bfa3e93f3796f11 Parents: 328d3f9 Author: Vineet Garg Authored: Mon Apr 9 15:17:11 2018 -0700 Committer: Vineet Garg Committed: Mon Apr 9 15:17:11 2018 -0700 -- accumulo-handler/pom.xml | 2 +- beeline/pom.xml | 2 +- classification/pom.xml | 2 +- cli/pom.xml | 2 +- common/pom.xml | 2 +- contrib/pom.xml | 2 +- druid-handler/pom.xml| 2 +- hbase-handler/pom.xml| 2 +- hcatalog/core/pom.xml| 2 +- hcatalog/hcatalog-pig-adapter/pom.xml| 2 +- hcatalog/pom.xml | 2 +- hcatalog/server-extensions/pom.xml | 2 +- hcatalog/streaming/pom.xml | 2 +- hcatalog/webhcat/java-client/pom.xml | 2 +- hcatalog/webhcat/svr/pom.xml | 2 +- hplsql/pom.xml | 2 +- itests/custom-serde/pom.xml | 2 +- itests/custom-udfs/pom.xml | 2 +- itests/custom-udfs/udf-classloader-udf1/pom.xml | 2 +- itests/custom-udfs/udf-classloader-udf2/pom.xml | 2 +- itests/custom-udfs/udf-classloader-util/pom.xml | 2 +- itests/custom-udfs/udf-vectorized-badexample/pom.xml | 2 +- itests/hcatalog-unit/pom.xml | 2 +- itests/hive-blobstore/pom.xml| 2 +- itests/hive-jmh/pom.xml | 2 +- itests/hive-minikdc/pom.xml | 2 +- itests/hive-unit-hadoop2/pom.xml | 2 +- itests/hive-unit/pom.xml | 2 +- itests/pom.xml | 4 ++-- itests/qtest-accumulo/pom.xml| 2 +- itests/qtest-druid/pom.xml | 2 +- itests/qtest-spark/pom.xml | 2 +- itests/qtest/pom.xml | 2 +- itests/test-serde/pom.xml| 2 +- itests/util/pom.xml | 2 +- jdbc-handler/pom.xml | 2 +- jdbc/pom.xml | 2 +- kryo-registrator/pom.xml | 2 +- llap-client/pom.xml | 2 +- llap-common/pom.xml | 2 +- llap-ext-client/pom.xml | 2 +- llap-server/pom.xml | 2 +- llap-tez/pom.xml | 2 +- metastore/pom.xml| 2 +- packaging/pom.xml| 2 +- pom.xml | 2 +- ql/pom.xml | 2 +- serde/pom.xml| 2 +- service-rpc/pom.xml | 2 +- service/pom.xml | 2 +- shims/0.23/pom.xml | 2 +- shims/aggregator/pom.xml | 2 +- shims/common/pom.xml | 2 +- shims/pom.xml| 2 +- shims/scheduler/pom.xml | 2 +- spark-client/pom.xml | 4 ++-- standalone-metastore/pom.xml | 2 +- testutils/pom.xml| 2 +- vector-code-gen/pom.xml | 2 +- 59 files changed, 61 insertions(+), 61 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/109c594a/accumulo-handler/pom.xml -- diff --git a/accumulo-handler/pom.xml b/accumulo-handler/pom.xml index edac1b1..b844c4a 100644 --- a/accumulo-handler/pom.xml +++ b/accumulo-handler/pom.xml @@ -19,7 +19,7 @@ org.apache.hive hive -3.0.0-SNAPSHOT +3.1.0-SNAPSHOT ../pom.xml http://git-wip-us.apache.org/repos/asf/hive/blob/109c594a/beeline/pom.xml -- diff --
hive git commit: HIVE-19014: utilize YARN-8028 (queue ACL check) in Hive Tez session pool (Sergey Shelukhin, reviewed by Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master 109c594a1 -> 76b696c26
HIVE-19014: utilize YARN-8028 (queue ACL check) in Hive Tez session pool
(Sergey Shelukhin, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/76b696c2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/76b696c2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/76b696c2
Branch: refs/heads/master
Commit: 76b696c266122851e9704b5cf4d6ffd55efe0240
Parents: 109c594
Author: Jason Dere
Authored: Mon Apr 9 16:15:09 2018 -0700
Committer: Jason Dere
Committed: Mon Apr 9 16:15:09 2018 -0700
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../java/org/apache/hadoop/hive/ql/Driver.java | 1 +
.../hadoop/hive/ql/exec/FunctionTask.java | 2 +-
.../hive/ql/exec/tez/TezSessionPoolManager.java | 60 ++--
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 13 +-
.../hive/ql/exec/tez/YarnQueueHelper.java | 143 +++
.../hive/ql/parse/DDLSemanticAnalyzer.java | 1 +
.../ql/udf/generic/GenericUDFLoggedInUser.java | 1 +
.../apache/hive/service/server/HiveServer2.java | 9 +-
9 files changed, 214 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/76b696c2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 0627c35..17b2485 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3011,6 +3011,9 @@ public class HiveConf extends Configuration {
"This flag is used in HiveServer2 to enable a user to use HiveServer2
without\n" +
"turning on Tez for HiveServer2. The user could potentially want to
run queries\n" +
"over Tez without the pool of sessions."),
+HIVE_SERVER2_TEZ_QUEUE_ACCESS_CHECK("hive.server2.tez.queue.access.check",
false,
+"Whether to check user access to explicitly specified YARN queues. " +
+ "yarn.resourcemanager.webapp.address must be configured to use
this."),
HIVE_SERVER2_TEZ_SESSION_LIFETIME("hive.server2.tez.session.lifetime",
"162h",
new TimeValidator(TimeUnit.HOURS),
"The lifetime of the Tez sessions launched by HS2 when default
sessions are enabled.\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/76b696c2/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 79db006..a88453c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -2045,6 +2045,7 @@ public class Driver implements IDriver {
SessionState ss = SessionState.get();
+ // TODO: should this use getUserFromAuthenticator?
hookContext = new PrivateHookContext(plan, queryState,
ctx.getPathToCS(), SessionState.get().getUserName(),
ss.getUserIpAddress(), InetAddress.getLocalHost().getHostAddress(),
operationId,
ss.getSessionId(), Thread.currentThread().getName(),
ss.isHiveServerQuery(), perfLogger, queryInfo, ctx);
http://git-wip-us.apache.org/repos/asf/hive/blob/76b696c2/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
index 1de333e..a0a90a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
@@ -185,7 +185,7 @@ public class FunctionTask extends Task {
funcName,
dbName,
className,
-SessionState.get().getUserName(),
+SessionState.get().getUserName(), // TODO: should this use
getUserFromAuthenticator?
PrincipalType.USER,
(int) (System.currentTimeMillis() / 1000),
org.apache.hadoop.hive.metastore.api.FunctionType.JAVA,
http://git-wip-us.apache.org/repos/asf/hive/blob/76b696c2/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
index a051f90..2633390 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
+++ b/ql/src/jav
hive git commit: HIVE-19014: utilize YARN-8028 (queue ACL check) in Hive Tez session pool (Sergey Shelukhin, reviewed by Jason Dere)
Repository: hive
Updated Branches:
refs/heads/branch-3 9ce42cba8 -> 1cd74b451
HIVE-19014: utilize YARN-8028 (queue ACL check) in Hive Tez session pool
(Sergey Shelukhin, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1cd74b45
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1cd74b45
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1cd74b45
Branch: refs/heads/branch-3
Commit: 1cd74b451c0aa707ed28b2531f72971e8904fade
Parents: 9ce42cb
Author: Jason Dere
Authored: Mon Apr 9 16:15:09 2018 -0700
Committer: Jason Dere
Committed: Mon Apr 9 16:27:49 2018 -0700
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../java/org/apache/hadoop/hive/ql/Driver.java | 1 +
.../hadoop/hive/ql/exec/FunctionTask.java | 2 +-
.../hive/ql/exec/tez/TezSessionPoolManager.java | 60 ++--
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 13 +-
.../hive/ql/exec/tez/YarnQueueHelper.java | 143 +++
.../hive/ql/parse/DDLSemanticAnalyzer.java | 1 +
.../ql/udf/generic/GenericUDFLoggedInUser.java | 1 +
.../apache/hive/service/server/HiveServer2.java | 9 +-
9 files changed, 214 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/1cd74b45/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 0627c35..17b2485 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3011,6 +3011,9 @@ public class HiveConf extends Configuration {
"This flag is used in HiveServer2 to enable a user to use HiveServer2
without\n" +
"turning on Tez for HiveServer2. The user could potentially want to
run queries\n" +
"over Tez without the pool of sessions."),
+HIVE_SERVER2_TEZ_QUEUE_ACCESS_CHECK("hive.server2.tez.queue.access.check",
false,
+"Whether to check user access to explicitly specified YARN queues. " +
+ "yarn.resourcemanager.webapp.address must be configured to use
this."),
HIVE_SERVER2_TEZ_SESSION_LIFETIME("hive.server2.tez.session.lifetime",
"162h",
new TimeValidator(TimeUnit.HOURS),
"The lifetime of the Tez sessions launched by HS2 when default
sessions are enabled.\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/1cd74b45/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 79db006..a88453c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -2045,6 +2045,7 @@ public class Driver implements IDriver {
SessionState ss = SessionState.get();
+ // TODO: should this use getUserFromAuthenticator?
hookContext = new PrivateHookContext(plan, queryState,
ctx.getPathToCS(), SessionState.get().getUserName(),
ss.getUserIpAddress(), InetAddress.getLocalHost().getHostAddress(),
operationId,
ss.getSessionId(), Thread.currentThread().getName(),
ss.isHiveServerQuery(), perfLogger, queryInfo, ctx);
http://git-wip-us.apache.org/repos/asf/hive/blob/1cd74b45/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
index 1de333e..a0a90a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
@@ -185,7 +185,7 @@ public class FunctionTask extends Task {
funcName,
dbName,
className,
-SessionState.get().getUserName(),
+SessionState.get().getUserName(), // TODO: should this use
getUserFromAuthenticator?
PrincipalType.USER,
(int) (System.currentTimeMillis() / 1000),
org.apache.hadoop.hive.metastore.api.FunctionType.JAVA,
http://git-wip-us.apache.org/repos/asf/hive/blob/1cd74b45/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
index a051f90..2633390 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
+++ b/ql/src
