hive git commit: HIVE-19079: Add extended query string to Spark job description (Sahil Takiar, reviewed by Aihua Xu) (addendum)

2018-06-06 Thread stakiar 
Repository: hive
Updated Branches:
  refs/heads/master 8e1e537bf -> 9615f24b9


HIVE-19079: Add extended query string to Spark job description (Sahil Takiar, 
reviewed by Aihua Xu) (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9615f24b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9615f24b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9615f24b

Branch: refs/heads/master
Commit: 9615f24b9c0baf68de3b24ed6366a277d44564cb
Parents: 8e1e537
Author: Sahil Takiar 
Authored: Wed Jun 6 12:55:46 2018 -0500
Committer: Sahil Takiar 
Committed: Wed Jun 6 12:55:46 2018 -0500

--
 .../hive/ql/exec/spark/TestHiveSparkClient.java   | 18 ++
 1 file changed, 18 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/9615f24b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestHiveSparkClient.java
--
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestHiveSparkClient.java 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestHiveSparkClient.java
index 239c098..b960508 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestHiveSparkClient.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestHiveSparkClient.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.hadoop.hive.ql.exec.spark;
 
 import org.apache.hadoop.fs.FileSystem;

hive git commit: HIVE-19079: Add extended query string to Spark job description (Sahil Takiar, reviewed by Aihua Xu)

2018-06-05 Thread stakiar 
Repository: hive
Updated Branches:
  refs/heads/master 3dcd36336 -> 98982e619


HIVE-19079: Add extended query string to Spark job description (Sahil Takiar, 
reviewed by Aihua Xu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/98982e61
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/98982e61
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/98982e61

Branch: refs/heads/master
Commit: 98982e6193ce05111d031bb8d54ce530ae41853f
Parents: 3dcd363
Author: Sahil Takiar 
Authored: Mon Apr 2 09:31:38 2018 -0700
Committer: Sahil Takiar 
Committed: Tue Jun 5 12:00:34 2018 -0500

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  2 +
 .../java/org/apache/hadoop/hive/ql/Driver.java  |  8 +-
 .../ql/exec/spark/LocalHiveSparkClient.java |  3 +
 .../ql/exec/spark/RemoteHiveSparkClient.java|  7 +-
 .../hive/ql/exec/spark/TestHiveSparkClient.java | 97 
 5 files changed, 115 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/98982e61/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 9004894..cd425aa 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1494,6 +1494,8 @@ public class HiveConf extends Configuration {
 HIVEQUERYID("hive.query.id", "",
 "ID for query being executed (might be multiple per a session)"),
 
+HIVESPARKJOBNAMELENGTH("hive.spark.jobname.length", 10, "max jobname 
length for Hive on " +
+"Spark queries"),
 HIVEJOBNAMELENGTH("hive.jobname.length", 50, "max jobname length"),
 
 // hive jar

http://git-wip-us.apache.org/repos/asf/hive/blob/98982e61/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 7d5e7d4..a3dcc3b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -2033,7 +2033,13 @@ public class Driver implements IDriver {
 perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_EXECUTE);
 
 boolean noName = StringUtils.isEmpty(conf.get(MRJobConfig.JOB_NAME));
-int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
+
+int maxlen;
+if ("spark".equals(conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE))) {
+  maxlen = conf.getIntVar(HiveConf.ConfVars.HIVESPARKJOBNAMELENGTH);
+} else {
+  maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
+}
 Metrics metrics = MetricsFactory.getInstance();
 
 String queryId = queryState.getQueryId();

http://git-wip-us.apache.org/repos/asf/hive/blob/98982e61/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
index dff3b0f..72ff53e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
@@ -163,6 +163,9 @@ public class LocalHiveSparkClient implements 
HiveSparkClient {
 // Execute generated plan.
 JavaPairRDD finalRDD = plan.generateGraph();
 
+// We get the query name for this SparkTask and set it to the description 
for the associated
+// Spark job; query names are guaranteed to be unique for each Spark job 
because the task id
+// is concatenated to the end of the query name
 sc.setJobGroup("queryId = " + sparkWork.getQueryId(), 
DagUtils.getQueryName(jobConf));
 
 // We use Spark RDD async action to submit job as it's the only way to get 
jobId now.

http://git-wip-us.apache.org/repos/asf/hive/blob/98982e61/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
index e8f39ae..d31a202 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hive.ql.exec.spark;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.