Repository: hive Updated Branches: refs/heads/branch-2.0 e83dd4163 -> 36f8b7590 refs/heads/master 01d06aa83 -> c0b04f7b0
HIVE-12357 : Allow user to set tez job name (Gunther Hagleitner, reviewed by Vikram Dixit K, Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c0b04f7b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c0b04f7b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c0b04f7b Branch: refs/heads/master Commit: c0b04f7b033cc4f2e7b2c105f0a5d0b2d0f9fb75 Parents: 01d06aa Author: Sergey Shelukhin <[email protected]> Authored: Wed Jan 27 13:45:38 2016 -0800 Committer: Sergey Shelukhin <[email protected]> Committed: Wed Jan 27 13:45:38 2016 -0800 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++ .../apache/hadoop/hive/ql/exec/tez/DagUtils.java | 16 ++++++++++++++++ .../org/apache/hadoop/hive/ql/exec/tez/TezTask.java | 10 ++++++---- .../org/apache/hadoop/hive/ql/hooks/ATSHook.java | 4 +++- 4 files changed, 28 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c0b04f7b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 74a8749..125d40c 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1207,6 +1207,9 @@ public class HiveConf extends Configuration { HIVETEZLOGLEVEL("hive.tez.log.level", "INFO", "The log level to use for tasks executing as part of the DAG.\n" + "Used only if hive.tez.java.opts is used to configure Java options."), + HIVEQUERYNAME ("hive.query.name", null, + "This named is used by Tez to set the dag name. This name in turn will appear on \n" + + "the Tez UI representing the work that was done."), HIVEOPTIMIZEBUCKETINGSORTING("hive.optimize.bucketingsorting", true, "Don't create a reducer for enforcing \n" + http://git-wip-us.apache.org/repos/asf/hive/blob/c0b04f7b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index aa44d5f..319447b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.mr.ExecMapper; @@ -1261,6 +1262,21 @@ public class DagUtils { } } + public String createDagName(Configuration conf, QueryPlan plan) { + String name = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYNAME); + + if (name == null) { + name = conf.get("mapred.job.name"); + } + + if (name == null) { + name = plan.getQueryId(); + } + + assert name != null; + return name; + } + private DagUtils() { // don't instantiate } http://git-wip-us.apache.org/repos/asf/hive/blob/c0b04f7b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 3cb7439..83defea 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -176,9 +176,8 @@ public class TezTask extends Task<TezWork> { if (driverContext.getCtx() == null) { boolean a = false; } - CallerContext callerContext = CallerContext.create("HIVE", - conf.getLogIdVar(SessionState.get().getSessionId()) + " " - + conf.getVar(HiveConf.ConfVars.HIVEQUERYID), + CallerContext callerContext = CallerContext.create( + "HIVE", queryPlan.getQueryId(), "HIVE_QUERY_ID", queryPlan.getQueryStr()); dag.setCallerContext(callerContext); @@ -320,7 +319,10 @@ public class TezTask extends Task<TezWork> { FileSystem fs = scratchDir.getFileSystem(conf); // the name of the dag is what is displayed in the AM/Job UI - DAG dag = DAG.create(work.getName()); + String dagName = utils.createDagName(conf, queryPlan); + + LOG.info("Dag name: " + dagName); + DAG dag = DAG.create(dagName); // set some info for the query JSONObject json = new JSONObject(new LinkedHashMap()).put("context", "Hive") http://git-wip-us.apache.org/repos/asf/hive/blob/c0b04f7b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java index 0be8b3c..f490161 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java @@ -50,13 +50,14 @@ public class ATSHook implements ExecuteWithHookContext { private static final Logger LOG = LoggerFactory.getLogger(ATSHook.class.getName()); private static final Object LOCK = new Object(); + private static final int VERSION = 2; private static ExecutorService executor; private static TimelineClient timelineClient; private enum EntityTypes { HIVE_QUERY_ID }; private enum EventTypes { QUERY_SUBMITTED, QUERY_COMPLETED }; private enum OtherInfoTypes { - QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, THREAD_NAME + QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, THREAD_NAME, VERSION }; private enum PrimaryFilterTypes { user, requestuser, operationid }; private static final int WAIT_TIME = 3; @@ -193,6 +194,7 @@ public class ATSHook implements ExecuteWithHookContext { atsEntity.addOtherInfo(OtherInfoTypes.MAPRED.name(), numMrJobs > 0); atsEntity.addOtherInfo(OtherInfoTypes.INVOKER_INFO.name(), logID); atsEntity.addOtherInfo(OtherInfoTypes.THREAD_NAME.name(), Thread.currentThread().getName()); + atsEntity.addOtherInfo(OtherInfoTypes.VERSION.name(), VERSION); return atsEntity; }
