Repository: hive
Updated Branches:
  refs/heads/master 776c7cd1f -> 04b303b69


HIVE-14922 : Add perf logging for post job completion steps (Ashutosh Chauhan 
via Pengcheng Xiong)

Signed-off-by: Ashutosh Chauhan <hashut...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/04b303b6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/04b303b6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/04b303b6

Branch: refs/heads/master
Commit: 04b303b6957a6b6f580ee10a4215b21071d87999
Parents: 776c7cd
Author: Ashutosh Chauhan <hashut...@apache.org>
Authored: Tue Oct 11 14:19:22 2016 -0700
Committer: Ashutosh Chauhan <hashut...@apache.org>
Committed: Wed Oct 12 08:23:09 2016 -0700

----------------------------------------------------------------------
 ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java  | 6 ++++--
 ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 7 +++++++
 ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java  | 4 ++++
 3 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/04b303b6/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 6d05f07..ec21cd6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -445,8 +445,10 @@ public class MoveTask extends Task<MoveWork> implements 
Serializable {
                 SessionState.get().getTxnMgr().getCurrentTxnId(), 
hasFollowingStatsTask(),
                 work.getLoadTableWork().getWriteType());
 
-            console.printInfo("\t Time taken to load dynamic partitions: "  +
-                (System.currentTimeMillis() - startTime)/1000.0 + " seconds");
+            String loadTime = "\t Time taken to load dynamic partitions: "  +
+                (System.currentTimeMillis() - startTime)/1000.0 + " seconds";
+            console.printInfo(loadTime);
+            LOG.info(loadTime);
 
             if (dp.size() == 0 && 
conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) {
               throw new HiveException("This query creates no partitions." +

http://git-wip-us.apache.org/repos/asf/hive/blob/04b303b6/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index fd64056..a9dbc3e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1402,16 +1402,23 @@ public final class Utilities {
       FileStatus[] statuses = HiveStatsUtils.getFileStatusRecurse(
           tmpPath, ((dpCtx == null) ? 1 : dpCtx.getNumDPCols()), fs);
       if(statuses != null && statuses.length > 0) {
+        PerfLogger perfLogger = SessionState.getPerfLogger();
+        perfLogger.PerfLogBegin("FileSinkOperator", 
"RemoveTempOrDuplicateFiles");
         // remove any tmp file or double-committed output files
         List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(fs, 
statuses, dpCtx, conf, hconf);
+        perfLogger.PerfLogEnd("FileSinkOperator", 
"RemoveTempOrDuplicateFiles");
         // create empty buckets if necessary
         if (emptyBuckets.size() > 0) {
+          perfLogger.PerfLogBegin("FileSinkOperator", "CreateEmptyBuckets");
           createEmptyBuckets(hconf, emptyBuckets, conf, reporter);
+          perfLogger.PerfLogEnd("FileSinkOperator", "CreateEmptyBuckets");
         }
 
         // move to the file destination
         log.info("Moving tmp dir: " + tmpPath + " to: " + specPath);
+        perfLogger.PerfLogBegin("FileSinkOperator", "RenameOrMoveFiles");
         Utilities.renameOrMoveFiles(fs, tmpPath, specPath);
+        perfLogger.PerfLogEnd("FileSinkOperator", "RenameOrMoveFiles");
       }
     } else {
       fs.delete(tmpPath, true);

http://git-wip-us.apache.org/repos/asf/hive/blob/04b303b6/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 691c3a8..dab4c6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -128,6 +128,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.tez.InPlaceUpdates;
 import org.apache.hadoop.hive.ql.index.HiveIndexHandler;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.metastore.SynchronizedMetaStoreClient;
 import 
org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils;
 import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
@@ -1567,6 +1568,8 @@ public class Hive {
         newPartPath = oldPartPath;
       }
       List<Path> newFiles = null;
+      PerfLogger perfLogger = SessionState.getPerfLogger();
+      perfLogger.PerfLogBegin("MoveTask", "FileMoves");
       if (replace || (oldPart == null && !isAcid)) {
         replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, 
getConf(),
             isSrcLocal);
@@ -1578,6 +1581,7 @@ public class Hive {
         FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
         Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, 
newFiles);
       }
+      perfLogger.PerfLogEnd("MoveTask", "FileMoves");
       Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, 
partSpec, newPartPath);
       alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), 
inheritTableSpecs, newPartPath.toString());
       validatePartition(newTPart);

Reply via email to