svn commit: r1549982 - /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java

[gunther]

Author: gunther
Date: Tue Dec 10 21:57:02 2013
New Revision: 1549982

URL: http://svn.apache.org/r1549982
Log:
HIVE-6001: Tez: UDFs are not properly localized (Gunther Hagleitner)

Modified:
    
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java

Modified: 
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: 
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1549982&r1=1549981&r2=1549982&view=diff
==============================================================================
--- 
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java 
(original)
+++ 
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java 
Tue Dec 10 21:57:02 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.plan.Ba
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.ReduceWork;
 import org.apache.hadoop.hive.ql.plan.TezWork.EdgeType;
+import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.stats.StatsFactory;
 import org.apache.hadoop.hive.ql.stats.StatsPublisher;
 import org.apache.hadoop.hive.shims.Hadoop20Shims.NullOutputCommitter;
@@ -417,10 +418,20 @@ public class DagUtils {
   public static List<LocalResource> localizeTempFiles(Configuration conf) 
throws IOException, LoginException {
     List<LocalResource> tmpResources = new ArrayList<LocalResource>();
 
+    String addedFiles = Utilities.getResourceFiles(conf, 
SessionState.ResourceType.FILE);
+    if (StringUtils.isNotBlank(addedFiles)) {
+      HiveConf.setVar(conf, ConfVars.HIVEADDEDFILES, addedFiles);
+    }
+    String addedJars = Utilities.getResourceFiles(conf, 
SessionState.ResourceType.JAR);
+    if (StringUtils.isNotBlank(addedJars)) {
+      HiveConf.setVar(conf, ConfVars.HIVEADDEDJARS, addedJars);
+    }
+    String addedArchives = Utilities.getResourceFiles(conf, 
SessionState.ResourceType.ARCHIVE);
+    if (StringUtils.isNotBlank(addedArchives)) {
+      HiveConf.setVar(conf, ConfVars.HIVEADDEDARCHIVES, addedArchives);
+    }
+
     String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS);
-    String addedJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEADDEDJARS);
-    String addedFiles = HiveConf.getVar(conf, 
HiveConf.ConfVars.HIVEADDEDFILES);
-    String addedArchives = HiveConf.getVar(conf, 
HiveConf.ConfVars.HIVEADDEDARCHIVES);
 
     // need to localize the additional jars and files
 
@@ -447,6 +458,9 @@ public class DagUtils {
     String allFiles = auxJars + "," + addedJars + "," + addedFiles + "," + 
addedArchives;
     String[] allFilesArr = allFiles.split(",");
     for (String file : allFilesArr) {
+      if (!StringUtils.isNotBlank(file)) {
+        continue;
+      }
       String hdfsFilePathStr = hdfsDirPathStr + "/" + 
getResourceBaseName(file);
       LocalResource localResource = localizeResource(new Path(file),
           new Path(hdfsFilePathStr), conf);