Repository: hive Updated Branches: refs/heads/master 2bf0d92d1 -> 2f7323396
HIVE-13128. NullScan fails on a secure setup. (Siddharth Seth, reviewed by Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2f732339 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2f732339 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2f732339 Branch: refs/heads/master Commit: 2f73233961e6f527f5460c124cc021a4a9628b01 Parents: 2bf0d92 Author: Siddharth Seth <ss...@apache.org> Authored: Wed Feb 24 10:31:41 2016 -0800 Committer: Siddharth Seth <ss...@apache.org> Committed: Wed Feb 24 10:31:41 2016 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/exec/tez/DagUtils.java | 5 ++--- .../hadoop/hive/ql/exec/tez/TezSessionState.java | 2 ++ .../org/apache/hadoop/hive/ql/exec/tez/TezTask.java | 7 ++----- .../apache/hadoop/hive/ql/io/NullScanFileSystem.java | 15 +++++++++++++++ 4 files changed, 21 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2f732339/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 39866a3..473dbd6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -754,7 +754,7 @@ public class DagUtils { URL resourceURL = ConverterUtils.getYarnUrlFromPath(file); long resourceSize = fstat.getLen(); long resourceModificationTime = fstat.getModificationTime(); - LOG.info("Resource modification time: " + resourceModificationTime); + LOG.info("Resource modification time: " + resourceModificationTime + " for " + file); LocalResource lr = Records.newRecord(LocalResource.class); lr.setResource(resourceURL); @@ -912,7 +912,7 @@ public class DagUtils { if (fstatus == null) { Path destDir = getDefaultDestDir(conf); - LOG.info("Jar dir is null/directory doesn't exist. Choosing HIVE_INSTALL_DIR - " + destDir); + LOG.info("Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - " + destDir); fstatus = validateTargetDir(destDir, conf); } @@ -1275,5 +1275,4 @@ public class DagUtils { private DagUtils() { // don't instantiate } - } http://git-wip-us.apache.org/repos/asf/hive/blob/2f732339/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index 6675f0d..3ea5ef9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -548,6 +548,8 @@ public class TezSessionState { private LocalResource createJarLocalResource(String localJarPath) throws IOException, LoginException, IllegalArgumentException, FileNotFoundException { + // TODO Reduce the number of lookups that happen here. This shouldn't go to HDFS for each call. + // The hiveJarDir can be determined once per client. FileStatus destDirStatus = utils.getHiveJarDirectory(conf); assert destDirStatus != null; Path destDirPath = destDirStatus.getPath(); http://git-wip-us.apache.org/repos/asf/hive/blob/2f732339/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 83defea..3789ce9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -173,9 +173,6 @@ public class TezTask extends Task<TezWork> { // next we translate the TezWork to a Tez DAG DAG dag = build(jobConf, work, scratchDir, appJarLr, additionalLr, ctx); - if (driverContext.getCtx() == null) { - boolean a = false; - } CallerContext callerContext = CallerContext.create( "HIVE", queryPlan.getQueryId(), "HIVE_QUERY_ID", queryPlan.getQueryStr()); @@ -460,10 +457,10 @@ public class TezTask extends Task<TezWork> { } catch (Exception e) { // In case of any other exception, retry. If this also fails, report original error and exit. try { - TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars, - true); console.printInfo("Dag submit failed due to " + e.getMessage() + " stack trace: " + Arrays.toString(e.getStackTrace()) + " retrying..."); + TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars, + true); dagClient = sessionState.getSession().submitDAG(dag); } catch (Exception retryException) { // we failed to submit after retrying. Destroy session and bail. http://git-wip-us.apache.org/repos/asf/hive/blob/2f732339/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java b/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java index 88e9445..5bfde18 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java @@ -29,6 +29,8 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; /** @@ -45,6 +47,8 @@ public class NullScanFileSystem extends FileSystem { return "nullscan"; } + private final Token<?>[] DEFAULT_EMPTY_TOKEN_ARRAY = new Token<?>[0]; + public NullScanFileSystem() { } @@ -113,4 +117,15 @@ public class NullScanFileSystem extends FileSystem { public FileStatus getFileStatus(Path f) throws IOException { return new FileStatus(0, false, 0, 0, 0, f); } + + @Override + public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws + IOException { + return DEFAULT_EMPTY_TOKEN_ARRAY; + } + + @Override + public Token<?> getDelegationToken(String renewer) throws IOException { + return null; + } } \ No newline at end of file
