Author: khorgath Date: Mon Jun 2 18:58:05 2014 New Revision: 1599308 URL: http://svn.apache.org/r1599308 Log: HIVE-7116 : HDFS FileSystem object cache causes permission issues in creating tmp directories (Vikram Dixit K via Thejas Nair) (from https://issues.apache.org/jira/secure/attachment/12647832/HIVE-7116.branch0.13.patch)
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1599308&r1=1599307&r2=1599308&view=diff ============================================================================== --- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original) +++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Mon Jun 2 18:58:05 2014 @@ -3396,9 +3396,24 @@ public final class Utilities { return createDirsWithPermission(conf, mkdir, fsPermission, recursive); } - public static boolean createDirsWithPermission(Configuration conf, Path mkdir, + private static void resetConfAndCloseFS (Configuration conf, boolean unsetUmask, + String origUmask, FileSystem fs) throws IOException { + if (unsetUmask) { + if (origUmask != null) { + conf.set("fs.permissions.umask-mode", origUmask); + } else { + conf.unset("fs.permissions.umask-mode"); + } + } + + fs.close(); + } + + public static boolean createDirsWithPermission(Configuration conf, Path mkdirPath, FsPermission fsPermission, boolean recursive) throws IOException { String origUmask = null; + LOG.debug("Create dirs " + mkdirPath + " with permission " + fsPermission + " recursive " + + recursive); if (recursive) { origUmask = conf.get("fs.permissions.umask-mode"); @@ -3406,15 +3421,17 @@ public final class Utilities { // all parents getting the fsPermission & !(022) permission instead of fsPermission conf.set("fs.permissions.umask-mode", "000"); } - - FileSystem fs = mkdir.getFileSystem(conf); - boolean retval = fs.mkdirs(mkdir, fsPermission); - - if (recursive) { - if (origUmask != null) { - conf.set("fs.permissions.umask-mode", origUmask); - } else { - conf.unset("fs.permissions.umask-mode"); + FileSystem fs = ShimLoader.getHadoopShims().getNonCachedFileSystem(mkdirPath.toUri(), conf); + boolean retval = false; + try { + retval = fs.mkdirs(mkdirPath, fsPermission); + resetConfAndCloseFS(conf, recursive, origUmask, fs); + } catch (IOException ioe) { + try { + resetConfAndCloseFS(conf, recursive, origUmask, fs); + } + catch (IOException e) { + // do nothing - double failure } } return retval; Modified: hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1599308&r1=1599307&r2=1599308&view=diff ============================================================================== --- hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original) +++ hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Mon Jun 2 18:58:05 2014 @@ -31,12 +31,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.Set; -import java.util.HashMap; import java.util.Map; +import java.util.Set; import javax.security.auth.Subject; import javax.security.auth.login.LoginException; @@ -53,25 +52,17 @@ import org.apache.hadoop.fs.ProxyFileSys import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; -import org.apache.hadoop.hive.shims.HadoopShims.DirectDecompressorShim; import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.FileInputFormat; -import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobContext; -import org.apache.hadoop.mapred.JobStatus; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.OutputCommitter; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.TaskAttemptContext; -import org.apache.hadoop.mapred.TaskCompletionEvent; -import org.apache.hadoop.mapred.TaskID; import org.apache.hadoop.mapred.TaskLogServlet; import org.apache.hadoop.mapred.lib.CombineFileInputFormat; import org.apache.hadoop.mapred.lib.CombineFileSplit; @@ -800,4 +791,15 @@ public class Hadoop20Shims implements Ha public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext context) { return context.getConfiguration(); } -} + + @Override + public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException { + boolean origDisableHDFSCache = + conf.getBoolean("fs." + uri.getScheme() + ".impl.disable.cache", false); + // hadoop-20 compatible flag. + conf.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true); + FileSystem fs = FileSystem.get(uri, conf); + conf.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", origDisableHDFSCache); + return fs; + } +} \ No newline at end of file Modified: hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1599308&r1=1599307&r2=1599308&view=diff ============================================================================== --- hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original) +++ hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Mon Jun 2 18:58:05 2014 @@ -420,4 +420,15 @@ public class Hadoop20SShims extends Hado public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext context) { return context.getConfiguration(); } + + @Override + public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException { + boolean origDisableHDFSCache = + conf.getBoolean("fs." + uri.getScheme() + ".impl.disable.cache", false); + // hadoop-1 compatible flag. + conf.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true); + FileSystem fs = FileSystem.get(uri, conf); + conf.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", origDisableHDFSCache); + return fs; + } } Modified: hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1599308&r1=1599307&r2=1599308&view=diff ============================================================================== --- hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original) +++ hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Mon Jun 2 18:58:05 2014 @@ -17,20 +17,17 @@ */ package org.apache.hadoop.hive.shims; +import java.io.FileNotFoundException; import java.io.IOException; -import java.lang.Integer; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URL; +import java.net.URI; import java.util.ArrayList; import java.util.Comparator; -import java.util.Iterator; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.HashMap; -import java.net.URI; -import java.nio.ByteBuffer; -import java.io.FileNotFoundException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -46,17 +43,13 @@ import org.apache.hadoop.fs.ProxyFileSys import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hive.shims.HadoopShims.ByteBufferPoolShim; -import org.apache.hadoop.hive.shims.HadoopShims.DirectCompressionType; -import org.apache.hadoop.hive.shims.HadoopShims.DirectDecompressorShim; -import org.apache.hadoop.hive.shims.HadoopShims.ZeroCopyReaderShim; import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.WebHCatJTShim23; +import org.apache.hadoop.mapred.lib.TotalOrderPartitioner; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobID; @@ -69,9 +62,8 @@ import org.apache.hadoop.mapreduce.task. import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.hadoop.mapreduce.util.HostUtil; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.util.Progressable; -import org.apache.hadoop.mapred.lib.TotalOrderPartitioner; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Progressable; import org.apache.tez.test.MiniTezCluster; /** @@ -576,4 +568,9 @@ public class Hadoop23Shims extends Hadoo public Configuration getConfiguration(org.apache.hadoop.mapreduce.JobContext context) { return context.getConfiguration(); } + + @Override + public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException { + return FileSystem.newInstance(uri, conf); + } } Modified: hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1599308&r1=1599307&r2=1599308&view=diff ============================================================================== --- hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original) +++ hive/branches/branch-0.13/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Mon Jun 2 18:58:05 2014 @@ -34,30 +34,23 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier; import org.apache.hadoop.hive.thrift.DelegationTokenSelector; import org.apache.hadoop.http.HtmlQuoting; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapred.FileInputFormat; -import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobContext; -import org.apache.hadoop.mapred.JobStatus; import org.apache.hadoop.mapred.OutputCommitter; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.TaskAttemptContext; -import org.apache.hadoop.mapred.TaskCompletionEvent; -import org.apache.hadoop.mapred.TaskID; import org.apache.hadoop.mapred.lib.CombineFileInputFormat; import org.apache.hadoop.mapred.lib.CombineFileSplit; import org.apache.hadoop.mapreduce.Job; @@ -589,7 +582,8 @@ public abstract class HadoopShimsSecure abstract public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception; @Override - abstract public org.apache.hadoop.mapreduce.TaskAttemptContext newTaskAttemptContext(Configuration conf, final Progressable progressable); + abstract public org.apache.hadoop.mapreduce.TaskAttemptContext newTaskAttemptContext( + Configuration conf, final Progressable progressable); @Override abstract public org.apache.hadoop.mapreduce.JobContext newJobContext(Job job); @@ -614,8 +608,11 @@ public abstract class HadoopShimsSecure @Override abstract public boolean moveToAppropriateTrash(FileSystem fs, Path path, Configuration conf) - throws IOException; + throws IOException; @Override abstract public FileSystem createProxyFileSystem(FileSystem fs, URI uri); + + @Override + abstract public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException; } Modified: hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1599308&r1=1599307&r2=1599308&view=diff ============================================================================== --- hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original) +++ hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Mon Jun 2 18:58:05 2014 @@ -27,7 +27,6 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -628,4 +627,7 @@ public interface HadoopShims { * Get configuration from JobContext */ public Configuration getConfiguration(JobContext context); + + public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException; + }