This is an automated email from the ASF dual-hosted git repository. ayushsaxena pushed a commit to branch branch-4.0 in repository https://gitbox.apache.org/repos/asf/hive.git
commit edb2192ac991e164f49c664d753922dcb3c5e040 Author: Ayush Saxena <[email protected]> AuthorDate: Thu May 30 21:59:54 2024 +0530 HIVE-28287: Attempt make the scratch directory writable before failing. (#5268). (Ayush Saxena, reviewed by Simhadri Govindappa) --- .../org/apache/hadoop/hive/ql/exec/Utilities.java | 24 ++++++++++++++++++---- .../apache/hadoop/hive/ql/exec/TestUtilities.java | 20 ++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 6d38f1d5afc..c300d268d90 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -4961,13 +4961,29 @@ public final class Utilities { "HDFS dir: " + rootHDFSDirPath + ", permission: " + currentHDFSDirPermission); } // If the root HDFS scratch dir already exists, make sure it is writeable. - if (!((currentHDFSDirPermission.toShort() & writableHDFSDirPermission - .toShort()) == writableHDFSDirPermission.toShort())) { - throw new RuntimeException("The dir: " + rootHDFSDirPath - + " on HDFS should be writable. Current permissions are: " + currentHDFSDirPermission); + if (!isWritable(currentHDFSDirPermission, writableHDFSDirPermission) && + !attemptMakePathWritable(fs, rootHDFSDirPath, + FsPermission.createImmutable((short) (currentHDFSDirPermission.toShort() | writableHDFSDirPermission.toShort())))) { + throw new RuntimeException( + "The dir: " + rootHDFSDirPath + " should be writable. Current permissions are: " + currentHDFSDirPermission); } } + private static boolean attemptMakePathWritable(FileSystem fs, Path path, FsPermission perm) { + try { + LOG.info("Attempting to set {} permissions on path {}", perm, path); + fs.setPermission(path, perm); + return isWritable(fs.getFileStatus(path).getPermission(), perm); + } catch (IOException e) { + return false; + } + } + + private static boolean isWritable(FsPermission currentHDFSDirPermission, FsPermission writableHDFSDirPermission) { + return (currentHDFSDirPermission.toShort() & writableHDFSDirPermission.toShort()) + == writableHDFSDirPermission.toShort(); + } + // Get the bucketing version stored in the string format public static int getBucketingVersion(final String versionStr) { int bucketingVersion = 1; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java index f87d6c40f17..7d47fa34e8c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java @@ -52,6 +52,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; @@ -75,6 +76,8 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.MRJobConfig; + +import com.google.common.io.Files; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -815,6 +818,23 @@ public class TestUtilities { assertEquals(expectedPathes, resultPathes); } + @Test + public void testSetPermissionsOnExistingDir() throws IOException { + File tmpDir = Files.createTempDir(); + Path path = new Path(tmpDir.getPath()); + HiveConf conf = new HiveConf(this.getClass()); + FileSystem fs = path.getFileSystem(conf); + fs.setPermission(path, new FsPermission((short) 00700)); + Utilities.ensurePathIsWritable(path, conf); + Assert.assertEquals((short) 0733, fs.getFileStatus(path).getPermission().toShort()); + + // Test with more open permissions than required, but still not writable, + // it should just make the directory writable without restricting the existing permissions + fs.setPermission(path, new FsPermission((short) 00755)); + Utilities.ensurePathIsWritable(path, conf); + Assert.assertEquals((short) 0777, fs.getFileStatus(path).getPermission().toShort()); + } + private FileStatus[] generateTestNotEmptyFileStatuses(String... fileNames) { return generateTestNotEmptyFileStatuses(null, fileNames); }
