subject:"hadoop git commit\: YARN\-9030. Log aggregation changes to handle filesystems which do not support setting permissions. \(Suma Shivaprasad via wangda\)"

[01/50] [abbrv] hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)

2018-12-05 Thread shv

Repository: hadoop
Updated Branches:
  refs/heads/HDFS-12943 b5497070a -> 47d726015


YARN-9030. Log aggregation changes to handle filesystems which do not support 
setting permissions. (Suma Shivaprasad via wangda)

Change-Id: I80f1e8196b8624e24d74494719fdedfd7061dced


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9de8e8d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9de8e8d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9de8e8d0

Branch: refs/heads/HDFS-12943
Commit: 9de8e8d0496a2628b63cc841b1fdee80e2912f7a
Parents: 4d8de7a
Author: Wangda Tan 
Authored: Wed Nov 21 17:28:37 2018 -0800
Committer: Wangda Tan 
Committed: Wed Nov 28 13:36:21 2018 -0800

--
 .../LogAggregationFileController.java   | 74 +++-
 1 file changed, 56 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de8e8d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index fe65288..e37308d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -109,6 +109,8 @@ public abstract class LogAggregationFileController {
   protected int retentionSize;
   protected String fileControllerName;
 
+  protected boolean fsSupportsChmod = true;
+
   public LogAggregationFileController() {}
 
   /**
@@ -250,7 +252,6 @@ public abstract class LogAggregationFileController {
* Verify and create the remote log directory.
*/
   public void verifyAndCreateRemoteLogDir() {
-boolean logPermError = true;
 // Checking the existence of the TLD
 FileSystem remoteFS = null;
 try {
@@ -264,14 +265,12 @@ public abstract class LogAggregationFileController {
 try {
   FsPermission perms =
   remoteFS.getFileStatus(remoteRootLogDir).getPermission();
-  if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) {
+  if (!perms.equals(TLDIR_PERMISSIONS)) {
 LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
 + "] already exist, but with incorrect permissions. "
 + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
 + "]." + " The cluster may have problems with multiple users.");
-logPermError = false;
-  } else {
-logPermError = true;
+
   }
 } catch (FileNotFoundException e) {
   remoteExists = false;
@@ -280,15 +279,26 @@ public abstract class LogAggregationFileController {
   "Failed to check permissions for dir ["
   + remoteRootLogDir + "]", e);
 }
+
+Path qualified =
+remoteRootLogDir.makeQualified(remoteFS.getUri(),
+remoteFS.getWorkingDirectory());
 if (!remoteExists) {
   LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
   + "] does not exist. Attempting to create it.");
   try {
-Path qualified =
-remoteRootLogDir.makeQualified(remoteFS.getUri(),
-remoteFS.getWorkingDirectory());
 remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS));
-remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+
+// Not possible to query FileSystem API to check if it supports
+// chmod, chown etc. Hence resorting to catching exceptions here.
+// Remove when FS APi is ready
+try {
+  remoteFS.setPermission(qualified, new 
FsPermission(TLDIR_PERMISSIONS));
+} catch ( UnsupportedOperationException use) {
+  LOG.info("Unable to set permissions for configured filesystem since"
+  + " it does not support this", remoteFS.getScheme());
+  fsSupportsChmod = false;
+}
 
 UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
 String primaryGroupName = null;
@@ -301,13 +311,31 @@ public abstract class LogAggregationFileController {
 }
 // set owner on the remote directory only if the primary group exists
 if (primaryGroupName != null) {
-  remoteFS.setOwner(qualified,
-  loginUser.getShortUserName(), primaryGroupName);
+

hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)

2018-11-28 Thread wangda

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.2 7a78bdf7b -> ee9deb6e9


YARN-9030. Log aggregation changes to handle filesystems which do not support 
setting permissions. (Suma Shivaprasad via wangda)

Change-Id: I80f1e8196b8624e24d74494719fdedfd7061dced
(cherry picked from commit 9de8e8d0496a2628b63cc841b1fdee80e2912f7a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee9deb6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee9deb6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee9deb6e

Branch: refs/heads/branch-3.2
Commit: ee9deb6e9ccebcf6ef2660e32b0d26f458b3ee5d
Parents: 7a78bdf
Author: Wangda Tan 
Authored: Wed Nov 21 17:28:37 2018 -0800
Committer: Wangda Tan 
Committed: Wed Nov 28 15:06:35 2018 -0800

--
 .../LogAggregationFileController.java   | 74 +++-
 1 file changed, 56 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee9deb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index fe65288..e37308d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -109,6 +109,8 @@ public abstract class LogAggregationFileController {
   protected int retentionSize;
   protected String fileControllerName;
 
+  protected boolean fsSupportsChmod = true;
+
   public LogAggregationFileController() {}
 
   /**
@@ -250,7 +252,6 @@ public abstract class LogAggregationFileController {
* Verify and create the remote log directory.
*/
   public void verifyAndCreateRemoteLogDir() {
-boolean logPermError = true;
 // Checking the existence of the TLD
 FileSystem remoteFS = null;
 try {
@@ -264,14 +265,12 @@ public abstract class LogAggregationFileController {
 try {
   FsPermission perms =
   remoteFS.getFileStatus(remoteRootLogDir).getPermission();
-  if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) {
+  if (!perms.equals(TLDIR_PERMISSIONS)) {
 LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
 + "] already exist, but with incorrect permissions. "
 + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
 + "]." + " The cluster may have problems with multiple users.");
-logPermError = false;
-  } else {
-logPermError = true;
+
   }
 } catch (FileNotFoundException e) {
   remoteExists = false;
@@ -280,15 +279,26 @@ public abstract class LogAggregationFileController {
   "Failed to check permissions for dir ["
   + remoteRootLogDir + "]", e);
 }
+
+Path qualified =
+remoteRootLogDir.makeQualified(remoteFS.getUri(),
+remoteFS.getWorkingDirectory());
 if (!remoteExists) {
   LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
   + "] does not exist. Attempting to create it.");
   try {
-Path qualified =
-remoteRootLogDir.makeQualified(remoteFS.getUri(),
-remoteFS.getWorkingDirectory());
 remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS));
-remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+
+// Not possible to query FileSystem API to check if it supports
+// chmod, chown etc. Hence resorting to catching exceptions here.
+// Remove when FS APi is ready
+try {
+  remoteFS.setPermission(qualified, new 
FsPermission(TLDIR_PERMISSIONS));
+} catch ( UnsupportedOperationException use) {
+  LOG.info("Unable to set permissions for configured filesystem since"
+  + " it does not support this", remoteFS.getScheme());
+  fsSupportsChmod = false;
+}
 
 UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
 String primaryGroupName = null;
@@ -301,13 +311,31 @@ public abstract class LogAggregationFileController {
 }
 // set owner on the remote directory only if the primary group exists
 if (primaryGroupName != null) {
-  remoteFS.setOwner(qualified,
-

[1/6] hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)

2018-11-28 Thread wangda

Repository: hadoop
Updated Branches:
  refs/heads/trunk 4d8de7ab6 -> fe7dab8ef


YARN-9030. Log aggregation changes to handle filesystems which do not support 
setting permissions. (Suma Shivaprasad via wangda)

Change-Id: I80f1e8196b8624e24d74494719fdedfd7061dced


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9de8e8d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9de8e8d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9de8e8d0

Branch: refs/heads/trunk
Commit: 9de8e8d0496a2628b63cc841b1fdee80e2912f7a
Parents: 4d8de7a
Author: Wangda Tan 
Authored: Wed Nov 21 17:28:37 2018 -0800
Committer: Wangda Tan 
Committed: Wed Nov 28 13:36:21 2018 -0800

--
 .../LogAggregationFileController.java   | 74 +++-
 1 file changed, 56 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de8e8d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index fe65288..e37308d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -109,6 +109,8 @@ public abstract class LogAggregationFileController {
   protected int retentionSize;
   protected String fileControllerName;
 
+  protected boolean fsSupportsChmod = true;
+
   public LogAggregationFileController() {}
 
   /**
@@ -250,7 +252,6 @@ public abstract class LogAggregationFileController {
* Verify and create the remote log directory.
*/
   public void verifyAndCreateRemoteLogDir() {
-boolean logPermError = true;
 // Checking the existence of the TLD
 FileSystem remoteFS = null;
 try {
@@ -264,14 +265,12 @@ public abstract class LogAggregationFileController {
 try {
   FsPermission perms =
   remoteFS.getFileStatus(remoteRootLogDir).getPermission();
-  if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) {
+  if (!perms.equals(TLDIR_PERMISSIONS)) {
 LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
 + "] already exist, but with incorrect permissions. "
 + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
 + "]." + " The cluster may have problems with multiple users.");
-logPermError = false;
-  } else {
-logPermError = true;
+
   }
 } catch (FileNotFoundException e) {
   remoteExists = false;
@@ -280,15 +279,26 @@ public abstract class LogAggregationFileController {
   "Failed to check permissions for dir ["
   + remoteRootLogDir + "]", e);
 }
+
+Path qualified =
+remoteRootLogDir.makeQualified(remoteFS.getUri(),
+remoteFS.getWorkingDirectory());
 if (!remoteExists) {
   LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
   + "] does not exist. Attempting to create it.");
   try {
-Path qualified =
-remoteRootLogDir.makeQualified(remoteFS.getUri(),
-remoteFS.getWorkingDirectory());
 remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS));
-remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+
+// Not possible to query FileSystem API to check if it supports
+// chmod, chown etc. Hence resorting to catching exceptions here.
+// Remove when FS APi is ready
+try {
+  remoteFS.setPermission(qualified, new 
FsPermission(TLDIR_PERMISSIONS));
+} catch ( UnsupportedOperationException use) {
+  LOG.info("Unable to set permissions for configured filesystem since"
+  + " it does not support this", remoteFS.getScheme());
+  fsSupportsChmod = false;
+}
 
 UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
 String primaryGroupName = null;
@@ -301,13 +311,31 @@ public abstract class LogAggregationFileController {
 }
 // set owner on the remote directory only if the primary group exists
 if (primaryGroupName != null) {
-  remoteFS.setOwner(qualified,
-  loginUser.getShortUserName(), primaryGroupName);
+  try {
+

[01/50] [abbrv] hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)

hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)

[1/6] hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)

3 matches

Site Navigation

Mail list logo

Footer information