[15/50] [abbrv] hadoop git commit: YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

[virajith]

YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0799fde3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0799fde3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0799fde3

Branch: refs/heads/HDFS-9806
Commit: 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1
Parents: b61144a
Author: Yufei Gu 
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: Yufei Gu 
Committed: Fri Oct 20 10:00:13 2017 -0700

--
 .../org/apache/hadoop/yarn/util/FSDownload.java| 17 +++--
 .../localizer/ResourceLocalizationService.java | 11 ++-
 2 files changed, 25 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable {
 } catch (URISyntaxException e) {
   throw new IOException("Invalid resource", e);
 }
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Starting to download " + sCopy);
+}
+
 createDir(destDirPath, cachePerms);
 final Path dst_work = new Path(destDirPath + "_tmp");
 createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable {
   unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
   changePermissions(dFinal.getFileSystem(conf), dFinal);
   files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+  if (LOG.isDebugEnabled()) {
+LOG.debug("File has been downloaded to " +
+new Path(destDirPath, sCopy.getName()));
+  }
 } catch (Exception e) {
   try {
 files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable {
   // APPLICATION:
   perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
 }
-LOG.debug("Changing permissions for path " + path
-+ " to perm " + perm);
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+}
+
 final FsPermission fPerm = perm;
 if (null == userUgi) {
   files.setPermission(path, perm);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends 
CompositeService
   // TODO handle failures, cancellation, requests by other containers
   LocalizedResource rsrc = request.getResource();
   LocalResourceRequest key = rsrc.getRequest();
-  LOG.info("Downloading public rsrc:" + key);
+  LOG.info("Downloading public resource: " + key);
   /*
* Here multiple containers may request the same resource. So we need
* to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends 
CompositeService
 + " Either queue is full or threadpool is shutdown.", re);
   }
 } else {
+  if (LOG.isDebugEnabled()) {
+LOG.debug("Skip 

[35/50] hadoop git commit: YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

[aengineer]

YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0799fde3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0799fde3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0799fde3

Branch: refs/heads/HDFS-7240
Commit: 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1
Parents: b61144a
Author: Yufei Gu 
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: Yufei Gu 
Committed: Fri Oct 20 10:00:13 2017 -0700

--
 .../org/apache/hadoop/yarn/util/FSDownload.java| 17 +++--
 .../localizer/ResourceLocalizationService.java | 11 ++-
 2 files changed, 25 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable {
 } catch (URISyntaxException e) {
   throw new IOException("Invalid resource", e);
 }
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Starting to download " + sCopy);
+}
+
 createDir(destDirPath, cachePerms);
 final Path dst_work = new Path(destDirPath + "_tmp");
 createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable {
   unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
   changePermissions(dFinal.getFileSystem(conf), dFinal);
   files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+  if (LOG.isDebugEnabled()) {
+LOG.debug("File has been downloaded to " +
+new Path(destDirPath, sCopy.getName()));
+  }
 } catch (Exception e) {
   try {
 files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable {
   // APPLICATION:
   perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
 }
-LOG.debug("Changing permissions for path " + path
-+ " to perm " + perm);
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+}
+
 final FsPermission fPerm = perm;
 if (null == userUgi) {
   files.setPermission(path, perm);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends 
CompositeService
   // TODO handle failures, cancellation, requests by other containers
   LocalizedResource rsrc = request.getResource();
   LocalResourceRequest key = rsrc.getRequest();
-  LOG.info("Downloading public rsrc:" + key);
+  LOG.info("Downloading public resource: " + key);
   /*
* Here multiple containers may request the same resource. So we need
* to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends 
CompositeService
 + " Either queue is full or threadpool is shutdown.", re);
   }
 } else {
+  if (LOG.isDebugEnabled()) {
+LOG.debug("Skip 

[03/50] [abbrv] hadoop git commit: YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

[templedf]

YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

(cherry picked from commit 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a0ef092
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a0ef092
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a0ef092

Branch: refs/heads/resource-types
Commit: 2a0ef092d3f63880b6437ef942caf1741b7d63c8
Parents: 96106b8
Author: Yufei Gu 
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: Yufei Gu 
Committed: Fri Oct 20 10:01:52 2017 -0700

--
 .../org/apache/hadoop/yarn/util/FSDownload.java| 17 +++--
 .../localizer/ResourceLocalizationService.java | 11 ++-
 2 files changed, 25 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a0ef092/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable {
 } catch (URISyntaxException e) {
   throw new IOException("Invalid resource", e);
 }
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Starting to download " + sCopy);
+}
+
 createDir(destDirPath, cachePerms);
 final Path dst_work = new Path(destDirPath + "_tmp");
 createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable {
   unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
   changePermissions(dFinal.getFileSystem(conf), dFinal);
   files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+  if (LOG.isDebugEnabled()) {
+LOG.debug("File has been downloaded to " +
+new Path(destDirPath, sCopy.getName()));
+  }
 } catch (Exception e) {
   try {
 files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable {
   // APPLICATION:
   perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
 }
-LOG.debug("Changing permissions for path " + path
-+ " to perm " + perm);
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+}
+
 final FsPermission fPerm = perm;
 if (null == userUgi) {
   files.setPermission(path, perm);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a0ef092/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends 
CompositeService
   // TODO handle failures, cancellation, requests by other containers
   LocalizedResource rsrc = request.getResource();
   LocalResourceRequest key = rsrc.getRequest();
-  LOG.info("Downloading public rsrc:" + key);
+  LOG.info("Downloading public resource: " + key);
   /*
* Here multiple containers may request the same resource. So we need
* to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends 
CompositeService
 + " Either queue is full or threadpool is shutdown.", re);
   }
 } else 

[52/54] [abbrv] hadoop git commit: YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

[vrushali]

YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

(cherry picked from commit 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/378fa3ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/378fa3ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/378fa3ac

Branch: refs/heads/branch-2
Commit: 378fa3ac7db79675db2f71c66ec7601eac5e204f
Parents: 4c4f28c
Author: Yufei Gu 
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: vrushali 
Committed: Fri Oct 20 11:49:34 2017 -0700

--
 .../org/apache/hadoop/yarn/util/FSDownload.java| 17 +++--
 .../localizer/ResourceLocalizationService.java | 11 ++-
 2 files changed, 25 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/378fa3ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable {
 } catch (URISyntaxException e) {
   throw new IOException("Invalid resource", e);
 }
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Starting to download " + sCopy);
+}
+
 createDir(destDirPath, cachePerms);
 final Path dst_work = new Path(destDirPath + "_tmp");
 createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable {
   unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
   changePermissions(dFinal.getFileSystem(conf), dFinal);
   files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+  if (LOG.isDebugEnabled()) {
+LOG.debug("File has been downloaded to " +
+new Path(destDirPath, sCopy.getName()));
+  }
 } catch (Exception e) {
   try {
 files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable {
   // APPLICATION:
   perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
 }
-LOG.debug("Changing permissions for path " + path
-+ " to perm " + perm);
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+}
+
 final FsPermission fPerm = perm;
 if (null == userUgi) {
   files.setPermission(path, perm);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/378fa3ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends 
CompositeService
   // TODO handle failures, cancellation, requests by other containers
   LocalizedResource rsrc = request.getResource();
   LocalResourceRequest key = rsrc.getRequest();
-  LOG.info("Downloading public rsrc:" + key);
+  LOG.info("Downloading public resource: " + key);
   /*
* Here multiple containers may request the same resource. So we need
* to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends 
CompositeService
 + " Either queue is full or threadpool is shutdown.", re);
   }
 } else {

[2/3] hadoop git commit: YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

[vrushali]

YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

(cherry picked from commit 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/378fa3ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/378fa3ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/378fa3ac

Branch: refs/heads/YARN-3368_branch2
Commit: 378fa3ac7db79675db2f71c66ec7601eac5e204f
Parents: 4c4f28c
Author: Yufei Gu 
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: vrushali 
Committed: Fri Oct 20 11:49:34 2017 -0700

--
 .../org/apache/hadoop/yarn/util/FSDownload.java| 17 +++--
 .../localizer/ResourceLocalizationService.java | 11 ++-
 2 files changed, 25 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/378fa3ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable {
 } catch (URISyntaxException e) {
   throw new IOException("Invalid resource", e);
 }
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Starting to download " + sCopy);
+}
+
 createDir(destDirPath, cachePerms);
 final Path dst_work = new Path(destDirPath + "_tmp");
 createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable {
   unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
   changePermissions(dFinal.getFileSystem(conf), dFinal);
   files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+  if (LOG.isDebugEnabled()) {
+LOG.debug("File has been downloaded to " +
+new Path(destDirPath, sCopy.getName()));
+  }
 } catch (Exception e) {
   try {
 files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable {
   // APPLICATION:
   perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
 }
-LOG.debug("Changing permissions for path " + path
-+ " to perm " + perm);
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+}
+
 final FsPermission fPerm = perm;
 if (null == userUgi) {
   files.setPermission(path, perm);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/378fa3ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends 
CompositeService
   // TODO handle failures, cancellation, requests by other containers
   LocalizedResource rsrc = request.getResource();
   LocalResourceRequest key = rsrc.getRequest();
-  LOG.info("Downloading public rsrc:" + key);
+  LOG.info("Downloading public resource: " + key);
   /*
* Here multiple containers may request the same resource. So we need
* to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends 
CompositeService
 + " Either queue is full or threadpool is shutdown.", re);
   }
 

hadoop git commit: YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

[yufei]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 01b2e8b9c -> c3946f29e


YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

(cherry picked from commit 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3946f29
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3946f29
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3946f29

Branch: refs/heads/branch-2
Commit: c3946f29e75fda1f0ae972b5967109fb9cf1bef3
Parents: 01b2e8b
Author: Yufei Gu 
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: Yufei Gu 
Committed: Fri Oct 20 10:02:35 2017 -0700

--
 .../org/apache/hadoop/yarn/util/FSDownload.java| 17 +++--
 .../localizer/ResourceLocalizationService.java | 11 ++-
 2 files changed, 25 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3946f29/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable {
 } catch (URISyntaxException e) {
   throw new IOException("Invalid resource", e);
 }
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Starting to download " + sCopy);
+}
+
 createDir(destDirPath, cachePerms);
 final Path dst_work = new Path(destDirPath + "_tmp");
 createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable {
   unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
   changePermissions(dFinal.getFileSystem(conf), dFinal);
   files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+  if (LOG.isDebugEnabled()) {
+LOG.debug("File has been downloaded to " +
+new Path(destDirPath, sCopy.getName()));
+  }
 } catch (Exception e) {
   try {
 files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable {
   // APPLICATION:
   perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
 }
-LOG.debug("Changing permissions for path " + path
-+ " to perm " + perm);
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+}
+
 final FsPermission fPerm = perm;
 if (null == userUgi) {
   files.setPermission(path, perm);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3946f29/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends 
CompositeService
   // TODO handle failures, cancellation, requests by other containers
   LocalizedResource rsrc = request.getResource();
   LocalResourceRequest key = rsrc.getRequest();
-  LOG.info("Downloading public rsrc:" + key);
+  LOG.info("Downloading public resource: " + key);
   /*
* Here multiple containers may request the same resource. So we need
* to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends 
CompositeService
 + " 

hadoop git commit: YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

[yufei]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 96106b8f5 -> 2a0ef092d


YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

(cherry picked from commit 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a0ef092
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a0ef092
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a0ef092

Branch: refs/heads/branch-3.0
Commit: 2a0ef092d3f63880b6437ef942caf1741b7d63c8
Parents: 96106b8
Author: Yufei Gu 
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: Yufei Gu 
Committed: Fri Oct 20 10:01:52 2017 -0700

--
 .../org/apache/hadoop/yarn/util/FSDownload.java| 17 +++--
 .../localizer/ResourceLocalizationService.java | 11 ++-
 2 files changed, 25 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a0ef092/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable {
 } catch (URISyntaxException e) {
   throw new IOException("Invalid resource", e);
 }
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Starting to download " + sCopy);
+}
+
 createDir(destDirPath, cachePerms);
 final Path dst_work = new Path(destDirPath + "_tmp");
 createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable {
   unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
   changePermissions(dFinal.getFileSystem(conf), dFinal);
   files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+  if (LOG.isDebugEnabled()) {
+LOG.debug("File has been downloaded to " +
+new Path(destDirPath, sCopy.getName()));
+  }
 } catch (Exception e) {
   try {
 files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable {
   // APPLICATION:
   perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
 }
-LOG.debug("Changing permissions for path " + path
-+ " to perm " + perm);
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+}
+
 final FsPermission fPerm = perm;
 if (null == userUgi) {
   files.setPermission(path, perm);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a0ef092/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends 
CompositeService
   // TODO handle failures, cancellation, requests by other containers
   LocalizedResource rsrc = request.getResource();
   LocalResourceRequest key = rsrc.getRequest();
-  LOG.info("Downloading public rsrc:" + key);
+  LOG.info("Downloading public resource: " + key);
   /*
* Here multiple containers may request the same resource. So we need
* to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends 
CompositeService
 + " 

hadoop git commit: YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)

[yufei]

Repository: hadoop
Updated Branches:
  refs/heads/trunk b61144a93 -> 0799fde35


YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0799fde3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0799fde3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0799fde3

Branch: refs/heads/trunk
Commit: 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1
Parents: b61144a
Author: Yufei Gu 
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: Yufei Gu 
Committed: Fri Oct 20 10:00:13 2017 -0700

--
 .../org/apache/hadoop/yarn/util/FSDownload.java| 17 +++--
 .../localizer/ResourceLocalizationService.java | 11 ++-
 2 files changed, 25 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable {
 } catch (URISyntaxException e) {
   throw new IOException("Invalid resource", e);
 }
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Starting to download " + sCopy);
+}
+
 createDir(destDirPath, cachePerms);
 final Path dst_work = new Path(destDirPath + "_tmp");
 createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable {
   unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
   changePermissions(dFinal.getFileSystem(conf), dFinal);
   files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+  if (LOG.isDebugEnabled()) {
+LOG.debug("File has been downloaded to " +
+new Path(destDirPath, sCopy.getName()));
+  }
 } catch (Exception e) {
   try {
 files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable {
   // APPLICATION:
   perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
 }
-LOG.debug("Changing permissions for path " + path
-+ " to perm " + perm);
+
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+}
+
 final FsPermission fPerm = perm;
 if (null == userUgi) {
   files.setPermission(path, perm);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends 
CompositeService
   // TODO handle failures, cancellation, requests by other containers
   LocalizedResource rsrc = request.getResource();
   LocalResourceRequest key = rsrc.getRequest();
-  LOG.info("Downloading public rsrc:" + key);
+  LOG.info("Downloading public resource: " + key);
   /*
* Here multiple containers may request the same resource. So we need
* to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends 
CompositeService
 + " Either queue is full or threadpool is shutdown.", re);
   }
 }