[03/50] [abbrv] hadoop git commit: HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi.

2018-08-25 Thread shv 
HDFS-13747. Statistic for list_located_status is incremented incorrectly by 
listStatusIterator. Contributed by Antal Mihalyi.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c67b0650
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c67b0650
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c67b0650

Branch: refs/heads/HDFS-12943
Commit: c67b0650ea10896c6289703595faef0d262c00b3
Parents: 1697a02
Author: Xiao Chen 
Authored: Thu Aug 16 23:13:10 2018 -0700
Committer: Xiao Chen 
Committed: Thu Aug 16 23:14:21 2018 -0700

--
 .../java/org/apache/hadoop/hdfs/DistributedFileSystem.java| 6 +-
 .../org/apache/hadoop/hdfs/TestDistributedFileSystem.java | 7 +++
 2 files changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c67b0650/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 70b3679..28c1e27 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1217,7 +1217,11 @@ public class DistributedFileSystem extends FileSystem
   thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME,
   needLocation);
   statistics.incrementReadOps(1);
-  storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS);
+  if (needLocation) {
+storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS);
+  } else {
+storageStatistics.incrementOpCounter(OpType.LIST_STATUS);
+  }
   if (thisListing == null) { // the directory does not exist
 throw new FileNotFoundException("File " + p + " does not exist.");
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c67b0650/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index f09255e..46323dd 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -706,6 +706,7 @@ public class TestDistributedFileSystem {
   // Iterative ls test
   long mkdirOp = getOpStatistics(OpType.MKDIRS);
   long listStatusOp = getOpStatistics(OpType.LIST_STATUS);
+  long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS);
   for (int i = 0; i < 10; i++) {
 Path p = new Path(dir, Integer.toString(i));
 fs.mkdirs(p);
@@ -729,6 +730,12 @@ public class TestDistributedFileSystem {
 checkStatistics(fs, readOps, ++writeOps, largeReadOps);
 checkOpStatistics(OpType.MKDIRS, mkdirOp);
 checkOpStatistics(OpType.LIST_STATUS, listStatusOp);
+
+fs.listLocatedStatus(dir);
+locatedListStatusOP++;
+readOps++;
+checkStatistics(fs, readOps, writeOps, largeReadOps);
+checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP);
   }
   
   opCount = getOpStatistics(OpType.GET_STATUS);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi.

2018-08-17 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 3532bd5c8 -> 06f0d5e25


HDFS-13747. Statistic for list_located_status is incremented incorrectly by 
listStatusIterator. Contributed by Antal Mihalyi.

(cherry picked from commit c67b0650ea10896c6289703595faef0d262c00b3)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06f0d5e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06f0d5e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06f0d5e2

Branch: refs/heads/branch-3.1
Commit: 06f0d5e257a3208f905f59019623a589825e4c8b
Parents: 3532bd5
Author: Xiao Chen 
Authored: Thu Aug 16 23:13:10 2018 -0700
Committer: Xiao Chen 
Committed: Thu Aug 16 23:15:48 2018 -0700

--
 .../java/org/apache/hadoop/hdfs/DistributedFileSystem.java| 6 +-
 .../org/apache/hadoop/hdfs/TestDistributedFileSystem.java | 7 +++
 2 files changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0d5e2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 3519c60..de05f82 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1217,7 +1217,11 @@ public class DistributedFileSystem extends FileSystem
   thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME,
   needLocation);
   statistics.incrementReadOps(1);
-  storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS);
+  if (needLocation) {
+storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS);
+  } else {
+storageStatistics.incrementOpCounter(OpType.LIST_STATUS);
+  }
   if (thisListing == null) { // the directory does not exist
 throw new FileNotFoundException("File " + p + " does not exist.");
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0d5e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 072ee9f..03e6c8a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -706,6 +706,7 @@ public class TestDistributedFileSystem {
   // Iterative ls test
   long mkdirOp = getOpStatistics(OpType.MKDIRS);
   long listStatusOp = getOpStatistics(OpType.LIST_STATUS);
+  long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS);
   for (int i = 0; i < 10; i++) {
 Path p = new Path(dir, Integer.toString(i));
 fs.mkdirs(p);
@@ -729,6 +730,12 @@ public class TestDistributedFileSystem {
 checkStatistics(fs, readOps, ++writeOps, largeReadOps);
 checkOpStatistics(OpType.MKDIRS, mkdirOp);
 checkOpStatistics(OpType.LIST_STATUS, listStatusOp);
+
+fs.listLocatedStatus(dir);
+locatedListStatusOP++;
+readOps++;
+checkStatistics(fs, readOps, writeOps, largeReadOps);
+checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP);
   }
   
   opCount = getOpStatistics(OpType.GET_STATUS);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi.

2018-08-17 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 90bf2d3b5 -> 185c8f2ab


HDFS-13747. Statistic for list_located_status is incremented incorrectly by 
listStatusIterator. Contributed by Antal Mihalyi.

(cherry picked from commit c67b0650ea10896c6289703595faef0d262c00b3)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/185c8f2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/185c8f2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/185c8f2a

Branch: refs/heads/branch-3.0
Commit: 185c8f2abc364e4941ca4d4522fb61b5b3f5f903
Parents: 90bf2d3
Author: Xiao Chen 
Authored: Thu Aug 16 23:13:10 2018 -0700
Committer: Xiao Chen 
Committed: Thu Aug 16 23:15:58 2018 -0700

--
 .../java/org/apache/hadoop/hdfs/DistributedFileSystem.java| 6 +-
 .../org/apache/hadoop/hdfs/TestDistributedFileSystem.java | 7 +++
 2 files changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/185c8f2a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 84d840f..9208e66 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1143,7 +1143,11 @@ public class DistributedFileSystem extends FileSystem
   thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME,
   needLocation);
   statistics.incrementReadOps(1);
-  storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS);
+  if (needLocation) {
+storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS);
+  } else {
+storageStatistics.incrementOpCounter(OpType.LIST_STATUS);
+  }
   if (thisListing == null) { // the directory does not exist
 throw new FileNotFoundException("File " + p + " does not exist.");
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/185c8f2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 072ee9f..03e6c8a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -706,6 +706,7 @@ public class TestDistributedFileSystem {
   // Iterative ls test
   long mkdirOp = getOpStatistics(OpType.MKDIRS);
   long listStatusOp = getOpStatistics(OpType.LIST_STATUS);
+  long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS);
   for (int i = 0; i < 10; i++) {
 Path p = new Path(dir, Integer.toString(i));
 fs.mkdirs(p);
@@ -729,6 +730,12 @@ public class TestDistributedFileSystem {
 checkStatistics(fs, readOps, ++writeOps, largeReadOps);
 checkOpStatistics(OpType.MKDIRS, mkdirOp);
 checkOpStatistics(OpType.LIST_STATUS, listStatusOp);
+
+fs.listLocatedStatus(dir);
+locatedListStatusOP++;
+readOps++;
+checkStatistics(fs, readOps, writeOps, largeReadOps);
+checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP);
   }
   
   opCount = getOpStatistics(OpType.GET_STATUS);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi.

2018-08-17 Thread xiao 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 1697a0230 -> c67b0650e


HDFS-13747. Statistic for list_located_status is incremented incorrectly by 
listStatusIterator. Contributed by Antal Mihalyi.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c67b0650
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c67b0650
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c67b0650

Branch: refs/heads/trunk
Commit: c67b0650ea10896c6289703595faef0d262c00b3
Parents: 1697a02
Author: Xiao Chen 
Authored: Thu Aug 16 23:13:10 2018 -0700
Committer: Xiao Chen 
Committed: Thu Aug 16 23:14:21 2018 -0700

--
 .../java/org/apache/hadoop/hdfs/DistributedFileSystem.java| 6 +-
 .../org/apache/hadoop/hdfs/TestDistributedFileSystem.java | 7 +++
 2 files changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c67b0650/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 70b3679..28c1e27 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1217,7 +1217,11 @@ public class DistributedFileSystem extends FileSystem
   thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME,
   needLocation);
   statistics.incrementReadOps(1);
-  storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS);
+  if (needLocation) {
+storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS);
+  } else {
+storageStatistics.incrementOpCounter(OpType.LIST_STATUS);
+  }
   if (thisListing == null) { // the directory does not exist
 throw new FileNotFoundException("File " + p + " does not exist.");
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c67b0650/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index f09255e..46323dd 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -706,6 +706,7 @@ public class TestDistributedFileSystem {
   // Iterative ls test
   long mkdirOp = getOpStatistics(OpType.MKDIRS);
   long listStatusOp = getOpStatistics(OpType.LIST_STATUS);
+  long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS);
   for (int i = 0; i < 10; i++) {
 Path p = new Path(dir, Integer.toString(i));
 fs.mkdirs(p);
@@ -729,6 +730,12 @@ public class TestDistributedFileSystem {
 checkStatistics(fs, readOps, ++writeOps, largeReadOps);
 checkOpStatistics(OpType.MKDIRS, mkdirOp);
 checkOpStatistics(OpType.LIST_STATUS, listStatusOp);
+
+fs.listLocatedStatus(dir);
+locatedListStatusOP++;
+readOps++;
+checkStatistics(fs, readOps, writeOps, largeReadOps);
+checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP);
   }
   
   opCount = getOpStatistics(OpType.GET_STATUS);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org