This is an automated email from the ASF dual-hosted git repository.
shashikant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 78ca8bf HDDS-4077. Incomplete OzoneFileSystem statistics (#1329)
78ca8bf is described below
commit 78ca8bfbcdb4ed9566c7906ef41fdf0f43a8bd47
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Aug 31 08:41:20 2020 +0200
HDDS-4077. Incomplete OzoneFileSystem statistics (#1329)
---
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 15 +++--
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 71 ++++++++++++++++++++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 71 ++++++++++++++++++++++
3 files changed, 153 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
index 06d1bd3..2b8803e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -357,15 +357,18 @@ public class TestOzoneFileInterfaces {
String dirPath = RandomStringUtils.randomAlphanumeric(5);
Path path = createPath("/" + dirPath);
paths.add(path);
+
+ long mkdirs = statistics.getLong(
+ StorageStatistics.CommonStatisticNames.OP_MKDIRS);
assertTrue("Makedirs returned with false for the path " + path,
fs.mkdirs(path));
+ assertCounter(++mkdirs, StorageStatistics.CommonStatisticNames.OP_MKDIRS);
long listObjects = statistics.getLong(Statistic.OBJECTS_LIST.getSymbol());
long omListStatus = omMetrics.getNumListStatus();
FileStatus[] statusList = fs.listStatus(createPath("/"));
assertEquals(1, statusList.length);
- assertEquals(++listObjects,
- statistics.getLong(Statistic.OBJECTS_LIST.getSymbol()).longValue());
+ assertCounter(++listObjects, Statistic.OBJECTS_LIST.getSymbol());
assertEquals(++omListStatus, omMetrics.getNumListStatus());
assertEquals(fs.getFileStatus(path), statusList[0]);
@@ -374,11 +377,11 @@ public class TestOzoneFileInterfaces {
paths.add(path);
assertTrue("Makedirs returned with false for the path " + path,
fs.mkdirs(path));
+ assertCounter(++mkdirs, StorageStatistics.CommonStatisticNames.OP_MKDIRS);
statusList = fs.listStatus(createPath("/"));
assertEquals(2, statusList.length);
- assertEquals(++listObjects,
- statistics.getLong(Statistic.OBJECTS_LIST.getSymbol()).longValue());
+ assertCounter(++listObjects, Statistic.OBJECTS_LIST.getSymbol());
assertEquals(++omListStatus, omMetrics.getNumListStatus());
for (Path p : paths) {
assertTrue(Arrays.asList(statusList).contains(fs.getFileStatus(p)));
@@ -528,4 +531,8 @@ public class TestOzoneFileInterfaces {
return status;
}
+
+ private void assertCounter(long value, String key) {
+ assertEquals(value, statistics.getLong(key).longValue());
+ }
}
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index 49f12f0..e4acabc 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -26,13 +26,16 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
@@ -692,6 +695,7 @@ public class BasicOzoneFileSystem extends FileSystem {
@Override
public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ incrementCounter(Statistic.INVOCATION_MKDIRS);
LOG.trace("mkdir() path:{} ", f);
String key = pathToKey(f);
if (isEmpty(key)) {
@@ -735,6 +739,73 @@ public class BasicOzoneFileSystem extends FileSystem {
return adapter.getDefaultReplication();
}
+ @Override
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs,
+ Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE);
+ super.copyFromLocalFile(delSrc, overwrite, srcs, dst);
+ }
+
+ @Override
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
+ Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE);
+ super.copyFromLocalFile(delSrc, overwrite, src, dst);
+ }
+
+ @Override
+ public boolean exists(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_EXISTS);
+ return super.exists(f);
+ }
+
+ @Override
+ public FileChecksum getFileChecksum(Path f, long length) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GET_FILE_CHECKSUM);
+ return super.getFileChecksum(f, length);
+ }
+
+ @Override
+ public FileStatus[] globStatus(Path pathPattern) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GLOB_STATUS);
+ return super.globStatus(pathPattern);
+ }
+
+ @Override
+ public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
+ throws IOException {
+ incrementCounter(Statistic.INVOCATION_GLOB_STATUS);
+ return super.globStatus(pathPattern, filter);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public boolean isDirectory(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_IS_DIRECTORY);
+ return super.isDirectory(f);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public boolean isFile(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_IS_FILE);
+ return super.isFile(f);
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive)
+ throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_FILES);
+ return super.listFiles(f, recursive);
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
+ throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS);
+ return super.listLocatedStatus(f);
+ }
+
/**
* Turn a path (relative or otherwise) into an Ozone key.
*
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 59aec47..015621c 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -24,12 +24,15 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
@@ -715,6 +718,7 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
@Override
public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ incrementCounter(Statistic.INVOCATION_MKDIRS);
LOG.trace("mkdir() path:{} ", f);
String key = pathToKey(f);
if (isEmpty(key)) {
@@ -764,6 +768,73 @@ public class BasicRootedOzoneFileSystem extends FileSystem
{
return adapter.getDefaultReplication();
}
+ @Override
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs,
+ Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE);
+ super.copyFromLocalFile(delSrc, overwrite, srcs, dst);
+ }
+
+ @Override
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
+ Path dst) throws IOException {
+ incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE);
+ super.copyFromLocalFile(delSrc, overwrite, src, dst);
+ }
+
+ @Override
+ public boolean exists(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_EXISTS);
+ return super.exists(f);
+ }
+
+ @Override
+ public FileChecksum getFileChecksum(Path f, long length) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GET_FILE_CHECKSUM);
+ return super.getFileChecksum(f, length);
+ }
+
+ @Override
+ public FileStatus[] globStatus(Path pathPattern) throws IOException {
+ incrementCounter(Statistic.INVOCATION_GLOB_STATUS);
+ return super.globStatus(pathPattern);
+ }
+
+ @Override
+ public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
+ throws IOException {
+ incrementCounter(Statistic.INVOCATION_GLOB_STATUS);
+ return super.globStatus(pathPattern, filter);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public boolean isDirectory(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_IS_DIRECTORY);
+ return super.isDirectory(f);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public boolean isFile(Path f) throws IOException {
+ incrementCounter(Statistic.INVOCATION_IS_FILE);
+ return super.isFile(f);
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive)
+ throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_FILES);
+ return super.listFiles(f, recursive);
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
+ throws IOException {
+ incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS);
+ return super.listLocatedStatus(f);
+ }
+
/**
* Turn a path (relative or otherwise) into an Ozone key.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
