HDFS-14095. EC: Track Erasure Coding commands in DFS statistics. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5347368 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5347368 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5347368 Branch: refs/heads/HDFS-12943 Commit: f534736867eed962899615ca1b7eb68bcf591d17 Parents: d0edd37 Author: Brahma Reddy Battula <bra...@apache.org> Authored: Fri Nov 30 00:18:27 2018 +0530 Committer: Brahma Reddy Battula <bra...@apache.org> Committed: Fri Nov 30 00:18:27 2018 +0530 ---------------------------------------------------------------------- .../hadoop/hdfs/DFSOpsCountStatistics.java | 9 +++ .../hadoop/hdfs/DistributedFileSystem.java | 18 ++++++ .../hadoop/hdfs/TestDistributedFileSystem.java | 63 +++++++++++++++++++- 3 files changed, 89 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java index 3dcf13b..b9852ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java @@ -41,6 +41,7 @@ public class DFSOpsCountStatistics extends StorageStatistics { /** This is for counting distributed file system operations. */ public enum OpType { + ADD_EC_POLICY("op_add_ec_policy"), ALLOW_SNAPSHOT("op_allow_snapshot"), APPEND(CommonStatisticNames.OP_APPEND), CONCAT("op_concat"), @@ -51,10 +52,15 @@ public class DFSOpsCountStatistics extends StorageStatistics { CREATE_SYM_LINK("op_create_symlink"), DELETE(CommonStatisticNames.OP_DELETE), DELETE_SNAPSHOT("op_delete_snapshot"), + DISABLE_EC_POLICY("op_disable_ec_policy"), DISALLOW_SNAPSHOT("op_disallow_snapshot"), + ENABLE_EC_POLICY("op_enable_ec_policy"), EXISTS(CommonStatisticNames.OP_EXISTS), GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"), GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY), + GET_EC_CODECS("op_get_ec_codecs"), + GET_EC_POLICY("op_get_ec_policy"), + GET_EC_POLICIES("op_get_ec_policies"), GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"), GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM), GET_FILE_LINK_STATUS("op_get_file_link_status"), @@ -76,11 +82,13 @@ public class DFSOpsCountStatistics extends StorageStatistics { REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL), REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES), REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL), + REMOVE_EC_POLICY("op_remove_ec_policy"), REMOVE_XATTR("op_remove_xattr"), RENAME(CommonStatisticNames.OP_RENAME), RENAME_SNAPSHOT("op_rename_snapshot"), RESOLVE_LINK("op_resolve_link"), SET_ACL(CommonStatisticNames.OP_SET_ACL), + SET_EC_POLICY("op_set_ec_policy"), SET_OWNER(CommonStatisticNames.OP_SET_OWNER), SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION), SET_REPLICATION("op_set_replication"), @@ -90,6 +98,7 @@ public class DFSOpsCountStatistics extends StorageStatistics { GET_SNAPSHOT_DIFF("op_get_snapshot_diff"), GET_SNAPSHOTTABLE_DIRECTORY_LIST("op_get_snapshottable_directory_list"), TRUNCATE(CommonStatisticNames.OP_TRUNCATE), + UNSET_EC_POLICY("op_unset_ec_policy"), UNSET_STORAGE_POLICY("op_unset_storage_policy"); private static final Map<String, OpType> SYMBOL_MAP = http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index ca1546c..7dd02bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2845,6 +2845,8 @@ public class DistributedFileSystem extends FileSystem */ public void setErasureCodingPolicy(final Path path, final String ecPolicyName) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.SET_EC_POLICY); Path absF = fixRelativePart(path); new FileSystemLinkResolver<Void>() { @Override @@ -2912,6 +2914,8 @@ public class DistributedFileSystem extends FileSystem */ public ErasureCodingPolicy getErasureCodingPolicy(final Path path) throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_EC_POLICY); Path absF = fixRelativePart(path); return new FileSystemLinkResolver<ErasureCodingPolicy>() { @Override @@ -2943,6 +2947,8 @@ public class DistributedFileSystem extends FileSystem */ public Collection<ErasureCodingPolicyInfo> getAllErasureCodingPolicies() throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_EC_POLICIES); return Arrays.asList(dfs.getErasureCodingPolicies()); } @@ -2955,6 +2961,8 @@ public class DistributedFileSystem extends FileSystem */ public Map<String, String> getAllErasureCodingCodecs() throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_EC_CODECS); return dfs.getErasureCodingCodecs(); } @@ -2971,6 +2979,8 @@ public class DistributedFileSystem extends FileSystem */ public AddErasureCodingPolicyResponse[] addErasureCodingPolicies( ErasureCodingPolicy[] policies) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.ADD_EC_POLICY); return dfs.addErasureCodingPolicies(policies); } @@ -2982,6 +2992,8 @@ public class DistributedFileSystem extends FileSystem */ public void removeErasureCodingPolicy(String ecPolicyName) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.REMOVE_EC_POLICY); dfs.removeErasureCodingPolicy(ecPolicyName); } @@ -2993,6 +3005,8 @@ public class DistributedFileSystem extends FileSystem */ public void enableErasureCodingPolicy(String ecPolicyName) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.ENABLE_EC_POLICY); dfs.enableErasureCodingPolicy(ecPolicyName); } @@ -3004,6 +3018,8 @@ public class DistributedFileSystem extends FileSystem */ public void disableErasureCodingPolicy(String ecPolicyName) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.DISABLE_EC_POLICY); dfs.disableErasureCodingPolicy(ecPolicyName); } @@ -3014,6 +3030,8 @@ public class DistributedFileSystem extends FileSystem * @throws IOException */ public void unsetErasureCodingPolicy(final Path path) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.UNSET_EC_POLICY); Path absF = fixRelativePart(path); new FileSystemLinkResolver<Void>() { @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5347368/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index cae0fbf..97ced7a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -773,7 +773,68 @@ public class TestDistributedFileSystem { } finally { if (cluster != null) cluster.shutdown(); } - + } + + @Test + public void testECStatistics() throws IOException { + try (MiniDFSCluster cluster = + new MiniDFSCluster.Builder(getTestConfiguration()).build()) { + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + Path dir = new Path("/test"); + dfs.mkdirs(dir); + int readOps = 0; + int writeOps = 0; + FileSystem.clearStatistics(); + + long opCount = getOpStatistics(OpType.ENABLE_EC_POLICY); + dfs.enableErasureCodingPolicy("RS-10-4-1024k"); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.ENABLE_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.SET_EC_POLICY); + dfs.setErasureCodingPolicy(dir, "RS-10-4-1024k"); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.SET_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.GET_EC_POLICY); + dfs.getErasureCodingPolicy(dir); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.UNSET_EC_POLICY); + dfs.unsetErasureCodingPolicy(dir); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.UNSET_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.GET_EC_POLICIES); + dfs.getAllErasureCodingPolicies(); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_EC_POLICIES, opCount + 1); + + opCount = getOpStatistics(OpType.GET_EC_CODECS); + dfs.getAllErasureCodingCodecs(); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_EC_CODECS, opCount + 1); + + ErasureCodingPolicy newPolicy = + new ErasureCodingPolicy(new ECSchema("rs", 5, 3), 1024 * 1024); + + opCount = getOpStatistics(OpType.ADD_EC_POLICY); + dfs.addErasureCodingPolicies(new ErasureCodingPolicy[] {newPolicy}); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.ADD_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.REMOVE_EC_POLICY); + dfs.removeErasureCodingPolicy("RS-5-3-1024k"); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.REMOVE_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.DISABLE_EC_POLICY); + dfs.disableErasureCodingPolicy("RS-10-4-1024k"); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.DISABLE_EC_POLICY, opCount + 1); + } } @SuppressWarnings("ThrowableResultOfMethodCallIgnored") --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
