[14/49] hadoop git commit: HDFS-14039. ec -listPolicies doesn't show correct state for the default policy when the default is not RS(6, 3). Contributed by Kitti Nanasi.
HDFS-14039. ec -listPolicies doesn't show correct state for the default policy
when the default is not RS(6,3). Contributed by Kitti Nanasi.
Signed-off-by: Xiao Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d99648c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d99648c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d99648c
Branch: refs/heads/HDFS-13891
Commit: 8d99648c203004045a9339ad27258092969145d6
Parents: 724c150
Author: Kitti Nanasi
Authored: Thu Nov 8 10:00:09 2018 -0800
Committer: Xiao Chen
Committed: Thu Nov 8 10:01:19 2018 -0800
--
.../namenode/ErasureCodingPolicyManager.java| 119 ++-
.../server/namenode/FSImageFormatProtobuf.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 2 +-
.../server/namenode/TestEnabledECPolicies.java | 103 +++-
.../hdfs/server/namenode/TestFSImage.java | 42 +--
.../server/namenode/TestNamenodeRetryCache.java | 2 +-
.../server/namenode/TestStripedINodeFile.java | 2 +-
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
8 files changed, 231 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index d2bf3af..57fa958 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -81,6 +82,15 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyInfo[] allPolicies;
/**
+ * All policies in the state as it will be persisted in the fsimage.
+ *
+ * The difference between persisted policies and all policies is that
+ * if a default policy is only enabled at startup,
+ * it will appear as disabled in the persisted policy list and in the
fsimage.
+ */
+ private Map allPersistedPolicies;
+
+ /**
* All enabled policies sorted by name for fast querying, including built-in
* policy, user defined policy.
*/
@@ -90,6 +100,7 @@ public final class ErasureCodingPolicyManager {
*/
private ErasureCodingPolicy[] enabledPolicies;
+ private String defaultPolicyName;
private volatile static ErasureCodingPolicyManager instance = null;
@@ -102,14 +113,11 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyManager() {}
- public void init(Configuration conf) {
-// Load erasure coding default policy
-final String defaultPolicyName = conf.getTrimmed(
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+ public void init(Configuration conf) throws IOException {
this.policiesByName = new TreeMap<>();
this.policiesByID = new TreeMap<>();
this.enabledPoliciesByName = new TreeMap<>();
+this.allPersistedPolicies = new TreeMap<>();
/**
* TODO: load user defined EC policy from fsImage HDFS-7859
@@ -125,31 +133,12 @@ public final class ErasureCodingPolicyManager {
final ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
policiesByName.put(policy.getName(), info);
policiesByID.put(policy.getId(), info);
+ allPersistedPolicies.put(policy.getId(),
+ new ErasureCodingPolicyInfo(policy));
}
-if (!defaultPolicyName.isEmpty()) {
- final ErasureCodingPolicyInfo info =
- policiesByName.get(defaultPolicyName);
- if (info == null) {
-String names = policiesByName.values()
-.stream().map((pi) -> pi.getPolicy().getName())
-.collect(Collectors.joining(", "));
-String msg = String.format("EC policy '%s' specified at %s is not a "
-+ "valid policy. Please choose from list of available "
-+ "policies: [%s]",
-defaultPolicyName,
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
-names);
-throw new HadoopIllegalArgumentException(msg);
- }
- info.setState(ErasureCodingPolicyStat
[15/50] [abbrv] hadoop git commit: HDFS-14039. ec -listPolicies doesn't show correct state for the default policy when the default is not RS(6, 3). Contributed by Kitti Nanasi.
HDFS-14039. ec -listPolicies doesn't show correct state for the default policy
when the default is not RS(6,3). Contributed by Kitti Nanasi.
Signed-off-by: Xiao Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d99648c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d99648c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d99648c
Branch: refs/heads/YARN-7402
Commit: 8d99648c203004045a9339ad27258092969145d6
Parents: 724c150
Author: Kitti Nanasi
Authored: Thu Nov 8 10:00:09 2018 -0800
Committer: Xiao Chen
Committed: Thu Nov 8 10:01:19 2018 -0800
--
.../namenode/ErasureCodingPolicyManager.java| 119 ++-
.../server/namenode/FSImageFormatProtobuf.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 2 +-
.../server/namenode/TestEnabledECPolicies.java | 103 +++-
.../hdfs/server/namenode/TestFSImage.java | 42 +--
.../server/namenode/TestNamenodeRetryCache.java | 2 +-
.../server/namenode/TestStripedINodeFile.java | 2 +-
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
8 files changed, 231 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index d2bf3af..57fa958 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -81,6 +82,15 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyInfo[] allPolicies;
/**
+ * All policies in the state as it will be persisted in the fsimage.
+ *
+ * The difference between persisted policies and all policies is that
+ * if a default policy is only enabled at startup,
+ * it will appear as disabled in the persisted policy list and in the
fsimage.
+ */
+ private Map allPersistedPolicies;
+
+ /**
* All enabled policies sorted by name for fast querying, including built-in
* policy, user defined policy.
*/
@@ -90,6 +100,7 @@ public final class ErasureCodingPolicyManager {
*/
private ErasureCodingPolicy[] enabledPolicies;
+ private String defaultPolicyName;
private volatile static ErasureCodingPolicyManager instance = null;
@@ -102,14 +113,11 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyManager() {}
- public void init(Configuration conf) {
-// Load erasure coding default policy
-final String defaultPolicyName = conf.getTrimmed(
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+ public void init(Configuration conf) throws IOException {
this.policiesByName = new TreeMap<>();
this.policiesByID = new TreeMap<>();
this.enabledPoliciesByName = new TreeMap<>();
+this.allPersistedPolicies = new TreeMap<>();
/**
* TODO: load user defined EC policy from fsImage HDFS-7859
@@ -125,31 +133,12 @@ public final class ErasureCodingPolicyManager {
final ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
policiesByName.put(policy.getName(), info);
policiesByID.put(policy.getId(), info);
+ allPersistedPolicies.put(policy.getId(),
+ new ErasureCodingPolicyInfo(policy));
}
-if (!defaultPolicyName.isEmpty()) {
- final ErasureCodingPolicyInfo info =
- policiesByName.get(defaultPolicyName);
- if (info == null) {
-String names = policiesByName.values()
-.stream().map((pi) -> pi.getPolicy().getName())
-.collect(Collectors.joining(", "));
-String msg = String.format("EC policy '%s' specified at %s is not a "
-+ "valid policy. Please choose from list of available "
-+ "policies: [%s]",
-defaultPolicyName,
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
-names);
-throw new HadoopIllegalArgumentException(msg);
- }
- info.setState(ErasureCodingPolicyState
[1/2] hadoop git commit: HDFS-14039. ec -listPolicies doesn't show correct state for the default policy when the default is not RS(6, 3). Contributed by Kitti Nanasi.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 be2a0a5e2 -> bbc94de47
HDFS-14039. ec -listPolicies doesn't show correct state for the default policy
when the default is not RS(6,3). Contributed by Kitti Nanasi.
Signed-off-by: Xiao Chen
(cherry picked from commit 8d99648c203004045a9339ad27258092969145d6)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
(cherry picked from commit 01b8197893c84fa24bb94945f85b3e2ebe8259a5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbc94de4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbc94de4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbc94de4
Branch: refs/heads/branch-3.0
Commit: bbc94de4779265a5cc02b6a4b9f903aeea9b8400
Parents: 76be351
Author: Kitti Nanasi
Authored: Thu Nov 8 10:00:09 2018 -0800
Committer: Xiao Chen
Committed: Thu Nov 8 10:22:32 2018 -0800
--
.../namenode/ErasureCodingPolicyManager.java| 119 ++-
.../server/namenode/FSImageFormatProtobuf.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 2 +-
.../server/namenode/TestEnabledECPolicies.java | 103 +++-
.../hdfs/server/namenode/TestFSImage.java | 42 +--
.../server/namenode/TestNamenodeRetryCache.java | 2 +-
.../server/namenode/TestStripedINodeFile.java | 2 +-
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
8 files changed, 231 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbc94de4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index e7de05b..b840480 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -80,6 +81,15 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyInfo[] allPolicies;
/**
+ * All policies in the state as it will be persisted in the fsimage.
+ *
+ * The difference between persisted policies and all policies is that
+ * if a default policy is only enabled at startup,
+ * it will appear as disabled in the persisted policy list and in the
fsimage.
+ */
+ private Map allPersistedPolicies;
+
+ /**
* All enabled policies sorted by name for fast querying, including built-in
* policy, user defined policy.
*/
@@ -89,6 +99,7 @@ public final class ErasureCodingPolicyManager {
*/
private ErasureCodingPolicy[] enabledPolicies;
+ private String defaultPolicyName;
private volatile static ErasureCodingPolicyManager instance = null;
@@ -101,14 +112,11 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyManager() {}
- public void init(Configuration conf) {
-// Load erasure coding default policy
-final String defaultPolicyName = conf.getTrimmed(
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+ public void init(Configuration conf) throws IOException {
this.policiesByName = new TreeMap<>();
this.policiesByID = new TreeMap<>();
this.enabledPoliciesByName = new TreeMap<>();
+this.allPersistedPolicies = new TreeMap<>();
/**
* TODO: load user defined EC policy from fsImage HDFS-7859
@@ -124,31 +132,12 @@ public final class ErasureCodingPolicyManager {
final ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
policiesByName.put(policy.getName(), info);
policiesByID.put(policy.getId(), info);
+ allPersistedPolicies.put(policy.getId(),
+ new ErasureCodingPolicyInfo(policy));
}
-if (!defaultPolicyName.isEmpty()) {
- final ErasureCodingPolicyInfo info =
- policiesByName.get(defaultPolicyName);
- if (info == null) {
-String names = policiesByName.values()
-.stream().map((pi) -> pi.getPolicy().getName())
-.collect(Collectors.joining(", "));
-String msg = String.format
hadoop git commit: HDFS-14039. ec -listPolicies doesn't show correct state for the default policy when the default is not RS(6, 3). Contributed by Kitti Nanasi.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 a3b61baf9 -> 01b819789
HDFS-14039. ec -listPolicies doesn't show correct state for the default policy
when the default is not RS(6,3). Contributed by Kitti Nanasi.
Signed-off-by: Xiao Chen
(cherry picked from commit 8d99648c203004045a9339ad27258092969145d6)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01b81978
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01b81978
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01b81978
Branch: refs/heads/branch-3.1
Commit: 01b8197893c84fa24bb94945f85b3e2ebe8259a5
Parents: a3b61ba
Author: Kitti Nanasi
Authored: Thu Nov 8 10:00:09 2018 -0800
Committer: Xiao Chen
Committed: Thu Nov 8 10:08:34 2018 -0800
--
.../namenode/ErasureCodingPolicyManager.java| 119 ++-
.../server/namenode/FSImageFormatProtobuf.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 2 +-
.../server/namenode/TestEnabledECPolicies.java | 103 +++-
.../hdfs/server/namenode/TestFSImage.java | 42 +--
.../server/namenode/TestNamenodeRetryCache.java | 2 +-
.../server/namenode/TestStripedINodeFile.java | 2 +-
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
8 files changed, 231 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01b81978/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index e7de05b..b840480 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -80,6 +81,15 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyInfo[] allPolicies;
/**
+ * All policies in the state as it will be persisted in the fsimage.
+ *
+ * The difference between persisted policies and all policies is that
+ * if a default policy is only enabled at startup,
+ * it will appear as disabled in the persisted policy list and in the
fsimage.
+ */
+ private Map allPersistedPolicies;
+
+ /**
* All enabled policies sorted by name for fast querying, including built-in
* policy, user defined policy.
*/
@@ -89,6 +99,7 @@ public final class ErasureCodingPolicyManager {
*/
private ErasureCodingPolicy[] enabledPolicies;
+ private String defaultPolicyName;
private volatile static ErasureCodingPolicyManager instance = null;
@@ -101,14 +112,11 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyManager() {}
- public void init(Configuration conf) {
-// Load erasure coding default policy
-final String defaultPolicyName = conf.getTrimmed(
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+ public void init(Configuration conf) throws IOException {
this.policiesByName = new TreeMap<>();
this.policiesByID = new TreeMap<>();
this.enabledPoliciesByName = new TreeMap<>();
+this.allPersistedPolicies = new TreeMap<>();
/**
* TODO: load user defined EC policy from fsImage HDFS-7859
@@ -124,31 +132,12 @@ public final class ErasureCodingPolicyManager {
final ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
policiesByName.put(policy.getName(), info);
policiesByID.put(policy.getId(), info);
+ allPersistedPolicies.put(policy.getId(),
+ new ErasureCodingPolicyInfo(policy));
}
-if (!defaultPolicyName.isEmpty()) {
- final ErasureCodingPolicyInfo info =
- policiesByName.get(defaultPolicyName);
- if (info == null) {
-String names = policiesByName.values()
-.stream().map((pi) -> pi.getPolicy().getName())
-.collect(Collectors.joining(", "));
-String msg = String.format("EC policy '%s' specified at %s is not a "
-+ "valid
hadoop git commit: HDFS-14039. ec -listPolicies doesn't show correct state for the default policy when the default is not RS(6, 3). Contributed by Kitti Nanasi.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 66715005f -> 57409874b
HDFS-14039. ec -listPolicies doesn't show correct state for the default policy
when the default is not RS(6,3). Contributed by Kitti Nanasi.
Signed-off-by: Xiao Chen
(cherry picked from commit 8d99648c203004045a9339ad27258092969145d6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57409874
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57409874
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57409874
Branch: refs/heads/branch-3.2
Commit: 57409874bb7ad6a4f9a68ee22dfe294d7126419f
Parents: 6671500
Author: Kitti Nanasi
Authored: Thu Nov 8 10:00:09 2018 -0800
Committer: Xiao Chen
Committed: Thu Nov 8 10:08:26 2018 -0800
--
.../namenode/ErasureCodingPolicyManager.java| 119 ++-
.../server/namenode/FSImageFormatProtobuf.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 2 +-
.../server/namenode/TestEnabledECPolicies.java | 103 +++-
.../hdfs/server/namenode/TestFSImage.java | 42 +--
.../server/namenode/TestNamenodeRetryCache.java | 2 +-
.../server/namenode/TestStripedINodeFile.java | 2 +-
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
8 files changed, 231 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57409874/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index d2bf3af..57fa958 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -81,6 +82,15 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyInfo[] allPolicies;
/**
+ * All policies in the state as it will be persisted in the fsimage.
+ *
+ * The difference between persisted policies and all policies is that
+ * if a default policy is only enabled at startup,
+ * it will appear as disabled in the persisted policy list and in the
fsimage.
+ */
+ private Map allPersistedPolicies;
+
+ /**
* All enabled policies sorted by name for fast querying, including built-in
* policy, user defined policy.
*/
@@ -90,6 +100,7 @@ public final class ErasureCodingPolicyManager {
*/
private ErasureCodingPolicy[] enabledPolicies;
+ private String defaultPolicyName;
private volatile static ErasureCodingPolicyManager instance = null;
@@ -102,14 +113,11 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyManager() {}
- public void init(Configuration conf) {
-// Load erasure coding default policy
-final String defaultPolicyName = conf.getTrimmed(
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+ public void init(Configuration conf) throws IOException {
this.policiesByName = new TreeMap<>();
this.policiesByID = new TreeMap<>();
this.enabledPoliciesByName = new TreeMap<>();
+this.allPersistedPolicies = new TreeMap<>();
/**
* TODO: load user defined EC policy from fsImage HDFS-7859
@@ -125,31 +133,12 @@ public final class ErasureCodingPolicyManager {
final ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
policiesByName.put(policy.getName(), info);
policiesByID.put(policy.getId(), info);
+ allPersistedPolicies.put(policy.getId(),
+ new ErasureCodingPolicyInfo(policy));
}
-if (!defaultPolicyName.isEmpty()) {
- final ErasureCodingPolicyInfo info =
- policiesByName.get(defaultPolicyName);
- if (info == null) {
-String names = policiesByName.values()
-.stream().map((pi) -> pi.getPolicy().getName())
-.collect(Collectors.joining(", "));
-String msg = String.format("EC policy '%s' specified at %s is not a "
-+ "valid policy. Please choose from list of available "
-+ "policies: [%s]",
-defaultPolicyName,
-DFSConfigKeys.DFS_NAMENODE_E
hadoop git commit: HDFS-14039. ec -listPolicies doesn't show correct state for the default policy when the default is not RS(6, 3). Contributed by Kitti Nanasi.
Repository: hadoop
Updated Branches:
refs/heads/trunk 724c15007 -> 8d99648c2
HDFS-14039. ec -listPolicies doesn't show correct state for the default policy
when the default is not RS(6,3). Contributed by Kitti Nanasi.
Signed-off-by: Xiao Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d99648c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d99648c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d99648c
Branch: refs/heads/trunk
Commit: 8d99648c203004045a9339ad27258092969145d6
Parents: 724c150
Author: Kitti Nanasi
Authored: Thu Nov 8 10:00:09 2018 -0800
Committer: Xiao Chen
Committed: Thu Nov 8 10:01:19 2018 -0800
--
.../namenode/ErasureCodingPolicyManager.java| 119 ++-
.../server/namenode/FSImageFormatProtobuf.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 2 +-
.../server/namenode/TestEnabledECPolicies.java | 103 +++-
.../hdfs/server/namenode/TestFSImage.java | 42 +--
.../server/namenode/TestNamenodeRetryCache.java | 2 +-
.../server/namenode/TestStripedINodeFile.java | 2 +-
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
8 files changed, 231 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d99648c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index d2bf3af..57fa958 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -81,6 +82,15 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyInfo[] allPolicies;
/**
+ * All policies in the state as it will be persisted in the fsimage.
+ *
+ * The difference between persisted policies and all policies is that
+ * if a default policy is only enabled at startup,
+ * it will appear as disabled in the persisted policy list and in the
fsimage.
+ */
+ private Map allPersistedPolicies;
+
+ /**
* All enabled policies sorted by name for fast querying, including built-in
* policy, user defined policy.
*/
@@ -90,6 +100,7 @@ public final class ErasureCodingPolicyManager {
*/
private ErasureCodingPolicy[] enabledPolicies;
+ private String defaultPolicyName;
private volatile static ErasureCodingPolicyManager instance = null;
@@ -102,14 +113,11 @@ public final class ErasureCodingPolicyManager {
private ErasureCodingPolicyManager() {}
- public void init(Configuration conf) {
-// Load erasure coding default policy
-final String defaultPolicyName = conf.getTrimmed(
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
+ public void init(Configuration conf) throws IOException {
this.policiesByName = new TreeMap<>();
this.policiesByID = new TreeMap<>();
this.enabledPoliciesByName = new TreeMap<>();
+this.allPersistedPolicies = new TreeMap<>();
/**
* TODO: load user defined EC policy from fsImage HDFS-7859
@@ -125,31 +133,12 @@ public final class ErasureCodingPolicyManager {
final ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
policiesByName.put(policy.getName(), info);
policiesByID.put(policy.getId(), info);
+ allPersistedPolicies.put(policy.getId(),
+ new ErasureCodingPolicyInfo(policy));
}
-if (!defaultPolicyName.isEmpty()) {
- final ErasureCodingPolicyInfo info =
- policiesByName.get(defaultPolicyName);
- if (info == null) {
-String names = policiesByName.values()
-.stream().map((pi) -> pi.getPolicy().getName())
-.collect(Collectors.joining(", "));
-String msg = String.format("EC policy '%s' specified at %s is not a "
-+ "valid policy. Please choose from list of available "
-+ "policies: [%s]",
-defaultPolicyName,
-DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
-names);
-throw new HadoopIllegalA
