Repository: hadoop Updated Branches: refs/heads/trunk de85f841d -> 30eceec34
HDFS-13806. EC: No error message for unsetting EC policy of the directory inherits the erasure coding policy from an ancestor directory. Contributed by Ayush Saxena. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30eceec3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30eceec3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30eceec3 Branch: refs/heads/trunk Commit: 30eceec3420fc6be00d3878ba787bd9518d3ca0e Parents: de85f84 Author: Vinayakumar B <vinayakum...@apache.org> Authored: Mon Sep 10 09:10:51 2018 +0530 Committer: Vinayakumar B <vinayakum...@apache.org> Committed: Mon Sep 10 09:10:51 2018 +0530 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +- .../hdfs/protocol/NoECPolicySetException.java | 37 ++++++++++++++++++++ .../server/namenode/FSDirErasureCodingOp.java | 4 +++ .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 7 ++++ .../TestUnsetAndChangeDirectoryEcPolicy.java | 23 +++++++++--- .../test/resources/testErasureCodingConf.xml | 24 +++++++++++++ 6 files changed, 92 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 61568d5..f4d11b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -131,6 +131,7 @@ import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; +import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType; @@ -2750,7 +2751,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, throw re.unwrapRemoteException(AccessControlException.class, SafeModeException.class, UnresolvedPathException.class, - FileNotFoundException.class); + FileNotFoundException.class, NoECPolicySetException.class); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java new file mode 100644 index 0000000..de3054a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.protocol; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + *Thrown when no EC policy is set explicitly on the directory. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class NoECPolicySetException extends IOException { + private static final long serialVersionUID = 1L; + + public NoECPolicySetException(String msg) { + super(msg); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 2ba840e..b0bc5e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableUtils; @@ -206,6 +207,9 @@ final class FSDirErasureCodingOp { } if (xAttrs != null) { fsn.getEditLog().logRemoveXAttrs(src, xAttrs, logRetryCache); + } else { + throw new NoECPolicySetException( + "No erasure coding policy explicitly set on " + src); } return fsd.getAuditFileInfo(iip); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 56d453b..903a1e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; import org.apache.hadoop.hdfs.util.ECPolicyLoader; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.tools.TableListing; @@ -424,6 +425,12 @@ public class ECAdmin extends Configured implements Tool { "non-empty directory will not automatically convert existing" + " files to replicated data."); } + } catch (NoECPolicySetException e) { + System.err.println(AdminHelper.prettifyException(e)); + System.err.println("Use '-setPolicy -path <PATH> -replicate' to enforce" + + " default replication policy irrespective of EC policy" + + " defined on parent."); + return 2; } catch (Exception e) { System.err.println(AdminHelper.prettifyException(e)); return 2; http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index aa14694..4f33ce7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; @@ -98,7 +99,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy { fs.mkdirs(dirPath); // Test unset a directory which has no EC policy - fs.unsetErasureCodingPolicy(dirPath); + try { + fs.unsetErasureCodingPolicy(dirPath); + fail(); + } catch (NoECPolicySetException e) { + } // Set EC policy on directory fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); @@ -126,8 +131,8 @@ public class TestUnsetAndChangeDirectoryEcPolicy { } /* - * Test nested directory with different EC policy. - */ + * Test nested directory with different EC policy. + */ @Test public void testNestedEcPolicy() throws Exception { final int numBlocks = 1; @@ -199,7 +204,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy { final Path replicateFilePath = new Path(rootPath, "rep_file"); // Test unset root path which has no EC policy - fs.unsetErasureCodingPolicy(rootPath); + try { + fs.unsetErasureCodingPolicy(rootPath); + fail(); + } catch (NoECPolicySetException e) { + } // Set EC policy on root path fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L); @@ -238,7 +247,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy { final ErasureCodingPolicy ec32Policy = SystemErasureCodingPolicies .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID); - fs.unsetErasureCodingPolicy(rootPath); + try { + fs.unsetErasureCodingPolicy(rootPath); + fail(); + } catch (NoECPolicySetException e) { + } fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); // Create RS(6,3) EC policy file DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L); http://git-wip-us.apache.org/repos/asf/hadoop/blob/30eceec3/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index b47d50f..34f5176 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -360,6 +360,30 @@ </test> <test> + <description>unsetPolicy : unset on non EC directory</description> + <test-commands> + <command>-fs NAMENODE -mkdir /ecdir</command> + <command>-fs NAMENODE -mkdir /ecdir/child</command> + <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm /ecdir/child/ecfile</command> + <command>-fs NAMENODE -rmdir /ecdir/child</command> + <command>-fs NAMENODE -rmdir /ecdir</command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>NoECPolicySetException: No erasure coding policy explicitly set on /ecdir/child</expected-output> + </comparator> + <comparator> + <type>SubstringComparator</type> + <expected-output>Use '-setPolicy -path <PATH> -replicate' to enforce default replication policy irrespective of EC policy defined on parent.</expected-output> + </comparator> + </comparators> + </test> + + <test> <description>unsetPolicy : unset policy on non-empty directory</description> <test-commands> <command>-fs NAMENODE -mkdir /ecdir</command> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
