mccormickt12 commented on code in PR #4181:
URL: https://github.com/apache/hadoop/pull/4181#discussion_r851530313
##########
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestNestedMountPoint.java:
##########
@@ -0,0 +1,345 @@
+package org.apache.hadoop.fs.viewfs;
+
+import java.net.URI;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestNestedMountPoint {
+ private InodeTree inodeTree;
+ private Configuration conf;
+ private String mtName;
+ private URI fsUri;
+
+ static class TestNestMountPointFileSystem {
+ public URI getUri() {
+ return uri;
+ }
+
+ private URI uri;
+
+ TestNestMountPointFileSystem(URI uri) {
+ this.uri = uri;
+ }
+ }
+
+ static class TestNestMountPointInternalFileSystem extends
TestNestMountPointFileSystem {
+ TestNestMountPointInternalFileSystem(URI uri) {
+ super(uri);
+ }
+ }
+
+ private static final URI LINKFALLBACK_TARGET = URI.create("hdfs://nn00");
+ private static final URI NN1_TARGET = URI.create("hdfs://nn01/a/b");
+ private static final URI NN2_TARGET = URI.create("hdfs://nn02/a/b/e");
+ private static final URI NN3_TARGET = URI.create("hdfs://nn03/a/b/c/d");
+ private static final URI NN4_TARGET = URI.create("hdfs://nn04/a/b/c/d/e");
+ private static final URI NN5_TARGET = URI.create("hdfs://nn05/b/c/d/e");
+ private static final URI NN6_TARGET = URI.create("hdfs://nn06/b/c/d/e/f");
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ mtName = TestNestedMountPoint.class.getName();
+ ConfigUtil.setIsNestedMountPointSupported(conf, true);
+ ConfigUtil.addLink(conf, mtName, "/a/b", NN1_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/a/b/e", NN2_TARGET);
Review Comment:
each dirlink, only has one nested mount point. Could could add one dirlink
that has multiple nested mount points under it.
##########
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestNestedMountPoint.java:
##########
@@ -0,0 +1,345 @@
+package org.apache.hadoop.fs.viewfs;
+
+import java.net.URI;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestNestedMountPoint {
+ private InodeTree inodeTree;
+ private Configuration conf;
+ private String mtName;
+ private URI fsUri;
+
+ static class TestNestMountPointFileSystem {
+ public URI getUri() {
+ return uri;
+ }
+
+ private URI uri;
+
+ TestNestMountPointFileSystem(URI uri) {
+ this.uri = uri;
+ }
+ }
+
+ static class TestNestMountPointInternalFileSystem extends
TestNestMountPointFileSystem {
+ TestNestMountPointInternalFileSystem(URI uri) {
+ super(uri);
+ }
+ }
+
+ private static final URI LINKFALLBACK_TARGET = URI.create("hdfs://nn00");
+ private static final URI NN1_TARGET = URI.create("hdfs://nn01/a/b");
+ private static final URI NN2_TARGET = URI.create("hdfs://nn02/a/b/e");
+ private static final URI NN3_TARGET = URI.create("hdfs://nn03/a/b/c/d");
+ private static final URI NN4_TARGET = URI.create("hdfs://nn04/a/b/c/d/e");
+ private static final URI NN5_TARGET = URI.create("hdfs://nn05/b/c/d/e");
+ private static final URI NN6_TARGET = URI.create("hdfs://nn06/b/c/d/e/f");
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ mtName = TestNestedMountPoint.class.getName();
+ ConfigUtil.setIsNestedMountPointSupported(conf, true);
+ ConfigUtil.addLink(conf, mtName, "/a/b", NN1_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/a/b/e", NN2_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/a/b/c/d", NN3_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/a/b/c/d/e", NN4_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/b/c/d/e", NN5_TARGET);
+ ConfigUtil.addLink(conf, mtName, "/b/c/d/e/f", NN6_TARGET);
+ ConfigUtil.addLinkFallback(conf, mtName, LINKFALLBACK_TARGET);
+
+ fsUri = new URI(FsConstants.VIEWFS_SCHEME, mtName, "/", null, null);
+
+ inodeTree = new
InodeTree<TestNestedMountPoint.TestNestMountPointFileSystem>(conf,
+ mtName, fsUri, false) {
+ @Override
+ protected Function<URI,
TestNestedMountPoint.TestNestMountPointFileSystem> initAndGetTargetFs() {
+ return new Function<URI, TestNestMountPointFileSystem>() {
+ @Override
+ public TestNestedMountPoint.TestNestMountPointFileSystem apply(URI
uri) {
+ return new TestNestMountPointFileSystem(uri);
+ }
+ };
+ }
+
+ // For intenral dir fs
+ @Override
+ protected TestNestedMountPoint.TestNestMountPointInternalFileSystem
getTargetFileSystem(
+ final INodeDir<TestNestedMountPoint.TestNestMountPointFileSystem>
dir) {
+ return new TestNestMountPointInternalFileSystem(fsUri);
+ }
+
+ @Override
+ protected TestNestedMountPoint.TestNestMountPointInternalFileSystem
getTargetFileSystem(
+ final String settings, final URI[] mergeFsURIList) {
+ return new TestNestMountPointInternalFileSystem(null);
+ }
+ };
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ inodeTree = null;
+ }
+
+ @Test
+ public void testPathResolveToLink() throws Exception {
+ // /a/b/c/d/e/f resolves to /a/b/c/d/e and /f
+ InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/e/f",
true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult.resolvedPath);
+ Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+ Assert.assertTrue(resolveResult.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+ // /a/b/c/d/e resolves to /a/b/c/d/e and /
+ InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d/e",
true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult2.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult2.resolvedPath);
+ Assert.assertEquals(new Path("/"), resolveResult2.remainingPath);
+ Assert.assertTrue(resolveResult2.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult2.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+ // /a/b/c/d/e/f/g/h/i resolves to /a/b/c/d/e and /f/g/h/i
+ InodeTree.ResolveResult resolveResult3 =
inodeTree.resolve("/a/b/c/d/e/f/g/h/i", true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult3.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult3.resolvedPath);
+ Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+ Assert.assertTrue(resolveResult3.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult3.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+ }
+
+ @Test
+ public void testPathResolveToLink_NotResolveLastComponent() throws Exception
{
+ // /a/b/c/d/e/f resolves to /a/b/c/d/e and /f
+ InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/e/f",
false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult.resolvedPath);
+ Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+ Assert.assertTrue(resolveResult.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+ // /a/b/c/d/e resolves to /a/b/c/d and /e
+ InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d/e",
false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult2.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult2.resolvedPath);
+ Assert.assertEquals(new Path("/e"), resolveResult2.remainingPath);
+ Assert.assertTrue(resolveResult2.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult2.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+ // /a/b/c/d/e/f/g/h/i resolves to /a/b/c/d/e and /f/g/h/i
+ InodeTree.ResolveResult resolveResult3 =
inodeTree.resolve("/a/b/c/d/e/f/g/h/i", false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult3.kind);
+ Assert.assertEquals("/a/b/c/d/e", resolveResult3.resolvedPath);
+ Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+ Assert.assertTrue(resolveResult3.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem)
resolveResult3.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+ }
+
+ @Test
+ public void testPathResolveToDirLink() throws Exception {
+ // /a/b/c/d/f resolves to /a/b/c/d, /f
+ InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/f",
true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult.resolvedPath);
+ Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+ Assert.assertTrue(resolveResult.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+ // /a/b/c/d resolves to /a/b/c/d and /
+ InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d",
true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult2.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult2.resolvedPath);
+ Assert.assertEquals(new Path("/"), resolveResult2.remainingPath);
+ Assert.assertTrue(resolveResult2.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult2.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+ // /a/b/c/d/f/g/h/i resolves to /a/b/c/d and /f/g/h/i
+ InodeTree.ResolveResult resolveResult3 =
inodeTree.resolve("/a/b/c/d/f/g/h/i", true);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult3.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult3.resolvedPath);
+ Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+ Assert.assertTrue(resolveResult3.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult3.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+ }
+
+ @Test
+ public void testPathResolveToDirLink_NotResolveLastComponent() throws
Exception {
+ // /a/b/c/d/f resolves to /a/b/c/d, /f
+ InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/f",
false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult.resolvedPath);
+ Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+ Assert.assertTrue(resolveResult.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+ // /a/b/c/d resolves to /a/b and /c/d
+ InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d",
false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult2.kind);
+ Assert.assertEquals("/a/b", resolveResult2.resolvedPath);
+ Assert.assertEquals(new Path("/c/d"), resolveResult2.remainingPath);
+ Assert.assertTrue(resolveResult2.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN1_TARGET, ((TestNestMountPointFileSystem)
resolveResult2.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+ // /a/b/c/d/f/g/h/i resolves to /a/b/c/d and /f/g/h/i
+ InodeTree.ResolveResult resolveResult3 =
inodeTree.resolve("/a/b/c/d/f/g/h/i", false);
+ Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR,
resolveResult3.kind);
+ Assert.assertEquals("/a/b/c/d", resolveResult3.resolvedPath);
+ Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+ Assert.assertTrue(resolveResult3.targetFileSystem instanceof
TestNestMountPointFileSystem);
+ Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem)
resolveResult3.targetFileSystem).getUri());
+ Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+ }
+
+ @Test
+ public void testMultiNestedMountPointsPathResolveToDirLink() throws
Exception {
+ // /a/b/f resolves to /a/b and /f
Review Comment:
Here you have a test that tests a top level dirlink correctly picks the
correct link instead of a nested mount point.
Since essentially unlimited nesting of these dirlinks is supported, can you
add another test that tests that a nested mount point also handles this
scenario correctly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
