chliang71 commented on a change in pull request #1988:
URL: https://github.com/apache/hadoop/pull/1988#discussion_r418732158
##########
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
##########
@@ -96,16 +96,49 @@ static AccessControlException readOnlyMountTable(final
String operation,
return readOnlyMountTable(operation, p.toString());
}
+ /**
+ * File system instance getter.
+ */
+ static class FsGetter {
+
+ /**
+ * Gets new file system instance of given uri.
+ */
+ public FileSystem getNewInstance(URI uri, Configuration conf)
+ throws IOException {
+ return FileSystem.newInstance(uri, conf);
+ }
+
+ /**
+ * Gets file system instance of given uri.
+ */
+ public FileSystem get(URI uri, Configuration conf) throws IOException {
+ return FileSystem.get(uri, conf);
+ }
+ }
+
+ /**
+ * Gets file system creator instance.
+ */
+ protected FsGetter fsGetter() {
Review comment:
why do need this method while we can just call new FsGetter() directly?
##########
File path:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFsOverloadScheme.java
##########
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.URI;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+
+/******************************************************************************
+ * This class is extended from the ViewFileSystem for the overloaded scheme
file
+ * system. The objective here is to handle multiple mounted file systems
+ * transparently. Mount link configurations and in-memory mount table
+ * building behaviors are inherited from ViewFileSystem. Unlike ViewFileSystem
+ * scheme (viewfs://), the users would be able to use any scheme.
+ *
+ * Example 1:
+ * If users want some of their existing cluster (hdfs://Cluster)
+ * data to mount with other hdfs and object store clusters(hdfs://NN1,
+ * o3fs://bucket1.volume1/, s3a://bucket1/)
+ *
+ * fs.viewfs.mounttable.Cluster./user = hdfs://NN1/user
+ * fs.viewfs.mounttable.Cluster./data = o3fs://bucket1.volume1/data
+ * fs.viewfs.mounttable.Cluster./backup = s3a://bucket1/backup/
+ *
+ * Op1: Create file hdfs://Cluster/user/fileA will go to hdfs://NN1/user/fileA
+ * Op2: Create file hdfs://Cluster/data/datafile will go to
+ * o3fs://bucket1.volume1/data/datafile
+ * Op3: Create file hdfs://Cluster/backup/data.zip will go to
+ * s3a://bucket1/backup/data.zip
+ *
+ * Example 2:
+ * If users want some of their existing cluster (s3a://bucketA/)
+ * data to mount with other hdfs and object store clusters
+ * (hdfs://NN1, o3fs://bucket1.volume1/)
+ *
+ * fs.viewfs.mounttable.bucketA./user = hdfs://NN1/user
+ * fs.viewfs.mounttable.bucketA./data = o3fs://bucket1.volume1/data
+ * fs.viewfs.mounttable.bucketA./salesDB = s3a://bucketA/salesDB/
+ *
+ * Op1: Create file s3a://bucketA/user/fileA will go to hdfs://NN1/user/fileA
+ * Op2: Create file s3a://bucketA/data/datafile will go to
+ * o3fs://bucket1.volume1/data/datafile
+ * Op3: Create file s3a://bucketA/salesDB/dbfile will go to
+ * s3a://bucketA/salesDB/dbfile
+ *****************************************************************************/
[email protected]({ "MapReduce", "HBase", "Hive" })
[email protected]
+public class ViewFsOverloadScheme extends ViewFileSystem {
Review comment:
a general comment is it would be good to have some logging in this class
(DEBUG level)
##########
File path:
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsOverloadSchemeLocalFileSystem.java
##########
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ *
+ * Test the TestViewFsOverloadSchemeLocalFS using a file with authority:
+ * file://mountTableName/ i.e, the authority is used to load a mount table.
+ */
+public class TestViewFsOverloadSchemeLocalFileSystem {
+ private static final String FILE = "file";
+ private static final Log LOG =
+ LogFactory.getLog(TestViewFsOverloadSchemeLocalFileSystem.class);
+ private FileSystem fsTarget;
+ private Configuration conf;
+ private Path targetTestRoot;
+ private FileSystemTestHelper fileSystemTestHelper;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.set(String.format("fs.%s.impl",
+ FILE),
+ ViewFsOverloadScheme.class.getName());
+ conf.set(String.format(
+ FsConstants.FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN,
+ FILE),
+ LocalFileSystem.class.getName());
+ fsTarget = new LocalFileSystem();
+ fsTarget.initialize(new URI("file:///"), conf);
+ fileSystemTestHelper = new FileSystemTestHelper();
+ // create the test root on local_fs
+ targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
+ fsTarget.delete(targetTestRoot, true);
+ fsTarget.mkdirs(targetTestRoot);
+ }
+
+ /**
+ * Tests write file and read file with ViewFSOverloadScheme.
+ */
+ @Test
+ public void testLocalTargetLinkWriteSimple() throws IOException {
+ LOG.info("Starting testLocalTargetLinkWriteSimple");
+ final String testString = "Hello Local!...";
+ final Path lfsRoot = new Path("/lfsRoot");
+ ConfigUtil.addLink(conf, lfsRoot.toString(),
+ URI.create(targetTestRoot + "/local"));
+ final FileSystem lViewFs = FileSystem.get(URI.create("file:///"), conf);
+
+ final Path testPath = new Path(lfsRoot, "test.txt");
+ final FSDataOutputStream fsDos = lViewFs.create(testPath);
+ try {
+ fsDos.writeUTF(testString);
+ } finally {
+ fsDos.close();
+ }
+
+ FSDataInputStream lViewIs = lViewFs.open(testPath);
+ try {
+ Assert.assertEquals(testString, lViewIs.readUTF());
+ } finally {
+ lViewIs.close();
+ }
+ }
+
+ /**
+ * Tests create file and delete file with ViewFSOverloadScheme.
+ */
+ @Test
+ public void testLocalFsCreateAndDelete() throws Exception {
+ LOG.info("Starting testLocalFsCreateAndDelete");
+ ConfigUtil.addLink(conf, "mt", "/lfsroot",
+ URI.create(targetTestRoot + "/wd2"));
+ final URI mountURI = URI.create("file://mt/");
+ final FileSystem lViewFS = FileSystem.get(mountURI, conf);
+ try {
+ Path testPath = new Path(mountURI.toString() + "/lfsroot/test");
+ lViewFS.create(testPath);
+ Assert.assertTrue(lViewFS.exists(testPath));
+ lViewFS.delete(testPath, true);
+ Assert.assertFalse(lViewFS.exists(testPath));
+ } finally {
+ lViewFS.close();
+ }
+ }
+
+ /**
+ * Tests root level file with linkMergeSlash with ViewFSOverloadScheme.
+ */
+ @Test
+ public void testLocalFsLinkSlashMerge() throws Exception {
+ LOG.info("Starting testLocalFSCreateAndDelete");
Review comment:
shouldn't this message be testLocalFsLinkSlashMerge? similar for the
other test below
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
