HADOOP-14730. Support protobuf FileStatus in AdlFileSystem.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55a181f8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55a181f8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55a181f8 Branch: refs/heads/HADOOP-13345 Commit: 55a181f845adcdcc9b008e9906ade1544fc220e4 Parents: 8d3fd81 Author: Chris Douglas <cdoug...@apache.org> Authored: Mon Aug 7 21:31:28 2017 -0700 Committer: Chris Douglas <cdoug...@apache.org> Committed: Mon Aug 7 21:31:28 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/adl/AdlFileStatus.java | 69 ++++++++++++++++++++ .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 27 ++------ .../apache/hadoop/fs/adl/TestGetFileStatus.java | 57 ++++++++-------- .../apache/hadoop/fs/adl/TestListStatus.java | 8 ++- 4 files changed, 105 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a181f8/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileStatus.java new file mode 100644 index 0000000..70c005d --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileStatus.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.fs.adl; + +import com.microsoft.azure.datalake.store.DirectoryEntry; +import com.microsoft.azure.datalake.store.DirectoryEntryType; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE; +import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_REPLICATION_FACTOR; + +/** + * Shim class supporting linking against 2.x clients. + */ +class AdlFileStatus extends FileStatus { + + private static final long serialVersionUID = 0x01fcbe5e; + + private boolean hasAcl = false; + + AdlFileStatus(DirectoryEntry entry, Path path, boolean hasAcl) { + this(entry, path, entry.user, entry.group, hasAcl); + } + + AdlFileStatus(DirectoryEntry entry, Path path, + String owner, String group, boolean hasAcl) { + super(entry.length, DirectoryEntryType.DIRECTORY == entry.type, + ADL_REPLICATION_FACTOR, ADL_BLOCK_SIZE, + entry.lastModifiedTime.getTime(), entry.lastAccessTime.getTime(), + new AdlPermission(hasAcl, Short.parseShort(entry.permission, 8)), + owner, group, null, path); + this.hasAcl = hasAcl; + } + + @Override + public boolean hasAcl() { + return hasAcl; + } + + @Override + public boolean equals(Object o) { + // satisfy findbugs + return super.equals(o); + } + + @Override + public int hashCode() { + // satisfy findbugs + return super.hashCode(); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a181f8/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 0de538e..76ce43e 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -29,7 +29,6 @@ import com.google.common.annotations.VisibleForTesting; import com.microsoft.azure.datalake.store.ADLStoreClient; import com.microsoft.azure.datalake.store.ADLStoreOptions; import com.microsoft.azure.datalake.store.DirectoryEntry; -import com.microsoft.azure.datalake.store.DirectoryEntryType; import com.microsoft.azure.datalake.store.IfExists; import com.microsoft.azure.datalake.store.LatencyTracker; import com.microsoft.azure.datalake.store.UserGroupRepresentation; @@ -606,30 +605,12 @@ public class AdlFileSystem extends FileSystem { } private FileStatus toFileStatus(final DirectoryEntry entry, final Path f) { - boolean isDirectory = entry.type == DirectoryEntryType.DIRECTORY; - long lastModificationData = entry.lastModifiedTime.getTime(); - long lastAccessTime = entry.lastAccessTime.getTime(); - // set aclBit from ADLS backend response if - // ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION is true. - final boolean aclBit = aclBitStatus ? entry.aclBit : false; - - FsPermission permission = new AdlPermission(aclBit, - Short.valueOf(entry.permission, 8)); - String user = entry.user; - String group = entry.group; - - FileStatus status; + Path p = makeQualified(f); + boolean aclBit = aclBitStatus ? entry.aclBit : false; if (overrideOwner) { - status = new FileStatus(entry.length, isDirectory, ADL_REPLICATION_FACTOR, - ADL_BLOCK_SIZE, lastModificationData, lastAccessTime, permission, - userName, "hdfs", this.makeQualified(f)); - } else { - status = new FileStatus(entry.length, isDirectory, ADL_REPLICATION_FACTOR, - ADL_BLOCK_SIZE, lastModificationData, lastAccessTime, permission, - user, group, this.makeQualified(f)); + return new AdlFileStatus(entry, p, userName, "hdfs", aclBit); } - - return status; + return new AdlFileStatus(entry, p, aclBit); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a181f8/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java index 0ea4b86..d9e22db 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java @@ -42,8 +42,8 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE; * org.apache.hadoop.fs.adl.live testing package. */ public class TestGetFileStatus extends AdlMockWebServer { - private static final Logger LOG = LoggerFactory - .getLogger(TestGetFileStatus.class); + private static final Logger LOG = + LoggerFactory.getLogger(TestGetFileStatus.class); @Test public void getFileStatusReturnsAsExpected() @@ -72,33 +72,30 @@ public class TestGetFileStatus extends AdlMockWebServer { fileStatus.isErasureCoded()); } - @Test - public void getFileStatusAclBit() - throws URISyntaxException, IOException { - // With ACLBIT set to true - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getGetFileStatusJSONResponse(true))); - long startTime = Time.monotonicNow(); - FileStatus fileStatus = getMockAdlFileSystem() - .getFileStatus(new Path("/test1/test2")); - long endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - Assert.assertTrue(fileStatus.isFile()); - Assert.assertEquals(true, fileStatus.getPermission().getAclBit()); - Assert.assertEquals(fileStatus.hasAcl(), - fileStatus.getPermission().getAclBit()); + @Test + public void getFileStatusAclBit() throws URISyntaxException, IOException { + // With ACLBIT set to true + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getGetFileStatusJSONResponse(true))); + long startTime = Time.monotonicNow(); + FileStatus fileStatus = getMockAdlFileSystem() + .getFileStatus(new Path("/test1/test2")); + long endTime = Time.monotonicNow(); + LOG.debug("Time : " + (endTime - startTime)); + Assert.assertTrue(fileStatus.isFile()); + Assert.assertTrue(fileStatus.hasAcl()); + Assert.assertTrue(fileStatus.getPermission().getAclBit()); - // With ACLBIT set to false - getMockServer().enqueue(new MockResponse().setResponseCode(200) - .setBody(TestADLResponseData.getGetFileStatusJSONResponse(false))); - startTime = Time.monotonicNow(); - fileStatus = getMockAdlFileSystem() - .getFileStatus(new Path("/test1/test2")); - endTime = Time.monotonicNow(); - LOG.debug("Time : " + (endTime - startTime)); - Assert.assertTrue(fileStatus.isFile()); - Assert.assertEquals(false, fileStatus.getPermission().getAclBit()); - Assert.assertEquals(fileStatus.hasAcl(), - fileStatus.getPermission().getAclBit()); - } + // With ACLBIT set to false + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getGetFileStatusJSONResponse(false))); + startTime = Time.monotonicNow(); + fileStatus = getMockAdlFileSystem() + .getFileStatus(new Path("/test1/test2")); + endTime = Time.monotonicNow(); + LOG.debug("Time : " + (endTime - startTime)); + Assert.assertTrue(fileStatus.isFile()); + Assert.assertFalse(fileStatus.hasAcl()); + Assert.assertFalse(fileStatus.getPermission().getAclBit()); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a181f8/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java index dac8886..db32476 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java @@ -102,7 +102,7 @@ public class TestListStatus extends AdlMockWebServer { } @Test - public void listStatusAclBit() + public void listStatusAcl() throws URISyntaxException, IOException { // With ACLBIT set to true getMockServer().enqueue(new MockResponse().setResponseCode(200) @@ -115,7 +115,8 @@ public class TestListStatus extends AdlMockWebServer { LOG.debug("Time : " + (endTime - startTime)); for (int i = 0; i < ls.length; i++) { Assert.assertTrue(ls[i].isDirectory()); - Assert.assertEquals(true, ls[i].getPermission().getAclBit()); + Assert.assertTrue(ls[i].hasAcl()); + Assert.assertTrue(ls[i].getPermission().getAclBit()); } // With ACLBIT set to false @@ -129,7 +130,8 @@ public class TestListStatus extends AdlMockWebServer { LOG.debug("Time : " + (endTime - startTime)); for (int i = 0; i < ls.length; i++) { Assert.assertTrue(ls[i].isDirectory()); - Assert.assertEquals(false, ls[i].getPermission().getAclBit()); + Assert.assertFalse(ls[i].hasAcl()); + Assert.assertFalse(ls[i].getPermission().getAclBit()); } } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
