FALCON-889 Windows azure replication fails with "wasb" as the scheme to an HDFS file system. Contributed by Chris Nauroth
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/60d33b61 Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/60d33b61 Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/60d33b61 Branch: refs/heads/master Commit: 60d33b615d90cdaea2a869c24b0157e9f5a798d4 Parents: fc3094a Author: Venkatesh Seetharam <venkat...@apache.org> Authored: Thu Nov 13 18:19:11 2014 -0800 Committer: Venkatesh Seetharam <venkat...@apache.org> Committed: Thu Nov 13 18:19:11 2014 -0800 ---------------------------------------------------------------------- CHANGES.txt | 3 ++ .../apache/falcon/entity/FileSystemStorage.java | 3 +- .../entity/parser/FeedEntityParserTest.java | 50 +++++++++++++++++++- 3 files changed, 54 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/60d33b61/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 443eb65..f20ef0d 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -144,6 +144,9 @@ Trunk (Unreleased) OPTIMIZATIONS BUG FIXES + FALCON-889 Windows azure replication fails with "wasb" as the scheme to an + HDFS file system (Chris Nauroth via Venkatesh Seetharam) + FALCON-885 RequestID is coming as null for all instance API calls (Venkatesh Seetharam) http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/60d33b61/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java index f7bd321..bbe274b 100644 --- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java +++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java @@ -273,10 +273,11 @@ public class FileSystemStorage extends Configured implements Storage { @Override public void validateACL(AccessControlList acl) throws FalconException { try { - FileSystem fileSystem = HadoopClientFactory.get().createProxiedFileSystem(getConf()); for (Location location : getLocations()) { String pathString = getRelativePath(location); Path path = new Path(pathString); + FileSystem fileSystem = + HadoopClientFactory.get().createProxiedFileSystem(path.toUri(), getConf()); if (fileSystem.exists(path)) { FileStatus fileStatus = fileSystem.getFileStatus(path); Set<String> groups = CurrentUser.getGroupNames(); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/60d33b61/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java index 8c67a7d..8e773cb 100644 --- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java +++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java @@ -30,7 +30,15 @@ import org.apache.falcon.entity.v0.Frequency; import org.apache.falcon.entity.v0.SchemaHelper; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.cluster.Interfacetype; -import org.apache.falcon.entity.v0.feed.*; +import org.apache.falcon.entity.v0.feed.ActionType; +import org.apache.falcon.entity.v0.feed.ClusterType; +import org.apache.falcon.entity.v0.feed.Feed; +import org.apache.falcon.entity.v0.feed.Location; +import org.apache.falcon.entity.v0.feed.LocationType; +import org.apache.falcon.entity.v0.feed.Locations; +import org.apache.falcon.entity.v0.feed.Partition; +import org.apache.falcon.entity.v0.feed.Partitions; +import org.apache.falcon.entity.v0.feed.Validity; import org.apache.falcon.group.FeedGroupMapTest; import org.apache.falcon.security.CurrentUser; import org.apache.falcon.util.StartupProperties; @@ -874,4 +882,44 @@ public class FeedEntityParserTest extends AbstractTestBase { } } } + + @Test + public void testValidateACLForArchiveReplication() throws Exception { + StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true"); + Assert.assertTrue(Boolean.valueOf( + StartupProperties.get().getProperty("falcon.security.authorization.enabled"))); + + CurrentUser.authenticate(USER); + try { + InputStream stream = this.getClass().getResourceAsStream(FEED_XML); + + // need a new parser since it caches authorization enabled flag + FeedEntityParser feedEntityParser = + (FeedEntityParser) EntityParserFactory.getParser(EntityType.FEED); + Feed feed = feedEntityParser.parse(stream); + + org.apache.falcon.entity.v0.feed.Cluster feedCluster = + FeedHelper.getCluster(feed, "backupCluster"); + Location location = new Location(); + location.setType(LocationType.DATA); + location.setPath( + "s3://falcontest...@hwxasvtesting.blob.core.windows.net/{YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}"); + Locations locations = new Locations(); + locations.getLocations().add(location); + feedCluster.setLocations(locations); + + Assert.assertNotNull(feed); + Assert.assertNotNull(feed.getACL()); + feed.getACL().setOwner(USER); + feed.getACL().setGroup(getPrimaryGroupName()); + + try { + feedEntityParser.validate(feed); + } catch (IllegalArgumentException e) { + // this is normal since AWS Secret Access Key is not specified as the password of a s3 URL + } + } finally { + StartupProperties.get().setProperty("falcon.security.authorization.enabled", "false"); + } + } }
