Repository: falcon Updated Branches: refs/heads/master d826852bf -> f9fdf0f7e
FALCON-2281 HiveDR fails with permissions denied Author: Sowmya Ramesh <[email protected]> Reviewers: Peeyush <[email protected]> Closes #366 from sowmyaramesh/FALCON-2281 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/f9fdf0f7 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/f9fdf0f7 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/f9fdf0f7 Branch: refs/heads/master Commit: f9fdf0f7e154958f9ae69cb488c395f4361d7d56 Parents: d826852 Author: Sowmya Ramesh <[email protected]> Authored: Thu Feb 16 14:44:10 2017 +0530 Committer: peeyush b <[email protected]> Committed: Thu Feb 16 14:44:10 2017 +0530 ---------------------------------------------------------------------- .../service/SharedLibraryHostingService.java | 21 ++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/f9fdf0f7/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java b/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java index b862111..9672272 100644 --- a/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java +++ b/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java @@ -54,8 +54,9 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener private static final String[] LIBS = StartupProperties.get().getProperty("shared.libs").split(","); - private static class FalconLibPath implements FalconPathFilter { + private static class FalconLibPath implements FalconPathFilter { private String[] shareLibs; + FalconLibPath(String[] libList) { this.shareLibs = Arrays.copyOf(libList, libList.length); } @@ -79,7 +80,8 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener } throw new IllegalArgumentException(path + " is not accepted!"); } - }; + } + private void pushExtensionArtifactsToCluster(final Cluster cluster, final FileSystem clusterFs) throws FalconException { @@ -95,6 +97,7 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener return; } + final String filterPath = "/apps/falcon/extensions/mirroring/"; Path extensionStorePath = store.getExtensionStorePath(); LOG.info("extensionStorePath :{}", extensionStorePath); FileSystem falconFileSystem = @@ -119,6 +122,12 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener LocatedFileStatus srcfileStatus = fileStatusListIterator.next(); Path filePath = Path.getPathWithoutSchemeAndAuthority(srcfileStatus.getPath()); + if (filePath != null && filePath.toString().startsWith(filterPath)) { + /* HiveDR uses filter path as store path in DRStatusStore, so skip it. Copy only the extension + artifacts */ + continue; + } + if (srcfileStatus.isDirectory()) { if (!clusterFs.exists(filePath)) { HadoopClientFactory.mkdirs(clusterFs, filePath, srcfileStatus.getPermission()); @@ -160,13 +169,13 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener nonFalconJarFilter); pushLibsToHDFS(fs, properties.getProperty("libext.paths"), libext, null); pushLibsToHDFS(fs, properties.getProperty("libext.feed.paths"), - new Path(libext, EntityType.FEED.name()) , null); + new Path(libext, EntityType.FEED.name()), null); pushLibsToHDFS(fs, properties.getProperty("libext.feed.replication.paths"), new Path(libext, EntityType.FEED.name() + "/replication"), null); pushLibsToHDFS(fs, properties.getProperty("libext.feed.retention.paths"), new Path(libext, EntityType.FEED.name() + "/retention"), null); pushLibsToHDFS(fs, properties.getProperty("libext.process.paths"), - new Path(libext, EntityType.PROCESS.name()) , null); + new Path(libext, EntityType.PROCESS.name()), null); } catch (IOException e) { throw new FalconException("Failed to copy shared libs to cluster" + cluster.getName(), e); } @@ -181,9 +190,9 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener LOG.debug("Copying libs from {}", src); createTargetPath(fs, target); - for(String srcPaths : src.split(",")) { + for (String srcPaths : src.split(",")) { File srcFile = new File(srcPaths); - File[] srcFiles = new File[] { srcFile }; + File[] srcFiles = new File[]{srcFile}; if (srcFiles != null) { if (srcFile.isDirectory()) { srcFiles = srcFile.listFiles();
