Repository: oozie Updated Branches: refs/heads/master 674a54ad2 -> 183f3faed
OOZIE-2215 Support glob in FS EL function (ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/183f3fae Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/183f3fae Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/183f3fae Branch: refs/heads/master Commit: 183f3faede81418018d5f94ad367aabff1f4975d Parents: 674a54a Author: egashira <[email protected]> Authored: Fri Jun 12 12:40:17 2015 -0700 Committer: egashira <[email protected]> Committed: Fri Jun 12 12:40:17 2015 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/FsELFunctions.java | 40 ++++++++++++++++++-- core/src/main/resources/oozie-default.xml | 2 +- .../oozie/action/hadoop/TestFsELFunctions.java | 6 +++ .../src/site/twiki/WorkflowFunctionalSpec.twiki | 2 +- release-log.txt | 5 ++- 5 files changed, 47 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/183f3fae/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java index f8af61c..801bfe6 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java @@ -25,9 +25,12 @@ import java.net.URISyntaxException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.DagELFunctions; +import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowJob; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.Services; import org.apache.oozie.service.HadoopAccessorService; @@ -71,10 +74,17 @@ public class FsELFunctions { * @throws Exception */ public static boolean fs_exists(String pathUri) throws Exception { - URI uri = new URI(pathUri); - String path = uri.getPath(); - FileSystem fs = getFileSystem(uri); - return fs.exists(new Path(path)); + Path path = new Path(pathUri); + FileSystem fs = getFileSystem(path.toUri()); + FileStatus[] pathArr; + try { + pathArr = fs.globStatus(path, new FSPathFilter()); + } + catch (ReachingGlobMaxException e) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS013", + "too many globbed files/dirs to do FS operation"); + } + return (pathArr != null && pathArr.length > 0); } /** @@ -157,4 +167,26 @@ public class FsELFunctions { return blockSize; } + static class FSPathFilter implements PathFilter { + int count = 0; + int globMax = Integer.MAX_VALUE; + public FSPathFilter() { + globMax = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX); + } + @Override + public boolean accept(Path p) { + count++; + if(count > globMax) { + throw new ReachingGlobMaxException(); + } + return true; + } + } + + /** + * ReachingGlobMaxException thrown when globbed file count exceeds the limit + */ + static class ReachingGlobMaxException extends RuntimeException { + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/183f3fae/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 8960073..4dc127e 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1679,7 +1679,7 @@ <property> <name>oozie.action.fs.glob.max</name> - <value>1000</value> + <value>50000</value> <description> Maximum number of globbed files. </description> http://git-wip-us.apache.org/repos/asf/oozie/blob/183f3fae/core/src/test/java/org/apache/oozie/action/hadoop/TestFsELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsELFunctions.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsELFunctions.java index c81fc61..6a2f5ff 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsELFunctions.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsELFunctions.java @@ -82,6 +82,9 @@ public class TestFsELFunctions extends XFsTestCase { conf.set("file1", file1); conf.set("file2", file2); conf.set("file3", "${file2}"); + conf.set("file4", getFsTestCaseDir()+"/file{1,2}"); + conf.set("file5", getFsTestCaseDir()+"/file*"); + conf.set("file6", getFsTestCaseDir()+"/file_*"); conf.set("dir", dir); LiteWorkflowApp def = @@ -110,6 +113,9 @@ public class TestFsELFunctions extends XFsTestCase { assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file1'))}", Boolean.class)); assertEquals(false, (boolean) eval.evaluate("${fs:exists(wf:conf('file2'))}", Boolean.class)); + assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file4'))}", Boolean.class)); + assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('file5'))}", Boolean.class)); + assertEquals(false, (boolean) eval.evaluate("${fs:exists(wf:conf('file6'))}", Boolean.class)); assertEquals(true, (boolean) eval.evaluate("${fs:exists(wf:conf('dir'))}", Boolean.class)); assertEquals(false, (boolean) eval.evaluate("${fs:isDir(wf:conf('file1'))}", Boolean.class)); assertEquals(0, (int) eval.evaluate("${fs:fileSize(wf:conf('file1'))}", Integer.class)); http://git-wip-us.apache.org/repos/asf/oozie/blob/183f3fae/docs/src/site/twiki/WorkflowFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki index 02dc65b..6b55117 100644 --- a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki +++ b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki @@ -2216,7 +2216,7 @@ For all the functions in this section the path must include the FS URI. For exam *boolean fs:exists(String path)* -It returns =true= or =false= depending if the specified path URI exists or not. +It returns =true= or =false= depending if the specified path URI exists or not. If a glob pattern is used for the URI, it returns true when there is at least one matching path. *boolean fs:isDir(String path)* http://git-wip-us.apache.org/repos/asf/oozie/blob/183f3fae/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index fe05679..ded886b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,11 +1,12 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2215 Support glob in FS EL function (ryota) OOZIE-2270 Audit log doesn't get loaded when "audit log" tab is clicked for coord job (puru) OOZIE-2257 BundleStatusTransitXCommand doesn't update last modified time (puru) +OOZIE-2260 Only set "java.io.tmpdir" to "./tmp" for the AM (ryota) --- Oozie 4.2.0 release (unreleased) +-- Oozie 4.2.0 release -OOZIE-2260 Only set "java.io.tmpdir" to "./tmp" for the AM (ryota) OOZIE-2240 add configuration to disable email attachment support (egashira via shwethags) OOZIE-1963 Create a Hive Server 2 example (qwertymaniac via shwethags) OOZIE-1993 Rerun fails during join in certain condition (shwethags)
