Repository: falcon Updated Branches: refs/heads/master 55bcc14af -> 75079c2e5
FALCON-1417 Make validity end date optional for feed / process. Contributed by Pragya Mittal. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/75079c2e Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/75079c2e Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/75079c2e Branch: refs/heads/master Commit: 75079c2e5166ef517e500109f0015fff6df47b90 Parents: 55bcc14 Author: Ajay Yadava <[email protected]> Authored: Tue Sep 22 19:52:49 2015 +0530 Committer: Ajay Yadava <[email protected]> Committed: Tue Sep 22 19:52:49 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../entity/parser/CrossEntityValidations.java | 7 +++++++ .../falcon/entity/parser/FeedEntityParser.java | 7 +++++++ .../entity/parser/ProcessEntityParser.java | 7 +++++++ .../java/org/apache/falcon/util/DateUtil.java | 4 ++++ .../entity/parser/FeedEntityParserTest.java | 8 ++++++++ .../entity/parser/ProcessEntityParserTest.java | 19 +++++++++++++++++++ 7 files changed, 54 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 445b56f..6836690 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,8 @@ Trunk (Unreleased) FALCON-1027 Falcon proxy user support(Sowmya Ramesh) IMPROVEMENTS + FALCON-1417 Make validity end date optional for feed / process(Pragya Mittal via Ajay Yadava) + FALCON-1434 Enhance schedule API to accept key-value properties(Pallavi Rao) FALCON-1426 Add documentation for Falcon Email notification (Peeyush Bishnoi via Pallavi Rao) http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java b/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java index 2696552..18ae754 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java @@ -28,6 +28,7 @@ import org.apache.falcon.entity.v0.process.Output; import org.apache.falcon.entity.v0.process.Process; import org.apache.falcon.entity.v0.process.Validity; import org.apache.falcon.expression.ExpressionHelper; +import org.apache.falcon.util.DateUtil; import java.util.Date; @@ -45,6 +46,12 @@ public final class CrossEntityValidations { String clusterName = cluster.getName(); org.apache.falcon.entity.v0.feed.Validity feedValidity = FeedHelper.getCluster(feed, clusterName).getValidity(); + + // Optinal end_date + if (feedValidity.getEnd() == null) { + feedValidity.setEnd(DateUtil.NEVER); + } + Date feedStart = feedValidity.getStart(); Date feedEnd = feedValidity.getEnd(); http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java index 992fc51..4f5599e 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java @@ -47,6 +47,7 @@ import org.apache.falcon.entity.v0.process.Process; import org.apache.falcon.expression.ExpressionHelper; import org.apache.falcon.group.FeedGroup; import org.apache.falcon.group.FeedGroupMap; +import org.apache.falcon.util.DateUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authorize.AuthorizationException; import org.slf4j.Logger; @@ -82,6 +83,12 @@ public class FeedEntityParser extends EntityParser<Feed> { validateACL(feed); for (Cluster cluster : feed.getClusters().getClusters()) { validateEntityExists(EntityType.CLUSTER, cluster.getName()); + + // Optinal end_date + if (cluster.getValidity().getEnd() == null) { + cluster.getValidity().setEnd(DateUtil.NEVER); + } + validateClusterValidity(cluster.getValidity().getStart(), cluster.getValidity().getEnd(), cluster.getName()); validateClusterHasRegistry(feed, cluster); http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java index 48a4286..10a5265 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/ProcessEntityParser.java @@ -40,6 +40,7 @@ import org.apache.falcon.entity.v0.process.Outputs; import org.apache.falcon.entity.v0.process.Process; import org.apache.falcon.expression.ExpressionHelper; import org.apache.falcon.hadoop.HadoopClientFactory; +import org.apache.falcon.util.DateUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -79,6 +80,12 @@ public class ProcessEntityParser extends EntityParser<Process> { + " is defined more than once for process: " + process.getName()); } validateEntityExists(EntityType.CLUSTER, clusterName); + + // Optinal end_date + if (cluster.getValidity().getEnd() == null) { + cluster.getValidity().setEnd(DateUtil.NEVER); + } + validateProcessValidity(cluster.getValidity().getStart(), cluster.getValidity().getEnd()); validateHDFSPaths(process, clusterName); validateProperties(process); http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/main/java/org/apache/falcon/util/DateUtil.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/util/DateUtil.java b/common/src/main/java/org/apache/falcon/util/DateUtil.java index e736340..f89ef64 100644 --- a/common/src/main/java/org/apache/falcon/util/DateUtil.java +++ b/common/src/main/java/org/apache/falcon/util/DateUtil.java @@ -27,6 +27,9 @@ import java.util.TimeZone; */ public final class DateUtil { + //Friday, April 16, 9999 7:12:55 AM UTC corresponding date + public static final Date NEVER = new Date(Long.parseLong("253379862775000")); + private DateUtil() {} public static Date getNextMinute(Date time) throws Exception { @@ -36,4 +39,5 @@ public final class DateUtil { insCal.add(Calendar.MINUTE, 1); return insCal.getTime(); } + } http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java index 1e9b72f..b6fdb13 100644 --- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java +++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java @@ -974,4 +974,12 @@ public class FeedEntityParserTest extends AbstractTestBase { } } + @Test + public void testFeedEndTimeOptional() throws Exception { + Feed feed = parser.parseAndValidate(ProcessEntityParserTest.class + .getResourceAsStream(FEED_XML)); + feed.getClusters().getClusters().get(0).getValidity().setEnd(null); + parser.validate(feed); + } + } http://git-wip-us.apache.org/repos/asf/falcon/blob/75079c2e/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java index a935db3..da22d56 100644 --- a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java +++ b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java @@ -21,6 +21,7 @@ package org.apache.falcon.entity.parser; import org.apache.falcon.FalconException; import org.apache.falcon.cluster.util.EmbeddedCluster; import org.apache.falcon.entity.AbstractTestBase; +import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.Frequency; import org.apache.falcon.entity.v0.SchemaHelper; @@ -601,4 +602,22 @@ public class ProcessEntityParserTest extends AbstractTestBase { } } + @Test + public void testProcessEndTimeOptional() throws FalconException { + Process process = parser.parseAndValidate((ProcessEntityParserTest.class + .getResourceAsStream(PROCESS_XML))); + process.getClusters().getClusters().get(0).getValidity().setEnd(null); + parser.validate(process); + } + + @Test + public void testProcessEndTime() throws FalconException { + Process process = parser.parseAndValidate((ProcessEntityParserTest.class + .getResourceAsStream(PROCESS_XML))); + String feedName = process.getInputs().getInputs().get(0).getFeed(); + Feed feedEntity = EntityUtil.getEntity(EntityType.FEED, feedName); + feedEntity.getClusters().getClusters().get(0).getValidity().setEnd(null); + process.getClusters().getClusters().get(0).getValidity().setEnd(null); + parser.validate(process); + } }
