Repository: falcon Updated Branches: refs/heads/master bb659d40b -> 3a8659611
FALCON-1449 Move getEntityProperties method to EntityUtil. Contributed by Ajay Yadava. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/3a865961 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/3a865961 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/3a865961 Branch: refs/heads/master Commit: 3a8659611c82b928ebb2abc7b5ec4629d4156402 Parents: bb659d4 Author: Ajay Yadava <[email protected]> Authored: Tue Sep 15 16:35:34 2015 +0530 Committer: Ajay Yadava <[email protected]> Committed: Tue Sep 15 16:35:34 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../org/apache/falcon/entity/EntityUtil.java | 40 +++++++++++++++ .../apache/falcon/entity/EntityUtilTest.java | 51 ++++++++++++++++++++ .../apache/falcon/oozie/OozieEntityBuilder.java | 44 ++--------------- .../OozieOrchestrationWorkflowBuilder.java | 6 +-- .../feed/FeedReplicationCoordinatorBuilder.java | 4 +- .../feed/FeedRetentionCoordinatorBuilder.java | 3 +- .../ProcessExecutionCoordinatorBuilder.java | 2 +- .../ProcessExecutionWorkflowBuilder.java | 2 +- 9 files changed, 106 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 3a00a1e..bd24a86 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,6 +8,8 @@ Trunk (Unreleased) FALCON-1027 Falcon proxy user support(Sowmya Ramesh) IMPROVEMENTS + FALCON-1449 Move getEntityProperties method to EntityUtil.(Ajay Yadava) + FALCON-1357 Update CHANGES.txt to change 0.7 branch to release.(Ajay Yadava) FALCON-1414 Add all fields in filterBy to the entity list output.(Narayan Periwal via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/common/src/main/java/org/apache/falcon/entity/EntityUtil.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java index 25d9008..2f05b1f 100644 --- a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java +++ b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java @@ -34,6 +34,7 @@ import org.apache.falcon.entity.v0.Frequency; import org.apache.falcon.entity.v0.SchemaHelper; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.cluster.ClusterLocationType; +import org.apache.falcon.entity.v0.cluster.Property; import org.apache.falcon.entity.v0.feed.ClusterType; import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.entity.v0.process.LateInput; @@ -66,6 +67,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.TimeZone; @@ -311,6 +313,44 @@ public final class EntityUtil { return startCal.getTime(); } + + public static Properties getEntityProperties(Entity myEntity) { + Properties properties = new Properties(); + switch (myEntity.getEntityType()) { + case CLUSTER: + org.apache.falcon.entity.v0.cluster.Properties clusterProps = ((Cluster) myEntity).getProperties(); + if (clusterProps != null) { + for (Property prop : clusterProps.getProperties()) { + properties.put(prop.getName(), prop.getValue()); + } + } + break; + + case FEED: + org.apache.falcon.entity.v0.feed.Properties feedProps = ((Feed) myEntity).getProperties(); + if (feedProps != null) { + for (org.apache.falcon.entity.v0.feed.Property prop : feedProps.getProperties()) { + properties.put(prop.getName(), prop.getValue()); + } + } + break; + + case PROCESS: + org.apache.falcon.entity.v0.process.Properties processProps = ((Process) myEntity).getProperties(); + if (processProps != null) { + for (org.apache.falcon.entity.v0.process.Property prop : processProps.getProperties()) { + properties.put(prop.getName(), prop.getValue()); + } + } + break; + + default: + throw new IllegalArgumentException("Unhandled entity type " + myEntity.getEntityType()); + } + return properties; + } + + public static int getInstanceSequence(Date startTime, Frequency frequency, TimeZone tz, Date instanceTime) { if (startTime.after(instanceTime)) { return -1; http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java index cfdc84d..f6a4679 100644 --- a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java +++ b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java @@ -28,6 +28,7 @@ import org.apache.falcon.entity.v0.Frequency; import org.apache.falcon.entity.v0.SchemaHelper; import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.entity.v0.feed.LateArrival; +import org.apache.falcon.entity.v0.feed.Property; import org.apache.falcon.entity.v0.process.Cluster; import org.apache.falcon.entity.v0.process.Process; import org.apache.falcon.hadoop.HadoopClientFactory; @@ -42,6 +43,7 @@ import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.Properties; import java.util.TimeZone; /** @@ -254,6 +256,55 @@ public class EntityUtilTest extends AbstractTestBase { } @Test + public void testGetFeedProperties() { + Feed feed = new Feed(); + org.apache.falcon.entity.v0.feed.Properties props = new org.apache.falcon.entity.v0.feed.Properties(); + Property queue = new Property(); + String name = "Q"; + String value = "head of Q division!"; + queue.setName(name); + queue.setValue(value); + props.getProperties().add(queue); + feed.setProperties(props); + Properties actual = EntityUtil.getEntityProperties(feed); + Assert.assertEquals(actual.size(), 1); + Assert.assertEquals(actual.getProperty(name), value); + } + + @Test + public void testGetProcessProperties() { + org.apache.falcon.entity.v0.cluster.Cluster cluster = new org.apache.falcon.entity.v0.cluster.Cluster(); + org.apache.falcon.entity.v0.cluster.Properties props = new org.apache.falcon.entity.v0.cluster.Properties(); + org.apache.falcon.entity.v0.cluster.Property priority = new org.apache.falcon.entity.v0.cluster.Property(); + String name = "priority"; + String value = "Sister of Moriarity!"; + priority.setName(name); + priority.setValue(value); + props.getProperties().add(priority); + cluster.setProperties(props); + Properties actual = EntityUtil.getEntityProperties(cluster); + Assert.assertEquals(actual.size(), 1); + Assert.assertEquals(actual.getProperty(name), value); + } + + @Test + public void testGetClusterProperties() { + Process process = new Process(); + org.apache.falcon.entity.v0.process.Properties props = new org.apache.falcon.entity.v0.process.Properties(); + org.apache.falcon.entity.v0.process.Property priority = new org.apache.falcon.entity.v0.process.Property(); + String name = "M"; + String value = "Minions!"; + priority.setName(name); + priority.setValue(value); + props.getProperties().add(priority); + process.setProperties(props); + Properties actual = EntityUtil.getEntityProperties(process); + Assert.assertEquals(actual.size(), 1); + Assert.assertEquals(actual.getProperty(name), value); + + } + + @Test public void testGetLateProcessFeed() throws FalconException { Feed feed = new Feed(); http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java index 9a6b14c..f9230da 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/OozieEntityBuilder.java @@ -23,11 +23,11 @@ import org.apache.commons.lang3.StringUtils; import org.apache.falcon.FalconException; import org.apache.falcon.entity.CatalogStorage; import org.apache.falcon.entity.ClusterHelper; +import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.cluster.ClusterLocationType; import org.apache.falcon.entity.v0.cluster.Interfacetype; -import org.apache.falcon.entity.v0.cluster.Property; import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.entity.v0.process.Output; import org.apache.falcon.entity.v0.process.Process; @@ -165,8 +165,8 @@ public abstract class OozieEntityBuilder<T extends Entity> { } } - protected Properties createAppProperties(Cluster cluster, String wfName) throws FalconException { - Properties properties = getEntityProperties(cluster); + protected Properties createAppProperties(Cluster cluster) throws FalconException { + Properties properties = EntityUtil.getEntityProperties(cluster); properties.setProperty(AbstractWorkflowEngine.NAME_NODE, ClusterHelper.getStorageUrl(cluster)); properties.setProperty(AbstractWorkflowEngine.JOB_TRACKER, ClusterHelper.getMREndPoint(cluster)); properties.setProperty("colo.name", cluster.getColo()); @@ -175,7 +175,7 @@ public abstract class OozieEntityBuilder<T extends Entity> { properties.setProperty(OozieClient.USE_SYSTEM_LIBPATH, "true"); } properties.setProperty("falcon.libpath", - ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath() + "/lib"); + ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath() + "/lib"); return properties; } @@ -215,42 +215,6 @@ public abstract class OozieEntityBuilder<T extends Entity> { return hiveConf; } - protected Properties getEntityProperties(Entity myEntity) { - Properties properties = new Properties(); - switch (myEntity.getEntityType()) { - case CLUSTER: - org.apache.falcon.entity.v0.cluster.Properties clusterProps = ((Cluster) myEntity).getProperties(); - if (clusterProps != null) { - for (Property prop : clusterProps.getProperties()) { - properties.put(prop.getName(), prop.getValue()); - } - } - break; - - case FEED: - org.apache.falcon.entity.v0.feed.Properties feedProps = ((Feed) myEntity).getProperties(); - if (feedProps != null) { - for (org.apache.falcon.entity.v0.feed.Property prop : feedProps.getProperties()) { - properties.put(prop.getName(), prop.getValue()); - } - } - break; - - case PROCESS: - org.apache.falcon.entity.v0.process.Properties processProps = ((Process) myEntity).getProperties(); - if (processProps != null) { - for (org.apache.falcon.entity.v0.process.Property prop : processProps.getProperties()) { - properties.put(prop.getName(), prop.getValue()); - } - } - break; - - default: - throw new IllegalArgumentException("Unhandled entity type " + myEntity.getEntityType()); - } - return properties; - } - protected void propagateCatalogTableProperties(Output output, CatalogStorage tableStorage, Properties props) { String prefix = "falcon_" + output.getName(); http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java index f7193a3..671ae6b 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/OozieOrchestrationWorkflowBuilder.java @@ -164,7 +164,7 @@ public abstract class OozieOrchestrationWorkflowBuilder<T extends Entity> extend protected void addAdditionalReplicationProperties(ACTION replicationAction) { List<String> args = replicationAction.getJava().getArg(); - Properties props = getEntityProperties(entity); + Properties props = EntityUtil.getEntityProperties(entity); for (ReplicationDistCpOption distcpOption : ReplicationDistCpOption.values()) { String propertyValue = props.getProperty(distcpOption.getName()); @@ -427,8 +427,8 @@ public abstract class OozieOrchestrationWorkflowBuilder<T extends Entity> extend props.put(MR_JOB_PRIORITY, "NORMAL"); //props in entity override the set props. - props.putAll(getEntityProperties(entity)); - props.putAll(createAppProperties(cluster, entity.getName())); + props.putAll(EntityUtil.getEntityProperties(entity)); + props.putAll(createAppProperties(cluster)); return props; } http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java index db08d60..b0e46f0 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java @@ -182,7 +182,7 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F propagateLateDataProperties(instancePaths, sourceStorage.getType().name(), props); // Add the custom properties set in feed. Else, dryrun won't catch any missing props. - props.putAll(getEntityProperties(entity)); + props.putAll(EntityUtil.getEntityProperties(entity)); workflow.setConfiguration(getConfig(props)); action.setWorkflow(workflow); @@ -336,7 +336,7 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F timeoutInMillis = THIRTY_MINUTES; } - Properties props = getEntityProperties(entity); + Properties props = EntityUtil.getEntityProperties(entity); String timeout = props.getProperty(TIMEOUT); if (timeout!=null) { try{ http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java index ce9ef9a..a4b5f35 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedRetentionCoordinatorBuilder.java @@ -21,6 +21,7 @@ package org.apache.falcon.oozie.feed; import org.apache.falcon.FalconException; import org.apache.falcon.LifeCycle; import org.apache.falcon.Tag; +import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.FeedHelper; import org.apache.falcon.entity.v0.Frequency.TimeUnit; import org.apache.falcon.entity.v0.SchemaHelper; @@ -78,7 +79,7 @@ public class FeedRetentionCoordinatorBuilder extends OozieCoordinatorBuilder<Fee workflow.setAppPath(getStoragePath(wfProps.getProperty(OozieEntityBuilder.ENTITY_PATH))); props.putAll(getProperties(coordPath, coordName)); // Add the custom properties set in feed. Else, dryrun won't catch any missing props. - props.putAll(getEntityProperties(entity)); + props.putAll(EntityUtil.getEntityProperties(entity)); workflow.setConfiguration(getConfig(props)); ACTION action = new ACTION(); action.setWorkflow(workflow); http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java index d6d42e1..7154e40 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionCoordinatorBuilder.java @@ -92,7 +92,7 @@ public class ProcessExecutionCoordinatorBuilder extends OozieCoordinatorBuilder< WORKFLOW wf = new WORKFLOW(); wf.setAppPath(getStoragePath(wfProps.getProperty(OozieEntityBuilder.ENTITY_PATH))); // Add the custom properties set in feed. Else, dryrun won't catch any missing props. - props.putAll(getEntityProperties(entity)); + props.putAll(EntityUtil.getEntityProperties(entity)); wf.setConfiguration(getConfig(props)); // set coord action to parent wf http://git-wip-us.apache.org/repos/asf/falcon/blob/3a865961/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java index ac436ca..3f49adb 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/process/ProcessExecutionWorkflowBuilder.java @@ -197,7 +197,7 @@ public abstract class ProcessExecutionWorkflowBuilder extends OozieOrchestration } protected void propagateEntityProperties(CONFIGURATION conf, List<String> paramList) { - Properties entityProperties = getEntityProperties(entity); + Properties entityProperties = EntityUtil.getEntityProperties(entity); // Propagate user defined properties to job configuration final List<org.apache.falcon.oozie.workflow.CONFIGURATION.Property> configuration = conf.getProperty();
