Repository: falcon Updated Branches: refs/heads/master ec4a273a8 -> ec3e49e39
FALCON-2084 HCatReplicationTest are failing in secure mode Removed the dependency on job-xml whose priority has changed because of some backward incompatible changes in Oozie - this patch will work with 4.2.0 and upcoming 4.3.0 Author: Venkat Ranganathan <[email protected]> Reviewers: "Balu Vellanki <[email protected]>, Ying Zheng <[email protected]>" Closes #231 from vrangan/FALCON-2084 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/ec3e49e3 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/ec3e49e3 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/ec3e49e3 Branch: refs/heads/master Commit: ec3e49e393c45ca580795e47fb085c8e58a8f789 Parents: ec4a273 Author: Venkat Ranganathan <[email protected]> Authored: Tue Jul 19 17:05:57 2016 -0700 Committer: bvellanki <[email protected]> Committed: Tue Jul 19 17:05:57 2016 -0700 ---------------------------------------------------------------------- .../feed/FeedReplicationCoordinatorBuilder.java | 5 --- .../feed/FeedReplicationWorkflowBuilder.java | 7 +--- .../feed/HCatReplicationWorkflowBuilder.java | 34 +++++++++++++++++++- .../main/resources/action/feed/table-export.xml | 1 - .../main/resources/action/feed/table-import.xml | 1 - .../feed/OozieFeedWorkflowBuilderTest.java | 8 ----- 6 files changed, 34 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java index b0e46f0..07d293c 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java @@ -275,11 +275,6 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F Path scriptPath = new Path(buildPath, "scripts"); copyHiveScript(fs, scriptPath, IMPORT_HQL); copyHiveScript(fs, scriptPath, EXPORT_HQL); - - // create hive conf to stagingDir - Path confPath = new Path(buildPath + "/conf"); - persistHiveConfiguration(fs, confPath, srcCluster, "falcon-source-"); - persistHiveConfiguration(fs, confPath, trgCluster, "falcon-target-"); } catch (IOException e) { throw new FalconException("Unable to create hive conf files", e); } http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java index 5a62130..010446b 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationWorkflowBuilder.java @@ -24,7 +24,6 @@ import org.apache.falcon.Tag; import org.apache.falcon.entity.ClusterHelper; import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.FeedHelper; -import org.apache.falcon.entity.HiveUtil; import org.apache.falcon.entity.store.ConfigurationStore; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.cluster.Cluster; @@ -79,11 +78,7 @@ public abstract class FeedReplicationWorkflowBuilder extends OozieOrchestrationW marshal(cluster, workflow, buildPath); Properties props = getProperties(buildPath, wfName); props.putAll(createDefaultConfiguration(cluster)); - if (EntityUtil.isTableStorageType(cluster, entity)) { - // todo: kludge send source hcat creds for coord dependency check to pass - props.putAll(HiveUtil.getHiveCredentials(srcCluster)); - props.putAll(HiveUtil.getHiveCredentials(cluster)); - } + props.putAll(getWorkflowProperties(entity)); props.putAll(FeedHelper.getUserWorkflowProperties(getLifecycle())); // Write out the config to config-default.xml http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java index 629485d..3da97d3 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/HCatReplicationWorkflowBuilder.java @@ -22,15 +22,18 @@ import org.apache.falcon.FalconException; import org.apache.falcon.Tag; import org.apache.falcon.entity.ClusterHelper; import org.apache.falcon.entity.EntityUtil; +import org.apache.falcon.entity.HiveUtil; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.oozie.workflow.ACTION; import org.apache.falcon.oozie.workflow.WORKFLOWAPP; +import org.apache.falcon.oozie.workflow.CONFIGURATION; import org.apache.falcon.util.OozieUtils; import org.apache.falcon.workflow.WorkflowExecutionArgs; import javax.xml.bind.JAXBElement; import java.util.Arrays; +import java.util.Map; import java.util.Properties; /** @@ -60,6 +63,15 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild //Add pre-processing if (shouldPreProcess()) { ACTION action = getPreProcessingAction(false, Tag.REPLICATION); + Properties hiveConf = HiveUtil.getHiveCredentials(src); + for (Map.Entry<Object, Object> e : hiveConf.entrySet()) { + CONFIGURATION.Property prop = new CONFIGURATION.Property(); + prop.setName((String) e.getKey()); + prop.setValue((String) e.getValue()); + LOG.info("Adding config to replication hive preprocessing action : key = {} value = {}", + e.getKey(), e.getValue()); + action.getJava().getConfiguration().getProperty().add(prop); + } addHDFSServersConfig(action, src, target); addTransition(action, EXPORT_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME); workflow.getDecisionOrForkOrJoin().add(action); @@ -72,6 +84,16 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild OozieUtils.unMarshalHiveAction(export); org.apache.falcon.oozie.hive.ACTION hiveExportAction = exportActionJaxbElement.getValue(); addHDFSServersConfig(hiveExportAction, src, target); + Properties hiveConf = HiveUtil.getHiveCredentials(src); + for (Map.Entry<Object, Object> e : hiveConf.entrySet()) { + org.apache.falcon.oozie.hive.CONFIGURATION.Property prop = + new org.apache.falcon.oozie.hive.CONFIGURATION.Property(); + prop.setName((String) e.getKey()); + prop.setValue((String) e.getValue()); + LOG.info("Adding config to replication hive export action : key = {} value = {}", + e.getKey(), e.getValue()); + hiveExportAction.getConfiguration().getProperty().add(prop); + } OozieUtils.marshalHiveAction(export, exportActionJaxbElement); addTransition(export, REPLICATION_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME); workflow.getDecisionOrForkOrJoin().add(export); @@ -89,6 +111,16 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild OozieUtils.unMarshalHiveAction(importAction); org.apache.falcon.oozie.hive.ACTION hiveImportAction = importActionJaxbElement.getValue(); addHDFSServersConfig(hiveImportAction, src, target); + Properties hiveConf2 = HiveUtil.getHiveCredentials(target); + for (Map.Entry<Object, Object> e : hiveConf2.entrySet()) { + org.apache.falcon.oozie.hive.CONFIGURATION.Property prop = + new org.apache.falcon.oozie.hive.CONFIGURATION.Property(); + prop.setName((String) e.getKey()); + prop.setValue((String) e.getValue()); + LOG.info("Adding config to replication hive import action : key = {} value = {}", + e.getKey(), e.getValue()); + hiveImportAction.getConfiguration().getProperty().add(prop); + } OozieUtils.marshalHiveAction(importAction, importActionJaxbElement); addTransition(importAction, CLEANUP_ACTION_NAME, FAIL_POSTPROCESS_ACTION_NAME); workflow.getDecisionOrForkOrJoin().add(importAction); @@ -133,8 +165,8 @@ public class HCatReplicationWorkflowBuilder extends FeedReplicationWorkflowBuild (org.apache.falcon.oozie.workflow.ACTION) object; String actionName = action.getName(); if (PREPROCESS_ACTION_NAME.equals(actionName)) { + // add reference to hive-site conf to each action - action.getJava().setJobXml("${wf:appPath()}/conf/falcon-source-hive-site.xml"); if (isSecurityEnabled) { // add a reference to credential in the action action.setCred(SOURCE_HIVE_CREDENTIAL_NAME); http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/resources/action/feed/table-export.xml ---------------------------------------------------------------------- diff --git a/oozie/src/main/resources/action/feed/table-export.xml b/oozie/src/main/resources/action/feed/table-export.xml index fcf1a1a..6bc214b 100644 --- a/oozie/src/main/resources/action/feed/table-export.xml +++ b/oozie/src/main/resources/action/feed/table-export.xml @@ -26,7 +26,6 @@ <prepare> <delete path="${distcpSourcePaths}"/> </prepare> - <job-xml>${wf:appPath()}/conf/falcon-source-hive-site.xml</job-xml> <configuration> <property> <name>mapred.job.queue.name</name> http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/main/resources/action/feed/table-import.xml ---------------------------------------------------------------------- diff --git a/oozie/src/main/resources/action/feed/table-import.xml b/oozie/src/main/resources/action/feed/table-import.xml index 6e9a073..450e68a 100644 --- a/oozie/src/main/resources/action/feed/table-import.xml +++ b/oozie/src/main/resources/action/feed/table-import.xml @@ -20,7 +20,6 @@ <hive xmlns="uri:oozie:hive-action:0.2"> <job-tracker>${falconTargetJobTracker}</job-tracker> <name-node>${falconTargetNameNode}</name-node> - <job-xml>${wf:appPath()}/conf/falcon-target-hive-site.xml</job-xml> <configuration> <property> <name>mapred.job.queue.name</name> http://git-wip-us.apache.org/repos/asf/falcon/blob/ec3e49e3/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java ---------------------------------------------------------------------- diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java index b0fc230..2040748 100644 --- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java +++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java @@ -561,9 +561,6 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { Assert.assertTrue(fs.exists(new Path(wfPath + "/scripts/falcon-table-export.hql"))); Assert.assertTrue(fs.exists(new Path(wfPath + "/scripts/falcon-table-import.hql"))); - Assert.assertTrue(fs.exists(new Path(wfPath + "/conf"))); - Assert.assertTrue(fs.exists(new Path(wfPath + "/conf/falcon-source-hive-site.xml"))); - Assert.assertTrue(fs.exists(new Path(wfPath + "/conf/falcon-target-hive-site.xml"))); HashMap<String, String> props = getCoordProperties(coord); @@ -626,11 +623,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { private void assertReplicationHCatCredentials(WORKFLOWAPP wf, String wfPath) throws IOException { FileSystem fs = trgMiniDFS.getFileSystem(); - Path hiveConfPath = new Path(wfPath, "conf/falcon-source-hive-site.xml"); - Assert.assertTrue(fs.exists(hiveConfPath)); - hiveConfPath = new Path(wfPath, "conf/falcon-target-hive-site.xml"); - Assert.assertTrue(fs.exists(hiveConfPath)); boolean isSecurityEnabled = SecurityUtil.isSecurityEnabled(); if (isSecurityEnabled) { @@ -651,7 +644,6 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase { } if ("recordsize".equals(actionName)) { - Assert.assertEquals(action.getJava().getJobXml(), "${wf:appPath()}/conf/falcon-source-hive-site.xml"); if (isSecurityEnabled) { Assert.assertNotNull(action.getCred()); Assert.assertEquals(action.getCred(), "falconSourceHiveAuth");
