Repository: falcon Updated Branches: refs/heads/master 3efffd4d9 -> 723f1f7f1
FALCON-2047 Use /apps/falcon/extensions/mirroring as base dir for HiveDR instead of /apps/data-mirroring/ Author: Sowmya Ramesh <[email protected]> Reviewers: "Venkat Ranganathan <[email protected]>, Balu Vellanki <[email protected]>, Peeyush B <[email protected]>, Praveen Adlakha <[email protected]>" Closes #252 from sowmyaramesh/FALCON-2047 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/723f1f7f Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/723f1f7f Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/723f1f7f Branch: refs/heads/master Commit: 723f1f7f1bb1cd052e81f757629f750393ae6a80 Parents: 3efffd4 Author: Sowmya Ramesh <[email protected]> Authored: Thu Aug 4 15:03:57 2016 -0700 Committer: Sowmya Ramesh <[email protected]> Committed: Thu Aug 4 15:03:57 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/falcon/hive/HiveDRTool.java | 17 +++++++++-------- .../org/apache/falcon/hive/util/DRStatusStore.java | 2 +- .../org/apache/falcon/hive/util/EventUtils.java | 2 +- .../falcon/extensions/store/ExtensionStore.java | 4 +--- .../src/test/resources/hive-mirroring-template.xml | 4 ++-- .../HdfsRecipe/hdfs-replication-template.xml | 2 +- .../HdfsRecipe/hdfs-replication.properties | 2 +- .../hive-disaster-recovery-template.xml | 2 +- .../hive-disaster-recovery-secure-template.xml | 2 +- falcon-ui/app/js/services/entity/entity-model.js | 4 ++-- src/main/assemblies/distributed-package.xml | 10 ++++++++++ src/main/assemblies/standalone-package.xml | 10 ++++++++++ .../src/test/resources/hive-mirroring-template.xml | 2 +- 13 files changed, 41 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java ---------------------------------------------------------------------- diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java index 2701e02..7c6133d 100644 --- a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java +++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java @@ -136,13 +136,6 @@ public class HiveDRTool extends Configured implements Tool { inputOptions = parseOptions(args); LOG.info("Input Options: {}", inputOptions); - // Update the source staging path - inputOptions.setSourceStagingPath(); - inputOptions.setTargetStagingPath(); - - LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath()); - LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath()); - Configuration sourceConf = FileUtils.getConfiguration(getConf(), inputOptions.getSourceWriteEP(), inputOptions.getSourceNNKerberosPrincipal()); sourceClusterFS = FileSystem.get(sourceConf); @@ -155,6 +148,14 @@ public class HiveDRTool extends Configured implements Tool { // init DR status store drStore = new HiveDRStatusStore(targetClusterFS); + + // Update the source staging path after initing DR status store + inputOptions.setSourceStagingPath(); + inputOptions.setTargetStagingPath(); + + LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath()); + LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath()); + eventSoucerUtil = new EventSourcerUtils(jobConf, inputOptions.shouldKeepHistory(), inputOptions.getJobName()); } @@ -310,7 +311,7 @@ public class HiveDRTool extends Configured implements Tool { } private Map<String, Long> getLastDBTableEvents(Path lastEventIdFile) throws Exception { - Map<String, Long> lastEventsIdMap = new HashMap<String, Long>(); + Map<String, Long> lastEventsIdMap = new HashMap<>(); BufferedReader in = new BufferedReader(new InputStreamReader(jobFS.open(lastEventIdFile))); try { String line; http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java ---------------------------------------------------------------------- diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java index cf6b7ad..86f50db 100644 --- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java +++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/DRStatusStore.java @@ -30,7 +30,7 @@ import java.util.List; */ public abstract class DRStatusStore { - public static final String BASE_DEFAULT_STORE_PATH = "/apps/data-mirroring/"; + public static final String BASE_DEFAULT_STORE_PATH = "/apps/falcon/extensions/mirroring"; public static final FsPermission DEFAULT_STORE_PERMISSION = new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.NONE); http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java ---------------------------------------------------------------------- diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java index 492c70e..74d0eb6 100644 --- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java +++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java @@ -50,7 +50,7 @@ import java.util.Map; import java.util.Properties; /** - * Utility class to handle Hive events for data-mirroring. + * Utility class to handle Hive events for hive-mirroring. */ public class EventUtils { private static final String DRIVER_NAME = "org.apache.hive.jdbc.HiveDriver"; http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index 4ca432a..b2cd7da 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -151,16 +151,14 @@ public final class ExtensionStore { for (FileStatus fileStatus : files) { if (fileStatus.getPath().getName().equalsIgnoreCase(LIBS_DIR)) { - if (fileStatus.getLen() != 0) { libsPath = Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath()); - } break; } } if (libsPath == null) { LOG.info("For extension " + extensionName + " there is no " - + LIBS_DIR + "at the extension store path " + storePath); + + LIBS_DIR + "at the extension store path " + extensionPath); return null; } else { return libsPath.toString(); http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/extensions/src/test/resources/hive-mirroring-template.xml ---------------------------------------------------------------------- diff --git a/extensions/src/test/resources/hive-mirroring-template.xml b/extensions/src/test/resources/hive-mirroring-template.xml index 9f28991..fe0fae1 100644 --- a/extensions/src/test/resources/hive-mirroring-template.xml +++ b/extensions/src/test/resources/hive-mirroring-template.xml @@ -37,8 +37,8 @@ <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> </properties> - <workflow name="##workflow.name##" engine="oozie" - path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> + <workflow name="##workflow.name##" engine="oozie" path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" + lib="##workflow.lib.path##"/> <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> <notification type="##notification.type##" to="##notification.receivers##"/> <ACL/> http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml index 46546ce..b9834bb 100644 --- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml +++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication-template.xml @@ -37,7 +37,7 @@ <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> </properties> - <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/data-mirroring/workflows/hdfs-replication-workflow.xml" lib="##workflow.lib.path##"/> + <workflow name="##falcon.recipe.workflow.name##" engine="oozie" path="/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml" lib="##workflow.lib.path##"/> <retry policy="##falcon.recipe.retry.policy##" delay="##falcon.recipe.retry.delay##" attempts="3"/> <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/> <ACL/> http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties index 6c715f3..a3a1e33 100644 --- a/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties +++ b/falcon-regression/merlin/src/test/resources/HdfsRecipe/hdfs-replication.properties @@ -25,7 +25,7 @@ falcon.recipe.name=hdfs-replication-monthly ##### Workflow properties falcon.recipe.workflow.name=hdfs-dr-workflow # Provide Wf absolute path. This can be HDFS or local FS path. If WF is on local FS it will be copied to HDFS -falcon.recipe.workflow.path=/apps/data-mirroring/workflows/hdfs-replication-workflow.xml +falcon.recipe.workflow.path=/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml ##### Cluster properties http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml index c11c2bb..bff8313 100644 --- a/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml +++ b/falcon-regression/merlin/src/test/resources/HiveDrRecipe/hive-disaster-recovery-template.xml @@ -38,7 +38,7 @@ </properties> <workflow name="##workflow.name##" engine="oozie" - path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> + path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/> <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/> <ACL/> http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml b/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml index c11c2bb..bff8313 100644 --- a/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml +++ b/falcon-regression/merlin/src/test/resources/HiveDrSecureRecipe/hive-disaster-recovery-secure-template.xml @@ -38,7 +38,7 @@ </properties> <workflow name="##workflow.name##" engine="oozie" - path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> + path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/> <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> <notification type="##falcon.recipe.notification.type##" to="##falcon.recipe.notification.receivers##"/> <ACL/> http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/falcon-ui/app/js/services/entity/entity-model.js ---------------------------------------------------------------------- diff --git a/falcon-ui/app/js/services/entity/entity-model.js b/falcon-ui/app/js/services/entity/entity-model.js index 48fa617..98d79c3 100644 --- a/falcon-ui/app/js/services/entity/entity-model.js +++ b/falcon-ui/app/js/services/entity/entity-model.js @@ -302,7 +302,7 @@ workflow: { _name: "hdfs-dr-workflow", _engine: "oozie", - _path: "/apps/data-mirroring/workflows/hdfs-replication-workflow.xml", + _path: "/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml", _lib: "" }, retry: { @@ -434,7 +434,7 @@ workflow: { _name: "falcon-dr-hive-workflow", _engine: "oozie", - _path: "/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml", + _path: "/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml", _lib: "" }, retry: { http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/src/main/assemblies/distributed-package.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml index 4683a81..09665dc 100644 --- a/src/main/assemblies/distributed-package.xml +++ b/src/main/assemblies/distributed-package.xml @@ -248,6 +248,16 @@ </fileSet> <fileSet> + <directory>../</directory> + <outputDirectory>extensions/mirroring</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0770</fileMode> + <directoryMode>0770</directoryMode> + </fileSet> + + <fileSet> <directory>../examples/app</directory> <outputDirectory>examples/app</outputDirectory> <directoryMode>0755</directoryMode> http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/src/main/assemblies/standalone-package.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml index cfc3a26..c3b2259 100644 --- a/src/main/assemblies/standalone-package.xml +++ b/src/main/assemblies/standalone-package.xml @@ -270,6 +270,16 @@ <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> + + <fileSet> + <directory>../</directory> + <outputDirectory>extensions/mirroring</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0770</fileMode> + <directoryMode>0770</directoryMode> + </fileSet> </fileSets> <files> http://git-wip-us.apache.org/repos/asf/falcon/blob/723f1f7f/webapp/src/test/resources/hive-mirroring-template.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/hive-mirroring-template.xml b/webapp/src/test/resources/hive-mirroring-template.xml index 9f28991..8d86408 100644 --- a/webapp/src/test/resources/hive-mirroring-template.xml +++ b/webapp/src/test/resources/hive-mirroring-template.xml @@ -38,7 +38,7 @@ </properties> <workflow name="##workflow.name##" engine="oozie" - path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> + path="/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml" lib="##workflow.lib.path##"/> <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> <notification type="##notification.type##" to="##notification.receivers##"/> <ACL/>
