Added: falcon/site/FalconNativeScheduler.html URL: http://svn.apache.org/viewvc/falcon/site/FalconNativeScheduler.html?rev=1730450&view=auto ============================================================================== --- falcon/site/FalconNativeScheduler.html (added) +++ falcon/site/FalconNativeScheduler.html Mon Feb 15 06:02:08 2016 @@ -0,0 +1,675 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-02-09 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160209" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Falcon - Falcon Native Scheduler</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft"> + <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> + <img src="images/apache-feather-tm.gif" alt="Falcon" height="45px"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Falcon"> + Falcon</a> + </li> + <li class="divider ">/</li> + <li class="">Falcon Native Scheduler</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-02-09</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Falcon</li> + + <li> + + <a href="index.html" title="About"> + <i class="none"></i> + About</a> + </li> + + <li> + + <a href="slides/falcon-overview.html" title="Overview"> + <i class="none"></i> + Overview</a> + </li> + + <li> + + <a href="slides/falcon-user-guide.html" title="User Guide"> + <i class="none"></i> + User Guide</a> + </li> + + <li> + + <a href="GettingStarted.html" title="Getting Started"> + <i class="none"></i> + Getting Started</a> + </li> + + <li> + + <a href="FalconDocumentation.html" title="Architecture"> + <i class="none"></i> + Architecture</a> + </li> + + <li> + + <a href="InstallationSteps.html" title="Installation"> + <i class="none"></i> + Installation</a> + </li> + + <li> + + <a href="OnBoarding.html" title="On Boarding"> + <i class="none"></i> + On Boarding</a> + </li> + + <li> + + <a href="Operability.html" title="Operability"> + <i class="none"></i> + Operability</a> + </li> + + <li> + + <a href="EntitySpecification.html" title="Entity Specification"> + <i class="none"></i> + Entity Specification</a> + </li> + + <li> + + <a href="FalconCLI.html" title="Client (Falcon CLI)"> + <i class="none"></i> + Client (Falcon CLI)</a> + </li> + + <li> + + <a href="restapi/ResourceList.html" title="Rest API"> + <i class="icon-chevron-right"></i> + Rest API</a> + </li> + + <li> + + <a href="HiveIntegration.html" title="Hive Integration"> + <i class="none"></i> + Hive Integration</a> + </li> + + <li> + + <a href="Security.html" title="Security"> + <i class="none"></i> + Security</a> + </li> + <li class="nav-header">Project Information</li> + + <li> + + <a href="project-info.html" title="Summary"> + <i class="none"></i> + Summary</a> + </li> + + <li> + + <a href="mail-lists.html" title="Mailing Lists"> + <i class="none"></i> + Mailing Lists</a> + </li> + + <li> + + <a href="http://webchat.freenode.net?channels=apachefalcon&uio=d4"; class="externalLink" title="IRC"> + <i class="none"></i> + IRC</a> + </li> + + <li> + + <a href="team-list.html" title="Team"> + <i class="none"></i> + Team</a> + </li> + + <li> + + <a href="issue-tracking.html" title="Issue Tracking"> + <i class="none"></i> + Issue Tracking</a> + </li> + + <li> + + <a href="source-repository.html" title="Source Repository"> + <i class="none"></i> + Source Repository</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Index"; class="externalLink" title="Wiki"> + <i class="none"></i> + Wiki</a> + </li> + + <li> + + <a href="license.html" title="License"> + <i class="none"></i> + License</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/News"; class="externalLink" title="News"> + <i class="none"></i> + News</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/PoweredBy"; class="externalLink" title="Powered by"> + <i class="none"></i> + Powered by</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Acknowledgements"; class="externalLink" title="Acknowledgements"> + <i class="none"></i> + Acknowledgements</a> + </li> + + <li> + + <a href="http://blogs.apache.org/falcon/"; class="externalLink" title="Blog"> + <i class="none"></i> + Blog</a> + </li> + <li class="nav-header">Releases</li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.8"; class="externalLink" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.7"; class="externalLink" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/falcon/0.6.1"; class="externalLink" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.6-incubating"; class="externalLink" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.5-incubating"; class="externalLink" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.4-incubating"; class="externalLink" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.3-incubating"; class="externalLink" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Roadmap"; class="externalLink" title="Coming soon"> + <i class="none"></i> + Coming soon</a> + </li> + <li class="nav-header">Documentation</li> + + <li> + + <a href="0.9/index.html" title="0.9 (Current)"> + <i class="none"></i> + 0.9 (Current)</a> + </li> + + <li> + + <a href="0.8/index.html" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="0.7/index.html" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="0.6.1/index.html" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="0.6-incubating/index.html" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="0.5-incubating/index.html" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="0.4-incubating/index.html" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="0.3-incubating/index.html" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + <li class="nav-header">ASF</li> + + <li> + + <a href="http://www.apache.org/foundation/how-it-works.html"; class="externalLink" title="How Apache Works"> + <i class="none"></i> + How Apache Works</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/"; class="externalLink" title="Foundation"> + <i class="none"></i> + Foundation</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/sponsorship.html"; class="externalLink" title="Sponsoring Apache"> + <i class="none"></i> + Sponsoring Apache</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/thanks.html"; class="externalLink" title="Thanks"> + <i class="none"></i> + Thanks</a> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <div class="section"> +<h2>Falcon Native Scheduler<a name="Falcon_Native_Scheduler"></a></h2></div> +<div class="section"> +<h3>Overview<a name="Overview"></a></h3> +<p>Falcon has been using Oozie as its scheduling engine. While the use of Oozie works reasonably well, there are scenarios where Oozie scheduling is proving to be a limiting factor. In its current form, Falcon relies on Oozie for both scheduling and for workflow execution, due to which the scheduling is limited to time based/cron based scheduling with additional gating conditions on data availability. Also, this imposes restrictions on datasets being periodic in nature. In order to offer better scheduling capabilities, Falcon comes with its own native scheduler.</p></div> +<div class="section"> +<h3>Capabilities<a name="Capabilities"></a></h3> +<p>The native scheduler will offer the capabilities offered by Oozie co-ordinator and more. The native scheduler will be built and released over the next few releases of Falcon giving users an opportunity to use it and provide feedback.</p> +<p>Currently, the native scheduler offers the following capabilities:</p> +<ol style="list-style-type: decimal"> +<li>Submit and schedule a Falcon process that runs periodically (without data dependency) - It could be a PIG script, oozie workflow, Hive (all the engine types currently supported).</li> +<li>Monitor/Query/Modify the scheduled process - All applicable entity APIs and instance APIs should work as it does now. Falcon provides data management functions for feeds declaratively. It allows users to represent feed locations as time-based partition directories on HDFS containing files.</li></ol> +<p><b>NOTE: Execution order is FIFO. LIFO and LAST_ONLY are not supported yet.</b></p> +<p>In the near future, Falcon scheduler will provide feature parity with Oozie scheduler and in subsequent releases will provide the following features:</p> +<ul> +<li>Periodic, cron-based, calendar-based scheduling.</li> +<li>Data availability based scheduling.</li> +<li>External trigger/notification based scheduling.</li> +<li>Support for periodic/a-periodic datasets.</li> +<li>Support for optional/mandatory datasets. Option to specify minumum/maximum/exactly-N instances of data to consume.</li> +<li>Handle dependencies across entities during re-run.</li></ul></div> +<div class="section"> +<h3>Configuring Native Scheduler<a name="Configuring_Native_Scheduler"></a></h3> +<p>You can enable native scheduler by making changes to <b><i>$FALCON_HOME/conf/startup.properties</i></b> as follows. You will need to restart Falcon Server for the changes to take effect.</p> +<div class="source"> +<pre> +*.dag.engine.impl=org.apache.falcon.workflow.engine.OozieDAGEngine +*.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ + org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ + org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.service.FeedSLAMonitoringService,\ + org.apache.falcon.service.LifecyclePolicyMap,\ + org.apache.falcon.state.store.service.FalconJPAService,\ + org.apache.falcon.entity.store.ConfigurationStore,\ + org.apache.falcon.rerun.service.RetryService,\ + org.apache.falcon.rerun.service.LateRunService,\ + org.apache.falcon.metadata.MetadataMappingService,\ + org.apache.falcon.service.LogCleanupService,\ + org.apache.falcon.service.GroupsService,\ + org.apache.falcon.service.ProxyUserService,\ + org.apache.falcon.notification.service.impl.JobCompletionService,\ + org.apache.falcon.notification.service.impl.SchedulerService,\ + org.apache.falcon.notification.service.impl.AlarmService,\ + org.apache.falcon.notification.service.impl.DataAvailabilityService,\ + org.apache.falcon.execution.FalconExecutionService + +</pre></div></div> +<div class="section"> +<h4>Making the Native Scheduler the default scheduler<a name="Making_the_Native_Scheduler_the_default_scheduler"></a></h4> +<p>To ensure backward compatibility, even when the native scheduler is enabled, the default scheduler is still Oozie. This means users will be scheduling entities on Oozie scheduler, by default. They will need to explicitly specify the scheduler as native, if they wish to schedule entities using native scheduler.</p> +<p><a href="#Scheduling_new_entities_on_Native_Scheduler">This section</a> has more details on how to schedule on either of the schedulers.</p> +<p>If you wish to make the Falcon Native Scheduler your default scheduler and remove Oozie as the scheduler, set the following property in <b><i>$FALCON_HOME/conf/startup.properties</i></b></p> +<div class="source"> +<pre> +## If you wish to use Falcon native scheduler as your default scheduler, set the workflow engine to FalconWorkflowEngine instead of OozieWorkflowEngine. ## +*.workflow.engine.impl=org.apache.falcon.workflow.engine.FalconWorkflowEngine + +</pre></div></div> +<div class="section"> +<h4>Configuring the state store for Native Scheduler<a name="Configuring_the_state_store_for_Native_Scheduler"></a></h4> +<p>You can configure statestore by making changes to <b><i>$FALCON_HOME/conf/statestore.properties</i></b> as follows. You will need to restart Falcon Server for the changes to take effect.</p> +<p>Falcon Server needs to maintain state of the entities and instances in a persistent store for the system to be recoverable. Since Prism only federates, it does not need to maintain any state information. Following properties need to be set in statestore.properties of Falcon Servers:</p> +<div class="source"> +<pre> +######### StateStore Properties ##### +*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore +*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver +*.falcon.statestore.jdbc.url=jdbc:derby:data/falcon.db +# StateStore credentials file where username,password and other properties can be stored securely. +# Set this credentials file permission 400 and make sure user who starts falcon should only have read permission. +# Give Absolute path to credentials file along with file name or put in classpath with file name statestore.credentials. +# Credentials file should be present either in given location or class path, otherwise falcon won't start. +*.falcon.statestore.credentials.file= +*.falcon.statestore.jdbc.username=sa +*.falcon.statestore.jdbc.password= +*.falcon.statestore.connection.data.source=org.apache.commons.dbcp.BasicDataSource +# Maximum number of active connections that can be allocated from this pool at the same time. +*.falcon.statestore.pool.max.active.conn=10 +*.falcon.statestore.connection.properties= +# Indicates the interval (in milliseconds) between eviction runs. +*.falcon.statestore.validate.db.connection.eviction.interval=300000 +## The number of objects to examine during each run of the idle object evictor thread. +*.falcon.statestore.validate.db.connection.eviction.num=10 +## Creates Falcon DB. +## If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP. +## If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. +*.falcon.statestore.create.db.schema=true + +</pre></div> +<p>The _*.falcon.statestore.jdbc.url_ property in statestore.properties determines the DB and data location. All other properties are common across RDBMS.</p> +<p><b>NOTE : Although multiple Falcon Servers can share a DB (not applicable for Derby DB), it is recommended that you have different DBs for different Falcon Servers for better performance.</b></p> +<p>You will need to create the state DB and tables before starting the Falcon Server. To create tables, a tool comes bundled with the Falcon installation. You can use the <i>falcon-db.sh</i> script to create tables in the DB. The script needs to be run only for Falcon Servers and can be run by any user that has execute permission on the script. The script picks up the DB connection details from <b><i>$FALCON_HOME/conf/statestore.properties</i></b>. Ensure that you have granted the right privileges to the user mentioned in statestore.properties_, so the tables can be created.</p> +<p>You can use the help command to get details on the sub-commands supported:</p> +<div class="source"> +<pre> +./bin/falcon-db.sh help +Hadoop home is set, adding libraries from '/Users/pallavi.rao/falcon/hadoop-2.6.0/bin/hadoop classpath' into falcon classpath +usage: + Falcon DB initialization tool currently supports Derby DB/ Mysql + + falcondb help : Display usage for all commands or specified command + + falcondb version : Show Falcon DB version information + + falcondb create <OPTIONS> : Create Falcon DB schema + -run Confirmation option regarding DB schema creation/upgrade + -sqlfile <arg> Generate SQL script instead of creating/upgrading the DB + schema + + falcondb upgrade <OPTIONS> : Upgrade Falcon DB schema + -run Confirmation option regarding DB schema creation/upgrade + -sqlfile <arg> Generate SQL script instead of creating/upgrading the DB + schema + + +</pre></div> +<p>Currently, MySQL and Derby are supported as state stores. We may extend support to other DBs in the future. Falcon has been tested against MySQL v5.5. If you are using MySQL ensure you also copy mysql-connector-java-<version>.jar under <b><i>$FALCON_HOME/server/webapp/falcon/WEB-INF/lib</i></b> and <b><i>$FALCON_HOME/client/lib</i></b></p></div> +<div class="section"> +<h5>Using Derby as the State Store<a name="Using_Derby_as_the_State_Store"></a></h5> +<p>Using Derby is ideal for QA and staging setup. Falcon comes bundled with a Derby connector and no explicit setup is required (although you can set it up) in terms creating the DB or tables. For example,</p> +<div class="source"> +<pre> *.falcon.statestore.jdbc.url=jdbc:derby:data/falcon.db;create=true +</pre></div> +<p>tells Falcon to use the Derby JDBC connector, with data directory, $FALCON_HOME/data/ and DB name 'falcon'. If <i>create=true</i> is specified, you will not need to create a DB up front; a database will be created if it does not exist.</p></div> +<div class="section"> +<h5>Using MySQL as the State Store<a name="Using_MySQL_as_the_State_Store"></a></h5> +<p>The jdbc.url property in statestore.properties determines the DB and data location. For example,</p> +<div class="source"> +<pre> *.falcon.statestore.jdbc.url=jdbc:mysql://localhost:3306/falcon +</pre></div> +<p>tells Falcon to use the MySQL JDBC connector, which is accessible @localhost:3306, with DB name 'falcon'.</p></div> +<div class="section"> +<h3>Scheduling new entities on Native Scheduler<a name="Scheduling_new_entities_on_Native_Scheduler"></a></h3> +<p>To schedule an entity (currently only process is supported) using the native scheduler, you need to specify the scheduler in the schedule command as shown below:</p> +<div class="source"> +<pre> +$FALCON_HOME/bin/falcon entity -type process -name <process name> -schedule -properties falcon.scheduler:native + +</pre></div> +<p>If Oozie is configured as the default scheduler, you can skip the scheduler option or explicitly set it to <i>oozie</i>, as shown below:</p> +<div class="source"> +<pre> +$FALCON_HOME/bin/falcon entity -type process -name <process name> -schedule +OR +$FALCON_HOME/bin/falcon entity -type process -name <process name> -schedule -properties falcon.scheduler:oozie + +</pre></div> +<p>If the native scheduler is configured as the default scheduler, then, you can omit the scheduler option, as shown below:</p> +<div class="source"> +<pre> +$FALCON_HOME/bin/falcon entity -type process -name <process name> -schedule + +</pre></div></div> +<div class="section"> +<h3>Migrating entities from Oozie Scheduler to Native Scheduler<a name="Migrating_entities_from_Oozie_Scheduler_to_Native_Scheduler"></a></h3> +<p>Currently, user will have to delete and re-create entities in order to move across schedulers. Attempting to schedule an already scheduled entity on a different scheduler will result in an error. Note that the history of instances prior to scheduling on native scheduler will not be available via the instance APIs. However, user can retrieve that information using metadata APIs. Native scheduler must be enabled before migrating entities to native scheduler.</p> +<p><a href="#Configuring_Native_Scheduler">Configuring Native Scheduler</a> has more details on how to enable native scheduler.</p></div> +<div class="section"> +<h4>Migrating from Oozie to Native Scheduler<a name="Migrating_from_Oozie_to_Native_Scheduler"></a></h4> +<p></p> +<ul> +<li>Delete the entity (process).</li></ul> +<div class="source"> +<pre>$FALCON_HOME/bin/falcon entity -type process -name <process name> -delete +</pre></div> +<p></p> +<ul> +<li>Submit the entity (process) with start time from where the Oozie scheduler left off.</li></ul> +<div class="source"> +<pre>$FALCON_HOME/bin/falcon entity -type process -submit <path to process xml> +</pre></div> +<p></p> +<ul> +<li>Schedule the entity on native scheduler.</li></ul> +<div class="source"> +<pre> $FALCON_HOME/bin/falcon entity -type process -name <process name> -schedule -properties falcon.scheduler:native +</pre></div></div> +<div class="section"> +<h4>Reverting to Oozie from Native Scheduler<a name="Reverting_to_Oozie_from_Native_Scheduler"></a></h4> +<p></p> +<ul> +<li>Delete the entity (process).</li></ul> +<div class="source"> +<pre>$FALCON_HOME/bin/falcon entity -type process -name <process name> -delete +</pre></div> +<p></p> +<ul> +<li>Submit the entity (process) with start time from where the Native scheduler left off.</li></ul> +<div class="source"> +<pre>$FALCON_HOME/bin/falcon entity -type process -submit <path to process xml> +</pre></div> +<p></p> +<ul> +<li>Schedule the entity on the default scheduler (Oozie).</li></ul> +<div class="source"> +<pre> $FALCON_HOME/bin/falcon entity -type process -name <process name> -schedule +</pre></div></div> +<div class="section"> +<h4>Differences in API responses between Oozie and Native Scheduler<a name="Differences_in_API_responses_between_Oozie_and_Native_Scheduler"></a></h4> +<p>Most API responses are similar whether the entity is scheduled via Oozie or via Native scheduler. However, there are a few exceptions and those are listed below.</p></div> +<div class="section"> +<h5>Rerun API<a name="Rerun_API"></a></h5> +<p>When a user performs a rerun using Oozie scheduler, Falcon directly reruns the workflow on Oozie and the instance will be moved to 'RUNNING'.</p> +<p>Example response:</p> +<div class="source"> +<pre> +$ falcon instance -rerun processMerlinOozie -start 2016-01-08T12:13Z -end 2016-01-08T12:15Z +Consolidated Status: SUCCEEDED + +Instances: +Instance Cluster SourceCluster Status Start End Details Log +----------------------------------------------------------------------------------------------- +2016-01-08T12:13Z ProcessMultipleClustersTest-corp-9706f068 - RUNNING 2016-01-08T13:03Z 2016-01-08T13:03Z - http://8RPCG32.corp.inmobi.com:11000/oozie?job=0001811-160104160825636-oozie-oozi-W +2016-01-08T12:13Z ProcessMultipleClustersTest-corp-0b270a1d - RUNNING 2016-01-08T13:03Z 2016-01-08T13:03Z - http://lda01:11000/oozie?job=0002247-160104115615658-oozie-oozi-W + +Additional Information: +Response: ua1/RERUN +ua2/RERUN +Request Id: ua1/871377866@qtp-630572412-35 - 7190c4c8-bacb-4639-8d48-c9e639f544da +ua2/1554129706@qtp-536122141-13 - bc18127b-1bf8-4ea1-99e6-b1f10ba3a441 + +</pre></div> +<p>However, when a user performs a rerun on native scheduler, the instance is scheduled again. This is done intentionally so as to not violate the number of instances running in parallel. Hence, the user will see the status of the instance as 'READY'.</p> +<p>Example response:</p> +<div class="source"> +<pre> +$ falcon instance -rerun ProcessMultipleClustersTest-agregator-coord16-8f55f59b -start 2016-01-08T12:13Z -end 2016-01-08T12:15Z +Consolidated Status: SUCCEEDED + +Instances: +Instance Cluster SourceCluster Status Start End Details Log +----------------------------------------------------------------------------------------------- +2016-01-08T12:13Z ProcessMultipleClustersTest-corp-9706f068 - READY 2016-01-08T13:03Z 2016-01-08T13:03Z - http://8RPCG32.corp.inmobi.com:11000/oozie?job=0001812-160104160825636-oozie-oozi-W + +2016-01-08T12:13Z ProcessMultipleClustersTest-corp-0b270a1d - READY 2016-01-08T13:03Z 2016-01-08T13:03Z - http://lda01:11000/oozie?job=0002248-160104115615658-oozie-oozi-W + +Additional Information: +Response: ua1/RERUN +ua2/RERUN +Request Id: ua1/871377866@qtp-630572412-35 - 8d118d4d-c0ef-4335-a9af-10364498ec4f +ua2/1554129706@qtp-536122141-13 - c2a3fc50-8b05-47ce-9c85-ca432b96d923 + +</pre></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2013-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html>
Modified: falcon/site/GettingStarted.html URL: http://svn.apache.org/viewvc/falcon/site/GettingStarted.html?rev=1730450&r1=1730449&r2=1730450&view=diff ============================================================================== --- falcon/site/GettingStarted.html (original) +++ falcon/site/GettingStarted.html Mon Feb 15 06:02:08 2016 @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2015-12-08 + | Generated by Apache Maven Doxia at 2016-02-09 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20151208" /> + <meta name="Date-Revision-yyyymmdd" content="20160209" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Apache Falcon - Data management and processing platform</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2015-12-08</li> + <li id="publishDate" class="pull-right">Last Published: 2016-02-09</li> </ul> </div> @@ -306,6 +306,13 @@ <li> + <a href="0.9/index.html" title="0.9 (Current)"> + <i class="none"></i> + 0.9 (Current)</a> + </li> + + <li> + <a href="0.8/index.html" title="0.8"> <i class="none"></i> 0.8</a> @@ -437,7 +444,7 @@ <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2013-2015 + <div class="row span12">Copyright © 2013-2016 <a href="http://www.apache.org";>Apache Software Foundation</a>. All Rights Reserved. Added: falcon/site/HDFSDR.html URL: http://svn.apache.org/viewvc/falcon/site/HDFSDR.html?rev=1730450&view=auto ============================================================================== --- falcon/site/HDFSDR.html (added) +++ falcon/site/HDFSDR.html Mon Feb 15 06:02:08 2016 @@ -0,0 +1,470 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-02-09 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160209" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Falcon - HDFS DR Recipe</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft"> + <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> + <img src="images/apache-feather-tm.gif" alt="Falcon" height="45px"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Falcon"> + Falcon</a> + </li> + <li class="divider ">/</li> + <li class="">HDFS DR Recipe</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-02-09</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Falcon</li> + + <li> + + <a href="index.html" title="About"> + <i class="none"></i> + About</a> + </li> + + <li> + + <a href="slides/falcon-overview.html" title="Overview"> + <i class="none"></i> + Overview</a> + </li> + + <li> + + <a href="slides/falcon-user-guide.html" title="User Guide"> + <i class="none"></i> + User Guide</a> + </li> + + <li> + + <a href="GettingStarted.html" title="Getting Started"> + <i class="none"></i> + Getting Started</a> + </li> + + <li> + + <a href="FalconDocumentation.html" title="Architecture"> + <i class="none"></i> + Architecture</a> + </li> + + <li> + + <a href="InstallationSteps.html" title="Installation"> + <i class="none"></i> + Installation</a> + </li> + + <li> + + <a href="OnBoarding.html" title="On Boarding"> + <i class="none"></i> + On Boarding</a> + </li> + + <li> + + <a href="Operability.html" title="Operability"> + <i class="none"></i> + Operability</a> + </li> + + <li> + + <a href="EntitySpecification.html" title="Entity Specification"> + <i class="none"></i> + Entity Specification</a> + </li> + + <li> + + <a href="FalconCLI.html" title="Client (Falcon CLI)"> + <i class="none"></i> + Client (Falcon CLI)</a> + </li> + + <li> + + <a href="restapi/ResourceList.html" title="Rest API"> + <i class="icon-chevron-right"></i> + Rest API</a> + </li> + + <li> + + <a href="HiveIntegration.html" title="Hive Integration"> + <i class="none"></i> + Hive Integration</a> + </li> + + <li> + + <a href="Security.html" title="Security"> + <i class="none"></i> + Security</a> + </li> + <li class="nav-header">Project Information</li> + + <li> + + <a href="project-info.html" title="Summary"> + <i class="none"></i> + Summary</a> + </li> + + <li> + + <a href="mail-lists.html" title="Mailing Lists"> + <i class="none"></i> + Mailing Lists</a> + </li> + + <li> + + <a href="http://webchat.freenode.net?channels=apachefalcon&uio=d4"; class="externalLink" title="IRC"> + <i class="none"></i> + IRC</a> + </li> + + <li> + + <a href="team-list.html" title="Team"> + <i class="none"></i> + Team</a> + </li> + + <li> + + <a href="issue-tracking.html" title="Issue Tracking"> + <i class="none"></i> + Issue Tracking</a> + </li> + + <li> + + <a href="source-repository.html" title="Source Repository"> + <i class="none"></i> + Source Repository</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Index"; class="externalLink" title="Wiki"> + <i class="none"></i> + Wiki</a> + </li> + + <li> + + <a href="license.html" title="License"> + <i class="none"></i> + License</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/News"; class="externalLink" title="News"> + <i class="none"></i> + News</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/PoweredBy"; class="externalLink" title="Powered by"> + <i class="none"></i> + Powered by</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Acknowledgements"; class="externalLink" title="Acknowledgements"> + <i class="none"></i> + Acknowledgements</a> + </li> + + <li> + + <a href="http://blogs.apache.org/falcon/"; class="externalLink" title="Blog"> + <i class="none"></i> + Blog</a> + </li> + <li class="nav-header">Releases</li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.8"; class="externalLink" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.7"; class="externalLink" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/falcon/0.6.1"; class="externalLink" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.6-incubating"; class="externalLink" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.5-incubating"; class="externalLink" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.4-incubating"; class="externalLink" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.3-incubating"; class="externalLink" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Roadmap"; class="externalLink" title="Coming soon"> + <i class="none"></i> + Coming soon</a> + </li> + <li class="nav-header">Documentation</li> + + <li> + + <a href="0.9/index.html" title="0.9 (Current)"> + <i class="none"></i> + 0.9 (Current)</a> + </li> + + <li> + + <a href="0.8/index.html" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="0.7/index.html" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="0.6.1/index.html" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="0.6-incubating/index.html" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="0.5-incubating/index.html" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="0.4-incubating/index.html" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="0.3-incubating/index.html" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + <li class="nav-header">ASF</li> + + <li> + + <a href="http://www.apache.org/foundation/how-it-works.html"; class="externalLink" title="How Apache Works"> + <i class="none"></i> + How Apache Works</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/"; class="externalLink" title="Foundation"> + <i class="none"></i> + Foundation</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/sponsorship.html"; class="externalLink" title="Sponsoring Apache"> + <i class="none"></i> + Sponsoring Apache</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/thanks.html"; class="externalLink" title="Thanks"> + <i class="none"></i> + Thanks</a> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <div class="section"> +<h2>HDFS DR Recipe<a name="HDFS_DR_Recipe"></a></h2></div> +<div class="section"> +<h3>Overview<a name="Overview"></a></h3> +<p>Falcon supports HDFS DR recipe to replicate data from source cluster to destination cluster.</p></div> +<div class="section"> +<h3>Usage<a name="Usage"></a></h3></div> +<div class="section"> +<h4>Setup cluster definition.<a name="Setup_cluster_definition."></a></h4> +<div class="source"> +<pre> + $FALCON_HOME/bin/falcon entity -submit -type cluster -file /cluster/definition.xml + +</pre></div></div> +<div class="section"> +<h4>Update recipes properties<a name="Update_recipes_properties"></a></h4> +<p>Copy HDFS replication recipe properties, workflow and template file from $FALCON_HOME/data-mirroring/hdfs-replication to the accessible directory path or to the recipe directory path (<b>falcon.recipe.path=<recipe directory path></b>). <b>"falcon.recipe.path"</b> must be specified in Falcon conf client.properties. Now update the copied recipe properties file with required attributes to replicate data from source cluster to destination cluster for HDFS DR.</p></div> +<div class="section"> +<h4>Submit HDFS DR recipe<a name="Submit_HDFS_DR_recipe"></a></h4> +<p>After updating the recipe properties file with required attributes in directory path or in falcon.recipe.path, there are two ways of submitting the HDFS DR recipe:</p> +<p></p> +<ul> +<li>1. Specify Falcon recipe properties file through recipe command line.</li></ul> +<div class="source"> +<pre> + $FALCON_HOME/bin/falcon recipe -name hdfs-replication -operation HDFS_REPLICATION + -properties /cluster/hdfs-replication.properties + +</pre></div> +<p></p> +<ul> +<li>2. Use Falcon recipe path specified in Falcon conf client.properties .</li></ul> +<div class="source"> +<pre> + $FALCON_HOME/bin/falcon recipe -name hdfs-replication -operation HDFS_REPLICATION + +</pre></div> +<p><b>Note:</b> Recipe properties file, workflow file and template file name must match to the recipe name, it must be unique and in the same directory.</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2013-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html> Added: falcon/site/HiveDR.html URL: http://svn.apache.org/viewvc/falcon/site/HiveDR.html?rev=1730450&view=auto ============================================================================== --- falcon/site/HiveDR.html (added) +++ falcon/site/HiveDR.html Mon Feb 15 06:02:08 2016 @@ -0,0 +1,503 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-02-09 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160209" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Falcon - Hive Disaster Recovery</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft"> + <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> + <img src="images/apache-feather-tm.gif" alt="Falcon" height="45px"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Falcon"> + Falcon</a> + </li> + <li class="divider ">/</li> + <li class="">Hive Disaster Recovery</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-02-09</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Falcon</li> + + <li> + + <a href="index.html" title="About"> + <i class="none"></i> + About</a> + </li> + + <li> + + <a href="slides/falcon-overview.html" title="Overview"> + <i class="none"></i> + Overview</a> + </li> + + <li> + + <a href="slides/falcon-user-guide.html" title="User Guide"> + <i class="none"></i> + User Guide</a> + </li> + + <li> + + <a href="GettingStarted.html" title="Getting Started"> + <i class="none"></i> + Getting Started</a> + </li> + + <li> + + <a href="FalconDocumentation.html" title="Architecture"> + <i class="none"></i> + Architecture</a> + </li> + + <li> + + <a href="InstallationSteps.html" title="Installation"> + <i class="none"></i> + Installation</a> + </li> + + <li> + + <a href="OnBoarding.html" title="On Boarding"> + <i class="none"></i> + On Boarding</a> + </li> + + <li> + + <a href="Operability.html" title="Operability"> + <i class="none"></i> + Operability</a> + </li> + + <li> + + <a href="EntitySpecification.html" title="Entity Specification"> + <i class="none"></i> + Entity Specification</a> + </li> + + <li> + + <a href="FalconCLI.html" title="Client (Falcon CLI)"> + <i class="none"></i> + Client (Falcon CLI)</a> + </li> + + <li> + + <a href="restapi/ResourceList.html" title="Rest API"> + <i class="icon-chevron-right"></i> + Rest API</a> + </li> + + <li> + + <a href="HiveIntegration.html" title="Hive Integration"> + <i class="none"></i> + Hive Integration</a> + </li> + + <li> + + <a href="Security.html" title="Security"> + <i class="none"></i> + Security</a> + </li> + <li class="nav-header">Project Information</li> + + <li> + + <a href="project-info.html" title="Summary"> + <i class="none"></i> + Summary</a> + </li> + + <li> + + <a href="mail-lists.html" title="Mailing Lists"> + <i class="none"></i> + Mailing Lists</a> + </li> + + <li> + + <a href="http://webchat.freenode.net?channels=apachefalcon&uio=d4"; class="externalLink" title="IRC"> + <i class="none"></i> + IRC</a> + </li> + + <li> + + <a href="team-list.html" title="Team"> + <i class="none"></i> + Team</a> + </li> + + <li> + + <a href="issue-tracking.html" title="Issue Tracking"> + <i class="none"></i> + Issue Tracking</a> + </li> + + <li> + + <a href="source-repository.html" title="Source Repository"> + <i class="none"></i> + Source Repository</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Index"; class="externalLink" title="Wiki"> + <i class="none"></i> + Wiki</a> + </li> + + <li> + + <a href="license.html" title="License"> + <i class="none"></i> + License</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/News"; class="externalLink" title="News"> + <i class="none"></i> + News</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/PoweredBy"; class="externalLink" title="Powered by"> + <i class="none"></i> + Powered by</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Acknowledgements"; class="externalLink" title="Acknowledgements"> + <i class="none"></i> + Acknowledgements</a> + </li> + + <li> + + <a href="http://blogs.apache.org/falcon/"; class="externalLink" title="Blog"> + <i class="none"></i> + Blog</a> + </li> + <li class="nav-header">Releases</li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.8"; class="externalLink" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.7"; class="externalLink" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/falcon/0.6.1"; class="externalLink" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.6-incubating"; class="externalLink" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.5-incubating"; class="externalLink" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.4-incubating"; class="externalLink" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.3-incubating"; class="externalLink" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Roadmap"; class="externalLink" title="Coming soon"> + <i class="none"></i> + Coming soon</a> + </li> + <li class="nav-header">Documentation</li> + + <li> + + <a href="0.9/index.html" title="0.9 (Current)"> + <i class="none"></i> + 0.9 (Current)</a> + </li> + + <li> + + <a href="0.8/index.html" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="0.7/index.html" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="0.6.1/index.html" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="0.6-incubating/index.html" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="0.5-incubating/index.html" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="0.4-incubating/index.html" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="0.3-incubating/index.html" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + <li class="nav-header">ASF</li> + + <li> + + <a href="http://www.apache.org/foundation/how-it-works.html"; class="externalLink" title="How Apache Works"> + <i class="none"></i> + How Apache Works</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/"; class="externalLink" title="Foundation"> + <i class="none"></i> + Foundation</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/sponsorship.html"; class="externalLink" title="Sponsoring Apache"> + <i class="none"></i> + Sponsoring Apache</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/thanks.html"; class="externalLink" title="Thanks"> + <i class="none"></i> + Thanks</a> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <div class="section"> +<h2>Hive Disaster Recovery<a name="Hive_Disaster_Recovery"></a></h2></div> +<div class="section"> +<h3>Overview<a name="Overview"></a></h3> +<p>Falcon provides feature to replicate Hive metadata and data events from source cluster to destination cluster. This is supported for secure and unsecure cluster through Falcon Recipes.</p></div> +<div class="section"> +<h3>Prerequisites<a name="Prerequisites"></a></h3> +<p>Following is the prerequisites to use Hive DR</p> +<p></p> +<ul> +<li><b>Hive 1.2.0+</b></li> +<li><b>Oozie 4.2.0+</b></li></ul> +<p><b>Note:</b> Set following properties in hive-site.xml for replicating the Hive events on source and destination Hive cluster:</p> +<div class="source"> +<pre> + <property> + <name>hive.metastore.event.listeners</name> + <value>org.apache.hive.hcatalog.listener.DbNotificationListener</value> + <description>event listeners that are notified of any metastore changes</description> + </property> + + <property> + <name>hive.metastore.dml.events</name> + <value>true</value> + </property> + +</pre></div></div> +<div class="section"> +<h3>Usage<a name="Usage"></a></h3></div> +<div class="section"> +<h4>Bootstrap<a name="Bootstrap"></a></h4> +<p>Perform initial bootstrap of Table and Database from source cluster to destination cluster</p> +<ul> +<li><b>Database Bootstrap</b></li></ul>For bootstrapping DB replication, first destination DB should be created. This step is expected, since DB replication definitions can be set up by users only on pre-existing DB’s. Second, Export all tables in the source db and Import it in the destination db, as described in Table bootstrap. +<p></p> +<ul> +<li><b>Table Bootstrap</b></li></ul>For bootstrapping table replication, essentially after having turned on the DbNotificationListener on the source db, perform an Export of the table, distcp the Export over to the destination warehouse and do an Import over there. Check the following <a class="externalLink" href="https://cwiki.apache.org/confluence/display/Hive/LanguageManual+ImportExport";>Hive Export-Import</a> for syntax details and examples. This will set up the destination table so that the events on the source cluster that modify the table will then be replicated.</div> +<div class="section"> +<h4>Setup cluster definition<a name="Setup_cluster_definition"></a></h4> +<div class="source"> +<pre> + $FALCON_HOME/bin/falcon entity -submit -type cluster -file /cluster/definition.xml + +</pre></div></div> +<div class="section"> +<h4>Update recipes properties<a name="Update_recipes_properties"></a></h4> +<p>Copy Hive DR recipe properties, workflow and template file from $FALCON_HOME/data-mirroring/hive-disaster-recovery to the accessible directory path or to the recipe directory path (<b>falcon.recipe.path=<recipe directory path></b>). <b>"falcon.recipe.path"</b> must be specified in Falcon conf client.properties. Now update the copied recipe properties file with required attributes to replicate metadata and data from source cluster to destination cluster for Hive DR.</p></div> +<div class="section"> +<h4>Submit Hive DR recipe<a name="Submit_Hive_DR_recipe"></a></h4> +<p>After updating the recipe properties file with required attributes in directory path or in falcon.recipe.path, there are two ways of submitting the Hive DR recipe:</p> +<p></p> +<ul> +<li>1. Specify Falcon recipe properties file through recipe command line.</li></ul> +<div class="source"> +<pre> + $FALCON_HOME/bin/falcon recipe -name hive-disaster-recovery -operation HIVE_DISASTER_RECOVERY + -properties /cluster/hive-disaster-recovery.properties + +</pre></div> +<p></p> +<ul> +<li>2. Use Falcon recipe path specified in Falcon conf client.properties .</li></ul> +<div class="source"> +<pre> + $FALCON_HOME/bin/falcon recipe -name hive-disaster-recovery -operation HIVE_DISASTER_RECOVERY + +</pre></div> +<p><b>Note:</b></p> +<ul> +<li>Recipe properties file, workflow file and template file name must match to the recipe name, it must be unique and in the same directory.</li> +<li>If kerberos security is enabled on cluster, use the secure templates for Hive DR from $FALCON_HOME/data-mirroring/hive-disaster-recovery .</li></ul></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2013-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html> Modified: falcon/site/HiveIntegration.html URL: http://svn.apache.org/viewvc/falcon/site/HiveIntegration.html?rev=1730450&r1=1730449&r2=1730450&view=diff ============================================================================== --- falcon/site/HiveIntegration.html (original) +++ falcon/site/HiveIntegration.html Mon Feb 15 06:02:08 2016 @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2015-12-08 + | Generated by Apache Maven Doxia at 2016-02-09 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20151208" /> + <meta name="Date-Revision-yyyymmdd" content="20160209" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Hive Integration</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2015-12-08</li> + <li id="publishDate" class="pull-right">Last Published: 2016-02-09</li> </ul> </div> @@ -306,6 +306,13 @@ <li> + <a href="0.9/index.html" title="0.9 (Current)"> + <i class="none"></i> + 0.9 (Current)</a> + </li> + + <li> + <a href="0.8/index.html" title="0.8"> <i class="none"></i> 0.8</a> @@ -775,7 +782,7 @@ store C into '$output_database.$output_t <footer> <div class="container-fluid"> - <div class="row span12">Copyright © 2013-2015 + <div class="row span12">Copyright © 2013-2016 <a href="http://www.apache.org";>Apache Software Foundation</a>. All Rights Reserved.
