http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/restapi/Vertices.html ---------------------------------------------------------------------- diff --git a/content/restapi/Vertices.html b/content/restapi/Vertices.html index 023f3b8..0c3b3f2 100644 --- a/content/restapi/Vertices.html +++ b/content/restapi/Vertices.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - GET api/metadata/lineage/vertices?key=:key&value=:value</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../../index.html" id="bannerLeft"> - <img src="../images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="../images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="../MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="../Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,28 +148,28 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="../restapi/ResourceList.html" title="Rest API"> <i class="icon-chevron-down"></i> Rest API</a> <ul class="nav nav-list"> - + <li> <a href="../restapi/ResourceList.html" title="Entity Resource"> <i class="icon-chevron-right"></i> Entity Resource</a> </li> - + <li> <a href="../restapi/ResourceList.html" title="Feed/Process Instances"> <i class="icon-chevron-right"></i> Feed/Process Instances</a> </li> - + <li> <a href="../restapi/ResourceList.html" title="Admin Resource"> @@ -239,13 +246,20 @@ </li> </ul> </li> - + <li> <a href="../restapi/ResourceList.html" title="Metadata Resource"> <i class="icon-chevron-right"></i> Metadata Resource</a> </li> + + <li> + + <a href="../restapi/ResourceList.html" title="Falcon Extensions"> + <i class="icon-chevron-right"></i> + Falcon Extensions</a> + </li> </ul> </li> @@ -258,6 +272,13 @@ <li> + <a href="../Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="../Security.html" title="Security"> <i class="none"></i> Security</a> @@ -351,6 +372,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -415,9 +443,16 @@ <li> - <a href="../0.9/index.html" title="0.9 (Current)"> + <a href="../0.10/index.html" title="0.10 (Current)"> <i class="none"></i> - 0.9 (Current)</a> + 0.10 (Current)</a> + </li> + + <li> + + <a href="../0.9/index.html" title="0.9"> + <i class="none"></i> + 0.9</a> </li> <li>
http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/source-repository.html ---------------------------------------------------------------------- diff --git a/content/source-repository.html b/content/source-repository.html index f793c4a..a9464b8 100644 --- a/content/source-repository.html +++ b/content/source-repository.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Source Repository</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,7 +148,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -158,6 +165,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -249,6 +263,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -313,9 +334,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/team-list.html ---------------------------------------------------------------------- diff --git a/content/team-list.html b/content/team-list.html index ce09772..4c95a5a 100644 --- a/content/team-list.html +++ b/content/team-list.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Team list</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,7 +148,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -158,6 +165,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -249,6 +263,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -313,9 +334,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> <i class="none"></i> - 0.9 (Current)</a> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> + <i class="none"></i> + 0.9</a> </li> <li> @@ -556,7 +584,7 @@ <td>Sowmya Ramesh</td> <td><a class="externalLink" href="mailto:[email protected]";>[email protected]</a></td> <td>Hortonworks</td> -<td>committer</td> +<td>PPMC</td> <td>-08:00</td> <td><span id="developer-15">-08:00</span></td></tr> <tr class="b"> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/pom.xml ---------------------------------------------------------------------- diff --git a/trunk/general/pom.xml b/trunk/general/pom.xml index ef5ba41..7bdad1c 100644 --- a/trunk/general/pom.xml +++ b/trunk/general/pom.xml @@ -22,10 +22,10 @@ <parent> <groupId>org.apache.falcon</groupId> <artifactId>falcon-website</artifactId> - <version>0.9-SNAPSHOT</version> + <version>0.11-SNAPSHOT</version> </parent> <artifactId>falcon-website-general</artifactId> - <version>0.9-SNAPSHOT</version> + <version>0.10</version> <packaging>war</packaging> <name>Apache Falcon - General</name> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/site.xml ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/site.xml b/trunk/general/src/site/site.xml index 7ef53eb..8f5f6f2 100644 --- a/trunk/general/src/site/site.xml +++ b/trunk/general/src/site/site.xml @@ -27,7 +27,7 @@ </skin> <bannerLeft> - <name>Falcon</name> + <name>Apache Falcon</name> <src>./images/falcon-logo.png</src> <href>http://falcon.apache.org/index.html</href> <width>200px</width> @@ -64,6 +64,7 @@ <item name="Architecture" href="./FalconDocumentation.html"/> <item name="Installation" href="./InstallationSteps.html"/> <item name="On Boarding" href="./OnBoarding.html"/> + <item name="Migrate to 0.10" href="./MigrationInstructions.html"/> <item name="Operability" href="./Operability.html"/> <item name="Entity Specification" href="./EntitySpecification.html"/> <item name="Client (Falcon CLI)" href="./falconcli/FalconCLI.html"/> @@ -83,6 +84,7 @@ <item href="./restapi/EntitySummary.html" name="Get instance summary of all entities" /> <item href="./restapi/EntityDependencies.html" name="Get the dependencies of the entity" /> <item href="./restapi/FeedLookup.html" name="Get feed for given path" /> + <item href="./restapi/FeedSLA.html" name="Get feed instances which missed SLA" /> </item> <item name="Feed/Process Instances" href="./restapi/ResourceList.html" collapse="true"> <item href="./restapi/InstanceRunning.html" name="List of running instances. " /> @@ -95,9 +97,12 @@ <item href="./restapi/InstanceRerun.html" name="Rerun a given instance " /> <item href="./restapi/InstanceLogs.html" name="Get logs of a given instance " /> <item href="./restapi/InstanceSummary.html" name="Return summary of instances for an entity " /> + <item href="./restapi/InstanceDependency.html" name="Return dependent instances of an instance " /> + <item href="./restapi/Triage.html" name="Triage a given instance " /> </item> <item name="Admin Resource" href="./restapi/ResourceList.html" collapse="true"> <item name="Stack Trace" href="./restapi/AdminStack.html" /> + <item name="Falcon Safemode" href="./restapi/AdminSafemode.html" /> <item name="Version" href="./restapi/AdminVersion.html" /> <item name="Configuration" href="./restapi/AdminConfig.html" /> </item> @@ -114,9 +119,26 @@ </item> <item name="Metadata Resource" href="./restapi/ResourceList.html" collapse="true"> <item href="./restapi/MetadataList.html" name="List of dimensions" /> + <item href="./restapi/MetadataRelations.html" name="List relations of a dimension" /> + </item> + <item name="Falcon Extensions" href="./restapi/ResourceList.html" collapse="true"> + <item href="./restapi/ExtensionEnumeration.html" name="Enumerate Falcon Extensions" /> + <item href="./restapi/ExtensionDescription.html" name="Describe an Extension" /> + <item href="./restapi/ExtensionDefinition.html" name="Get JSON Extension properties" /> + <item href="./restapi/ExtensionList.html" name="List Extension jobs" /> + <item href="./restapi/ExtensionInstances.html" name="List Instances of Extension job" /> + <item href="./restapi/ExtensionValidate.html" name="Validate Extension Job" /> + <item href="./restapi/ExtensionSubmit.html" name="Submit Extension Job" /> + <item href="./restapi/ExtensionSubmitAndSchedule.html" name="Submit and schedule extension job" /> + <item href="./restapi/ExtensionUpdate.html" name="Update Extension Job" /> + <item href="./restapi/ExtensionSchedule.html" name="Schedule Extension Job" /> + <item href="./restapi/ExtensionSuspend.html" name="Suspend Extension Job" /> + <item href="./restapi/ExtensionResume.html" name="Resume Extension Job" /> + <item href="./restapi/ExtensionDelete.html" name="Delete Extension Job" /> </item> </item> <item name="Hive Integration" href="./HiveIntegration.html"/> + <item name="Server side Extensions" href="./Extensions.html"/> <item name="Security" href="./Security.html"/> </menu> @@ -136,6 +158,7 @@ </menu> <menu name="Releases"> + <item name="0.10" href="http://www.apache.org/dyn/closer.lua/falcon/0.10"/> <item name="0.9" href="http://www.apache.org/dyn/closer.lua/falcon/0.9"/> <item name="0.8" href="http://www.apache.org/dyn/closer.lua/falcon/0.8"/> <item name="0.7" href="http://www.apache.org/dyn/closer.lua/falcon/0.7"/> @@ -149,7 +172,8 @@ <menu name="Documentation"> <!-- current points to latest release --> - <item name="0.9 (Current)" href="./0.9/index.html"/> + <item name="0.10 (Current)" href="./0.10/index.html"/> + <item name="0.9" href="./0.9/index.html"/> <item name="0.8" href="./0.8/index.html"/> <item name="0.7" href="./0.7/index.html"/> <item name="0.6.1" href="./0.6.1/index.html"/> @@ -167,8 +191,8 @@ </menu> <footer> - é 2011-2014 The Apache Software Foundation. Apache Falcon, Falcon, Apache, the Apache feather logo, + é 2011-2016 The Apache Software Foundation. Apache Falcon, Falcon, Apache, the Apache feather logo, and the Apache Falcon project logo are trademarks of The Apache Software Foundation. </footer> </body> -</project> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/Configuration.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/Configuration.twiki b/trunk/general/src/site/twiki/Configuration.twiki index 0df094f..98acb83 100644 --- a/trunk/general/src/site/twiki/Configuration.twiki +++ b/trunk/general/src/site/twiki/Configuration.twiki @@ -47,6 +47,9 @@ before any commands are executed. The following environment variables are availa # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. #export FALCON_EXPANDED_WEBAPP_DIR= + +# Any additional classpath elements to be added to the Falcon server/client classpath +#export FALCON_EXTRA_CLASS_PATH= </verbatim> ---++Advanced Configurations @@ -87,7 +90,6 @@ export FALCON_SERVER_OPTS="-Djava.awt.headless=true -Djava.security.krb5.realm= </verbatim> ---+++Activemq - * falcon server starts embedded active mq. To control this behaviour, set the following system properties using -D option in environment variable FALCON_OPTS: * falcon.embeddedmq=<true/false> - Should server start embedded active mq, default true @@ -95,14 +97,334 @@ option in environment variable FALCON_OPTS: * falcon.embeddedmq.data=<path> - Data path for embedded active mq, default {package dir}/logs/data ---+++Falcon System Notifications -Some Falcon features such as late data handling, retries, metadata service, depend on JMS notifications sent when the Oozie workflow completes. These system notifications are sent as part of Falcon Post Processing action. Given that the post processing action is also a job, it is prone to failures and in case of failures, Falcon is blind to the status of the workflow. To alleviate this problem and make the notifications more reliable, you can enable Oozie's JMS notification feature and disable Falcon post-processing notification by making the following changes: - * In Falcon runtime.properties, set *.falcon.jms.notification.enabled to false. This will turn off JMS notification in post-processing. - * Copy notification related properties in oozie/conf/oozie-site.xml to oozie-site.xml of the Oozie installation. Restart Oozie so changes get reflected. -*NOTE : If you disable Falcon post-processing JMS notification and not enable Oozie JMS notification, features such as failure retry, late data handling and metadata service will be disabled for all entities on the server.* +Some Falcon features such as late data handling, retries, metadata service, depend on JMS notifications sent when the +Oozie workflow completes. Falcon listens to Oozie notification via JMS. You need to enable Oozie JMS notification as +explained below. Falcon post processing feature continues to only send user notifications so enabling Oozie +JMS notification is important. + +---+++Enable Oozie JMS notification + + * Please add/change the following properties in oozie-site.xml in the oozie installation dir. + +<verbatim> + <property> + <name>oozie.jms.producer.connection.properties</name> + <value>java.naming.factory.initial#org.apache.activemq.jndi.ActiveMQInitialContextFactory;java.naming.provider.url#tcp://<activemq-host>:<port></value> + </property> + + <property> + <name>oozie.service.EventHandlerService.event.listeners</name> + <value>org.apache.oozie.jms.JMSJobEventListener</value> + </property> + + <property> + <name>oozie.service.JMSTopicService.topic.name</name> + <value>WORKFLOW=ENTITY.TOPIC,COORDINATOR=ENTITY.TOPIC</value> + </property> + + <property> + <name>oozie.service.JMSTopicService.topic.prefix</name> + <value>FALCON.</value> + </property> + + <!-- add org.apache.oozie.service.JMSAccessorService to the other existing services if any --> + <property> + <name>oozie.services.ext</name> + <value>org.apache.oozie.service.JMSAccessorService,org.apache.oozie.service.PartitionDependencyManagerService,org.apache.oozie.service.HCatAccessorService</value> + </property> +</verbatim> + + * In falcon startup.properties, set JMS broker url to be the same as the one set in oozie-site.xml property + oozie.jms.producer.connection.properties (see above) + +<verbatim> + *.broker.url=tcp://<activemq-host>:<port> +</verbatim> + +---+++Configuring Oozie for Falcon + +Falcon uses HCatalog for data availability notification when Hive tables are replicated. Make the following configuration +changes to Oozie to ensure Hive table replication in Falcon: + + * Stop the Oozie service on all Falcon clusters. Run the following commands on the Oozie host machine. + +<verbatim> +su - $OOZIE_USER + +<oozie-install-dir>/bin/oozie-stop.sh + +where $OOZIE_USER is the Oozie user. For example, oozie. +</verbatim> + + * Copy each cluster's hadoop conf directory to a different location. For example, if you have two clusters, copy one to /etc/hadoop/conf-1 and the other to /etc/hadoop/conf-2. + + * For each oozie-site.xml file, modify the oozie.service.HadoopAccessorService.hadoop.configurations property, specifying clusters, the RPC ports of the NameNodes, and HostManagers accordingly. For example, if Falcon connects to three clusters, specify: + +<verbatim> + +<property> + <name>oozie.service.HadoopAccessorService.hadoop.configurations</name> + <value>*=/etc/hadoop/conf,$NameNode:$rpcPortNN=$hadoopConfDir1,$ResourceManager1:$rpcPortRM=$hadoopConfDir1,$NameNode2=$hadoopConfDir2,$ResourceManager2:$rpcPortRM=$hadoopConfDir2,$NameNode3 :$rpcPortNN =$hadoopConfDir3,$ResourceManager3 :$rpcPortRM =$hadoopConfDir3</value> + <description> + Comma separated AUTHORITY=HADOOP_CONF_DIR, where AUTHORITY is the HOST:PORT of + the Hadoop service (JobTracker, HDFS). The wildcard '*' configuration is + used when there is no exact match for an authority. The HADOOP_CONF_DIR contains + the relevant Hadoop *-site.xml files. If the path is relative is looked within + the Oozie configuration directory; though the path can be absolute (i.e. to point + to Hadoop client conf/ directories in the local filesystem. + </description> +</property> + +</verbatim> + + * Add the following properties to the /etc/oozie/conf/oozie-site.xml file: + +<verbatim> + +<property> + <name>oozie.service.ProxyUserService.proxyuser.falcon.hosts</name> + <value>*</value> +</property> + +<property> + <name>oozie.service.ProxyUserService.proxyuser.falcon.groups</name> + <value>*</value> +</property> + +<property> + <name>oozie.service.URIHandlerService.uri.handlers</name> + <value>org.apache.oozie.dependency.FSURIHandler, org.apache.oozie.dependency.HCatURIHandler</value> +</property> + +<property> + <name>oozie.services.ext</name> + <value>org.apache.oozie.service.JMSAccessorService, org.apache.oozie.service.PartitionDependencyManagerService, + org.apache.oozie.service.HCatAccessorService</value> +</property> + +<!-- Coord EL Functions Properties --> + +<property> + <name>oozie.service.ELService.ext.functions.coord-job-submit-instances</name> + <value>now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo, + today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo, + yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo, + currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo, + lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo, + currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo, + lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo, + formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, + latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, + future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo + </value> +</property> + +<property> + <name>oozie.service.ELService.ext.functions.coord-action-create-inst</name> + <value>now=org.apache.oozie.extensions.OozieELExtensions#ph2_now_inst, + today=org.apache.oozie.extensions.OozieELExtensions#ph2_today_inst, + yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday_inst, + currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth_inst, + lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth_inst, + currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear_inst, + lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear_inst, + latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, + future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, + formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, + user=org.apache.oozie.coord.CoordELFunctions#coord_user + </value> +</property> + +<property> +<name>oozie.service.ELService.ext.functions.coord-action-start</name> +<value> +now=org.apache.oozie.extensions.OozieELExtensions#ph2_now, +today=org.apache.oozie.extensions.OozieELExtensions#ph2_today, +yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday, +currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth, +lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth, +currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear, +lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear, +latest=org.apache.oozie.coord.CoordELFunctions#ph3_coord_latest, +future=org.apache.oozie.coord.CoordELFunctions#ph3_coord_future, +dataIn=org.apache.oozie.extensions.OozieELExtensions#ph3_dataIn, +instanceTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_nominalTime, +dateOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateOffset, +formatTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_formatTime, +user=org.apache.oozie.coord.CoordELFunctions#coord_user +</value> +</property> + +<property> + <name>oozie.service.ELService.ext.functions.coord-sla-submit</name> + <value> + instanceTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_fixed, + user=org.apache.oozie.coord.CoordELFunctions#coord_user + </value> +</property> + +<property> + <name>oozie.service.ELService.ext.functions.coord-sla-create</name> + <value> + instanceTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_nominalTime, + user=org.apache.oozie.coord.CoordELFunctions#coord_user + </value> +</property> + +</verbatim> + + * Copy the existing Oozie WAR file to <oozie-install-dir>/oozie.war. This will ensure that all existing items in the WAR file are still present after the current update. + +<verbatim> +su - root +cp $CATALINA_BASE/webapps/oozie.war <oozie-install-dir>/oozie.war + +where $CATALINA_BASE is the path for the Oozie web app. By default, $CATALINA_BASE is: <oozie-install-dir> +</verbatim> + + * Add the Falcon EL extensions to Oozie. + +Copy the extension JAR files provided with the Falcon Server to a temporary directory on the Oozie server. For example, if your standalone Falcon Server is on the same machine as your Oozie server, you can just copy the JAR files. + +<verbatim> + +mkdir /tmp/falcon-oozie-jars +cp <falcon-install-dir>/oozie/ext/falcon-oozie-el-extension-<$version>.jar /tmp/falcon-oozie-jars +cp /tmp/falcon-oozie-jars/falcon-oozie-el-extension-<$version>.jar <oozie-install-dir>/libext + +</verbatim> + + * Package the Oozie WAR file as the Oozie user + +<verbatim> +su - $OOZIE_USER +cd <oozie-install-dir>/bin +./oozie-setup.sh prepare-war + +Where $OOZIE_USER is the Oozie user. For example, oozie. +</verbatim> + + * Start the Oozie service on all Falcon clusters. Run these commands on the Oozie host machine. + +<verbatim> +su - $OOZIE_USER +<oozie-install-dir>/bin/oozie-start.sh + +Where $OOZIE_USER is the Oozie user. For example, oozie. +</verbatim> + ---+++Enabling Falcon Native Scheudler -You can either choose to schedule entities using Oozie's coordinator or using Falcon's native scheduler. To be able to schedule entities natively on Falcon, you will need to add some additional properties to <verbatim>$FALCON_HOME/conf/startup.properties</verbatim> before starting the Falcon Server. For details on the same, refer to [[FalconNativeScheduler][Falcon Native Scheduler]] +You can either choose to schedule entities using Oozie's coordinator or using Falcon's native scheduler. To be able to +schedule entities natively on Falcon, you will need to add some additional properties +to <verbatim>$FALCON_HOME/conf/startup.properties</verbatim> before starting the Falcon Server. +For details on the same, refer to [[FalconNativeScheduler][Falcon Native Scheduler]] + +---+++Titan GraphDB backend +GraphDB backend needs to be configured to properly start Falcon server. +You can either choose to use 5.0.73 version of berkeleydb (the default for Falcon for the last few releases) or 1.1.x or later version HBase as the backend database. +Falcon in its release distributions will have the titan storage plugins for both BerkeleyDB and HBase. + +----++++Using BerkeleyDB backend +Falcon distributions may not package berkeley db artifacts (je-5.0.73.jar) based on build profiles. +If Berkeley DB is not packaged, you can download the Berkeley DB jar file from the URL: +<verbatim>http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip</verbatim>. +The following properties describe an example berkeley db graph storage backend that can be specified in the configuration file +<verbatim>$FALCON_HOME/conf/startup.properties</verbatim>. + +<verbatim> +# Graph Storage +*.falcon.graph.storage.directory=${user.dir}/target/graphdb +*.falcon.graph.storage.backend=berkeleyje +*.falcon.graph.serialize.path=${user.dir}/target/graphdb +</verbatim> + +---++++Using HBase backend + +To use HBase as the backend it is recommended that a HBase cluster be provisioned with distributed mode configuration primarily because of the support of kerberos enabled clusters and HA considerations. Based on build profile, a standalone hbase version can be packaged with the Falcon binary distribution. Along with this, a template for <verbatim>hbase-site.xml</verbatim> is provided, which can be used to start the standalone mode HBase enviornment for development/testing purposes. + +---++++ Basic configuration + +<verbatim> +##### Falcon startup.properties +*.falcon.graph.storage.backend=hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +*.falcon.graph.storage.hostname=<ZooKeeper Quorum> +</verbatim> + +HBase configuration file (hbase-site.xml) and hbase libraries need to be added to classpath when Falcon starts up. The following must be appended to the environment variable <verbatim>FALCON_EXTRA_CLASS_PATH<verbatim> in <verbatim>$FALCON_HOME/bin/falcon-env.sh</verbatim>. Additionally the correct hbase client libraries need to be added. For example, +<verbatim> +export FALCON_EXTRA_CLASS_PATH=`${HBASE_HOME}/bin/hbase classpath` +</verbatim> + +Table name +We recommend that in the startup config the tablename for titan storage be named <verbatim>falcon_titan<verbatim> so that multiple applications using Titan can share the same HBase cluster. This can be set by specifying the tablename using the startup property given below. The default value is shown. + +<verbatim> +*.falcon.graph.storage.hbase.table=falcon_titan +</verbatim> + +---++++Starting standalone HBase for testing + +HBase can be started in stand alone mode for testing as a backend for Titan. The following steps outline the config changes required: +<verbatim> +1. Build Falcon as below to package hbase binaries + $ export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m" && mvn clean assembly:assembly -Ppackage-standalone-hbase +2. Configure HBase + a. When falcon tar file is expanded, HBase binaries are under ${FALCON_HOME}/hbase + b. Copy ${FALCON_HOME}/conf/hbase-site.xml.template into hbase conf dir in ${FALCON_HOME}/hbase/conf/hbase-site.xml + c. Set {hbase_home} property to point to a local dir + d. Standalone HBase starts zookeeper on the default port (2181). This port can be changed by adding the following to hbase-site.xml + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>2223</value> + </property> + + <property> + <name>hbase.zookeeper.quorum</name> + <value>localhost</value> + </property> + e. set JAVA_HOME to point to Java 1.7 or above + f. Start hbase as ${FALCON_HOME}/hbase/bin/start-hbase.sh +3. Configure Falcon + a. In ${FALCON_HOME}/conf/startup.properties, uncomment the following to enable HBase as the backend + *.falcon.graph.storage.backend=hbase + ### specify the zookeeper host and port name with which standalone hbase is started (see step 2) + ### by default, it will be localhost and port 2181 + *.falcon.graph.storage.hostname=<zookeeper-host-name>:<zookeeper-host-port> + *.falcon.graph.serialize.path=${user.dir}/target/graphdb + *.falcon.graph.storage.hbase.table=falcon_titan + *.falcon.graph.storage.transactions=false +4. Add HBase jars to Falcon classpath in ${FALCON_HOME}/conf/falcon-env.sh as: + FALCON_EXTRA_CLASS_PATH=`${FALCON_HOME}/hbase/bin/hbase classpath` +5. Set the following in ${FALCON_HOME}/conf/startup.properties to disable SSL if needed + *.falcon.enableTLS=false +6. Start Falcon +</verbatim> + +---++++Permissions + +When Falcon is configured with HBase as the storage backend Titan needs to have sufficient authorizations to create and access an HBase table. In a secure cluster it may be necessary to grant permissions to the <verbatim>falcon</verbatim> user for the <verbatim>falcon_titan</verbatim> table (or whateven tablename was specified for the property <verbatim>*.falcon.graph.storage.hbase.table</verbatim> + +With Ranger, a policy can be configured for <verbatim>falcon_titan</verbatim>. + +Without Ranger, HBase shell can be used to set the permissions. + +<verbatim> + su hbase + kinit -k -t <hbase keytab> <hbase principal> + echo "grant 'falcon', 'RWXCA', 'falcon_titan'" | hbase shell +</verbatim> + +---++++Advanced configuration + +HBase storage backend support in Titan has a few other configurations and they can be set in <verbatim>$FALCON_HOME/conf/startup.properties</verbatim>, by prefixing the Titan property with <verbatim>*.falcon.graph</verbatim> prefix. + +Please Refer to <verbatim>http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html#_storage</verbatim> for generic storage properties, <verbaim>http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html#_storage_berkeleydb</verbatim> for berkeley db properties and <verbatim>http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html#_storage_hbase</verbatim> for hbase storage backend properties. + + ---+++Adding Extension Libraries http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/DataReplicationAzure.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/DataReplicationAzure.twiki b/trunk/general/src/site/twiki/DataReplicationAzure.twiki new file mode 100644 index 0000000..24e543b --- /dev/null +++ b/trunk/general/src/site/twiki/DataReplicationAzure.twiki @@ -0,0 +1,61 @@ +---+ Data Replication between On-premise Hadoop Clusters and Azure Cloud + +---++ Overview +Falcon provides an easy way to replicate data between on-premise Hadoop clusters and Azure cloud. +With this feature, users would be able to build a hybrid data pipeline, +e.g. processing sensitive data on-premises for privacy and compliance reasons +while leverage cloud for elastic scale and online services (e.g. Azure machine learning) with non-sensitive data. + +---++ Use Case +1. Copy data from on-premise Hadoop clusters to Azure cloud +2. Copy data from Azure cloud to on-premise Hadoop clusters +3. Copy data within Azure cloud (i.e. from one Azure location to another). + +---++ Usage +---+++ Set Up Azure Blob Credentials +To move data to/from Azure blobs, we need to add Azure blob credentials in HDFS. +This can be done by adding the credential property through Ambari HDFS configs, and HDFS needs to be restarted after adding the credential. +You can also add the credential property to core-site.xml directly, but make sure you restart HDFS from command line instead of Ambari. +Otherwise, Ambari will take the previous HDFS configuration without your Azure blob credentials. +<verbatim> +<property> + <name>fs.azure.account.key.{AZURE_BLOB_ACCOUNT_NAME}.blob.core.windows.net</name> + <value>{AZURE_BLOB_ACCOUNT_KEY}</value> +</property> +</verbatim> + +To verify you set up Azure credential properly, you can check if you are able to access Azure blob through HDFS, e.g. +<verbatim> +hadoop fs Âls wasb://{AZURE_BLOB_CONTAINER}@{AZURE_BLOB_ACCOUNT_NAME}.blob.core.windows.net/ +</verbatim> + +---+++ Replication Feed +[[EntitySpecification][Falcon replication feed]] can be used for data replication to/from Azure cloud. +You can specify WASB (i.e. Windows Azure Storage Blob) url in source or target locations. +See below for an example of data replication from Hadoop cluster to Azure blob. +Note that the clusters for the source and the target need to be different. +Analogously, if you want to copy data from Azure blob, you can add Azure blob location to the source. +<verbatim> +<?xml version="1.0" encoding="UTF-8"?> +<feed name="AzureReplication" xmlns="uri:falcon:feed:0.1"> + <frequency>months(1)</frequency> + <clusters> + <cluster name="SampleCluster1" type="source"> + <validity start="2010-06-01T00:00Z" end="2010-06-02T00:00Z"/> + <retention limit="days(90)" action="delete"/> + </cluster> + <cluster name="SampleCluster2" type="target"> + <validity start="2010-06-01T00:00Z" end="2010-06-02T00:00Z"/> + <retention limit="days(90)" action="delete"/> + <locations> + <location type="data" path="wasb://[email protected]/replicated-${YEAR}-${MONTH}"/> + </locations> + </cluster> + </clusters> + <locations> + <location type="data" path="/apps/falcon/demo/data-${YEAR}-${MONTH}" /> + </locations> + <ACL owner="ambari-qa" group="users" permission="0755"/> + <schema location="hcat" provider="hcat"/> +</feed> +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/Embedded-mode.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/Embedded-mode.twiki b/trunk/general/src/site/twiki/Embedded-mode.twiki index d5c37a1..47acab4 100644 --- a/trunk/general/src/site/twiki/Embedded-mode.twiki +++ b/trunk/general/src/site/twiki/Embedded-mode.twiki @@ -178,6 +178,7 @@ Submit and schedule the process: <verbatim> $bin/falcon entity -submitAndSchedule -type process -file examples/entity/filesystem/oozie-mr-process.xml $bin/falcon entity -submitAndSchedule -type process -file examples/entity/filesystem/pig-process.xml +$bin/falcon entity -submitAndSchedule -type process -file examples/entity/spark/spark-process.xml </verbatim> Generate input data: <verbatim> @@ -189,7 +190,7 @@ $bin/falcon instance -status -type process -name oozie-mr-process -start 2013-11 </verbatim> HCat based example entities are in examples/entity/hcat. - +Spark based example entities are in examples/entity/spark. ---+++Stopping Falcon Server <verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/EntitySpecification.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/EntitySpecification.twiki b/trunk/general/src/site/twiki/EntitySpecification.twiki index d08c3a3..9f9e210 100644 --- a/trunk/general/src/site/twiki/EntitySpecification.twiki +++ b/trunk/general/src/site/twiki/EntitySpecification.twiki @@ -52,6 +52,11 @@ A registry interface specifies the interface for metadata catalog, such as Hive Falcon uses this interface to register/de-register partitions for a given database and table. Also, uses this information to schedule data availability events based on partitions in the workflow engine. Although Hive metastore supports both RPC and HTTP, Falcon comes with an implementation for RPC over thrift. +For Hive HA mode, make sure the uris are separated with comma and you only add protocol "thrift://" at the beginning. +See below for an example of Hive HA mode: +<verbatim> +<interface type="registry" endpoint="thrift://c6402.ambari.apache.org:9083,c6403.ambari.apache.org:9083" version="0.11.0" /> +</verbatim> <verbatim> <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.6" /> @@ -682,8 +687,7 @@ Example workflow configuration: ---+++ Optional Inputs -User can mention one or more inputs as optional inputs. In such cases the job does not wait on those inputs which are -mentioned as optional. If they are present it considers them otherwise continue with the compulsory ones. +User can mention one or more inputs as optional inputs. In such cases the job does not wait on those inputs which are mentioned as optional. If they are present it considers them otherwise continues with the mandatory ones. If some instances of the optional feed are present for the given data window, those are considered and passed on to the process. While checking for presence of an feed instance, Falcon looks for __availabilityFlag__ in the directory, if specified in the feed definition. If no __availabilityFlag__ is specified, presence of the instance directory is treated as indication of availability of data. Example: <verbatim> <feed name="feed1"> @@ -829,13 +833,13 @@ be in lib folder inside the workflow path. The properties defined in the cluster and cluster properties(nameNode and jobTracker) will also be available for the workflow. -There are 3 engines supported today. +There are 4 engines supported today. ---++++ Oozie As part of oozie workflow engine support, users can embed a oozie workflow. -Refer to oozie [[http://oozie.apache.org/docs/4.0.1/DG_Overview.html][workflow overview]] and -[[http://oozie.apache.org/docs/4.0.1/WorkflowFunctionalSpec.html][workflow specification]] for details. +Refer to oozie [[http://oozie.apache.org/docs/4.2.0/DG_Overview.html][workflow overview]] and +[[http://oozie.apache.org/docs/4.2.0/WorkflowFunctionalSpec.html][workflow specification]] for details. Syntax: <verbatim> @@ -898,6 +902,54 @@ This defines the workflow engine to be hive and the hive script is defined at Feeds with Hive table storage will send one more parameter apart from the general ones: <verbatim>$input_filter</verbatim> +---++++ Spark +Falcon also adds the Spark engine as part of Spark Integration which enables users to run the Java/Python Spark application as a process. +When "spark" workflow engine is mentioned spark related parameters must be provided through <spark-attributes> +Examples: +<verbatim> +<process name="spark-process"> +... + <workflow engine="spark" path="/resources/action"> + <spark-attributes> + <master>local</master> + <name>Spark WordCount</name> + <class>org.examples.WordCount</class> + <jar>/resources/action/lib/spark-application.jar</jar> + <spark-opts>--num-executors 1 --driver-memory 512m</spark-opts> + </spark-attributes> +... +</process> +</verbatim> + +This defines the workflow engine to be spark and Java/Python Spark application must be defined with "jar" option that need to be executed. +There is flexibility to override the Spark master through process entity either to "yarn-client" or "yarn-cluster", if spark interface is already defined in cluster entity. +Input and Output data to the Spark application will be set as argument when Spark workflow will be generated, if input and output feed entity is defined in the process entity. +In the set of arguments, first argument will always correspond to input feed, second argument will always correspond to output feed and then user's provided argument will be set. + + +For running the Spark SQL process entity, that read and write the data stored on Hive, the datanucleus jars under the $HIVE_HOME/lib directory and hive-site.xml +under $SPARK_HOME/conf/ directory need to be available on the driver and all executors launched by the YARN cluster. +The convenient way to do this is adding them through the --jars option and --file option of the spark-opts attribute. +Example: +<verbatim> +<process name="spark-process"> +... + <workflow engine="spark" path="/resources/action"> + <spark-attributes> + <master>local</master> + <name>Spark SQL</name> + <class>org.examples.SparkSQLProcessTable</class> + <jar>/resources/action/lib/spark-application.jar</jar> + <spark-opts>--num-executors 1 --driver-memory 512m --jars /usr/local/hive/lib/datanucleus-rdbms.jar,/usr/local/hive/lib/datanucleus-core.jar,/usr/local/hive/lib/datanucleus-api-jdo.jar --files /usr/local/spark/conf/hive-site.xml</spark-opts> + </spark-attributes> +... +</process> +</verbatim> + +Input and Output to the Spark SQL application will be set as argument when Spark workflow will be generated, if input and output feed entity is defined in the process entity. +If input feed is of table type, then input table partition, table name and database name will be set as input arguments. If output feed is of table type, then output table partition, table name and database name will be set as output arguments. +Once input and output arguments is set, then user's provided argument will be set. + ---+++ Retry Retry policy defines how the workflow failures should be handled. Three retry policies are defined: periodic, exp-backoff(exponential backoff) and final. Depending on the delay and number of attempts, the workflow is re-tried after specific intervals. If user sets the onTimeout attribute to "true", retries will happen for TIMED_OUT instances. Syntax: @@ -923,10 +975,10 @@ The workflow is re-tried after 10 mins, 20 mins and 30 mins. With exponential ba To enable retries for instances for feeds, user will have to set the following properties in runtime.properties <verbatim> -falcon.recipe.retry.policy=periodic -falcon.recipe.retry.delay=minutes(30) -falcon.recipe.retry.attempts=3 -falcon.recipe.retry.onTimeout=false +falcon.retry.policy=periodic +falcon.retry.delay=minutes(30) +falcon.retry.attempts=3 +falcon.retry.onTimeout=false <verbatim> ---+++ Late data Late data handling defines how the late data should be handled. Each feed is defined with a late cut-off value which specifies the time till which late data is valid. For example, late cut-off of hours(6) means that data for nth hour can get delayed by upto 6 hours. Late data specification in process defines how this late data is handled. http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/Extensions.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/Extensions.twiki b/trunk/general/src/site/twiki/Extensions.twiki new file mode 100644 index 0000000..a3fed4e --- /dev/null +++ b/trunk/general/src/site/twiki/Extensions.twiki @@ -0,0 +1,65 @@ +---+ Falcon Extensions + +---++ Overview + +A Falcon extension is a static process template with parameterized workflow to realize a specific use case and enable non-programmers to capture and re-use very complex business logic. Extensions are defined in server space. Objective of the extension is to solve a standard data management function that can be invoked as a tool using the standard Falcon features (REST API, CLI and UI access) supporting standard falcon features. + +For example: + + * Replicating directories from one HDFS cluster to another (not timed partitions) + * Replicating hive metadata (database, table, views, etc.) + * Replicating between HDFS and Hive - either way + * Data masking etc. + +---++ Proposal + +Falcon provides a Process abstraction that encapsulates the configuration for a user workflow with scheduling controls. All extensions can be modeled as a Process and its dependent feeds with in Falcon which executes the user +workflow periodically. The process and its associated workflow are parameterized. The user will provide properties which are <name, value> pairs that are substituted by falcon before scheduling it. Falcon translates these extensions +as a process entity by replacing the parameters in the workflow definition. + +---++ Falcon extension artifacts to manage extensions + +Extension artifacts are published in addons/extensions. Artifacts are expected to be installed on HDFS at "extension.store.uri" path defined in startup properties. Each extension is expected to ahve the below artifacts + * json file under META directory lists all the required and optional parameters/arguments for scheduling extension job + * process entity template to be scheduled under resources directory + * parameterized workflow under resources directory + * required libs under the libs directory + * README describing the functionality achieved by extension + +REST API and CLI support has been added for extension artifact management on HDFS. Please Refer to [[falconcli/FalconCLI][Falcon CLI]] and [[restapi/ResourceList][REST API]] for more details. + +---++ CLI and REST API support +REST APIs and CLI support has been added to manage extension jobs and instances. + +Please Refer to [[falconcli/FalconCLI][Falcon CLI]] and [[restapi/ResourceList][REST API]] for more details on usage of CLI and REST API's for extension jobs and instances management. + +---++ Metrics +HDFS mirroring and Hive mirroring extensions will capture the replication metrics like TIMETAKEN, BYTESCOPIED, COPY (number of files copied) for an instance and populate to the GraphDB. + +---++ Sample extensions + +Sample extensions are published in addons/extensions + +---++ Types of extensions + * [[HDFSMirroring][HDFS mirroring extension]] + * [[HiveMirroring][Hive mirroring extension]] + * [[HdfsSnapshotMirroring][HDFS snapshot based mirroring]] + +---++ Packaging and installation + +This feature is enabled by default but could be disabled by removing the following from startup properties: +<verbatim> +config name: *.application.services +config value: org.apache.falcon.extensions.ExtensionService +</verbatim> + +ExtensionService should be added before ConfigurationStore in startup properties for application services configuration. +For manual installation user is expected to update "extension.store.uri" property defined in startup properties with +HDFS path where the extension artifacts will be copied to. +Extension artifacts in addons/extensions are packaged in falcon. For manual installation once the Falcon Server is setup user is expected to copy the extension artifacts under {falcon-server-dir}/extensions to HDFS at "extension.store.uri" path defined in startup properties and then restart Falcon. + +---++ Migration +Recipes framework and HDFS mirroring capability was added in Apache Falcon 0.6.0 release and it was client side logic. With 0.10 release its moved to server side and renamed as server side extensions. Client side recipes only had CLI support and expected certain pre steps to get it working. This is no longer required in 0.10 release as new CLI and REST API support has been provided. + +Migrating to 0.10 release and above is not backward compatible for Recipes. If user is migrating to 0.10 release and above then old Recipe setup and CLI's won't work. For manual installation user is expected to copy Extension artifacts to HDFS. Please refer "Packaging and installation" section above for more details. +Please Refer to [[falconcli/FalconCLI][Falcon CLI]] and [[restapi/ResourceList][REST API]] for more details on usage of CLI and REST API's for extension jobs and instances management. http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/FalconDocumentation.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/FalconDocumentation.twiki b/trunk/general/src/site/twiki/FalconDocumentation.twiki index 122435a..fe1c0de 100644 --- a/trunk/general/src/site/twiki/FalconDocumentation.twiki +++ b/trunk/general/src/site/twiki/FalconDocumentation.twiki @@ -2,18 +2,20 @@ * <a href="#Architecture">Architecture</a> * <a href="#Control_flow">Control flow</a> * <a href="#Modes_Of_Deployment">Modes Of Deployment</a> + * <a href="#Configuring_Falcon">Configuring Falcon</a> * <a href="#Entity_Management_actions">Entity Management actions</a> * <a href="#Instance_Management_actions">Instance Management actions</a> * <a href="#Retention">Retention</a> * <a href="#Replication">Replication</a> * <a href="#Cross_entity_validations">Cross entity validations</a> + * <a href="#Updating_cluster_entity_definition">Updating cluster entity</a> * <a href="#Updating_process_and_feed_definition">Updating process and feed definition</a> * <a href="#Handling_late_input_data">Handling late input data</a> * <a href="#Idempotency">Idempotency</a> * <a href="#Falcon_EL_Expressions">Falcon EL Expressions</a> * <a href="#Lineage">Lineage</a> * <a href="#Security">Security</a> - * <a href="#Recipes">Recipes</a> + * <a href="#Extensions">Extensions</a> * <a href="#Monitoring">Monitoring</a> * <a href="#Email_Notification">Email Notification</a> * <a href="#Backwards_Compatibility">Backwards Compatibility Instructions</a> @@ -91,6 +93,46 @@ Startup property file in both falcon & prism server need to be configured with t ---++++ Prism Setup <img src="PrismSetup.png" height="400" width="600" /> + +---+++ Safe Mode +Safemode is useful when the admin wants to prevent Falcon users from scheduling entities in the workflow engine. This can happen when + * Hadoop clusters are being upgraded. + * Falcon cluster entities are being updated. + +When in Safemode, users can only perform limited operations. To be specific, + * Users can perform read operations on entities subject to authorization. + * Superuser can perform cluster entity update operation. + * Suspend/Kill of individual instances will be allowed if users want to suspend specific job operations. + * Suspend operation on schedulable entities will be allowed. This is because, a user might want to suspend entities during rolling-upgrade to handle jobs incompatible with updated versions. + * All other operations are not allowed. To enumerate, + * All entity submit, submitAndSchedule operations are not allowed. + * Entity operations not allowed are : update, schedule, touch, delete, submit, submitAndSchedule, resume + * Instance operations not allowed are : rerun, resume + +---++++ Getting into/out-of Safemode. +The Falcon process user can specify whether to start Falcon in safemode with the following command: +<verbatim> +<falcon-server>/bin/falcon-start -setsafemode <true/false> +</verbatim> + +A superuser or admin-user can set Falcon server into/outof safemode using CLI or RestAPI. A user is considered superuser if they +owns the Falcon process or belong to group specified in startup property falcon.security.authorization.superusergroup. A user is +considered admin user if they are listed under startup property falcon.security.authorization.admin.users, OR they belong to group +listed under startup property falcon.security.authorization.admin.groups. + +<verbatim> +## CLI +<falcon-server>/bin/falcon admin [-setsafemode <true/false>] + +## RestAPI +GET http://falcon-server:15000/api/admin/setSafeMode/true +OR +GET http://falcon-server:15000/api/admin/setSafeMode/false +</verbatim> + +*NOTE* User can find if FalconServer is in safemode or not, by calling the [[restapi/AdminVersion][Admin Version]] API. Once +server is set to safemode, this state is persisted during restarts. It has to be unset explicitly if user wants to exit safemode. + ---+++ Configuration Store Configuration store is file system based store that the Falcon system maintains where the entity definitions @@ -156,6 +198,9 @@ Examples: <table uri="catalog:tgt_demo_db:customer_bcp#ds=${YEAR}-${MONTH}-${DAY}-${HOUR}" /> </verbatim> +---++ Configuring Falcon + +Configuring Falcon is detailed in [[Configuration][Configuration]]. ---++ Entity Management actions All the following operation can also be done using [[restapi/ResourceList][Falcon's RESTful API]]. @@ -205,9 +250,14 @@ entity from the falcon configuration store. Delete operation on an entity would no dependent entities on the deleted entity. ---+++ Update -Update operation allows an already submitted/scheduled entity to be updated. Cluster update is currently -not allowed. Feed update can cause cascading update to all the processes already scheduled. Process update triggers -update in falcon if entity is updated. The following set of actions are performed in scheduler to realize an update: +Update operation allows an already submitted/scheduled entity to be updated. Feed update can cause cascading update to +all the processes already scheduled. Process update triggers update in falcon if entity is scheduled. + +Cluster update will require user to update dependent Feed and Process entities that are already scheduled. +Cluster update needs to be performed in safemode. We provide a CLI command for the user to update the scheduled +dependent entities after cluster update and exiting safemode. + +The following set of actions are performed in scheduler to realize an update: * Update the old scheduled entity to set the end time to "now" * Schedule as per the new process/feed definition with the start time as "now" @@ -266,6 +316,12 @@ to false in runtime.properties. With the integration of Hive, Falcon also provides retention for tables in Hive catalog. +When a feed is scheduled Falcon kicks off the retention policy immediately. When job runs, it deletes everything that's eligible for eviction - eligibility criteria is the date pattern on the partition and NOT creation date. +For e.g. if the retention limit is 90 days then retention job consistently deletes files older than 90 days. + +For retention, Falcon expects data to be in dated partitions. When the retention job is kicked off, it discovers data that needs to be evicted based on retention policy. It gets the location from the feed and uses pattern matching +to find the pattern to get the list of data for the feed, then gets the date from the data path. If the data path date is beyond the retention limit it's deleted. As this uses pattern matching it is not time consuming and hence doesn't introduce performance overhead. + ---+++ Example: If retention period is 10 hours, and the policy kicks in at time 't', the data retained by system is essentially the one after or equal to t-10h . Any data before t-10h is removed from the system. @@ -391,9 +447,11 @@ cluster, (no dirty reads) ---+++ Archival as Replication -Falcon allows users to archive data from on-premice to cloud, either Azure WASB or S3. +Falcon allows users to archive data from on-premise to cloud, either Azure WASB or S3. It uses the underlying replication for archiving data from source to target. The archival URI is specified as the overridden location for the target cluster. +Note that for data replication between on-premise and Azure cloud, Azure credentials need to be added to core-site.xml. +Please refer to [[DataReplicationAzure][AzureDataReplication]] for details and examples. *Example:* <verbatim> @@ -565,10 +623,37 @@ Failure to follow any of the above rules would result in a process submission fa present in the system for the specified time period, the process can be submitted and scheduled, but all instances created would result in a WAITING state unless data is actually provided in the cluster. +---++ Updating cluster entity definition +Cluster entities can be updated when the user wants to change their interface endpoints or properties, +e.g. hadoop clusters updated from unsecure to secure; hadoop cluster moved from non high-availability to high-availability, etc. + +In these scenarios, user would want to change the cluster entity to reflect updated interface endpoints or properties. +Updating cluster would require cascading update to dependent feed/process jobs scheduled on this cluster. So Falcon only allows +Cluster update when + * Falcon server is in safemode. + * The update is requested by superuser + * The underlying namenode or workflow engine referenced by interface URI is the same. It is only the URI that has changed to reflect secure/HA environments. + +Cluster entity should be updated by superuser using following CLI command. +<verbatim> +bash$ falcon entity -type cluster -name primaryCluster -update -file ~/primary-updated.xml +</verbatim> + +Once the cluster entity is updated, user should exit FalconServer from safemode and update the scheduled entities that are +dependent on this Cluster. In case of an error during update, user should address the root cause of failure and retry +the command. For example : if the cluster has 10 dependent entities and the updateClusterDependents command failed +after updating 6th entity, rerun of this command will only update entities 7 to 10. +<verbatim> +bash$ falcon entity -updateClusterDependents -cluster primaryCluster +</verbatim> +Please Refer to [[falconcli/FalconCLI][Falcon CLI]] for more details on usage of CLI commands. ---++ Updating process and feed definition -Any changes in feed/process can be done by updating its definition. After the update, any new workflows which are to be scheduled after the update call will pick up the new changes. Feed/process name and start time can't be updated. Updating a process triggers updates to the workflow that is triggered in the workflow engine. Updating feed updates feed workflows like retention, replication etc. and also updates the processes that reference the feed. +Any changes in feed/process can be done by updating its definition. After the update, any new workflows which are to +be scheduled after the update call will pick up the new changes. Feed/process name and start time can't be updated. +Updating a process triggers updates to the workflow that is triggered in the workflow engine. Updating feed updates +feed workflows like retention, replication etc. and also updates the processes that reference the feed. ---++ Handling late input data @@ -732,13 +817,13 @@ lifecycle policies such as replication and retention. Security is detailed in [[Security][Security]]. ----++ Recipes +---++ Extensions -Recipes is detailed in [[Recipes][Recipes]]. +Extensions is detailed in [[Extensions][Extensions]]. ---++ Monitoring -Monitoring and Operationalizing Falcon is detailed in [[Operability][Operability]]. +Monitoring and Operationalizing Falcon is detailed in [[Operability][Operability]] and [[FeedSLAMonitoring][Falcon Feed SLA Monitoring]]. ---++ Email Notification Notification for instance completion in Falcon is defined in [[FalconEmailNotification][Falcon Email Notification]]. http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/FalconNativeScheduler.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/FalconNativeScheduler.twiki b/trunk/general/src/site/twiki/FalconNativeScheduler.twiki index 9ffc5e9..1f51739 100644 --- a/trunk/general/src/site/twiki/FalconNativeScheduler.twiki +++ b/trunk/general/src/site/twiki/FalconNativeScheduler.twiki @@ -29,7 +29,7 @@ You can enable native scheduler by making changes to __$FALCON_HOME/conf/startup org.apache.falcon.service.ProcessSubscriberService,\ org.apache.falcon.service.FeedSLAMonitoringService,\ org.apache.falcon.service.LifecyclePolicyMap,\ - org.apache.falcon.state.store.service.FalconJPAService,\ + org.apache.falcon.service.FalconJPAService,\ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/FeedSLAMonitoring.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/FeedSLAMonitoring.twiki b/trunk/general/src/site/twiki/FeedSLAMonitoring.twiki new file mode 100644 index 0000000..88132ce --- /dev/null +++ b/trunk/general/src/site/twiki/FeedSLAMonitoring.twiki @@ -0,0 +1,27 @@ +---++Falcon Feed SLA Monitoring + +Feed SLA monitoring allows you to monitor the feed ,It keeps track of the instances of the feed which are running and stores them in the db. + +Feed SLA monitoring service requires FalconJPAService to be up.Following are the values we need to set to run FeedSLAMonitoring - +In startup.properties : + +*.application.services= org.apache.falcon.state.store.service.FalconJPAService, + org.apache.falcon.service.FeedSLAMonitoringService + +These properties are required for FalconJPAService in statestore.properties: + + * *falcon.state.store.impl* - org.apache.falcon.state.store.jdbc.JDBCStateStore + * *falcon.statestore.jdbc.driver* -org.apache.derby.jdbc.EmbeddedDriver + * *falcon.statestore.jdbc.url* -jdbc:derby:target/test-data/data.db;create=true + * *falcon.statestore.connection.data.source* -org.apache.commons.dbcp.BasicDataSource + Maximum number of active connections that can be allocated from this pool at the same time. + * *falcon.statestore.pool.max.active.conn* -10 + * *falcon.statestore.connection.properties* - + Indicates the interval (in milliseconds) between eviction runs. + * *falcon.statestore.validate.db.connection.eviction.interval* -300000 + The number of objects to examine during each run of the idle object evictor thread. + * *falcon.statestore.validate.db.connection.eviction.num* -10 + Creates Falcon DB. If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP.If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. + * *falcon.statestore.create.db.schema* -false + +Note: First time we have to manually create the schema in production as we have set falcon.statestore.create.db.schema = false \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/GraphiteMetricCollection.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/GraphiteMetricCollection.twiki b/trunk/general/src/site/twiki/GraphiteMetricCollection.twiki new file mode 100644 index 0000000..c76e68d --- /dev/null +++ b/trunk/general/src/site/twiki/GraphiteMetricCollection.twiki @@ -0,0 +1,22 @@ +---++Graphite Metric Collection + +Graphite Metric Collection currently allows to collect the following metrics at process level : + +1. Processing time the process spent in the running state in seconds (workflow_end_time - workflow_start_time) +2. Wait time that the process spent in the waiting/ready state. (workflow_start_time - workflow_nominal_time) +3. Number of instances that are failed for a process. + +To send data to graphite we need to intialize metricNotificationService in startup.properties: + +*.application.services= org.apache.falcon.metrics.MetricNotificationService, + + +Add following properties for graphiteNotificationPlugin : + +Graphite properties + * *.falcon.graphite.hostname=localhost + * *.falcon.graphite.port=2003 + * *.falcon.graphite.frequency=1 + * *.falcon.graphite.prefix=falcon + +The falcon.graphite.frequency is in seconds and all the time that is being sent to graphite is in seconds. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/HDFSMirroring.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/HDFSMirroring.twiki b/trunk/general/src/site/twiki/HDFSMirroring.twiki new file mode 100644 index 0000000..a810947 --- /dev/null +++ b/trunk/general/src/site/twiki/HDFSMirroring.twiki @@ -0,0 +1,27 @@ +---+ HDFS mirroring Extension +---++ Overview +Falcon supports HDFS mirroring extension to replicate data from source cluster to destination cluster. This extension implements replicating arbitrary directories on HDFS and piggy backs on replication solution in Falcon which uses the DistCp tool. It also allows users to replicate data from on-premise to cloud, either Azure WASB or S3. + +---++ Use Case +* Copy directories between HDFS clusters with out dated partitions +* Archive directories from HDFS to Cloud. Ex: S3, Azure WASB + +---++ Limitations +As the data volume and number of files grow, this can get inefficient. + +---++ Usage +---+++ Setup source and destination clusters + <verbatim> + $FALCON_HOME/bin/falcon entity -submit -type cluster -file /cluster/definition.xml + </verbatim> + +---+++ HDFS mirroring extension properties + Extension artifacts are expected to be installed on HDFS at the path specified by "extension.store.uri" in startup properties. hdfs-mirroring-properties.json file located at "<extension.store.uri>/hdfs-mirroring/META/hdfs-mirroring-properties.json" lists all the required and optional parameters/arguments for scheduling HDFS mirroring job. + +---+++ Submit and schedule HDFS mirroring extension + + <verbatim> + $FALCON_HOME/bin/falcon extension -submitAndSchedule -extensionName hdfs-mirroring -file /process/definition.xml + </verbatim> + + Please Refer to [[falconcli/FalconCLI][Falcon CLI]] and [[restapi/ResourceList][REST API]] for more details on usage of CLI and REST API's. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/general/src/site/twiki/HdfsSnapshotMirroring.twiki ---------------------------------------------------------------------- diff --git a/trunk/general/src/site/twiki/HdfsSnapshotMirroring.twiki b/trunk/general/src/site/twiki/HdfsSnapshotMirroring.twiki new file mode 100644 index 0000000..ec4f16c --- /dev/null +++ b/trunk/general/src/site/twiki/HdfsSnapshotMirroring.twiki @@ -0,0 +1,93 @@ +---+HDFS Snapshot based Mirroring + +---++Overview +HDFS snapshots are very cost effective to create ( cost is O(1) excluding iNode lookup time). Once created, it is very +efficient to find modifications relative to a snapshot and copy over these modifications for disaster recovery (DR). +This makes for cost effective HDFS mirroring. + +---++Prerequisites +Following is the prerequisite to use HDFS Snapshot based Mirrroring. + + * Hadoop version 2.7.0 or higher. + * User submitting and scheduling falcon snapshot based mirroring job should have permission to create and manage snapshots on both source and target directories. + +---++ Use Case +Create and manage snapshots on source/target directories. Mirror data from source to target for disaster +recovery using these snapshots. Perform retention on the snapshots created on source and target. + + +---++ Usage + +---+++ Setup + * Submit a source cluster and target cluster entities to Falcon. + <verbatim> + $FALCON_HOME/bin/falcon entity -submit -type cluster -file source-cluster-definition.xml + $FALCON_HOME/bin/falcon entity -submit -type cluster -file target-cluster-definition.xml + </verbatim> + * Ensure that source directory on source cluster and target directory on target cluster exists. + * Ensure that these dirs are snapshot-able by user submitting extension. You can find more [[https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-hdfs/HdfsSnapshots.html][information on snapshots here]]. + +---+++ HDFS Snapshot based mirroring extension properties + Extension artifacts are expected to be installed on HDFS at the path specified by "extension.store.uri" in startup properties. + hdfs-snapshot-mirroring-properties.json file located at "<extension.store.uri>/hdfs-snapshot-mirroring/META/hdfs-snapshot-mirroring-properties.json" + lists all the required and optional parameters/arguments for scheduling the mirroring job. + + Here is a sample set of properties, + <verbatim> + ## Job Properties + jobName=hdfs-snapshot-test + jobClusterName=backupCluster + jobValidityStart=2016-01-01T00:00Z + jobValidityEnd=2016-04-01T00:00Z + jobFrequency=hours(12) + jobTimezone=UTC + [email protected] + jobRetryPolicy=periodic + jobRetryDelay=minutes(30) + jobRetryAttempts=3 + + ## Job owner + jobAclOwner=ambari-qa + jobAclGroup=users + jobAclPermission=* + + ## Source information + sourceCluster=primaryCluster + sourceSnapshotDir=/apps/falcon/snapshots/source/ + sourceSnapshotRetentionPolicy=delete + sourceSnapshotRetentionAgeLimit=days(15) + sourceSnapshotRetentionNumber=10 + + ## Target information + targetCluster=backupCluster + targetSnapshotDir=/apps/falcon/snapshots/target/ + targetSnapshotRetentionPolicy=delete + targetSnapshotRetentionAgeLimit=months(6) + targetSnapshotRetentionNumber=20 + + ## Distcp properties + distcpMaxMaps=1 + distcpMapBandwidth=100 + tdeEncryptionEnabled=false + </verbatim> + + +The above properties ensure Falcon hdfs snapshot based mirroring extension does the following every 12 hours. + * Create snapshot on dir /apps/falcon/snapshots/source/ on primaryCluster. + * DistCP data from /apps/falcon/snapshots/source/ on primaryCluster to /apps/falcon/snapshots/target/ on backupCluster. + * Create snapshot on dir /apps/falcon/snapshots/target/ on backupCluster. + * Perform retention job on source and target. + * Maintain at least N latest snapshots and delete all other snapshots older than specified age limit. + * Today, only "delete" policy is supported for snapshot retention. + +*Note:* +When TDE encryption is enabled on source/target directories, DistCP ignores the snapshots and treats it like a regular +replication. While user may not get the performance benefit of using snapshot based DistCP, the extension is still useful +for creating and maintaining snapshots. + +---+++ Submit and schedule HDFS snapshot mirroring extension +User can submit extension using CLI or RestAPI. CLI command looks as follows + <verbatim> + $FALCON_HOME/bin/falcon extension -submitAndSchedule -extensionName hdfs-snapshot-mirroring -file propeties-file.txt + </verbatim> + Please Refer to [[falconcli/FalconCLI][Falcon CLI]] and [[restapi/ResourceList][REST API]] for more details on usage of CLI and REST API's. \ No newline at end of file
