http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/Extensions.html ---------------------------------------------------------------------- diff --git a/content/Extensions.html b/content/Extensions.html new file mode 100644 index 0000000..81aeea2 --- /dev/null +++ b/content/Extensions.html @@ -0,0 +1,521 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-08-08 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Falcon - Falcon Extensions</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft"> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> + <img src="images/apache-feather-tm.gif" alt="Falcon" height="45px"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Falcon"> + Falcon</a> + </li> + <li class="divider ">/</li> + <li class="">Falcon Extensions</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Falcon</li> + + <li> + + <a href="index.html" title="About"> + <i class="none"></i> + About</a> + </li> + + <li> + + <a href="slides/falcon-overview.html" title="Overview"> + <i class="none"></i> + Overview</a> + </li> + + <li> + + <a href="slides/falcon-user-guide.html" title="User Guide"> + <i class="none"></i> + User Guide</a> + </li> + + <li> + + <a href="GettingStarted.html" title="Getting Started"> + <i class="none"></i> + Getting Started</a> + </li> + + <li> + + <a href="FalconDocumentation.html" title="Architecture"> + <i class="none"></i> + Architecture</a> + </li> + + <li> + + <a href="InstallationSteps.html" title="Installation"> + <i class="none"></i> + Installation</a> + </li> + + <li> + + <a href="OnBoarding.html" title="On Boarding"> + <i class="none"></i> + On Boarding</a> + </li> + + <li> + + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + + <a href="Operability.html" title="Operability"> + <i class="none"></i> + Operability</a> + </li> + + <li> + + <a href="EntitySpecification.html" title="Entity Specification"> + <i class="none"></i> + Entity Specification</a> + </li> + + <li> + + <a href="falconcli/FalconCLI.html" title="Client (Falcon CLI)"> + <i class="none"></i> + Client (Falcon CLI)</a> + </li> + + <li> + + <a href="restapi/ResourceList.html" title="Rest API"> + <i class="icon-chevron-right"></i> + Rest API</a> + </li> + + <li> + + <a href="HiveIntegration.html" title="Hive Integration"> + <i class="none"></i> + Hive Integration</a> + </li> + + <li class="active"> + + <a href="#"><i class="none"></i>Server side Extensions</a> + </li> + + <li> + + <a href="Security.html" title="Security"> + <i class="none"></i> + Security</a> + </li> + <li class="nav-header">Project Information</li> + + <li> + + <a href="project-info.html" title="Summary"> + <i class="none"></i> + Summary</a> + </li> + + <li> + + <a href="mail-lists.html" title="Mailing Lists"> + <i class="none"></i> + Mailing Lists</a> + </li> + + <li> + + <a href="http://webchat.freenode.net?channels=apachefalcon&uio=d4"; class="externalLink" title="IRC"> + <i class="none"></i> + IRC</a> + </li> + + <li> + + <a href="team-list.html" title="Team"> + <i class="none"></i> + Team</a> + </li> + + <li> + + <a href="issue-tracking.html" title="Issue Tracking"> + <i class="none"></i> + Issue Tracking</a> + </li> + + <li> + + <a href="source-repository.html" title="Source Repository"> + <i class="none"></i> + Source Repository</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Index"; class="externalLink" title="Wiki"> + <i class="none"></i> + Wiki</a> + </li> + + <li> + + <a href="license.html" title="License"> + <i class="none"></i> + License</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/News"; class="externalLink" title="News"> + <i class="none"></i> + News</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/PoweredBy"; class="externalLink" title="Powered by"> + <i class="none"></i> + Powered by</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Acknowledgements"; class="externalLink" title="Acknowledgements"> + <i class="none"></i> + Acknowledgements</a> + </li> + + <li> + + <a href="http://blogs.apache.org/falcon/"; class="externalLink" title="Blog"> + <i class="none"></i> + Blog</a> + </li> + <li class="nav-header">Releases</li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> + <i class="none"></i> + 0.9</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.8"; class="externalLink" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.7"; class="externalLink" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/falcon/0.6.1"; class="externalLink" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.6-incubating"; class="externalLink" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.5-incubating"; class="externalLink" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.4-incubating"; class="externalLink" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.3-incubating"; class="externalLink" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Roadmap"; class="externalLink" title="Coming soon"> + <i class="none"></i> + Coming soon</a> + </li> + <li class="nav-header">Documentation</li> + + <li> + + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> + <i class="none"></i> + 0.9</a> + </li> + + <li> + + <a href="0.8/index.html" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="0.7/index.html" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="0.6.1/index.html" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="0.6-incubating/index.html" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="0.5-incubating/index.html" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="0.4-incubating/index.html" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="0.3-incubating/index.html" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + <li class="nav-header">ASF</li> + + <li> + + <a href="http://www.apache.org/foundation/how-it-works.html"; class="externalLink" title="How Apache Works"> + <i class="none"></i> + How Apache Works</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/"; class="externalLink" title="Foundation"> + <i class="none"></i> + Foundation</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/sponsorship.html"; class="externalLink" title="Sponsoring Apache"> + <i class="none"></i> + Sponsoring Apache</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/thanks.html"; class="externalLink" title="Thanks"> + <i class="none"></i> + Thanks</a> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <div class="section"> +<h2>Falcon Extensions<a name="Falcon_Extensions"></a></h2></div> +<div class="section"> +<h3>Overview<a name="Overview"></a></h3> +<p>A Falcon extension is a static process template with parameterized workflow to realize a specific use case and enable non-programmers to capture and re-use very complex business logic. Extensions are defined in server space. Objective of the extension is to solve a standard data management function that can be invoked as a tool using the standard Falcon features (REST API, CLI and UI access) supporting standard falcon features.</p> +<p>For example:</p> +<p></p> +<ul> +<li>Replicating directories from one HDFS cluster to another (not timed partitions)</li> +<li>Replicating hive metadata (database, table, views, etc.)</li> +<li>Replicating between HDFS and Hive - either way</li> +<li>Data masking etc.</li></ul></div> +<div class="section"> +<h3>Proposal<a name="Proposal"></a></h3> +<p>Falcon provides a Process abstraction that encapsulates the configuration for a user workflow with scheduling controls. All extensions can be modeled as a Process and its dependent feeds with in Falcon which executes the user workflow periodically. The process and its associated workflow are parameterized. The user will provide properties which are <name, value> pairs that are substituted by falcon before scheduling it. Falcon translates these extensions as a process entity by replacing the parameters in the workflow definition.</p></div> +<div class="section"> +<h3>Falcon extension artifacts to manage extensions<a name="Falcon_extension_artifacts_to_manage_extensions"></a></h3> +<p>Extension artifacts are published in addons/extensions. Artifacts are expected to be installed on HDFS at "extension.store.uri" path defined in startup properties. Each extension is expected to ahve the below artifacts</p> +<ul> +<li>json file under META directory lists all the required and optional parameters/arguments for scheduling extension job</li> +<li>process entity template to be scheduled under resources directory</li> +<li>parameterized workflow under resources directory</li> +<li>required libs under the libs directory</li> +<li>README describing the functionality achieved by extension</li></ul> +<p>REST API and CLI support has been added for extension artifact management on HDFS. Please Refer to <a href="./Falconcli/FalconCLI.html">Falcon CLI</a> and <a href="./Restapi/ResourceList.html">REST API</a> for more details.</p></div> +<div class="section"> +<h3>CLI and REST API support<a name="CLI_and_REST_API_support"></a></h3> +<p>REST APIs and CLI support has been added to manage extension jobs and instances.</p> +<p>Please Refer to <a href="./Falconcli/FalconCLI.html">Falcon CLI</a> and <a href="./Restapi/ResourceList.html">REST API</a> for more details on usage of CLI and REST API's for extension jobs and instances management.</p></div> +<div class="section"> +<h3>Metrics<a name="Metrics"></a></h3> +<p>HDFS mirroring and Hive mirroring extensions will capture the replication metrics like TIMETAKEN, BYTESCOPIED, COPY (number of files copied) for an instance and populate to the GraphDB.</p></div> +<div class="section"> +<h3>Sample extensions<a name="Sample_extensions"></a></h3> +<p>Sample extensions are published in addons/extensions</p></div> +<div class="section"> +<h3>Types of extensions<a name="Types_of_extensions"></a></h3> +<p></p> +<ul> +<li><a href="./HDFSMirroring.html">HDFS mirroring extension</a></li> +<li><a href="./HiveMirroring.html">Hive mirroring extension</a></li> +<li><a href="./HdfsSnapshotMirroring.html">HDFS snapshot based mirroring</a></li></ul></div> +<div class="section"> +<h3>Packaging and installation<a name="Packaging_and_installation"></a></h3> +<p>This feature is enabled by default but could be disabled by removing the following from startup properties:</p> +<div class="source"> +<pre> +config name: *.application.services +config value: org.apache.falcon.extensions.ExtensionService + +</pre></div> +<p><a href="./ExtensionService.html">ExtensionService</a> should be added before <a href="./ConfigurationStore.html">ConfigurationStore</a> in startup properties for application services configuration. For manual installation user is expected to update "extension.store.uri" property defined in startup properties with HDFS path where the extension artifacts will be copied to. Extension artifacts in addons/extensions are packaged in falcon. For manual installation once the Falcon Server is setup user is expected to copy the extension artifacts under {falcon-server-dir}/extensions to HDFS at "extension.store.uri" path defined in startup properties and then restart Falcon.</p></div> +<div class="section"> +<h3>Migration<a name="Migration"></a></h3> +<p>Recipes framework and HDFS mirroring capability was added in Apache Falcon 0.6.0 release and it was client side logic. With 0.10 release its moved to server side and renamed as server side extensions. Client side recipes only had CLI support and expected certain pre steps to get it working. This is no longer required in 0.10 release as new CLI and REST API support has been provided.</p> +<p>Migrating to 0.10 release and above is not backward compatible for Recipes. If user is migrating to 0.10 release and above then old Recipe setup and CLI's won't work. For manual installation user is expected to copy Extension artifacts to HDFS. Please refer "Packaging and installation" section above for more details. Please Refer to <a href="./Falconcli/FalconCLI.html">Falcon CLI</a> and <a href="./Restapi/ResourceList.html">REST API</a> for more details on usage of CLI and REST API's for extension jobs and instances management.</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2013-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html>
http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/FalconCLI.html ---------------------------------------------------------------------- diff --git a/content/FalconCLI.html b/content/FalconCLI.html index b13a685..d1f03d9 100644 --- a/content/FalconCLI.html +++ b/content/FalconCLI.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - FalconCLI</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,7 +148,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -158,6 +165,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -251,6 +265,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -315,9 +336,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/FalconDocumentation.html ---------------------------------------------------------------------- diff --git a/content/FalconDocumentation.html b/content/FalconDocumentation.html index 432880e..3369514 100644 --- a/content/FalconDocumentation.html +++ b/content/FalconDocumentation.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Contents</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -121,6 +121,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -139,7 +146,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -156,6 +163,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -249,6 +263,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -313,9 +334,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> <i class="none"></i> - 0.9 (Current)</a> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> + <i class="none"></i> + 0.9</a> </li> <li> @@ -422,18 +450,20 @@ <li><a href="#Architecture">Architecture</a></li> <li><a href="#Control_flow">Control flow</a></li> <li><a href="#Modes_Of_Deployment">Modes Of Deployment</a></li> +<li><a href="#Configuring_Falcon">Configuring Falcon</a></li> <li><a href="#Entity_Management_actions">Entity Management actions</a></li> <li><a href="#Instance_Management_actions">Instance Management actions</a></li> <li><a href="#Retention">Retention</a></li> <li><a href="#Replication">Replication</a></li> <li><a href="#Cross_entity_validations">Cross entity validations</a></li> +<li><a href="#Updating_cluster_entity_definition">Updating cluster entity</a></li> <li><a href="#Updating_process_and_feed_definition">Updating process and feed definition</a></li> <li><a href="#Handling_late_input_data">Handling late input data</a></li> <li><a href="#Idempotency">Idempotency</a></li> <li><a href="#Falcon_EL_Expressions">Falcon EL Expressions</a></li> <li><a href="#Lineage">Lineage</a></li> <li><a href="#Security">Security</a></li> -<li><a href="#Recipes">Recipes</a></li> +<li><a href="#Extensions">Extensions</a></li> <li><a href="#Monitoring">Monitoring</a></li> <li><a href="#Email_Notification">Email Notification</a></li> <li><a href="#Backwards_Compatibility">Backwards Compatibility Instructions</a></li> @@ -478,6 +508,44 @@ <h5>Prism Setup<a name="Prism_Setup"></a></h5> <p><img src="PrismSetup.png" alt="" /></p></div> <div class="section"> +<h4>Safe Mode<a name="Safe_Mode"></a></h4> +<p>Safemode is useful when the admin wants to prevent Falcon users from scheduling entities in the workflow engine. This can happen when</p> +<ul> +<li>Hadoop clusters are being upgraded.</li> +<li>Falcon cluster entities are being updated.</li></ul> +<p>When in Safemode, users can only perform limited operations. To be specific,</p> +<ul> +<li>Users can perform read operations on entities subject to authorization.</li> +<li>Superuser can perform cluster entity update operation.</li> +<li>Suspend/Kill of individual instances will be allowed if users want to suspend specific job operations.</li> +<li>Suspend operation on schedulable entities will be allowed. This is because, a user might want to suspend entities during rolling-upgrade to handle jobs incompatible with updated versions.</li> +<li>All other operations are not allowed. To enumerate, +<ul> +<li>All entity submit, submitAndSchedule operations are not allowed.</li> +<li>Entity operations not allowed are : update, schedule, touch, delete, submit, submitAndSchedule, resume</li> +<li>Instance operations not allowed are : rerun, resume</li></ul></li></ul></div> +<div class="section"> +<h5>Getting into/out-of Safemode.<a name="Getting_intoout-of_Safemode."></a></h5> +<p>The Falcon process user can specify whether to start Falcon in safemode with the following command:</p> +<div class="source"> +<pre> +<falcon-server>/bin/falcon-start -setsafemode <true/false> + +</pre></div> +<p>A superuser or admin-user can set Falcon server into/outof safemode using CLI or RestAPI. A user is considered superuser if they owns the Falcon process or belong to group specified in startup property falcon.security.authorization.superusergroup. A user is considered admin user if they are listed under startup property falcon.security.authorization.admin.users, OR they belong to group listed under startup property falcon.security.authorization.admin.groups.</p> +<div class="source"> +<pre> +## CLI +<falcon-server>/bin/falcon admin [-setsafemode <true/false>] + +## RestAPI +GET http://falcon-server:15000/api/admin/setSafeMode/true +OR +GET http://falcon-server:15000/api/admin/setSafeMode/false + +</pre></div> +<p><b>NOTE</b> User can find if <a href="./FalconServer.html">FalconServer</a> is in safemode or not, by calling the <a href="./Restapi/AdminVersion.html">Admin Version</a> API. Once server is set to safemode, this state is persisted during restarts. It has to be unset explicitly if user wants to exit safemode.</p></div> +<div class="section"> <h4>Configuration Store<a name="Configuration_Store"></a></h4> <p>Configuration store is file system based store that the Falcon system maintains where the entity definitions are stored. File System used for the configuration store can either be a local file system or HDFS file system. It is recommended that the store be maintained outside of the system where Falcon is deployed. This is needed for handling issues relating to disk failures or other permanent failures of the system where Falcon is deployed. Configuration store also maintains an archive location where prior versions of the configuration or deleted configurations are maintained. They are never accessed by the Falcon system and they merely serve to track historical changes to the entity definitions.</p></div> <div class="section"> @@ -524,6 +592,9 @@ catalog:$database-name:$table-name#partition-key=partition-value);partition-key= </pre></div></div> <div class="section"> +<h3>Configuring Falcon<a name="Configuring_Falcon"></a></h3> +<p>Configuring Falcon is detailed in <a href="./Configuration.html">Configuration</a>.</p></div> +<div class="section"> <h3>Entity Management actions<a name="Entity_Management_actions"></a></h3> <p>All the following operation can also be done using <a href="./Restapi/ResourceList.html">Falcon's RESTful API</a>.</p></div> <div class="section"> @@ -556,7 +627,9 @@ catalog:$database-name:$table-name#partition-key=partition-value);partition-key= <p>Delete operation on the entity removes any scheduled activity on the workflow engine, besides removing the entity from the falcon configuration store. Delete operation on an entity would only succeed if there are no dependent entities on the deleted entity.</p></div> <div class="section"> <h4>Update<a name="Update"></a></h4> -<p>Update operation allows an already submitted/scheduled entity to be updated. Cluster update is currently not allowed. Feed update can cause cascading update to all the processes already scheduled. Process update triggers update in falcon if entity is updated. The following set of actions are performed in scheduler to realize an update:</p> +<p>Update operation allows an already submitted/scheduled entity to be updated. Feed update can cause cascading update to all the processes already scheduled. Process update triggers update in falcon if entity is scheduled.</p> +<p>Cluster update will require user to update dependent Feed and Process entities that are already scheduled. Cluster update needs to be performed in safemode. We provide a CLI command for the user to update the scheduled dependent entities after cluster update and exiting safemode.</p> +<p>The following set of actions are performed in scheduler to realize an update:</p> <ul> <li>Update the old scheduled entity to set the end time to "now"</li> <li>Schedule as per the new process/feed definition with the start time as "now"</li></ul></div> @@ -603,7 +676,9 @@ catalog:$database-name:$table-name#partition-key=partition-value);partition-key= </pre></div> <p>The 'limit' attribute can be specified in units of minutes/hours/days/months, and a corresponding numeric value can be attached to it. It essentially instructs the system to retain data till the time specified in the attribute spanning backwards in time, from now. Any data older than that is erased from the system. By default, Falcon runs retention jobs up to the cluster validity end time. This causes the instances created within the endTime and "endTime - retentionLimit" to be retained forever. If the users do not want to retain any instances of the feed past the cluster validity end time, user should set property "falcon.retention.keep.instances.beyond.validity" to false in runtime.properties.</p> -<p>With the integration of Hive, Falcon also provides retention for tables in Hive catalog.</p></div> +<p>With the integration of Hive, Falcon also provides retention for tables in Hive catalog.</p> +<p>When a feed is scheduled Falcon kicks off the retention policy immediately. When job runs, it deletes everything that's eligible for eviction - eligibility criteria is the date pattern on the partition and NOT creation date. For e.g. if the retention limit is 90 days then retention job consistently deletes files older than 90 days.</p> +<p>For retention, Falcon expects data to be in dated partitions. When the retention job is kicked off, it discovers data that needs to be evicted based on retention policy. It gets the location from the feed and uses pattern matching to find the pattern to get the list of data for the feed, then gets the date from the data path. If the data path date is beyond the retention limit it's deleted. As this uses pattern matching it is not time consuming and hence doesn't introduce performance overhead.</p></div> <div class="section"> <h4>Example:<a name="Example:"></a></h4> <p>If retention period is 10 hours, and the policy kicks in at time 't', the data retained by system is essentially the one after or equal to t-10h . Any data before t-10h is removed from the system.</p> @@ -694,7 +769,7 @@ catalog:$database-name:$table-name#partition-key=partition-value);partition-key= <li>The partition is not complete and hence not visible to users until all the data is committed on the secondary</li></ul>cluster, (no dirty reads)</div> <div class="section"> <h4>Archival as Replication<a name="Archival_as_Replication"></a></h4> -<p>Falcon allows users to archive data from on-premice to cloud, either Azure WASB or S3. It uses the underlying replication for archiving data from source to target. The archival URI is specified as the overridden location for the target cluster.</p> +<p>Falcon allows users to archive data from on-premise to cloud, either Azure WASB or S3. It uses the underlying replication for archiving data from source to target. The archival URI is specified as the overridden location for the target cluster. Note that for data replication between on-premise and Azure cloud, Azure credentials need to be added to core-site.xml. Please refer to <a href="./DataReplicationAzure.html">AzureDataReplication</a> for details and examples.</p> <p><b>Example:</b></p> <div class="source"> <pre> @@ -830,6 +905,27 @@ validity start="2009-01-01T00:00Z" end="2012-12-31T23:59Z" . <p>Failure to follow any of the above rules would result in a process submission failure.</p> <p><b>NOTE:</b> Even though the above check ensures that the timelines are not anachronistic, if the input data is not present in the system for the specified time period, the process can be submitted and scheduled, but all instances created would result in a WAITING state unless data is actually provided in the cluster.</p></div> <div class="section"> +<h3>Updating cluster entity definition<a name="Updating_cluster_entity_definition"></a></h3> +<p>Cluster entities can be updated when the user wants to change their interface endpoints or properties, e.g. hadoop clusters updated from unsecure to secure; hadoop cluster moved from non high-availability to high-availability, etc.</p> +<p>In these scenarios, user would want to change the cluster entity to reflect updated interface endpoints or properties. Updating cluster would require cascading update to dependent feed/process jobs scheduled on this cluster. So Falcon only allows Cluster update when</p> +<ul> +<li>Falcon server is in safemode.</li> +<li>The update is requested by superuser</li> +<li>The underlying namenode or workflow engine referenced by interface URI is the same. It is only the URI that has changed to reflect secure/HA environments.</li></ul> +<p>Cluster entity should be updated by superuser using following CLI command.</p> +<div class="source"> +<pre> +bash$ falcon entity -type cluster -name primaryCluster -update -file ~/primary-updated.xml + +</pre></div> +<p>Once the cluster entity is updated, user should exit <a href="./FalconServer.html">FalconServer</a> from safemode and update the scheduled entities that are dependent on this Cluster. In case of an error during update, user should address the root cause of failure and retry the command. For example : if the cluster has 10 dependent entities and the updateClusterDependents command failed after updating 6th entity, rerun of this command will only update entities 7 to 10.</p> +<div class="source"> +<pre> +bash$ falcon entity -updateClusterDependents -cluster primaryCluster + +</pre></div> +<p>Please Refer to <a href="./Falconcli/FalconCLI.html">Falcon CLI</a> for more details on usage of CLI commands.</p></div> +<div class="section"> <h3>Updating process and feed definition<a name="Updating_process_and_feed_definition"></a></h3> <p>Any changes in feed/process can be done by updating its definition. After the update, any new workflows which are to be scheduled after the update call will pick up the new changes. Feed/process name and start time can't be updated. Updating a process triggers updates to the workflow that is triggered in the workflow engine. Updating feed updates feed workflows like retention, replication etc. and also updates the processes that reference the feed.</p></div> <div class="section"> @@ -962,11 +1058,11 @@ config value: org.apache.falcon.metadata.MetadataMappingService <h3>Security<a name="Security"></a></h3> <p>Security is detailed in <a href="./Security.html">Security</a>.</p></div> <div class="section"> -<h3>Recipes<a name="Recipes"></a></h3> -<p>Recipes is detailed in <a href="./Recipes.html">Recipes</a>.</p></div> +<h3>Extensions<a name="Extensions"></a></h3> +<p>Extensions is detailed in <a href="./Extensions.html">Extensions</a>.</p></div> <div class="section"> <h3>Monitoring<a name="Monitoring"></a></h3> -<p>Monitoring and Operationalizing Falcon is detailed in <a href="./Operability.html">Operability</a>.</p></div> +<p>Monitoring and Operationalizing Falcon is detailed in <a href="./Operability.html">Operability</a> and <a href="./FeedSLAMonitoring.html">Falcon Feed SLA Monitoring</a>.</p></div> <div class="section"> <h3>Email Notification<a name="Email_Notification"></a></h3> <p>Notification for instance completion in Falcon is defined in <a href="./FalconEmailNotification.html">Falcon Email Notification</a>.</p></div> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/FalconEmailNotification.html ---------------------------------------------------------------------- diff --git a/content/FalconEmailNotification.html b/content/FalconEmailNotification.html index d9dbcc5..a3425f9 100644 --- a/content/FalconEmailNotification.html +++ b/content/FalconEmailNotification.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Falcon Email Notification</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,7 +148,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -158,6 +165,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -251,6 +265,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -315,9 +336,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/FalconNativeScheduler.html ---------------------------------------------------------------------- diff --git a/content/FalconNativeScheduler.html b/content/FalconNativeScheduler.html index 49da51d..b1a8c90 100644 --- a/content/FalconNativeScheduler.html +++ b/content/FalconNativeScheduler.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Falcon Native Scheduler</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,7 +148,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -158,6 +165,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -251,6 +265,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -315,9 +336,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li> @@ -449,7 +477,7 @@ org.apache.falcon.service.ProcessSubscriberService,\ org.apache.falcon.service.FeedSLAMonitoringService,\ org.apache.falcon.service.LifecyclePolicyMap,\ - org.apache.falcon.state.store.service.FalconJPAService,\ + org.apache.falcon.service.FalconJPAService,\ org.apache.falcon.entity.store.ConfigurationStore,\ org.apache.falcon.rerun.service.RetryService,\ org.apache.falcon.rerun.service.LateRunService,\ http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/FeedSLAMonitoring.html ---------------------------------------------------------------------- diff --git a/content/FeedSLAMonitoring.html b/content/FeedSLAMonitoring.html new file mode 100644 index 0000000..82db958 --- /dev/null +++ b/content/FeedSLAMonitoring.html @@ -0,0 +1,489 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-08-08 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Falcon - Falcon Feed SLA Monitoring</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft"> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> + <img src="images/apache-feather-tm.gif" alt="Falcon" height="45px"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Falcon"> + Falcon</a> + </li> + <li class="divider ">/</li> + <li class="">Falcon Feed SLA Monitoring</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Falcon</li> + + <li> + + <a href="index.html" title="About"> + <i class="none"></i> + About</a> + </li> + + <li> + + <a href="slides/falcon-overview.html" title="Overview"> + <i class="none"></i> + Overview</a> + </li> + + <li> + + <a href="slides/falcon-user-guide.html" title="User Guide"> + <i class="none"></i> + User Guide</a> + </li> + + <li> + + <a href="GettingStarted.html" title="Getting Started"> + <i class="none"></i> + Getting Started</a> + </li> + + <li> + + <a href="FalconDocumentation.html" title="Architecture"> + <i class="none"></i> + Architecture</a> + </li> + + <li> + + <a href="InstallationSteps.html" title="Installation"> + <i class="none"></i> + Installation</a> + </li> + + <li> + + <a href="OnBoarding.html" title="On Boarding"> + <i class="none"></i> + On Boarding</a> + </li> + + <li> + + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + + <a href="Operability.html" title="Operability"> + <i class="none"></i> + Operability</a> + </li> + + <li> + + <a href="EntitySpecification.html" title="Entity Specification"> + <i class="none"></i> + Entity Specification</a> + </li> + + <li> + + <a href="falconcli/FalconCLI.html" title="Client (Falcon CLI)"> + <i class="none"></i> + Client (Falcon CLI)</a> + </li> + + <li> + + <a href="restapi/ResourceList.html" title="Rest API"> + <i class="icon-chevron-right"></i> + Rest API</a> + </li> + + <li> + + <a href="HiveIntegration.html" title="Hive Integration"> + <i class="none"></i> + Hive Integration</a> + </li> + + <li> + + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + + <a href="Security.html" title="Security"> + <i class="none"></i> + Security</a> + </li> + <li class="nav-header">Project Information</li> + + <li> + + <a href="project-info.html" title="Summary"> + <i class="none"></i> + Summary</a> + </li> + + <li> + + <a href="mail-lists.html" title="Mailing Lists"> + <i class="none"></i> + Mailing Lists</a> + </li> + + <li> + + <a href="http://webchat.freenode.net?channels=apachefalcon&uio=d4"; class="externalLink" title="IRC"> + <i class="none"></i> + IRC</a> + </li> + + <li> + + <a href="team-list.html" title="Team"> + <i class="none"></i> + Team</a> + </li> + + <li> + + <a href="issue-tracking.html" title="Issue Tracking"> + <i class="none"></i> + Issue Tracking</a> + </li> + + <li> + + <a href="source-repository.html" title="Source Repository"> + <i class="none"></i> + Source Repository</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Index"; class="externalLink" title="Wiki"> + <i class="none"></i> + Wiki</a> + </li> + + <li> + + <a href="license.html" title="License"> + <i class="none"></i> + License</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/News"; class="externalLink" title="News"> + <i class="none"></i> + News</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/PoweredBy"; class="externalLink" title="Powered by"> + <i class="none"></i> + Powered by</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Acknowledgements"; class="externalLink" title="Acknowledgements"> + <i class="none"></i> + Acknowledgements</a> + </li> + + <li> + + <a href="http://blogs.apache.org/falcon/"; class="externalLink" title="Blog"> + <i class="none"></i> + Blog</a> + </li> + <li class="nav-header">Releases</li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> + <i class="none"></i> + 0.9</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.8"; class="externalLink" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.7"; class="externalLink" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/falcon/0.6.1"; class="externalLink" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.6-incubating"; class="externalLink" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.5-incubating"; class="externalLink" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.4-incubating"; class="externalLink" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.3-incubating"; class="externalLink" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Roadmap"; class="externalLink" title="Coming soon"> + <i class="none"></i> + Coming soon</a> + </li> + <li class="nav-header">Documentation</li> + + <li> + + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> + <i class="none"></i> + 0.9</a> + </li> + + <li> + + <a href="0.8/index.html" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="0.7/index.html" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="0.6.1/index.html" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="0.6-incubating/index.html" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="0.5-incubating/index.html" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="0.4-incubating/index.html" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="0.3-incubating/index.html" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + <li class="nav-header">ASF</li> + + <li> + + <a href="http://www.apache.org/foundation/how-it-works.html"; class="externalLink" title="How Apache Works"> + <i class="none"></i> + How Apache Works</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/"; class="externalLink" title="Foundation"> + <i class="none"></i> + Foundation</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/sponsorship.html"; class="externalLink" title="Sponsoring Apache"> + <i class="none"></i> + Sponsoring Apache</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/thanks.html"; class="externalLink" title="Thanks"> + <i class="none"></i> + Thanks</a> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <div class="section"> +<h3>Falcon Feed SLA Monitoring<a name="Falcon_Feed_SLA_Monitoring"></a></h3> +<p>Feed SLA monitoring allows you to monitor the feed ,It keeps track of the instances of the feed which are running and stores them in the db.</p> +<p>Feed SLA monitoring service requires FalconJPAService to be up.Following are the values we need to set to run FeedSLAMonitoring - In startup.properties :</p> +<p>*.application.services= org.apache.falcon.state.store.service.FalconJPAService, org.apache.falcon.service.FeedSLAMonitoringService</p> +<p>These properties are required for FalconJPAService in statestore.properties:</p> +<p></p> +<ul> +<li><b>falcon.state.store.impl</b> - org.apache.falcon.state.store.jdbc.JDBCStateStore</li> +<li><b>falcon.statestore.jdbc.driver</b> -org.apache.derby.jdbc.EmbeddedDriver</li> +<li><b>falcon.statestore.jdbc.url</b> -jdbc:derby:target/test-data/data.db;create=true</li> +<li><b>falcon.statestore.connection.data.source</b> -org.apache.commons.dbcp.BasicDataSource</li></ul>Maximum number of active connections that can be allocated from this pool at the same time. +<ul> +<li><b>falcon.statestore.pool.max.active.conn</b> -10</li> +<li><b>falcon.statestore.connection.properties</b> -</li></ul>Indicates the interval (in milliseconds) between eviction runs. +<ul> +<li><b>falcon.statestore.validate.db.connection.eviction.interval</b> -300000</li></ul>The number of objects to examine during each run of the idle object evictor thread. +<ul> +<li><b>falcon.statestore.validate.db.connection.eviction.num</b> -10</li></ul>Creates Falcon DB. If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP.If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. +<ul> +<li><b>falcon.statestore.create.db.schema</b> -false</li></ul> +<p>Note: First time we have to manually create the schema in production as we have set falcon.statestore.create.db.schema = false</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2013-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/GettingStarted.html ---------------------------------------------------------------------- diff --git a/content/GettingStarted.html b/content/GettingStarted.html index 1f4d9f4..3424214 100644 --- a/content/GettingStarted.html +++ b/content/GettingStarted.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Apache Falcon - Data management and processing platform</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -121,6 +121,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -139,7 +146,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -156,6 +163,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -249,6 +263,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -313,9 +334,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li>
