http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/HiveMirroring.html ---------------------------------------------------------------------- diff --git a/content/HiveMirroring.html b/content/HiveMirroring.html new file mode 100644 index 0000000..875ff3a --- /dev/null +++ b/content/HiveMirroring.html @@ -0,0 +1,528 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-08-08 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Falcon - Hive Mirroring</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarDisabled"> + + + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft"> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> + <img src="images/apache-feather-tm.gif" alt="Falcon" height="45px"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Falcon"> + Falcon</a> + </li> + <li class="divider ">/</li> + <li class="">Hive Mirroring</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span3"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Falcon</li> + + <li> + + <a href="index.html" title="About"> + <i class="none"></i> + About</a> + </li> + + <li> + + <a href="slides/falcon-overview.html" title="Overview"> + <i class="none"></i> + Overview</a> + </li> + + <li> + + <a href="slides/falcon-user-guide.html" title="User Guide"> + <i class="none"></i> + User Guide</a> + </li> + + <li> + + <a href="GettingStarted.html" title="Getting Started"> + <i class="none"></i> + Getting Started</a> + </li> + + <li> + + <a href="FalconDocumentation.html" title="Architecture"> + <i class="none"></i> + Architecture</a> + </li> + + <li> + + <a href="InstallationSteps.html" title="Installation"> + <i class="none"></i> + Installation</a> + </li> + + <li> + + <a href="OnBoarding.html" title="On Boarding"> + <i class="none"></i> + On Boarding</a> + </li> + + <li> + + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + + <a href="Operability.html" title="Operability"> + <i class="none"></i> + Operability</a> + </li> + + <li> + + <a href="EntitySpecification.html" title="Entity Specification"> + <i class="none"></i> + Entity Specification</a> + </li> + + <li> + + <a href="falconcli/FalconCLI.html" title="Client (Falcon CLI)"> + <i class="none"></i> + Client (Falcon CLI)</a> + </li> + + <li> + + <a href="restapi/ResourceList.html" title="Rest API"> + <i class="icon-chevron-right"></i> + Rest API</a> + </li> + + <li> + + <a href="HiveIntegration.html" title="Hive Integration"> + <i class="none"></i> + Hive Integration</a> + </li> + + <li> + + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + + <a href="Security.html" title="Security"> + <i class="none"></i> + Security</a> + </li> + <li class="nav-header">Project Information</li> + + <li> + + <a href="project-info.html" title="Summary"> + <i class="none"></i> + Summary</a> + </li> + + <li> + + <a href="mail-lists.html" title="Mailing Lists"> + <i class="none"></i> + Mailing Lists</a> + </li> + + <li> + + <a href="http://webchat.freenode.net?channels=apachefalcon&uio=d4"; class="externalLink" title="IRC"> + <i class="none"></i> + IRC</a> + </li> + + <li> + + <a href="team-list.html" title="Team"> + <i class="none"></i> + Team</a> + </li> + + <li> + + <a href="issue-tracking.html" title="Issue Tracking"> + <i class="none"></i> + Issue Tracking</a> + </li> + + <li> + + <a href="source-repository.html" title="Source Repository"> + <i class="none"></i> + Source Repository</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Index"; class="externalLink" title="Wiki"> + <i class="none"></i> + Wiki</a> + </li> + + <li> + + <a href="license.html" title="License"> + <i class="none"></i> + License</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/News"; class="externalLink" title="News"> + <i class="none"></i> + News</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/PoweredBy"; class="externalLink" title="Powered by"> + <i class="none"></i> + Powered by</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Acknowledgements"; class="externalLink" title="Acknowledgements"> + <i class="none"></i> + Acknowledgements</a> + </li> + + <li> + + <a href="http://blogs.apache.org/falcon/"; class="externalLink" title="Blog"> + <i class="none"></i> + Blog</a> + </li> + <li class="nav-header">Releases</li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> + <i class="none"></i> + 0.9</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.8"; class="externalLink" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.7"; class="externalLink" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/falcon/0.6.1"; class="externalLink" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.6-incubating"; class="externalLink" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.5-incubating"; class="externalLink" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.4-incubating"; class="externalLink" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="http://archive.apache.org/dist/incubator/falcon/0.3-incubating"; class="externalLink" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + + <li> + + <a href="https://cwiki.apache.org/confluence/display/FALCON/Roadmap"; class="externalLink" title="Coming soon"> + <i class="none"></i> + Coming soon</a> + </li> + <li class="nav-header">Documentation</li> + + <li> + + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> + <i class="none"></i> + 0.9</a> + </li> + + <li> + + <a href="0.8/index.html" title="0.8"> + <i class="none"></i> + 0.8</a> + </li> + + <li> + + <a href="0.7/index.html" title="0.7"> + <i class="none"></i> + 0.7</a> + </li> + + <li> + + <a href="0.6.1/index.html" title="0.6.1"> + <i class="none"></i> + 0.6.1</a> + </li> + + <li> + + <a href="0.6-incubating/index.html" title="0.6-incubating"> + <i class="none"></i> + 0.6-incubating</a> + </li> + + <li> + + <a href="0.5-incubating/index.html" title="0.5-incubating"> + <i class="none"></i> + 0.5-incubating</a> + </li> + + <li> + + <a href="0.4-incubating/index.html" title="0.4-incubating"> + <i class="none"></i> + 0.4-incubating</a> + </li> + + <li> + + <a href="0.3-incubating/index.html" title="0.3-incubating"> + <i class="none"></i> + 0.3-incubating</a> + </li> + <li class="nav-header">ASF</li> + + <li> + + <a href="http://www.apache.org/foundation/how-it-works.html"; class="externalLink" title="How Apache Works"> + <i class="none"></i> + How Apache Works</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/"; class="externalLink" title="Foundation"> + <i class="none"></i> + Foundation</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/sponsorship.html"; class="externalLink" title="Sponsoring Apache"> + <i class="none"></i> + Sponsoring Apache</a> + </li> + + <li> + + <a href="http://www.apache.org/foundation/thanks.html"; class="externalLink" title="Thanks"> + <i class="none"></i> + Thanks</a> + </li> + </ul> + + + + <hr class="divider" /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span9" > + + <div class="section"> +<h2>Hive Mirroring<a name="Hive_Mirroring"></a></h2></div> +<div class="section"> +<h3>Overview<a name="Overview"></a></h3> +<p>Falcon provides feature to replicate Hive metadata and data events from source cluster to destination cluster. This is supported for both secure and unsecure cluster through Falcon extensions.</p></div> +<div class="section"> +<h3>Prerequisites<a name="Prerequisites"></a></h3> +<p>Following is the prerequisites to use Hive Mirrroring</p> +<p></p> +<ul> +<li><b>Hive 1.2.0+</b></li> +<li><b>Oozie 4.2.0+</b></li></ul> +<p><b>Note:</b> Set following properties in hive-site.xml for replicating the Hive events on source and destination Hive cluster:</p> +<div class="source"> +<pre> + <property> + <name>hive.metastore.event.listeners</name> + <value>org.apache.hive.hcatalog.listener.DbNotificationListener</value> + <description>event listeners that are notified of any metastore changes</description> + </property> + + <property> + <name>hive.metastore.dml.events</name> + <value>true</value> + </property> + +</pre></div></div> +<div class="section"> +<h3>Use Case<a name="Use_Case"></a></h3> +<p>* Replicate data/metadata of Hive DB & table from source to target cluster</p></div> +<div class="section"> +<h3>Limitations<a name="Limitations"></a></h3> +<p>* Currently Hive doesn't support create database, roles, views, offline tables, direct HDFS writes without registering with metadata and Database/Table name mapping replication events. Hence Hive mirroring extension cannot be used to replicate above mentioned events between warehouses.</p></div> +<div class="section"> +<h3>Usage<a name="Usage"></a></h3></div> +<div class="section"> +<h4>Bootstrap<a name="Bootstrap"></a></h4> +<p>Perform initial bootstrap of Table and Database from source cluster to destination cluster</p> +<ul> +<li><b>Database Bootstrap</b></li></ul>For bootstrapping DB replication, first destination DB should be created. This step is expected, since DB replication definitions can be set up by users only on pre-existing DB’s. Second, Export all tables in the source db and Import it in the destination db, as described in Table bootstrap. +<p></p> +<ul> +<li><b>Table Bootstrap</b></li></ul>For bootstrapping table replication, essentially after having turned on the DbNotificationListener on the source db, perform an Export of the table, distcp the Export over to the destination warehouse and do an Import over there. Check the following <a class="externalLink" href="https://cwiki.apache.org/confluence/display/Hive/LanguageManual+ImportExport";>Hive Export-Import</a> for syntax details and examples. This will set up the destination table so that the events on the source cluster that modify the table will then be replicated.</div> +<div class="section"> +<h4>Setup source and destination clusters<a name="Setup_source_and_destination_clusters"></a></h4> +<div class="source"> +<pre> + $FALCON_HOME/bin/falcon entity -submit -type cluster -file /cluster/definition.xml + +</pre></div></div> +<div class="section"> +<h4>Hive mirroring extension properties<a name="Hive_mirroring_extension_properties"></a></h4> +<p>Extension artifacts are expected to be installed on HDFS at the path specified by "extension.store.uri" in startup properties. hive-mirroring-properties.json file located at "<extension.store.uri>/hive-mirroring/META/hive-mirroring-properties.json" lists all the required and optional parameters/arguments for scheduling Hive mirroring job.</p></div> +<div class="section"> +<h4>Submit and schedule Hive mirroring extension<a name="Submit_and_schedule_Hive_mirroring_extension"></a></h4> +<div class="source"> +<pre> + $FALCON_HOME/bin/falcon extension -submitAndSchedule -extensionName hive-mirroring -file /process/definition.xml + +</pre></div> +<p>Please Refer to <a href="./Falconcli/FalconCLI.html">Falcon CLI</a> and <a href="./Restapi/ResourceList.html">REST API</a> for more details on usage of CLI and REST API's.</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row span12">Copyright © 2013-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + + </div> + </footer> + </body> +</html>
http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/ImportExport.html ---------------------------------------------------------------------- diff --git a/content/ImportExport.html b/content/ImportExport.html index 01c8fb7..61dede1 100644 --- a/content/ImportExport.html +++ b/content/ImportExport.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Falcon Data Import and Export</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,7 +148,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -158,6 +165,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -251,6 +265,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -315,9 +336,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/InstallationSteps.html ---------------------------------------------------------------------- diff --git a/content/InstallationSteps.html b/content/InstallationSteps.html index 10d512c..93765da 100644 --- a/content/InstallationSteps.html +++ b/content/InstallationSteps.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Building & Installing Falcon</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -121,6 +121,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -139,7 +146,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -156,6 +163,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -249,6 +263,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -313,9 +334,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> <i class="none"></i> - 0.9 (Current)</a> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> + <i class="none"></i> + 0.9</a> </li> <li> @@ -436,13 +464,16 @@ $git clone https://git-wip-us.apache.org/repos/asf/falcon.git falcon <h4>Step 2 - Build Falcon<a name="Step_2_-_Build_Falcon"></a></h4> <div class="source"> <pre> -$cd falcon -$export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m -noverify" && mvn clean install +$ cd falcon +$ export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=256m -noverify" +$ mvn clean install + </pre></div> <p>It builds and installs the package into the local repository, for use as a dependency in other projects locally.</p> -<p>[optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for a specific version of Hadoop]</p> -<p><b>NOTE:</b> Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards [optionally -Doozie.version=<<oozie version>> can be appended to build with a specific version of Oozie. Oozie versions >= 4 are supported] NOTE: Falcon builds with JDK 1.7/1.8 using -noverify option To compile Falcon with Hive Replication, optionally "-P hadoop-2,hivedr" can be appended. For this Hive >= 1.2.0 and Oozie >= 4.2.0 should be available.</p></div> +<p>[optionally -Dhadoop.version=<<hadoop.version>> can be appended to build for a specific version of hadoop]</p> +<p><b>Note 1:</b> Falcon drops support for Hadoop-1 and only supports Hadoop-2 from Falcon 0.6 onwards Falcon build with JDK 1.7 using -noverify option</p> +<p><b>Note 2:</b> To compile Falcon with addon extensions, append additional profiles to build command using syntax -P<<profile1,profile2>> For Hive Mirroring extension, use profile"hivedr". Hive >= 1.2.0 and Oozie >= 4.2.0 is required For HDFS Snapshot mirroring extension, use profile "hdfs-snapshot-mirroring". Hadoop >= 2.7.0 is required For ADF integration, use profile "adf"</p></div> <div class="section"> <h4>Step 3 - Package and Deploy Falcon<a name="Step_3_-_Package_and_Deploy_Falcon"></a></h4> <p>Once the build successfully completes, artifacts can be packaged for deployment using the assembly plugin. The Assembly Plugin for Maven is primarily intended to allow users to aggregate the project output along with its dependencies, modules, site documentation, and other files into a single distributable archive. There are two basic ways in which you can deploy Falcon - Embedded mode(also known as Stand Alone Mode) and Distributed mode. Your next steps will vary based on the mode in which you want to deploy Falcon.</p> @@ -464,6 +495,8 @@ $src/bin/package.sh <<hadoop-version>> <<oozie-version>> >> ex. src/bin/package.sh 2.5.0 4.0.0 >> Falcon package is available in <<falcon home>>/target/apache-falcon-<<version>>-bin.tar.gz >> Oozie package is available in <<falcon home>>/target/oozie-4.0.1-distro.tar.gz +>> __IMPORTANT: You need to download the je-5.0.73 version from http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip and extract je-5.0.73 under the Falcon webapp directory or provision an HBase cluster for use as Falcon graphdb backend DB. + Depending on the Graphdb backend choice, update the startup.properties appropriately.__ </pre></div> <p><b>NOTE:</b> If you have a separate Apache Oozie installation, you will need to follow some additional steps:</p> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/MigrationInstructions.html ---------------------------------------------------------------------- diff --git a/content/MigrationInstructions.html b/content/MigrationInstructions.html index 778a014..af5faf9 100644 --- a/content/MigrationInstructions.html +++ b/content/MigrationInstructions.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Migration Instructions</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -121,6 +121,11 @@ On Boarding</a> </li> + <li class="active"> + + <a href="#"><i class="none"></i>Migrate to 0.10</a> + </li> + <li> <a href="Operability.html" title="Operability"> @@ -141,7 +146,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -158,6 +163,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -251,6 +263,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -315,9 +334,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li> @@ -420,16 +446,29 @@ <div class="section"> <h2>Migration Instructions<a name="Migration_Instructions"></a></h2></div> <div class="section"> -<h3>Migrate from 0.5-incubating to 0.6-incubating<a name="Migrate_from_0.5-incubating_to_0.6-incubating"></a></h3> -<p>This is a placeholder wiki for migration instructions from falcon 0.5-incubating to 0.6-incubating.</p></div> -<div class="section"> -<h4>Update Entities<a name="Update_Entities"></a></h4></div> +<h3>Migrate from 0.9 to 0.10<a name="Migrate_from_0.9_to_0.10"></a></h3> +<p>FALCON-1333 (Instance Search feature) requires Falcon to use titan-berkeleyje version 0.5.4 to support indexing. Up until version 0.9 - Falcon used titan-berkeleyje-jre6 version 0.4.2. GraphDB created by version 0.4.2 cannot be read by version 0.5.4. The solution is to migrate the GraphDB to be compatible with Falcon 0.10 release. Please make sure that no falcon server is running while performing the migration.</p></div> <div class="section"> -<h4>Change cluster dir permissions<a name="Change_cluster_dir_permissions"></a></h4></div> +<h4>1. Install Falcon 0.10<a name="a1._Install_Falcon_0.10"></a></h4> +<p>Install Falcon 0.10 by following the <a href="./InstallationSteps.html">Installation Steps</a>. Do not start the falcon server yet. The tool to migrate graphDB is packaged with 0.10 Falcon server in falcon-common-0.10.jar.</p></div> <div class="section"> -<h4>Enable/Disable TLS<a name="EnableDisable_TLS"></a></h4></div> +<h4>2. Export GraphDB to JSON file using Falcon 0.9<a name="a2._Export_GraphDB_to_JSON_file_using_Falcon_0.9"></a></h4> +<p>Please run the following command to generate the JSON file.</p> +<div class="source"> +<pre> + $FALCON_HOME/bin/graphdbutil.sh export <<java_home> <<hadoop_home>> <<falcon_0.9_home>> <<path_to_falcon-common-0.10.jar>> /jsonFile/dir/ + +</pre></div> +<p>This command will create /jsonFile/dir/instanceMetadata.json</p></div> <div class="section"> -<h4>Authorization<a name="Authorization"></a></h4></div> +<h4>3. Import GraphDB from JSON file using Falcon 0.10<a name="a3._Import_GraphDB_from_JSON_file_using_Falcon_0.10"></a></h4> +<p>Please run the following command to import graphDB the JSON file. The location of graphDB will be based on property "*.falcon.graph.storage.directory" set in startup.properties file.</p> +<div class="source"> +<pre> + $FALCON_HOME/bin/graphdbutil.sh export <<java_home> <<hadoop_home>> <<falcon_0.10_home>> <<path_to_falcon-common-0.10.jar>> /jsonFile/dir/ + +</pre></div> +<p>This command will import from /jsonFile/dir/instanceMetadata.json, now start the Falcon 0.10 server.</p></div> </div> </div> </div> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/OnBoarding.html ---------------------------------------------------------------------- diff --git a/content/OnBoarding.html b/content/OnBoarding.html index 456f8b5..29e3c85 100644 --- a/content/OnBoarding.html +++ b/content/OnBoarding.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Contents</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -121,6 +121,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -139,7 +146,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -156,6 +163,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -249,6 +263,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -313,9 +334,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/Operability.html ---------------------------------------------------------------------- diff --git a/content/Operability.html b/content/Operability.html index a27265c..b259b20 100644 --- a/content/Operability.html +++ b/content/Operability.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Operationalizing Falcon</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -121,6 +121,13 @@ On Boarding</a> </li> + <li> + + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + <li class="active"> <a href="#"><i class="none"></i>Operability</a> @@ -139,7 +146,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -156,6 +163,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -249,6 +263,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -313,9 +334,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> <i class="none"></i> - 0.9 (Current)</a> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> + <i class="none"></i> + 0.9</a> </li> <li> @@ -469,10 +497,128 @@ <p>Users may parse the metric.log or capture these events from custom monitoring frameworks and can plot various graphs or send alerts according to their requirements.</p></div> <div class="section"> <h3>Notifications<a name="Notifications"></a></h3> -<p>Falcon creates a JMS topic for every process/feed that is scheduled in Falcon. The implementation class and the broker url of the JMS engine are read from the dependent cluster's definition. Users may register consumers on the required topic to check the availability or status of feed instances.</p> -<p>For a given process that is scheduled, the name of the topic is same as the process name. Falcon sends a Map message for every feed produced by the instance of a process to the JMS topic. The JMS MapMessage sent to a topic has the following properties: entityName, feedNames, feedInstancePath, workflowId, runId, nominalTime, timeStamp, brokerUrl, brokerImplClass, entityType, operation, logFile, topicName, status, brokerTTL;</p> -<p>For a given feed that is scheduled, the name of the topic is same as the feed name. Falcon sends a map message for every feed instance that is deleted/archived/replicated depending upon the retention policy set in the feed definition. The JMS MapMessage sent to a topic has the following properties: entityName, feedNames, feedInstancePath, workflowId, runId, nominalTime, timeStamp, brokerUrl, brokerImplClass, entityType, operation, logFile, topicName, status, brokerTTL;</p> -<p>The JMS messages are automatically purged after a certain period (default 3 days) by the Falcon JMS house-keeping service.TTL (Time-to-live) for JMS message can be configured in the Falcon's startup.properties file.</p></div> +<p>Falcon has two types of notifications - System and User notifications.</p></div> +<div class="section"> +<h4>System notifications<a name="System_notifications"></a></h4> +<p>The System notifications are internally generated and used by Falcon to monitor the Falcon orchestrated workflow jobs. By default, Falcon starts an ActiveMQ embedded JMS server on Falcon machine on port 61616 as a daemon. Alternatively, users can make Falcon to use an existing JMS server instead of starting an embedded instance by doing the following 2 steps:</p> +<p></p> +<ul> +<li>Setting the property broker.url in the startup.properties as below</li></ul> +<div class="source"> +<pre> + *.broker.url=tcp://jms-server-host:61616 + +</pre></div> +<p></p> +<ul> +<li>Set the system property falcon.embeddedmq to false as below</li></ul> +<div class="source"> +<pre> + <FALCON-INSTALL-DIR>/bin/falcon-start -Dfalcon.embeddedmq=false + +</pre></div> +<p>Falcon uses FALCON.ENTITY.TOPIC to publish system notifications. This topic and the Map Message fields are internal and could change between releases.</p></div> +<div class="section"> +<h4>User notifications<a name="User_notifications"></a></h4> +<p>Falcon, in addition to the FALCON.ENTITY.TOPIC, also creates a JMS topic for every process/feed that is scheduled in Falcon as part of User notification. To enable User notifications, the broker url and implementation class of the JMS engine need to be specified in the cluster definition associated with the feed/process. Users may register consumers on the required topic to check the availability or status of feed instances. The User notification JMS broker instance can be same as the System notification or different.</p> +<p>The name of the JMS topic is same as the process/feed name. Falcon sends a map message for every feed instance that is created/deleted/replicated/imported/exported to the JMS topic. The JMS Map Message sent to a topic has the following fields:</p> +<p></p> +<ol style="list-style-type: decimal"> +<li>cluster - name of the current cluster the feed/process is dependent on.</li> +<li>entityType - type of the entity (feed or process).</li> +<li>entityName - name of the entity.</li> +<li>nominalTime - instance time (or data date).</li> +<li>operation - operation like generate, delete, replicate, import, export.</li> +<li>feedNames - name of the feeds which are generated/replicated/deleted/imported/exported.</li> +<li>feedInstancePaths - comma separated feed instance paths.</li> +<li>workflowId - current workflow-id of the instance.</li> +<li>workflowUser - user who owns the feed instance (i.e partition).</li> +<li>runId - current run-id of the instance.</li> +<li>status - status of the user workflow instance.</li> +<li>timeStamp - current timestamp.</li> +<li>logDir - log dir where lineage can be recorded.</li></ol> +<p>The JMS messages are automatically purged after a certain period (default 3 days) by the Falcon JMS house-keeping service. TTL (Time-to-live) for JMS message can be configured in the Falcon's startup.properties file.</p> +<p>The following example shows how to enable and read user notification by connecting to the JMS broker.</p> +<p>First, specify the JMS broker url in the cluster definition XML as shown below.</p> +<div class="source"> +<pre> + +<?xml version="1.0"?> +<!-- filename : primaryCluster.xml --> +<cluster colo="USWestOregon" description="oregonHadoopCluster" name="primaryCluster" xmlns="uri:falcon:cluster:0.1"> + <interfaces> + ... + ... + <interface type="messaging" endpoint="tcp://user-jms-broker-host:61616?daemon=true" version="5.1.6" /> + ... + </interfaces> +</cluster> + + +</pre></div> +<p>Next, use a JMS consumer (example below in Java) to read the message from the topic with the name FALCON.<feed-or-process-name></p> +<div class="source"> +<pre> +import org.apache.activemq.ActiveMQConnectionFactory; +import org.apache.activemq.command.ActiveMQMapMessage; +import javax.jms.ConnectionFactory; +import javax.jms.Connection; +import javax.jms.MessageConsumer; +import javax.jms.Topic; +import javax.jms.Session; +import javax.jms.TopicSession; + +public class FalconUserJMSClient { + public static void main(String[] args)throws Exception { + // Note: specify the JMS broker URL + String brokerUrl = "tcp://localhost:61616"; + + ConnectionFactory connectionFactory = new ActiveMQConnectionFactory(brokerUrl); + Connection connection = connectionFactory.createConnection(); + connection.setClientID("Falcon User JMS Consumer"); + TopicSession session = (TopicSession) connection.createSession(false, Session.AUTO_ACKNOWLEDGE); + try { + + // Note: the topic name for the feed will be FALCON.<feed-name> + Topic falconTopic = session.createTopic("FALCON.feed-sample"); + MessageConsumer consumer = session.createConsumer(falconTopic); + connection.start(); + while (true) { + ActiveMQMapMessage msg = (ActiveMQMapMessage) consumer.receive(); + System.out.println("cluster : " + msg.getString("cluster")); + System.out.println("entityType : " + msg.getString("entityType")); + System.out.println("entityName : " + msg.getString("entityName")); + System.out.println("nominalTime : " + msg.getString("nominalTime")); + System.out.println("operation : " + msg.getString("operation")); + + System.out.println("feedNames : " + msg.getString("feedNames")); + System.out.println("feedInstancePaths : " + msg.getString("feedInstancePaths")); + + System.out.println("workflowId : " + msg.getString("workflowId")); + System.out.println("workflowUser : " + msg.getString("workflowUser")); + System.out.println("runId : " + msg.getString("runId")); + System.out.println("status : " + msg.getString("status")); + System.out.println("timeStamp : " + msg.getString("timeStamp")); + System.out.println("logDir : " + msg.getString("logDir")); + + System.out.println("brokerUrl : " + msg.getString("brokerUrl")); + System.out.println("brokerImplClass : " + msg.getString("brokerImplClass")); + System.out.println("logFile : " + msg.getString("logFile")); + System.out.println("topicName : " + msg.getString("topicName")); + System.out.println("brokerTTL : " + msg.getString("brokerTTL")); + } + } finally { + if (session != null) { + session.close(); + } + if (connection != null) { + connection.close(); + } + } + } +} + +</pre></div></div> <div class="section"> <h3>Alerts<a name="Alerts"></a></h3> <p>Falcon generates alerts for unrecoverable errors into a log file by default. Users can view these alerts in the alerts.log file, by default this file is created under ${user.dir}/logs/ directory.</p> @@ -481,6 +627,9 @@ <h3>Audits<a name="Audits"></a></h3> <p>Falcon audits all user activity and captures them into a log file by default. Users can view these audits in the audit.log file, by default this file is created under ${user.dir}/logs/ directory.</p> <p>Users may also extend the Falcon Audit plugin to send audits to systems like Apache Argus, etc. by extending org.apache.falcon.plugin.AuditingPlugin interface.</p></div> +<div class="section"> +<h3>Metrics Collection In Graphite<a name="Metrics_Collection_In_Graphite"></a></h3> +<p>Falcon has support to send metrics to graphite more details regarding this can be found on <a href="./GraphiteMetricCollection.html">Graphite Metric Collection</a></p></div> </div> </div> </div> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/Recipes.html ---------------------------------------------------------------------- diff --git a/content/Recipes.html b/content/Recipes.html index bdf1966..f6d728c 100644 --- a/content/Recipes.html +++ b/content/Recipes.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Falcon Recipes</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,7 +148,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -158,6 +165,13 @@ <li> + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="Security.html" title="Security"> <i class="none"></i> Security</a> @@ -251,6 +265,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -315,9 +336,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/Security.html ---------------------------------------------------------------------- diff --git a/content/Security.html b/content/Security.html index 6accf55..445ed2e 100644 --- a/content/Security.html +++ b/content/Security.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Securing Falcon</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../index.html" id="bannerLeft"> - <img src="images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,7 +148,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="restapi/ResourceList.html" title="Rest API"> @@ -156,6 +163,13 @@ Hive Integration</a> </li> + <li> + + <a href="Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + <li class="active"> <a href="#"><i class="none"></i>Security</a> @@ -249,6 +263,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -313,9 +334,16 @@ <li> - <a href="0.9/index.html" title="0.9 (Current)"> + <a href="0.10/index.html" title="0.10 (Current)"> <i class="none"></i> - 0.9 (Current)</a> + 0.10 (Current)</a> + </li> + + <li> + + <a href="0.9/index.html" title="0.9"> + <i class="none"></i> + 0.9</a> </li> <li> @@ -419,7 +447,18 @@ <h2>Securing Falcon<a name="Securing_Falcon"></a></h2></div> <div class="section"> <h3>Overview<a name="Overview"></a></h3> -<p>Apache Falcon enforces authentication and authorization which are detailed below. Falcon also provides transport level security ensuring data confidentiality and integrity.</p></div> +<p>Apache Falcon provides the following security features:</p> +<ul> +<li>Support credential provider alias for passwords used in Falcon server.</li> +<li>Support authentication to identify proper users.</li> +<li>Support authorization to specify resource access permission for users or groups.</li> +<li>Support SSL to provide transport level security for data confidentiality and integrity.</li></ul></div> +<div class="section"> +<h3>Credential Provider Alias for Passwords<a name="Credential_Provider_Alias_for_Passwords"></a></h3> +<p>Server-side configuration properties (i.e. startup.properties) contain passwords and other sensitive information. In addition to specifying properties in plain text, we provide the user an option to use credential provider alias in the property file.</p> +<p>Take SMTP password for example. The user can store the password in a <a class="externalLink" href="http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CommandsManual.html#credential";>Hadoop credential provider</a> with the alias name <i>SMTPPasswordAlias</i>. In startup.properties where SMTP password is needed, the user can refer to its alias name <i>SMTPPasswordAlias</i> instead of providing the real password.</p> +<p>The alias property to be resolved through Hadoop credential provider should have the format: <i>credential.provider.alias.for.[property-key]</i>. For example, <i>credential.provider.alias.for.falcon.email.smtp.password=SMTPPasswordAlias</i> for SMTP password. Falcon server, during the start, will automatically retrieve the real password provided the alias name.</p> +<p>The user can specify the provider path with the property key <i>credential.provider.path</i>, e.g. <i>credential.provider.path=jceks://file/tmp/test.jceks</i>. If not specified, Falcon will use the default Hadoop credential provider path in core-site.xml.</p></div> <div class="section"> <h3>Authentication (User Identity)<a name="Authentication_User_Identity"></a></h3> <p>Apache Falcon enforces authentication on protected resources. Once authentication has been established it sets a signed HTTP Cookie that contains an authentication token with the user name, user principal, authentication type and expiration time.</p> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/content/falconcli/CommonCLI.html ---------------------------------------------------------------------- diff --git a/content/falconcli/CommonCLI.html b/content/falconcli/CommonCLI.html index 25bd858..1a156ca 100644 --- a/content/falconcli/CommonCLI.html +++ b/content/falconcli/CommonCLI.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-03-11 + | Generated by Apache Maven Doxia at 2016-08-08 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160311" /> + <meta name="Date-Revision-yyyymmdd" content="20160808" /> <meta http-equiv="Content-Language" content="en" /> <title>Falcon - Common CLI Options</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" /> @@ -30,7 +30,7 @@ <div id="banner"> <div class="pull-left"> <a href="../../index.html" id="bannerLeft"> - <img src="../images/falcon-logo.png" alt="Falcon" width="200px" height="45px"/> + <img src="../images/falcon-logo.png" alt="Apache Falcon" width="200px" height="45px"/> </a> </div> <div class="pull-right"> <a href="http://www.apache.org"; id="bannerRight"> @@ -58,7 +58,7 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-03-11</li> + <li id="publishDate" class="pull-right">Last Published: 2016-08-08</li> </ul> </div> @@ -123,6 +123,13 @@ <li> + <a href="../MigrationInstructions.html" title="Migrate to 0.10"> + <i class="none"></i> + Migrate to 0.10</a> + </li> + + <li> + <a href="../Operability.html" title="Operability"> <i class="none"></i> Operability</a> @@ -141,7 +148,7 @@ <i class="none"></i> Client (Falcon CLI)</a> </li> - + <li> <a href="../restapi/ResourceList.html" title="Rest API"> @@ -158,6 +165,13 @@ <li> + <a href="../Extensions.html" title="Server side Extensions"> + <i class="none"></i> + Server side Extensions</a> + </li> + + <li> + <a href="../Security.html" title="Security"> <i class="none"></i> Security</a> @@ -251,6 +265,13 @@ <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.10"; class="externalLink" title="0.10"> + <i class="none"></i> + 0.10</a> + </li> + + <li> + <a href="http://www.apache.org/dyn/closer.lua/falcon/0.9"; class="externalLink" title="0.9"> <i class="none"></i> 0.9</a> @@ -315,9 +336,16 @@ <li> - <a href="../0.9/index.html" title="0.9 (Current)"> + <a href="../0.10/index.html" title="0.10 (Current)"> + <i class="none"></i> + 0.10 (Current)</a> + </li> + + <li> + + <a href="../0.9/index.html" title="0.9"> <i class="none"></i> - 0.9 (Current)</a> + 0.9</a> </li> <li>
