Included Apache Incubator disclaimer and other updates to project documentation
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/commit/320f4d41 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/tree/320f4d41 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/diff/320f4d41 Branch: refs/heads/asf-site Commit: 320f4d412df19cb2d8bab041bebf3ce486133d7b Parents: ba4c65f Author: Hemanth Yamijala <[email protected]> Authored: Thu Jun 30 11:54:31 2016 +0530 Committer: Hemanth Yamijala <[email protected]> Committed: Thu Jun 30 11:54:31 2016 +0530 ---------------------------------------------------------------------- Architecture.html | 8 +- Authentication-Authorization.html | 376 +++++++++++ Bridge-Falcon.html | 25 +- Bridge-Hive.html | 63 +- Bridge-Sqoop.html | 8 +- Configuration.html | 52 +- HighAvailability.html | 8 +- InstallationSteps.html | 74 ++- Notification-Entity.html | 8 +- QuickStart.html | 10 +- Repository.html | 8 +- Search.html | 26 +- Security.html | 88 ++- StormAtlasHook.html | 8 +- TypeSystem.html | 8 +- api/application.wadl | 256 ++++++-- api/index.html | 6 +- api/resource_AdminResource.html | 27 +- api/resource_DataSetLineageResource.html | 234 +++++++ api/resource_EntityResource.html | 8 +- api/resource_EntityService.html | 376 +++++++++++ api/resource_HiveLineageResource.html | 230 ------- api/resource_LineageResource.html | 234 +++++++ api/resource_MetadataDiscoveryResource.html | 8 +- api/resource_RexsterGraphResource.html | 353 ----------- api/resource_TaxonomyService.html | 758 +++++++++++++++++++++++ api/resource_TypesResource.html | 8 +- api/rest.html | 6 +- index.html | 12 +- issue-tracking.html | 8 +- license.html | 8 +- mail-lists.html | 8 +- project-info.html | 8 +- source-repository.html | 8 +- team-list.html | 8 +- 35 files changed, 2548 insertions(+), 786 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Architecture.html ---------------------------------------------------------------------- diff --git a/Architecture.html b/Architecture.html index 480aae0..64a3749 100644 --- a/Architecture.html +++ b/Architecture.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Architecture</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Authentication-Authorization.html ---------------------------------------------------------------------- diff --git a/Authentication-Authorization.html b/Authentication-Authorization.html new file mode 100644 index 0000000..17971e7 --- /dev/null +++ b/Authentication-Authorization.html @@ -0,0 +1,376 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-06-30 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache Atlas – Authentication & Authorization in Apache Atlas.</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarEnabled"> + + + + + + <div id="topbar" class="navbar navbar-fixed-top "> + <div class="navbar-inner"> + <div class="container" style="width: 68%;"><div class="nav-collapse"> + + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Atlas <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="index.html" title="About">About</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="Wiki">Wiki</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="News">News</a> +</li> + + <li> <a href="https://git-wip-us.apache.org/repos/asf/incubator-atlas.git"; title="Git">Git</a> +</li> + + <li> <a href="https://issues.apache.org/jira/browse/ATLAS"; title="Jira">Jira</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS/PoweredBy"; title="Powered by">Powered by</a> +</li> + + <li> <a href="http://blogs.apache.org/atlas/"; title="Blog">Blog</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Information <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="project-info.html" title="Summary">Summary</a> +</li> + + <li> <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="http://webchat.freenode.net?channels=apacheatlas&uio=d4"; title="IRC">IRC</a> +</li> + + <li> <a href="team-list.html" title="Team">Team</a> +</li> + + <li> <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="license.html" title="License">License</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Releases <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/dyn/closer.cgi/incubator/atlas/0.6.0-incubating/"; title="0.6-incubating">0.6-incubating</a> +</li> + + <li> <a href="http://www.apache.org/dyn/closer.cgi/incubator/atlas/0.5.0-incubating/"; title="0.5-incubating">0.5-incubating</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="0.6.0-incubating/index.html" title="0.6-incubating">0.6-incubating</a> +</li> + + <li> <a href="0.5.0-incubating/index.html" title="0.5-incubating">0.5-incubating</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/foundation/how-it-works.html"; title="How Apache Works">How Apache Works</a> +</li> + + <li> <a href="http://www.apache.org/foundation/"; title="Foundation">Foundation</a> +</li> + + <li> <a href="http://www.apache.org/foundation/sponsorship.html"; title="Sponsoring Apache">Sponsoring Apache</a> +</li> + + <li> <a href="http://www.apache.org/foundation/thanks.html"; title="Thanks">Thanks</a> +</li> + </ul> + </li> + </ul> + + <form id="search-form" action="http://www.google.com/search"; method="get" class="navbar-search pull-right" > + + <input value="http://atlas.incubator.apache.org"; name="sitesearch" type="hidden"/> + <input class="search-query" name="q" id="query" type="text" /> +</form> +<script type="text/javascript" src="http://www.google.com/coop/cse/brand?form=search-form";></script> + + + + + + <iframe src="http://www.facebook.com/plugins/like.php?href=http://atlas.incubator.apache.org/atlas-docs&send=false&layout=button_count&show-faces=false&action=like&colorscheme=dark"; + scrolling="no" frameborder="0" + style="border:none; width:80px; height:20px; margin-top: 10px;" class="pull-right" ></iframe> + + <script type="text/javascript" src="https://apis.google.com/js/plusone.js";></script> + + <ul class="nav pull-right"><li style="margin-top: 10px;"> + + <div class="g-plusone" data-href="http://atlas.incubator.apache.org/atlas-docs"; data-size="medium" width="60px" align="right" ></div> + + </li></ul> + + + </div> + + </div> + </div> + </div> + + <div class="container"> + <div id="banner"> + <div class="pull-left"> + <a href=".." id="bannerLeft"> + <img src="images/atlas-logo.png" alt="Apache Atlas" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://incubator.apache.org"; id="bannerRight"> + <img src="images/apache-incubator-logo.png" alt="Apache Incubator"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Atlas"> + Atlas</a> + </li> + <li class="divider ">/</li> + <li class="">Authentication & Authorization in Apache Atlas.</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> + + </ul> + </div> + + + + <div id="bodyColumn" > + + <div class="section"> +<h3><a name="Authentication__Authorization_in_Apache_Atlas."></a>Authentication & Authorization in Apache Atlas.</h3></div> +<div class="section"> +<h4><a name="Authentication"></a>Authentication</h4> +<p>Atlas supports following authentication methods</p> +<p></p> +<ul> +<li><b>File</b></li> +<li><b>Kerberos</b></li> +<li><b>LDAP</b></li></ul> +<p>Following properties should be set true to enable the authentication of that type in <tt>atlas-application.properties</tt> file.</p> +<div class="source"> +<pre> +atlas.authentication.method.kerberos=true|false +atlas.authentication.method.ldap=true|false +atlas.authentication.method.file=true|false + +</pre></div> +<p>If two or more authentication methods are set to true, then the authentication falls back to the latter method if the earlier one fails. For example if Kerberos authentication is set to true and ldap authentication is also set to true then, if for a request without kerberos principal and keytab LDAP authentication will be used as a fallback scenario.</p></div> +<div class="section"> +<h5><a name="FILE_method."></a>FILE method.</h5> +<p>File authentication requires users' login details in users credentials file in the format specified below and the file path should set to property <tt>atlas.authentication.method.file.filename</tt> in <tt>atlas-application.properties</tt>.</p> +<div class="source"> +<pre> +atlas.authentication.method.file=true +atlas.authentication.method.file.filename=${sys:atlas.home}/conf/users-credentials.properties + +</pre></div> +<p>The users credentials file should have below format</p> +<div class="source"> +<pre> +username=group::sha256-password + +</pre></div> +<p>For e.g.</p> +<div class="source"> +<pre> +admin=ADMIN::e7cf3ef4f17c3999a94f2c6f612e8a888e5b1026878e4e19398b23bd38ec221a + +</pre></div> +<p>Users group can be either <b>ADMIN</b>, <b>DATA_STEWARD</b> OR <b>DATA_SCIENTIST</b></p> +<p><b>Note</b>:-password is encoded with sha256 encoding method and can be generated using unix tool.</p> +<p>For e.g.</p> +<div class="source"> +<pre> +echo -n "Password" | sha256sum +e7cf3ef4f17c3999a94f2c6f612e8a888e5b1026878e4e19398b23bd38ec221a - + +</pre></div></div> +<div class="section"> +<h5><a name="Kerberos_Method."></a>Kerberos Method.</h5> +<p>To enable the authentication in Kerberos mode in Atlas, set the property <tt>atlas.authentication.method.kerberos</tt> to true in <tt>atlas-application.properties</tt></p> +<div class="source"> +<pre> +atlas.authentication.method.kerberos = true + +</pre></div> +<p>Also following properties should be set.</p> +<div class="source"> +<pre> +atlas.authentication.method.kerberos.principal=<principal>/<fqdn>@EXAMPLE.COM +atlas.authentication.method.kerberos.keytab = /<key tab filepath>.keytab +atlas.authentication.method.kerberos.name.rules = RULE:[2:$1@$0]([email protected])s/.*/atlas/ + +</pre></div></div> +<div class="section"> +<h5><a name="LDAP_Method."></a>LDAP Method.</h5> +<p>To enable the authentication in LDAP mode in Atlas, set the property <tt>atlas.authentication.method.ldap</tt> to true and also set Ldap type to property <tt>atlas.authentication.method.ldap.type</tt> to LDAP or AD in <tt>atlas-application.properties</tt>. Use AD if connecting to Active Directory.</p> +<div class="source"> +<pre> +atlas.authentication.method.ldap=true +atlas.authentication.method.ldap.type=ldap|ad + +</pre></div> +<p>For LDAP or AD the following configuration needs to be set in atlas application properties.</p> +<p><b>Active Directory</b></p> +<div class="source"> +<pre> +atlas.authentication.method.ldap.ad.domain= example.com +atlas.authentication.method.ldap.ad.url=ldap://<AD server ip>:389 +atlas.authentication.method.ldap.ad.base.dn=DC=example,DC=com +atlas.authentication.method.ldap.ad.bind.dn=CN=Administrator,CN=Users,DC=example,DC=com +atlas.authentication.method.ldap.ad.bind.password=<password> +atlas.authentication.method.ldap.ad.referral=ignore +atlas.authentication.method.ldap.ad.user.searchfilter=(sAMAccountName={0}) +atlas.authentication.method.ldap.ad.default.role=ROLE_USER + +</pre></div> +<p><b>LDAP Directroy</b></p> +<div class="source"> +<pre> +atlas.authentication.method.ldap.url=ldap://<Ldap server ip>:389 +atlas.authentication.method.ldap.userDNpattern=uid={0],ou=users,dc=example,dc=com +atlas.authentication.method.ldap.groupSearchBase=dc=example,dc=com +atlas.authentication.method.ldap.groupSearchFilter=(member=cn={0},ou=users,dc=example,dc=com +atlas.authentication.method.ldap.groupRoleAttribute=cn +atlas.authentication.method.ldap.base.dn=dc=example,dc=com +atlas.authentication.method.ldap.bind.dn=cn=Manager,dc=example,dc=com +atlas.authentication.method.ldap.bind.password=<password> +atlas.authentication.method.ldap.referral=ignore +atlas.authentication.method.ldap.user.searchfilter=(uid={0}) +atlas.authentication.method.ldap.default.role=ROLE_USER + +</pre></div></div> +<div class="section"> +<h4><a name="Authorization"></a>Authorization</h4></div> +<div class="section"> +<h5><a name="Atlas_Authorization_Methods_SimpleRanger"></a>Atlas Authorization Methods [Simple/Ranger]</h5> +<p>To set authorization in atlas, update the <tt>atlas.authorizer.impl</tt> properties in <tt>atlas-application.properties</tt></p> +<ul> +<li><b>Simple</b></li> +<li><b>Ranger</b></li></ul> +<div class="source"> +<pre> +atlas.authorizer.impl=simple | ranger | <Qualified Authorizer Class Name> + +</pre></div></div> +<div class="section"> +<h5><a name="Simple_Authorizer."></a>Simple Authorizer.</h5> +<p>In Simple Authorizer the policy store file is configured locally. The path of policy store file is set in <tt>atlas.auth.policy.file</tt> property of <tt>atlas-application.properties</tt></p> +<div class="source"> +<pre> +atlas.auth.policy.file={{conf_dir}}/policy-store.txt + +</pre></div> +<p>The policy store file format is as follows:</p> +<div class="source"> +<pre> +Policy_Name;;User_Name:Operations_Allowed;;Group_Name:Operations_Allowed;;Resource_Type:Resource_Name + +</pre></div> +<p>eg. of admin policy:</p> +<div class="source"> +<pre> +adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:*,taxonomy:*,term:* + +</pre></div> +<p>Note : The User_Name, Group_Name and Operations_Allowed are comma(,) separated lists.</p> +<p>Authorizer Resource Types:</p> +<ul> +<li>Operation</li> +<li>Type</li> +<li>Entity</li> +<li>Taxonomy</li> +<li>Term</li> +<li>Unknown</li></ul> +<p>Operations_Allowed are r = read, w = write, u = update, d = delete</p></div> +<div class="section"> +<h5><a name="Ranger_Authorizer."></a>Ranger Authorizer.</h5> +<p>Ranger Authorizer is enabled by activating Atlas-Ranger plugin from Ambari.</p> +<p>For more details visit the <a class="externalLink" href="http://ranger.apache.org/";>Apache-Ranger documentation</a>.</p></div> + </div> + </div> + + <hr/> + + <footer> + <div class="container"> + <div class="row span12">Copyright © 2015-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + <p id="poweredBy" class="pull-right"> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </p> + + </div> + </footer> + </body> +</html> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Bridge-Falcon.html ---------------------------------------------------------------------- diff --git a/Bridge-Falcon.html b/Bridge-Falcon.html index df7f952..50f9761 100644 --- a/Bridge-Falcon.html +++ b/Bridge-Falcon.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Falcon Atlas Bridge</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> @@ -206,18 +206,26 @@ <p>The default falcon modelling is available in org.apache.atlas.falcon.model.FalconDataModelGenerator. It defines the following types:</p> <div class="source"> <pre> -falcon_process(ClassType) - super types [Process] - attributes [timestamp, owned-by, tags] +falcon_cluster(ClassType) - super types [Infrastructure] - attributes [timestamp, colo, owner, tags] +falcon_feed(ClassType) - super types [DataSet] - attributes [timestamp, stored-in, owner, groups, tags] +falcon_feed_creation(ClassType) - super types [Process] - attributes [timestamp, stored-in, owner] +falcon_feed_replication(ClassType) - super types [Process] - attributes [timestamp, owner] +falcon_process(ClassType) - super types [Process] - attributes [timestamp, runs-on, owner, tags, pipelines, workflow-properties] </pre></div> <p>One falcon_process entity is created for every cluster that the falcon process is defined for.</p> -<p>The entities are created and de-duped using unique qualified name. They provide namespace and can be used for querying/lineage as well. The unique attributes are:</p> +<p>The entities are created and de-duped using unique qualifiedName attribute. They provide namespace and can be used for querying/lineage as well. The unique attributes are:</p> <ul> -<li>falcon_process - attribute name - <process name>@<cluster name></li></ul></div> +<li>falcon_process - <process name>@<cluster name></li> +<li>falcon_cluster - <cluster name></li> +<li>falcon_feed - <feed name>@<cluster name></li> +<li>falcon_feed_creation - <feed name></li> +<li>falcon_feed_replication - <feed name></li></ul></div> <div class="section"> <h3><a name="Falcon_Hook"></a>Falcon Hook</h3> <p>Falcon supports listeners on falcon entity submission. This is used to add entities in Atlas using the model defined in org.apache.atlas.falcon.model.FalconDataModelGenerator. The hook submits the request to a thread pool executor to avoid blocking the command execution. The thread submits the entities as message to the notification server and atlas server reads these messages and registers the entities.</p> <ul> -<li>Add 'org.apache.falcon.atlas.service.AtlasService' to application.services in <falcon-conf>/startup.properties</li> +<li>Add 'org.apache.atlas.falcon.service.AtlasService' to application.services in <falcon-conf>/startup.properties</li> <li>Link falcon hook jars in falcon classpath - 'ln -s <atlas-home>/hook/falcon/* <falcon-home>/server/webapp/falcon/WEB-INF/lib/'</li> <li>In <falcon_conf>/falcon-env.sh, set an environment variable as follows:</li></ul> <div class="source"> @@ -238,7 +246,6 @@ falcon_process(ClassType) - super types [Process] - attributes [timestamp, owned <h3><a name="Limitations"></a>Limitations</h3> <p></p> <ul> -<li>Only the process entity creation is currently handled. This model will be expanded to include all Falcon metadata</li> <li>In falcon cluster entity, cluster name used should be uniform across components like hive, falcon, sqoop etc. If used with ambari, ambari cluster name should be used for cluster entity</li></ul></div> </div> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Bridge-Hive.html ---------------------------------------------------------------------- diff --git a/Bridge-Hive.html b/Bridge-Hive.html index 95d391a..b0a54c3 100644 --- a/Bridge-Hive.html +++ b/Bridge-Hive.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Hive Atlas Bridge</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> @@ -206,43 +206,42 @@ <p>The default hive modelling is available in org.apache.atlas.hive.model.HiveDataModelGenerator. It defines the following types:</p> <div class="source"> <pre> -hive_object_type(EnumType) - values [GLOBAL, DATABASE, TABLE, PARTITION, COLUMN] -hive_resource_type(EnumType) - values [JAR, FILE, ARCHIVE] -hive_principal_type(EnumType) - values [USER, ROLE, GROUP] hive_db(ClassType) - super types [Referenceable] - attributes [name, clusterName, description, locationUri, parameters, ownerName, ownerType] +hive_storagedesc(ClassType) - super types [Referenceable] - attributes [cols, location, inputFormat, outputFormat, compressed, numBuckets, serdeInfo, bucketCols, sortCols, parameters, storedAsSubDirectories] +hive_column(ClassType) - super types [Referenceable] - attributes [name, type, comment, table] +hive_table(ClassType) - super types [DataSet] - attributes [name, db, owner, createTime, lastAccessTime, comment, retention, sd, partitionKeys, columns, aliases, parameters, viewOriginalText, viewExpandedText, tableType, temporary] +hive_process(ClassType) - super types [Process] - attributes [name, startTime, endTime, userName, operationType, queryText, queryPlan, queryId] +hive_principal_type(EnumType) - values [USER, ROLE, GROUP] hive_order(StructType) - attributes [col, order] -hive_resourceuri(StructType) - attributes [resourceType, uri] hive_serde(StructType) - attributes [name, serializationLib, parameters] -hive_type(ClassType) - super types [] - attributes [name, type1, type2, fields] -hive_storagedesc(ClassType) - super types [Referenceable] - attributes [cols, location, inputFormat, outputFormat, compressed, numBuckets, serdeInfo, bucketCols, sortCols, parameters, storedAsSubDirectories] -hive_role(ClassType) - super types [] - attributes [roleName, createTime, ownerName] -hive_column(ClassType) - super types [Referenceable] - attributes [name, type, comment] -hive_table(ClassType) - super types [DataSet] - attributes [tableName, db, owner, createTime, lastAccessTime, comment, retention, sd, partitionKeys, columns, parameters, viewOriginalText, viewExpandedText, tableType, temporary] -hive_partition(ClassType) - super types [Referenceable] - attributes [values, table, createTime, lastAccessTime, sd, columns, parameters] -hive_process(ClassType) - super types [Process] - attributes [startTime, endTime, userName, operationType, queryText, queryPlan, queryId, queryGraph] </pre></div> -<p>The entities are created and de-duped using unique qualified name. They provide namespace and can be used for querying/lineage as well. Note that dbName and tableName should be in lower case. clusterName is explained below.</p> +<p>The entities are created and de-duped using unique qualified name. They provide namespace and can be used for querying/lineage as well. Note that dbName, tableName and columnName should be in lower case. clusterName is explained below.</p> <ul> <li>hive_db - attribute qualifiedName - <dbName>@<clusterName></li> -<li>hive_table - attribute name - <dbName>.<tableName>@<clusterName></li> +<li>hive_table - attribute qualifiedName - <dbName>.<tableName>@<clusterName></li> <li>hive_column - attribute qualifiedName - <dbName>.<tableName>.<columnName>@<clusterName></li> -<li>hive_partition - attribute qualifiedName - <dbName>.<tableName>.<partitionValues('-' separated)>@<clusterName></li> <li>hive_process - attribute name - <queryString> - trimmed query string in lower case</li></ul></div> <div class="section"> <h3><a name="Importing_Hive_Metadata"></a>Importing Hive Metadata</h3> -<p>org.apache.atlas.hive.bridge.HiveMetaStoreBridge imports the hive metadata into Atlas using the model defined in org.apache.atlas.hive.model.HiveDataModelGenerator. import-hive.sh command can be used to facilitate this. Set the following configuration in <atlas-conf>/atlas-application.properties and set environment variable $HIVE_CONF_DIR to the hive conf directory:</p> +<p>org.apache.atlas.hive.bridge.HiveMetaStoreBridge imports the Hive metadata into Atlas using the model defined in org.apache.atlas.hive.model.HiveDataModelGenerator.</p> +<p>import-hive.sh command can be used to facilitate this. The script needs Hadoop and Hive classpath jars.</p> +<p>* For Hadoop jars, please make sure that the environment variable HADOOP_CLASSPATH is set. Another way is to set HADOOP_HOME to point to root directory of your Hadoop installation</p> +<p>* Similarly, for Hive jars, set HIVE_HOME to the root of Hive installation</p> +<p>* Set environment variable HIVE_CONF_DIR to Hive configuration directory</p> <div class="source"> <pre> - <property> - <name>atlas.cluster.name</name> - <value>primary</value> - </property> - + Usage: <atlas package>/bin/import-hive.sh + </pre></div> -<p>Usage: <atlas package>/bin/import-hive.sh. The logs are in <atlas package>/logs/import-hive.log</p> +<p>The logs are in <atlas package>/logs/import-hive.log</p> <p>If you you are importing metadata in a kerberized cluster you need to run the command like this:</p> -<p><atlas package>/bin/import-hive.sh -Dsun.security.jgss.debug=true -Djavax.security.auth.useSubjectCredsOnly=false -Djava.security.krb5.conf=[krb5.conf location] -Djava.security.auth.login.config=[jaas.conf location]</p> +<div class="source"> +<pre> +<atlas package>/bin/import-hive.sh -Dsun.security.jgss.debug=true -Djavax.security.auth.useSubjectCredsOnly=false -Djava.security.krb5.conf=[krb5.conf location] -Djava.security.auth.login.config=[jaas.conf location] + +</pre></div> +<p></p> <ul> <li>krb5.conf is typically found at /etc/krb5.conf</li> <li>for details about jaas.conf and a suggested location see the <a href="./Security.html">atlas security documentation</a></li></ul></div> @@ -273,7 +272,7 @@ hive_process(ClassType) - super types [Process] - attributes [startTime, endTime <li>Copy <atlas-conf>/atlas-application.properties to the hive conf directory.</li></ul> <p>The following properties in <atlas-conf>/atlas-application.properties control the thread pool and notification details:</p> <ul> -<li>atlas.hook.hive.synchronous - boolean, true to run the hook synchronously. default false</li> +<li>atlas.hook.hive.synchronous - boolean, true to run the hook synchronously. default false. Recommended to be set to false to avoid delays in hive query completion.</li> <li>atlas.hook.hive.numRetries - number of retries for notification failure. default 3</li> <li>atlas.hook.hive.minThreads - core number of threads. default 5</li> <li>atlas.hook.hive.maxThreads - maximum number of threads. default 5</li> @@ -285,7 +284,15 @@ hive_process(ClassType) - super types [Process] - attributes [startTime, endTime <p></p> <ul> <li>Since database name, table name and column names are case insensitive in hive, the corresponding names in entities are lowercase. So, any search APIs should use lowercase while querying on the entity names</li> -<li>Only the following hive operations are captured by hive hook currently - create database, create table, create view, CTAS, load, import, export, query, alter database, alter table(except alter table replace columns and alter table change column position), alter view (except replacing and changing column position)</li></ul></div> +<li>The following hive operations are captured by hive hook currently +<ul> +<li>create database</li> +<li>create table/view, create table as select</li> +<li>load, import, export</li> +<li>DMLs (insert)</li> +<li>alter database</li> +<li>alter table (skewed table information, stored as, protection is not supported)</li> +<li>alter view</li></ul></li></ul></div> </div> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Bridge-Sqoop.html ---------------------------------------------------------------------- diff --git a/Bridge-Sqoop.html b/Bridge-Sqoop.html index 7e9764f..607be2e 100644 --- a/Bridge-Sqoop.html +++ b/Bridge-Sqoop.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Sqoop Atlas Bridge</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Configuration.html ---------------------------------------------------------------------- diff --git a/Configuration.html b/Configuration.html index b8f74c5..55b4e08 100644 --- a/Configuration.html +++ b/Configuration.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Configuring Apache Atlas - Application Properties</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> @@ -244,7 +244,7 @@ zookeeper.znode.parent=/hbase-unsecure echo "grant 'atlas', 'RWXCA', 'titan'" | hbase shell </pre></div> -<p>Note that HBase is included in the distribution so that a standalone instance of HBase can be started as the default storage backend for the graph repository.</p></div> +<p>Note that if the embedded-hbase-solr profile is used then HBase is included in the distribution so that a standalone instance of HBase can be started as the default storage backend for the graph repository. Using the embedded-hbase-solr profile will configure Atlas so that HBase instance will be started and stopped along with the Atlas server by default. To use the embedded-hbase-solr profile please see "Building Atlas" in the <a href="./InstallationSteps.html">Installation Steps</a> section.</p></div> <div class="section"> <h4><a name="Graph_Search_Index"></a>Graph Search Index</h4> <p>This section sets up the graph db - titan - to use an search indexing system. The example configuration below sets up to use an embedded Elastic search indexing system.</p> @@ -266,7 +266,8 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000 atlas.graph.index.search.solr.mode=cloud atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181 -</pre></div></div> +</pre></div> +<p>Also note that if the embedded-hbase-solr profile is used then Solr is included in the distribution so that a standalone instance of Solr can be started as the default search indexing backend. Using the embedded-hbase-solr profile will configure Atlas so that the standalone Solr instance will be started and stopped along with the Atlas server by default. To use the embedded-hbase-solr profile please see "Building Atlas" in the <a href="./InstallationSteps.html">Installation Steps</a> section.</p></div> <div class="section"> <h4><a name="Choosing_between_Persistence_and_Indexing_Backends"></a>Choosing between Persistence and Indexing Backends</h4> <p>Refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html";>http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html</a> and <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html";>http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html</a> for choosing between the persistence backends. BerkeleyDB is suitable for smaller data sets in the range of upto 10 million vertices with ACID gurantees. HBase on the other hand doesnt provide ACID guarantees but is able to scale for larger graphs. HBase also provides HA inherently.</p></div> @@ -317,6 +318,30 @@ atlas.kafka.hook.group.id=atlas <pre> atlas.kafka.entities.group.id=<consumer id> +</pre></div> +<p>These configuration parameters are useful for setting up Kafka topics via Atlas provided scripts, described in the <a href="./InstallationSteps.html">Installation Steps</a> page.</p> +<div class="source"> +<pre> +# Whether to create the topics automatically, default is true. +# Comma separated list of topics to be created, default is "ATLAS_HOOK,ATLAS_ENTITES" +atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES +# Number of replicas for the Atlas topics, default is 1. Increase for higher resilience to Kafka failures. +atlas.notification.replicas=1 +# Enable the below two properties if Kafka is running in Kerberized mode. +# Set this to the service principal representing the Kafka service +atlas.notification.kafka.service.principal=kafka/[email protected] +# Set this to the location of the keytab file for Kafka +#atlas.notification.kafka.keytab.location=/etc/security/keytabs/kafka.service.keytab + +</pre></div> +<p>These configuration parameters are useful for saving messages in case there are issues in reaching Kafka for sending messages.</p> +<div class="source"> +<pre> +# Whether to save messages that failed to be sent to Kafka, default is true +atlas.notification.log.failed.messages=true +# If saving messages is enabled, the file name to save them to. This file will be created under the log directory of the hook's host component - like HiveServer2 +atlas.notification.failed.messages.filename=atlas_hook_failed_messages.log + </pre></div></div> <div class="section"> <h3><a name="Client_Configs"></a>Client Configs</h3> @@ -387,6 +412,21 @@ atlas.client.ha.sleep.interval.ms=5000 atlas.server.run.setup.on.start=false </pre></div></div> +<div class="section"> +<h3><a name="Performance_configuration_items"></a>Performance configuration items</h3> +<p>The following properties can be used to tune performance of Atlas under specific circumstances:</p> +<div class="source"> +<pre> +# The number of times Atlas code tries to acquire a lock (to ensure consistency) while committing a transaction. +# This should be related to the amount of concurrency expected to be supported by the server. For e.g. with retries set to 10, upto 100 threads can concurrently create types in the Atlas system. +# If this is set to a low value (default is 3), concurrent operations might fail with a PermanentLockingException. +atlas.graph.storage.lock.retries=10 + +# Milliseconds to wait before evicting a cached entry. This should be > atlas.graph.storage.lock.wait-time x atlas.graph.storage.lock.retries +# If this is set to a low value (default is 10000), warnings on transactions taking too long will occur in the Atlas application log. +atlas.graph.storage.cache.db-cache-time=120000 + +</pre></div></div> </div> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/HighAvailability.html ---------------------------------------------------------------------- diff --git a/HighAvailability.html b/HighAvailability.html index f91e088..006209a 100644 --- a/HighAvailability.html +++ b/HighAvailability.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Fault Tolerance and High Availability Options</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/InstallationSteps.html ---------------------------------------------------------------------- diff --git a/InstallationSteps.html b/InstallationSteps.html index 45c05e1..4d1ae37 100644 --- a/InstallationSteps.html +++ b/InstallationSteps.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Building & Installing Apache Atlas</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> @@ -209,7 +209,7 @@ git clone https://git-wip-us.apache.org/repos/asf/incubator-atlas.git atlas cd atlas -export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=512m" && mvn clean install +export MAVEN_OPTS="-Xmx1536m -XX:MaxPermSize=512m" && mvn clean install </pre></div> <p>Once the build successfully completes, artifacts can be packaged for deployment.</p> @@ -220,6 +220,39 @@ mvn clean package -Pdist </pre></div> +<p>To build a distribution that configures Atlas for external HBase and Solr, build with the external-hbase-solr profile.</p> +<div class="source"> +<pre> + +mvn clean package -Pdist,external-hbase-solr + + +</pre></div> +<p>Note that when the external-hbase-solr profile is used the following steps need to be completed to make Atlas functional.</p> +<ul> +<li>Configure atlas.graph.storage.hostname (see "Graph persistence engine - HBase" in the <a href="./Configuration.html">Configuration</a> section).</li> +<li>Configure atlas.graph.index.search.solr.zookeeper-url (see "Graph Search Index - Solr" in the <a href="./Configuration.html">Configuration</a> section).</li> +<li>Set HBASE_CONF_DIR to point to a valid HBase config directory (see "Graph persistence engine - HBase" in the <a href="./Configuration.html">Configuration</a> section).</li> +<li>Create the SOLR indices (see "Graph Search Index - Solr" in the <a href="./Configuration.html">Configuration</a> section).</li></ul> +<p>To build a distribution that packages HBase and Solr, build with the embedded-hbase-solr profile.</p> +<div class="source"> +<pre> + +mvn clean package -Pdist,embedded-hbase-solr + + +</pre></div> +<p>Using the embedded-hbase-solr profile will configure Atlas so that an HBase instance and a Solr instance will be started and stopped along with the Atlas server by default.</p> +<p>Atlas also supports building a distribution that can use BerkeleyDB and Elastic search as the graph and index backends. To build a distribution that is configured for these backends, build with the berkeley-elasticsearch profile.</p> +<div class="source"> +<pre> + +mvn clean package -Pdist,berkeley-elasticsearch + + +</pre></div> +<p>An additional step is required for the binary built using this profile to be used along with the Atlas distribution. Due to licensing requirements, Atlas does not bundle the BerkeleyDB Java Edition in the tarball.</p> +<p>You can download the Berkeley DB jar file from the URL: <verbatim><a class="externalLink" href="http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip";>http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip</a></verbatim> and copy the je-5.0.73.jar to the ${atlas_home}/libext directory.</p> <p>Tar can be found in atlas/distro/target/apache-atlas-${project.version}-bin.tar.gz</p> <p>Tar is structured as follows</p> <div class="source"> @@ -254,15 +287,20 @@ mvn clean package -Pdist |- server |- webapp |- atlas.war +|- solr + |- bin + ... |- README -|- NOTICE.txt -|- LICENSE.txt +|- NOTICE +|- LICENSE |- DISCLAIMER.txt |- CHANGES.txt </pre></div> -<p>Note that HBase is included in the distribution so that a standalone instance of HBase can be started as the default storage backend for the graph repository. During Atlas installation the conf/hbase/hbase-site.xml.template gets expanded and moved to hbase/conf/hbase-site.xml for the initial standalone HBase configuration. To configure ATLAS graph persistence for a different HBase instance, please see "Graph persistence engine - HBase" in the <a href="./Configuration.html">Configuration</a> section.</p></div> +<p>Note that if the embedded-hbase-solr profile is specified for the build then HBase and Solr are included in the distribution.</p> +<p>In this case, a standalone instance of HBase can be started as the default storage backend for the graph repository. During Atlas installation the conf/hbase/hbase-site.xml.template gets expanded and moved to hbase/conf/hbase-site.xml for the initial standalone HBase configuration. To configure ATLAS graph persistence for a different HBase instance, please see "Graph persistence engine - HBase" in the <a href="./Configuration.html">Configuration</a> section.</p> +<p>Also, a standalone instance of Solr can be started as the default search indexing backend. To configure ATLAS search indexing for a different Solr instance please see "Graph Search Index - Solr" in the <a href="./Configuration.html">Configuration</a> section.</p></div> <div class="section"> <h4><a name="Installing__Running_Atlas"></a>Installing & Running Atlas</h4></div> <div class="section"> @@ -354,6 +392,13 @@ export ATLAS_SERVER_OPTS="-Djava.awt.headless=true -Djava.security.krb5.rea <ul> <li>3 or 5 ZooKeeper nodes</li> <li>Atleast 3 RegionServer nodes. It would be ideal to run the DataNodes on the same hosts as the Region servers for data locality.</li></ul> +<p>HBase tablename in Titan can be set using the following configuration in ATLAS_HOME/conf/atlas-application.properties:</p> +<div class="source"> +<pre> +atlas.graph.storage.hbase.table=apache_atlas_titan +atlas.audit.hbase.tablename=apache_atlas_entity_audit + +</pre></div> <p><b>Configuring SOLR as the Indexing Backend for the Graph Repository</b></p> <p>By default, Atlas uses Titan as the graph repository and is the only graph repository implementation available currently. For configuring Titan to work with Solr, please follow the instructions below</p> <p></p> @@ -372,16 +417,17 @@ export ATLAS_SERVER_OPTS="-Djava.awt.headless=true -Djava.security.krb5.rea </pre></div> <p></p> <ul> -<li>Run the following commands from SOLR_HOME directory to create collections in Solr corresponding to the indexes that Atlas uses. In the case that the ATLAS and SOLR instance are on 2 different hosts,</li></ul>first copy the required configuration files from ATLAS_HOME/conf/solr on the ATLAS instance host to the Solr instance host. SOLR_CONF in the below mentioned commands refer to the directory where the solr configuration files have been copied to on Solr host: +<li>Run the following commands from SOLR_BIN (e.g. $SOLR_HOME/bin) directory to create collections in Solr corresponding to the indexes that Atlas uses. In the case that the ATLAS and SOLR instance are on 2 different hosts,</li></ul>first copy the required configuration files from ATLAS_HOME/conf/solr on the ATLAS instance host to the Solr instance host. SOLR_CONF in the below mentioned commands refer to the directory where the solr configuration files have been copied to on Solr host: <div class="source"> <pre> - bin/solr create -c vertex_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor - bin/solr create -c edge_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor - bin/solr create -c fulltext_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor + $SOLR_BIN/solr create -c vertex_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor + $SOLR_BIN/solr create -c edge_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor + $SOLR_BIN/solr create -c fulltext_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor </pre></div> <p>Note: If numShards and replicationFactor are not specified, they default to 1 which suffices if you are trying out solr with ATLAS on a single node instance. Otherwise specify numShards according to the number of hosts that are in the Solr cluster and the maxShardsPerNode configuration. The number of shards cannot exceed the total number of Solr nodes in your !SolrCloud cluster.</p> <p>The number of replicas (replicationFactor) can be set according to the redundancy required.</p> +<p>Also note that solr will automatically be called to create the indexes when the Atlas server is started if the SOLR_BIN and SOLR_CONF environment variables are set and the search indexing backend is set to 'solr5'.</p> <p></p> <ul> <li>Change ATLAS configuration to point to the Solr instance setup. Please make sure the following configurations are set to the below values in ATLAS_HOME/conf/atlas-application.properties</li></ul> @@ -396,7 +442,9 @@ export ATLAS_SERVER_OPTS="-Djava.awt.headless=true -Djava.security.krb5.rea <ul> <li>Restart Atlas</li></ul> <p>For more information on Titan solr configuration , please refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/solr.htm";>http://s3.thinkaurelius.com/docs/titan/0.5.4/solr.htm</a></p> -<p>Pre-requisites for running Solr in cloud mode * Memory - Solr is both memory and CPU intensive. Make sure the server running Solr has adequate memory, CPU and disk. Solr works well with 32GB RAM. Plan to provide as much memory as possible to Solr process * Disk - If the number of entities that need to be stored are large, plan to have at least 500 GB free space in the volume where Solr is going to store the index data * SolrCloud has support for replication and sharding. It is highly recommended to use SolrCloud with at least two Solr nodes running on different servers with replication enabled. If using SolrCloud, then you also need ZooKeeper installed and configured with 3 or 5 ZooKeeper nodes</p></div> +<p>Pre-requisites for running Solr in cloud mode * Memory - Solr is both memory and CPU intensive. Make sure the server running Solr has adequate memory, CPU and disk. Solr works well with 32GB RAM. Plan to provide as much memory as possible to Solr process * Disk - If the number of entities that need to be stored are large, plan to have at least 500 GB free space in the volume where Solr is going to store the index data * SolrCloud has support for replication and sharding. It is highly recommended to use SolrCloud with at least two Solr nodes running on different servers with replication enabled. If using SolrCloud, then you also need ZooKeeper installed and configured with 3 or 5 ZooKeeper nodes</p> +<p><b>Configuring Kafka Topics</b></p> +<p>Atlas uses Kafka to ingest metadata from other components at runtime. This is described in the <a href="./Architecture.html">Architecture page</a> in more detail. Depending on the configuration of Kafka, sometimes you might need to setup the topics explicitly before using Atlas. To do so, Atlas provides a script <tt>bin/atlas_kafka_setup.py</tt> which can be run from the Atlas server. In some environments, the hooks might start getting used first before Atlas server itself is setup. In such cases, the topics can be run on the hosts where hooks are installed using a similar script <tt>hook-bin/atlas_kafka_setup_hook.py</tt>. Both these use configuration in <tt>atlas-application.properties</tt> for setting up the topics. Please refer to the <a href="./Configuration.html">Configuration page</a> for these details.</p></div> <div class="section"> <h5><a name="Setting_up_Atlas"></a>Setting up Atlas</h5> <p>There are a few steps that setup dependencies of Atlas. One such example is setting up the Titan schema in the storage backend of choice. In a simple single server setup, these are automatically setup with default configuration when the server first accesses these dependencies.</p> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Notification-Entity.html ---------------------------------------------------------------------- diff --git a/Notification-Entity.html b/Notification-Entity.html index 0471908..0ba4536 100644 --- a/Notification-Entity.html +++ b/Notification-Entity.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Entity Change Notifications</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/QuickStart.html ---------------------------------------------------------------------- diff --git a/QuickStart.html b/QuickStart.html index 8f32fe4..6aea598 100644 --- a/QuickStart.html +++ b/QuickStart.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Quick Start Guide</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> @@ -221,7 +221,7 @@ <p>A simple dashboard with search is available.</p> <p></p> <ul> -<li><a class="externalLink" href="http://localhost:21000/dashboard";>http://localhost:21000/dashboard</a></li></ul></div> +<li><a class="externalLink" href="http://localhost:21000/";>http://localhost:21000/</a></li></ul></div> </div> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Repository.html ---------------------------------------------------------------------- diff --git a/Repository.html b/Repository.html index ed0f43d..3649cad 100644 --- a/Repository.html +++ b/Repository.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Repository</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Search.html ---------------------------------------------------------------------- diff --git a/Search.html b/Search.html index ac79c42..58eca56 100644 --- a/Search.html +++ b/Search.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Search</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> @@ -214,7 +214,7 @@ queryWithPath: query ~ opt(WITHPATH) query: rep1sep(singleQuery, opt(COMMA)) -singleQuery: singleQrySrc ~ opt(loopExpression) ~ opt(selectClause) +singleQuery: singleQrySrc ~ opt(loopExpression) ~ opt(selectClause) ~ opt(orderby) ~ opt(limitOffset) singleQrySrc = FROM ~ fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) | WHERE ~ (expr ^? notIdExpression) | @@ -223,6 +223,14 @@ singleQrySrc = FROM ~ fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) | fromSrc: identifier ~ AS ~ alias | identifier +orderby: ORDERBY ~ order ~ opt (sortOrder) + +limitOffset: LIMIT ~ lmt ~ opt (offset) + +offset: OFFSET ~ offsetValue + +sortOrder = ASC | DESC + loopExpression: LOOP ~ (LPAREN ~> query <~ RPAREN) ~ opt(intConstant <~ TIMES) ~ opt(AS ~> alias) selectClause: SELECT ~ rep1sep(selectExpression, COMMA) @@ -273,6 +281,9 @@ literal: booleanConstant | <ul> <li>The <i>WithPath</i> clause can be used with transitive closure queries to retrieve the Path that</li></ul>connects the two related Entities. (We also provide a higher level interface for Closure Queries see scaladoc for 'org.apache.atlas.query.ClosureQuery') <ul> +<li>ORDERBY is optional. Orderby clause should be specified in single quote ('). When order by clause is specified case insensitive sorting is done in ascending order.</li></ul>For sorting in descending order specify 'DESC' after order by clause. If no order by is specified then no default sorting is applied. +<ul> +<li>LIMIT is optional. It limits the maximum number of objects to be fetched starting from specified optional offset. If no offset is specified count starts from beginning.</li> <li>There are couple of Predicate functions different from SQL: <ul> <li><i>is</i> or <i>isa</i>can be used to filter Entities that have a particular Trait.</li> @@ -284,6 +295,11 @@ literal: booleanConstant | <ul> <li>from DB</li> <li>DB where name="Reporting" select name, owner</li> +<li>DB where name="Reporting" select name, owner orderby 'name'</li> +<li>DB where name="Reporting" select name limit 10</li> +<li>DB where name="Reporting" select name, owner limit 10 offset 0</li> +<li>DB where name="Reporting" select name, owner orderby 'name' limit 10 offset 5</li> +<li>DB where name="Reporting" select name, owner orderby 'name' desc limit 10 offset 5</li> <li>DB has name</li> <li>DB is JdbcAccess</li> <li>Column where Column isa PII</li> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/Security.html ---------------------------------------------------------------------- diff --git a/Security.html b/Security.html index 9649d81..a69c6b0 100644 --- a/Security.html +++ b/Security.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Security Features of Apache Atlas</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> @@ -244,28 +244,76 @@ <p>Note that when Atlas is configured with HBase as the storage backend in a secure cluster, the graph db (titan) needs sufficient user permissions to be able to create and access an HBase table. To grant the appropriate permissions see <a href="./Configuration.html">Graph persistence engine - Hbase</a>.</p></div> <div class="section"> <h4><a name="JAAS_configuration"></a>JAAS configuration</h4> -<p>When Atlas is configured with HBase as the storage backend in a secure cluster, a JAAS configuration file should be created and specified so that the HBase client can attempt to SASL-authenticate.</p> +<p>In a secure cluster, some of the components (such as Kafka) that Atlas interacts with, require Atlas to authenticate itself to them using JAAS. The following properties are used to set up appropriate JAAS Configuration.</p> <p></p> <ul> -<li>Create Atlas JAAS configuration file (e.g. /etc/atlas/conf/atlas-jaas.conf).</li></ul> +<li><code>atlas.jaas.<code>client-id<code>.loginModuleName<code> - the authentication method used by the component (for example, com.sun.security.auth.module.Krb5LoginModule)</li> +<li><code>atlas.jaas.<code>client-id<code>.loginModuleControlFlag<code> (required|requisite|sufficient|optional) [default: required]</li> +<li><code>atlas.jaas.<code>client-id<code>.option.useKeyTab<code> (true|false)</li> +<li><code>atlas.jaas.<code>client-id<code>.option.storeKey<code> (true | false)</li> +<li><code>atlas.jaas.<code>client-id<code>.option.serviceName<code> - service name of server component</li> +<li><code>atlas.jaas.<code>client-id<code>.option.keyTab<code> = <atlas keytab></li> +<li><code>atlas.jaas.<code>client-id<code>.option.principal<code> = <atlas principal></li></ul> +<p>For example, the following property settings in jaas-application.properties file</p> <div class="source"> <pre> - Client { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - useTicketCache=false - storeKey=true - doNotPrompt=false - keyTab="<atlas keytab>" - principal="<atlas principal>"; - }; + +atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.KafkaClient.loginModuleControlFlag = required +atlas.jaas.KafkaClient.option.useKeyTab = true +atlas.jaas.KafkaClient.option.storeKey = true +atlas.jaas.KafkaClient.option.serviceName = kafka +atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/kafka_client.keytab +atlas.jaas.KafkaClient.option.principal = [email protected] + +atlas.jaas.MyClient.0.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.MyClient.0.loginModuleControlFlag = required +atlas.jaas.MyClient.0.option.useKeyTab = true +atlas.jaas.MyClient.0.option.storeKey = true +atlas.jaas.MyClient.0.option.serviceName = kafka +atlas.jaas.MyClient.0.option.keyTab = /etc/security/keytabs/kafka_client.keytab +atlas.jaas.MyClient.0.option.principal = [email protected] + +atlas.jaas.MyClient.1.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.MyClient.1.loginModuleControlFlag = optional +atlas.jaas.MyClient.1.option.useKeyTab = true +atlas.jaas.MyClient.1.option.storeKey = true +atlas.jaas.MyClient.1.option.serviceName = kafka +atlas.jaas.MyClient.1.option.keyTab = /etc/security/keytabs/kafka_client.keytab +atlas.jaas.MyClient.1.option.principal = [email protected] + </pre></div> -<p></p> -<ul> -<li>Update Atlas ATLAS_OPTS to include ‘java.security.auth.login.config’ set to the above Atlas JAAS configuration file. -<ul> -<li>For example, <code>-Djava.security.auth.login.config=/etc/atlas/conf/atlas-jaas.conf</code></li></ul></li></ul></div> +<p>will set the JAAS configuration that is equivalent to the following jaas.conf file entries.</p> +<div class="source"> +<pre> + +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + serviceName=kafka + keyTab="/etc/security/keytabs/kafka_client.keytab" + principal="[email protected]"; +}; +MyClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + serviceName=kafka keyTab="/etc/security/keytabs/kafka_client.keytab" + principal="[email protected]"; +}; +MyClient { + com.sun.security.auth.module.Krb5LoginModule optional + useKeyTab=true + storeKey=true + serviceName=kafka + keyTab="/etc/security/keytabs/kafka_client.keytab" + principal="[email protected]"; +}; + + +</pre></div></div> <div class="section"> <h4><a name="SPNEGO-based_HTTP_Authentication"></a>SPNEGO-based HTTP Authentication</h4> <p>HTTP access to the Atlas platform can be secured by enabling the platform's SPNEGO support. There are currently two supported authentication mechanisms:</p> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/StormAtlasHook.html ---------------------------------------------------------------------- diff --git a/StormAtlasHook.html b/StormAtlasHook.html index b6c3099..4141e74 100644 --- a/StormAtlasHook.html +++ b/StormAtlasHook.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Storm Atlas Bridge</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/320f4d41/TypeSystem.html ---------------------------------------------------------------------- diff --git a/TypeSystem.html b/TypeSystem.html index 6c9b46b..c6be939 100644 --- a/TypeSystem.html +++ b/TypeSystem.html @@ -1,13 +1,13 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2016-04-25 + | Generated by Apache Maven Doxia at 2016-06-30 | Rendered using Apache Maven Fluido Skin 1.3.0 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160425" /> + <meta name="Date-Revision-yyyymmdd" content="20160630" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache Atlas – Type System</title> <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> @@ -189,8 +189,8 @@ - <li id="publishDate" class="pull-right">Last Published: 2016-04-25</li> <li class="divider pull-right">|</li> - <li id="projectVersion" class="pull-right">Version: 0.7-incubating-SNAPSHOT</li> + <li id="publishDate" class="pull-right">Last Published: 2016-06-30</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.8-incubating-SNAPSHOT</li> </ul> </div>
