http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/60041d8d/0.7.0-incubating/Configuration.html ---------------------------------------------------------------------- diff --git a/0.7.0-incubating/Configuration.html b/0.7.0-incubating/Configuration.html new file mode 100644 index 0000000..58f157a --- /dev/null +++ b/0.7.0-incubating/Configuration.html @@ -0,0 +1,459 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-07-09 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160709" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache Atlas – Configuring Apache Atlas - Application Properties</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarEnabled"> + + + + + + <div id="topbar" class="navbar navbar-fixed-top "> + <div class="navbar-inner"> + <div class="container" style="width: 68%;"><div class="nav-collapse"> + + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Atlas <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="index.html" title="About">About</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="Wiki">Wiki</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="News">News</a> +</li> + + <li> <a href="https://git-wip-us.apache.org/repos/asf/incubator-atlas.git"; title="Git">Git</a> +</li> + + <li> <a href="https://issues.apache.org/jira/browse/ATLAS"; title="Jira">Jira</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS/PoweredBy"; title="Powered by">Powered by</a> +</li> + + <li> <a href="http://blogs.apache.org/atlas/"; title="Blog">Blog</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Information <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="project-info.html" title="Summary">Summary</a> +</li> + + <li> <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="http://webchat.freenode.net?channels=apacheatlas&uio=d4"; title="IRC">IRC</a> +</li> + + <li> <a href="team-list.html" title="Team">Team</a> +</li> + + <li> <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="license.html" title="License">License</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Releases <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/dyn/closer.cgi/incubator/atlas/0.7.0-incubating/"; title="0.7-incubating">0.7-incubating</a> +</li> + + <li> <a href="http://archive.apache.org/dist/incubator/atlas/0.6.0-incubating/"; title="0.6-incubating">0.6-incubating</a> +</li> + + <li> <a href="http://archive.apache.org/dist/incubator/atlas/0.5.0-incubating/"; title="0.5-incubating">0.5-incubating</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="index.html" title="0.7-incubating">0.7-incubating</a> +</li> + + <li> <a href="../0.6.0-incubating/index.html" title="0.6-incubating">0.6-incubating</a> +</li> + + <li> <a href="../0.5.0-incubating/index.html" title="0.5-incubating">0.5-incubating</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/foundation/how-it-works.html"; title="How Apache Works">How Apache Works</a> +</li> + + <li> <a href="http://www.apache.org/foundation/"; title="Foundation">Foundation</a> +</li> + + <li> <a href="http://www.apache.org/foundation/sponsorship.html"; title="Sponsoring Apache">Sponsoring Apache</a> +</li> + + <li> <a href="http://www.apache.org/foundation/thanks.html"; title="Thanks">Thanks</a> +</li> + </ul> + </li> + </ul> + + <form id="search-form" action="http://www.google.com/search"; method="get" class="navbar-search pull-right" > + + <input value="http://atlas.incubator.apache.org"; name="sitesearch" type="hidden"/> + <input class="search-query" name="q" id="query" type="text" /> +</form> +<script type="text/javascript" src="http://www.google.com/coop/cse/brand?form=search-form";></script> + + + + + + <iframe src="http://www.facebook.com/plugins/like.php?href=http://atlas.incubator.apache.org/atlas-docs&send=false&layout=button_count&show-faces=false&action=like&colorscheme=dark"; + scrolling="no" frameborder="0" + style="border:none; width:80px; height:20px; margin-top: 10px;" class="pull-right" ></iframe> + + <script type="text/javascript" src="https://apis.google.com/js/plusone.js";></script> + + <ul class="nav pull-right"><li style="margin-top: 10px;"> + + <div class="g-plusone" data-href="http://atlas.incubator.apache.org/atlas-docs"; data-size="medium" width="60px" align="right" ></div> + + </li></ul> + + + </div> + + </div> + </div> + </div> + + <div class="container"> + <div id="banner"> + <div class="pull-left"> + <a href=".." id="bannerLeft"> + <img src="images/atlas-logo.png" alt="Apache Atlas" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://incubator.apache.org"; id="bannerRight"> + <img src="images/apache-incubator-logo.png" alt="Apache Incubator"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Atlas"> + Atlas</a> + </li> + <li class="divider ">/</li> + <li class="">Configuring Apache Atlas - Application Properties</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-07-09</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.7-incubating</li> + + </ul> + </div> + + + + <div id="bodyColumn" > + + <div class="section"> +<h2><a name="Configuring_Apache_Atlas_-_Application_Properties"></a>Configuring Apache Atlas - Application Properties</h2> +<p>All configuration in Atlas uses java properties style configuration. The main configuration file is atlas-application.properties which is in the <b>conf</b> dir at the deployed location. It consists of the following sections:</p></div> +<div class="section"> +<h3><a name="Graph_Configs"></a>Graph Configs</h3></div> +<div class="section"> +<h4><a name="Graph_persistence_engine"></a>Graph persistence engine</h4> +<p>This section sets up the graph db - titan - to use a persistence engine. Please refer to <a href="http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html";>link</a> for more details. The example below uses BerkeleyDBJE.</p> +<div class="source"> +<pre> +atlas.graph.storage.backend=berkeleyje +atlas.graph.storage.directory=data/berkley + +</pre></div></div> +<div class="section"> +<h5><a name="Graph_persistence_engine_-_Hbase"></a>Graph persistence engine - Hbase</h5> +<p>Basic configuration</p> +<div class="source"> +<pre> +atlas.graph.storage.backend=hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +atlas.graph.storage.hostname=<ZooKeeper Quorum> + +</pre></div> +<p>HBASE_CONF_DIR environment variable needs to be set to point to the Hbase client configuration directory which is added to classpath when Atlas starts up. hbase-site.xml needs to have the following properties set according to the cluster setup</p> +<div class="source"> +<pre> +#Set below to /hbase-secure if the Hbase server is setup in secure mode +zookeeper.znode.parent=/hbase-unsecure + +</pre></div> +<p>Advanced configuration</p> +<p># If you are planning to use any of the configs mentioned below, they need to be prefixed with "atlas.graph." to take effect in ATLAS Refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html#_storage_hbase";>http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html#_storage_hbase</a></p> +<p>Permissions</p> +<p>When Atlas is configured with HBase as the storage backend the graph db (titan) needs sufficient user permissions to be able to create and access an HBase table. In a secure cluster it may be necessary to grant permissions to the 'atlas' user for the 'titan' table.</p> +<p>With Ranger, a policy can be configured for 'titan'.</p> +<p>Without Ranger, HBase shell can be used to set the permissions.</p> +<div class="source"> +<pre> + su hbase + kinit -k -t <hbase keytab> <hbase principal> + echo "grant 'atlas', 'RWXCA', 'titan'" | hbase shell + +</pre></div> +<p>Note that if the embedded-hbase-solr profile is used then HBase is included in the distribution so that a standalone instance of HBase can be started as the default storage backend for the graph repository. Using the embedded-hbase-solr profile will configure Atlas so that HBase instance will be started and stopped along with the Atlas server by default. To use the embedded-hbase-solr profile please see "Building Atlas" in the <a href="./InstallationSteps.html">Installation Steps</a> section.</p></div> +<div class="section"> +<h4><a name="Graph_Search_Index"></a>Graph Search Index</h4> +<p>This section sets up the graph db - titan - to use an search indexing system. The example configuration below sets up to use an embedded Elastic search indexing system.</p> +<div class="source"> +<pre> +atlas.graph.index.search.backend=elasticsearch +atlas.graph.index.search.directory=data/es +atlas.graph.index.search.elasticsearch.client-only=false +atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +</pre></div></div> +<div class="section"> +<h5><a name="Graph_Search_Index_-_Solr"></a>Graph Search Index - Solr</h5> +<p>Please note that Solr installation in Cloud mode is a prerequisite before configuring Solr as the search indexing backend. Refer <a href="./InstallationSteps.html">InstallationSteps</a> section for Solr installation/configuration.</p> +<div class="source"> +<pre> + atlas.graph.index.search.backend=solr5 + atlas.graph.index.search.solr.mode=cloud + atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181 + +</pre></div> +<p>Also note that if the embedded-hbase-solr profile is used then Solr is included in the distribution so that a standalone instance of Solr can be started as the default search indexing backend. Using the embedded-hbase-solr profile will configure Atlas so that the standalone Solr instance will be started and stopped along with the Atlas server by default. To use the embedded-hbase-solr profile please see "Building Atlas" in the <a href="./InstallationSteps.html">Installation Steps</a> section.</p></div> +<div class="section"> +<h4><a name="Choosing_between_Persistence_and_Indexing_Backends"></a>Choosing between Persistence and Indexing Backends</h4> +<p>Refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html";>http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html</a> and <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html";>http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html</a> for choosing between the persistence backends. BerkeleyDB is suitable for smaller data sets in the range of upto 10 million vertices with ACID gurantees. HBase on the other hand doesnt provide ACID guarantees but is able to scale for larger graphs. HBase also provides HA inherently.</p></div> +<div class="section"> +<h4><a name="Choosing_between_Persistence_Backends"></a>Choosing between Persistence Backends</h4> +<p>Refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html";>http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html</a> and <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html";>http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html</a> for choosing between the persistence backends. BerkeleyDB is suitable for smaller data sets in the range of upto 10 million vertices with ACID gurantees. HBase on the other hand doesnt provide ACID guarantees but is able to scale for larger graphs. HBase also provides HA inherently.</p></div> +<div class="section"> +<h4><a name="Choosing_between_Indexing_Backends"></a>Choosing between Indexing Backends</h4> +<p>Refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/elasticsearch.html";>http://s3.thinkaurelius.com/docs/titan/0.5.4/elasticsearch.html</a> and <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/solr.html";>http://s3.thinkaurelius.com/docs/titan/0.5.4/solr.html</a> for choosing between ElasticSearch and Solr. Solr in cloud mode is the recommended setup.</p></div> +<div class="section"> +<h4><a name="Switching_Persistence_Backend"></a>Switching Persistence Backend</h4> +<p>For switching the storage backend from BerkeleyDB to HBase and vice versa, refer the documentation for "Graph Persistence Engine" described above and restart ATLAS. The data in the indexing backend needs to be cleared else there will be discrepancies between the storage and indexing backend which could result in errors during the search. ElasticSearch runs by default in embedded mode and the data could easily be cleared by deleting the ATLAS_HOME/data/es directory. For Solr, the collections which were created during ATLAS Installation - vertex_index, edge_index, fulltext_index could be deleted which will cleanup the indexes</p></div> +<div class="section"> +<h4><a name="Switching_Index_Backend"></a>Switching Index Backend</h4> +<p>Switching the Index backend requires clearing the persistence backend data. Otherwise there will be discrepancies between the persistence and index backends since switching the indexing backend means index data will be lost. This leads to "Fulltext" queries not working on the existing data For clearing the data for BerkeleyDB, delete the ATLAS_HOME/data/berkeley directory For clearing the data for HBase, in Hbase shell, run 'disable titan' and 'drop titan'</p></div> +<div class="section"> +<h3><a name="Lineage_Configs"></a>Lineage Configs</h3> +<p>The higher layer services like lineage, schema, etc. are driven by the type system and this section encodes the specific types for the hive data model.</p> +<p># This models reflects the base super types for Data and Process</p> +<div class="source"> +<pre> +atlas.lineage.hive.table.type.name=DataSet +atlas.lineage.hive.process.type.name=Process +atlas.lineage.hive.process.inputs.name=inputs +atlas.lineage.hive.process.outputs.name=outputs + +## Schema +atlas.lineage.hive.table.schema.query=hive_table where name=?, columns + +</pre></div></div> +<div class="section"> +<h3><a name="Notification_Configs"></a>Notification Configs</h3> +<p>Refer <a class="externalLink" href="http://kafka.apache.org/documentation.html#configuration";>http://kafka.apache.org/documentation.html#configuration</a> for Kafka configuration. All Kafka configs should be prefixed with 'atlas.kafka.'</p> +<div class="source"> +<pre> +atlas.notification.embedded=true +atlas.kafka.data=${sys:atlas.home}/data/kafka +atlas.kafka.zookeeper.connect=localhost:9026 +atlas.kafka.bootstrap.servers=localhost:9027 +atlas.kafka.zookeeper.session.timeout.ms=400 +atlas.kafka.zookeeper.sync.time.ms=20 +atlas.kafka.auto.commit.interval.ms=1000 +atlas.kafka.hook.group.id=atlas + +</pre></div> +<p>Note that Kafka group ids are specified for a specific topic. The Kafka group id configuration for entity notifications is 'atlas.kafka.entities.group.id'</p> +<div class="source"> +<pre> +atlas.kafka.entities.group.id=<consumer id> + +</pre></div> +<p>These configuration parameters are useful for setting up Kafka topics via Atlas provided scripts, described in the <a href="./InstallationSteps.html">Installation Steps</a> page.</p> +<div class="source"> +<pre> +# Whether to create the topics automatically, default is true. +# Comma separated list of topics to be created, default is "ATLAS_HOOK,ATLAS_ENTITES" +atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES +# Number of replicas for the Atlas topics, default is 1. Increase for higher resilience to Kafka failures. +atlas.notification.replicas=1 +# Enable the below two properties if Kafka is running in Kerberized mode. +# Set this to the service principal representing the Kafka service +atlas.notification.kafka.service.principal=kafka/[email protected] +# Set this to the location of the keytab file for Kafka +#atlas.notification.kafka.keytab.location=/etc/security/keytabs/kafka.service.keytab + +</pre></div> +<p>These configuration parameters are useful for saving messages in case there are issues in reaching Kafka for sending messages.</p> +<div class="source"> +<pre> +# Whether to save messages that failed to be sent to Kafka, default is true +atlas.notification.log.failed.messages=true +# If saving messages is enabled, the file name to save them to. This file will be created under the log directory of the hook's host component - like HiveServer2 +atlas.notification.failed.messages.filename=atlas_hook_failed_messages.log + +</pre></div></div> +<div class="section"> +<h3><a name="Client_Configs"></a>Client Configs</h3> +<div class="source"> +<pre> +atlas.client.readTimeoutMSecs=60000 +atlas.client.connectTimeoutMSecs=60000 +atlas.rest.address=<http/https>://<atlas-fqdn>:<atlas port> - default http://localhost:21000 + +</pre></div></div> +<div class="section"> +<h3><a name="Security_Properties"></a>Security Properties</h3></div> +<div class="section"> +<h4><a name="SSL_config"></a>SSL config</h4> +<p>The following property is used to toggle the SSL feature.</p> +<div class="source"> +<pre> +atlas.enableTLS=false + +</pre></div></div> +<div class="section"> +<h3><a name="High_Availability_Properties"></a>High Availability Properties</h3> +<p>The following properties describe High Availability related configuration options:</p> +<div class="source"> +<pre> +# Set the following property to true, to enable High Availability. Default = false. +atlas.server.ha.enabled=true + +# Define a unique set of strings to identify each instance that should run an Atlas Web Service instance as a comma separated list. +atlas.server.ids=id1,id2 +# For each string defined above, define the host and port on which Atlas server binds to. +atlas.server.address.id1=host1.company.com:21000 +atlas.server.address.id2=host2.company.com:31000 + +# Specify Zookeeper properties needed for HA. +# Specify the list of services running Zookeeper servers as a comma separated list. +atlas.server.ha.zookeeper.connect=zk1.company.com:2181,zk2.company.com:2181,zk3.company.com:2181 +# Specify how many times should connection try to be established with a Zookeeper cluster, in case of any connection issues. +atlas.server.ha.zookeeper.num.retries=3 +# Specify how much time should the server wait before attempting connections to Zookeeper, in case of any connection issues. +atlas.server.ha.zookeeper.retry.sleeptime.ms=1000 +# Specify how long a session to Zookeeper should last without inactiviy to be deemed as unreachable. +atlas.server.ha.zookeeper.session.timeout.ms=20000 + +# Specify the scheme and the identity to be used for setting up ACLs on nodes created in Zookeeper for HA. +# The format of these options is <scheme>:<identity>. For more information refer to http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#sc_ZooKeeperAccessControl. +# The 'acl' option allows to specify a scheme, identity pair to setup an ACL for. +atlas.server.ha.zookeeper.acl=auth:sasl:[email protected] +# The 'auth' option specifies the authentication that should be used for connecting to Zookeeper. +atlas.server.ha.zookeeper.auth=sasl:[email protected] + +# Since Zookeeper is a shared service that is typically used by many components, +# it is preferable for each component to set its znodes under a namespace. +# Specify the namespace under which the znodes should be written. Default = /apache_atlas +atlas.server.ha.zookeeper.zkroot=/apache_atlas + +# Specify number of times a client should retry with an instance before selecting another active instance, or failing an operation. +atlas.client.ha.retries=4 +# Specify interval between retries for a client. +atlas.client.ha.sleep.interval.ms=5000 + +</pre></div></div> +<div class="section"> +<h3><a name="Server_Properties"></a>Server Properties</h3> +<div class="source"> +<pre> +# Set the following property to true, to enable the setup steps to run on each server start. Default = false. +atlas.server.run.setup.on.start=false + +</pre></div></div> +<div class="section"> +<h3><a name="Performance_configuration_items"></a>Performance configuration items</h3> +<p>The following properties can be used to tune performance of Atlas under specific circumstances:</p> +<div class="source"> +<pre> +# The number of times Atlas code tries to acquire a lock (to ensure consistency) while committing a transaction. +# This should be related to the amount of concurrency expected to be supported by the server. For e.g. with retries set to 10, upto 100 threads can concurrently create types in the Atlas system. +# If this is set to a low value (default is 3), concurrent operations might fail with a PermanentLockingException. +atlas.graph.storage.lock.retries=10 + +# Milliseconds to wait before evicting a cached entry. This should be > atlas.graph.storage.lock.wait-time x atlas.graph.storage.lock.retries +# If this is set to a low value (default is 10000), warnings on transactions taking too long will occur in the Atlas application log. +atlas.graph.storage.cache.db-cache-time=120000 + +</pre></div></div> + </div> + </div> + + <hr/> + + <footer> + <div class="container"> + <div class="row span12">Copyright © 2015-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + <p id="poweredBy" class="pull-right"> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </p> + + </div> + </footer> + </body> +</html>
http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/60041d8d/0.7.0-incubating/HighAvailability.html ---------------------------------------------------------------------- diff --git a/0.7.0-incubating/HighAvailability.html b/0.7.0-incubating/HighAvailability.html new file mode 100644 index 0000000..f7edaf6 --- /dev/null +++ b/0.7.0-incubating/HighAvailability.html @@ -0,0 +1,405 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-07-09 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160709" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache Atlas – Fault Tolerance and High Availability Options</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarEnabled"> + + + + + + <div id="topbar" class="navbar navbar-fixed-top "> + <div class="navbar-inner"> + <div class="container" style="width: 68%;"><div class="nav-collapse"> + + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Atlas <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="index.html" title="About">About</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="Wiki">Wiki</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="News">News</a> +</li> + + <li> <a href="https://git-wip-us.apache.org/repos/asf/incubator-atlas.git"; title="Git">Git</a> +</li> + + <li> <a href="https://issues.apache.org/jira/browse/ATLAS"; title="Jira">Jira</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS/PoweredBy"; title="Powered by">Powered by</a> +</li> + + <li> <a href="http://blogs.apache.org/atlas/"; title="Blog">Blog</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Information <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="project-info.html" title="Summary">Summary</a> +</li> + + <li> <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="http://webchat.freenode.net?channels=apacheatlas&uio=d4"; title="IRC">IRC</a> +</li> + + <li> <a href="team-list.html" title="Team">Team</a> +</li> + + <li> <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="license.html" title="License">License</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Releases <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/dyn/closer.cgi/incubator/atlas/0.7.0-incubating/"; title="0.7-incubating">0.7-incubating</a> +</li> + + <li> <a href="http://archive.apache.org/dist/incubator/atlas/0.6.0-incubating/"; title="0.6-incubating">0.6-incubating</a> +</li> + + <li> <a href="http://archive.apache.org/dist/incubator/atlas/0.5.0-incubating/"; title="0.5-incubating">0.5-incubating</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="index.html" title="0.7-incubating">0.7-incubating</a> +</li> + + <li> <a href="../0.6.0-incubating/index.html" title="0.6-incubating">0.6-incubating</a> +</li> + + <li> <a href="../0.5.0-incubating/index.html" title="0.5-incubating">0.5-incubating</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/foundation/how-it-works.html"; title="How Apache Works">How Apache Works</a> +</li> + + <li> <a href="http://www.apache.org/foundation/"; title="Foundation">Foundation</a> +</li> + + <li> <a href="http://www.apache.org/foundation/sponsorship.html"; title="Sponsoring Apache">Sponsoring Apache</a> +</li> + + <li> <a href="http://www.apache.org/foundation/thanks.html"; title="Thanks">Thanks</a> +</li> + </ul> + </li> + </ul> + + <form id="search-form" action="http://www.google.com/search"; method="get" class="navbar-search pull-right" > + + <input value="http://atlas.incubator.apache.org"; name="sitesearch" type="hidden"/> + <input class="search-query" name="q" id="query" type="text" /> +</form> +<script type="text/javascript" src="http://www.google.com/coop/cse/brand?form=search-form";></script> + + + + + + <iframe src="http://www.facebook.com/plugins/like.php?href=http://atlas.incubator.apache.org/atlas-docs&send=false&layout=button_count&show-faces=false&action=like&colorscheme=dark"; + scrolling="no" frameborder="0" + style="border:none; width:80px; height:20px; margin-top: 10px;" class="pull-right" ></iframe> + + <script type="text/javascript" src="https://apis.google.com/js/plusone.js";></script> + + <ul class="nav pull-right"><li style="margin-top: 10px;"> + + <div class="g-plusone" data-href="http://atlas.incubator.apache.org/atlas-docs"; data-size="medium" width="60px" align="right" ></div> + + </li></ul> + + + </div> + + </div> + </div> + </div> + + <div class="container"> + <div id="banner"> + <div class="pull-left"> + <a href=".." id="bannerLeft"> + <img src="images/atlas-logo.png" alt="Apache Atlas" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://incubator.apache.org"; id="bannerRight"> + <img src="images/apache-incubator-logo.png" alt="Apache Incubator"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Atlas"> + Atlas</a> + </li> + <li class="divider ">/</li> + <li class="">Fault Tolerance and High Availability Options</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-07-09</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.7-incubating</li> + + </ul> + </div> + + + + <div id="bodyColumn" > + + <div class="section"> +<h2><a name="Fault_Tolerance_and_High_Availability_Options"></a>Fault Tolerance and High Availability Options</h2></div> +<div class="section"> +<h3><a name="Introduction"></a>Introduction</h3> +<p>Apache Atlas uses and interacts with a variety of systems to provide metadata management and data lineage to data administrators. By choosing and configuring these dependencies appropriately, it is possible to achieve a high degree of service availability with Atlas. This document describes the state of high availability support in Atlas, including its capabilities and current limitations, and also the configuration required for achieving this level of high availability.</p> +<p><a href="./Architecture.html">The architecture page</a> in the wiki gives an overview of the various components that make up Atlas. The options mentioned below for various components derive context from the above page, and would be worthwhile to review before proceeding to read this page.</p></div> +<div class="section"> +<h3><a name="Atlas_Web_Service"></a>Atlas Web Service</h3> +<p>Currently, the Atlas Web Service has a limitation that it can only have one active instance at a time. In earlier releases of Atlas, a backup instance could be provisioned and kept available. However, a manual failover was required to make this backup instance active.</p> +<p>From this release, Atlas will support multiple instances of the Atlas Web service in an active/passive configuration with automated failover. This means that users can deploy and start multiple instances of the Atlas Web Service on different physical hosts at the same time. One of these instances will be automatically selected as an 'active' instance to service user requests. The others will automatically be deemed 'passive'. If the 'active' instance becomes unavailable either because it is deliberately stopped, or due to unexpected failures, one of the other instances will automatically be elected as an 'active' instance and start to service user requests.</p> +<p>An 'active' instance is the only instance that can respond to user requests correctly. It can create, delete, modify or respond to queries on metadata objects. A 'passive' instance will accept user requests, but will redirect them using HTTP redirect to the currently known 'active' instance. Specifically, a passive instance will not itself respond to any queries on metadata objects. However, all instances (both active and passive), will respond to admin requests that return information about that instance.</p> +<p>When configured in a High Availability mode, users can get the following operational benefits:</p> +<p></p> +<ul> +<li><b>Uninterrupted service during maintenance intervals</b>: If an active instance of the Atlas Web Service needs to be brought down for maintenance, another instance would automatically become active and can service requests.</li> +<li><b>Uninterrupted service in event of unexpected failures</b>: If an active instance of the Atlas Web Service fails due to software or hardware errors, another instance would automatically become active and can service requests.</li></ul> +<p>In the following sub-sections, we describe the steps required to setup High Availability for the Atlas Web Service. We also describe how the deployment and client can be designed to take advantage of this capability. Finally, we describe a few details of the underlying implementation.</p></div> +<div class="section"> +<h4><a name="Setting_up_the_High_Availability_feature_in_Atlas"></a>Setting up the High Availability feature in Atlas</h4> +<p>The following pre-requisites must be met for setting up the High Availability feature.</p> +<p></p> +<ul> +<li>Ensure that you install Apache Zookeeper on a cluster of machines (a minimum of 3 servers is recommended for production).</li> +<li>Select 2 or more physical machines to run the Atlas Web Service instances on. These machines define what we refer to as a 'server ensemble' for Atlas.</li></ul> +<p>To setup High Availability in Atlas, a few configuration options must be defined in the <tt>atlas-application.properties</tt> file. While the complete list of configuration items are defined in the <a href="./Configuration.html">Configuration Page</a>, this section lists a few of the main options.</p> +<p></p> +<ul> +<li>High Availability is an optional feature in Atlas. Hence, it must be enabled by setting the configuration option <tt>atlas.server.ha.enabled</tt> to true.</li> +<li>Next, define a list of identifiers, one for each physical machine you have selected for the Atlas Web Service instance. These identifiers can be simple strings like <tt>id1</tt>, <tt>id2</tt> etc. They should be unique and should not contain a comma.</li> +<li>Define a comma separated list of these identifiers as the value of the option <tt>atlas.server.ids</tt>.</li> +<li>For each physical machine, list the IP Address/hostname and port as the value of the configuration <tt>atlas.server.address.id</tt>, where <tt>id</tt> refers to the identifier string for this physical machine. +<ul> +<li>For e.g., if you have selected 2 machines with hostnames <tt>host1.company.com</tt> and <tt>host2.company.com</tt>, you can define the configuration options as below:</li></ul></li></ul> +<div class="source"> +<pre> + atlas.server.ids=id1,id2 + atlas.server.address.id1=host1.company.com:21000 + atlas.server.address.id2=host2.company.com:21000 + +</pre></div> +<p></p> +<ul> +<li>Define the Zookeeper quorum which will be used by the Atlas High Availability feature.</li></ul> +<div class="source"> +<pre> + atlas.server.ha.zookeeper.connect=zk1.company.com:2181,zk2.company.com:2181,zk3.company.com:2181 + +</pre></div> +<p></p> +<ul> +<li>You can review other configuration options that are defined for the High Availability feature, and set them up as desired in the <tt>atlas-application.properties</tt> file.</li> +<li>For production environments, the components that Atlas depends on must also be set up in High Availability mode. This is described in detail in the following sections. Follow those instructions to setup and configure them.</li> +<li>Install the Atlas software on the selected physical machines.</li> +<li>Copy the <tt>atlas-application.properties</tt> file created using the steps above to the configuration directory of all the machines.</li> +<li>Start the dependent components.</li> +<li>Start each instance of the Atlas Web Service.</li></ul> +<p>To verify that High Availability is working, run the following script on each of the instances where Atlas Web Service is installed.</p> +<div class="source"> +<pre> +$ATLAS_HOME/bin/atlas_admin.py -status + +</pre></div> +<p>This script can print one of the values below as response:</p> +<p></p> +<ul> +<li><b>ACTIVE</b>: This instance is active and can respond to user requests.</li> +<li><b>PASSIVE</b>: This instance is PASSIVE. It will redirect any user requests it receives to the current active instance.</li> +<li><b>BECOMING_ACTIVE</b>: This would be printed if the server is transitioning to become an ACTIVE instance. The server cannot service any metadata user requests in this state.</li> +<li><b>BECOMING_PASSIVE</b>: This would be printed if the server is transitioning to become a PASSIVE instance. The server cannot service any metadata user requests in this state.</li></ul> +<p>Under normal operating circumstances, only one of these instances should print the value <b>ACTIVE</b> as response to the script, and the others would print <b>PASSIVE</b>.</p></div> +<div class="section"> +<h4><a name="Configuring_clients_to_use_the_High_Availability_feature"></a>Configuring clients to use the High Availability feature</h4> +<p>The Atlas Web Service can be accessed in two ways:</p> +<p></p> +<ul> +<li><b>Using the Atlas Web UI</b>: This is a browser based client that can be used to query the metadata stored in Atlas.</li> +<li><b>Using the Atlas REST API</b>: As Atlas exposes a RESTful API, one can use any standard REST client including libraries in other applications. In fact, Atlas ships with a client called AtlasClient that can be used as an example to build REST client access.</li></ul> +<p>In order to take advantage of the High Availability feature in the clients, there are two options possible.</p></div> +<div class="section"> +<h5><a name="Using_an_intermediate_proxy"></a>Using an intermediate proxy</h5> +<p>The simplest solution to enable highly available access to Atlas is to install and configure some intermediate proxy that has a capability to transparently switch services based on status. One such proxy solution is <a class="externalLink" href="http://www.haproxy.org/";>HAProxy</a>.</p> +<p>Here is an example HAProxy configuration that can be used. Note this is provided for illustration only, and not as a recommended production configuration. For that, please refer to the HAProxy documentation for appropriate instructions.</p> +<div class="source"> +<pre> +frontend atlas_fe + bind *:41000 + default_backend atlas_be + +backend atlas_be + mode http + option httpchk get /api/atlas/admin/status + http-check expect string ACTIVE + balance roundrobin + server host1_21000 host1:21000 check + server host2_21000 host2:21000 check backup + +listen atlas + bind localhost:42000 + +</pre></div> +<p>The above configuration binds HAProxy to listen on port 41000 for incoming client connections. It then routes the connections to either of the hosts host1 or host2 depending on a HTTP status check. The status check is done using a HTTP GET on the REST URL <tt>/api/atlas/admin/status</tt>, and is deemed successful only if the HTTP response contains the string ACTIVE.</p></div> +<div class="section"> +<h5><a name="Using_automatic_detection_of_active_instance"></a>Using automatic detection of active instance</h5> +<p>If one does not want to setup and manage a separate proxy, then the other option to use the High Availability feature is to build a client application that is capable of detecting status and retrying operations. In such a setting, the client application can be launched with the URLs of all Atlas Web Service instances that form the ensemble. The client should then call the REST URL <tt>/api/atlas/admin/status</tt> on each of these to determine which is the active instance. The response from the Active instance would be of the form <tt>{Status:ACTIVE}</tt>. Also, when the client faces any exceptions in the course of an operation, it should again determine which of the remaining URLs is active and retry the operation.</p> +<p>The AtlasClient class that ships with Atlas can be used as an example client library that implements the logic for working with an ensemble and selecting the right Active server instance.</p> +<p>Utilities in Atlas, like <tt>quick_start.py</tt> and <tt>import-hive.sh</tt> can be configured to run with multiple server URLs. When launched in this mode, the AtlasClient automatically selects and works with the current active instance. If a proxy is set up in between, then its address can be used when running quick_start.py or import-hive.sh.</p></div> +<div class="section"> +<h4><a name="Implementation_Details_of_Atlas_High_Availability"></a>Implementation Details of Atlas High Availability</h4> +<p>The Atlas High Availability work is tracked under the master JIRA <a class="externalLink" href="https://issues.apache.org/jira/browse/ATLAS-510";>ATLAS-510</a>. The JIRAs filed under it have detailed information about how the High Availability feature has been implemented. At a high level the following points can be called out:</p> +<p></p> +<ul> +<li>The automatic selection of an Active instance, as well as automatic failover to a new Active instance happen through a leader election algorithm.</li> +<li>For leader election, we use the <a class="externalLink" href="http://curator.apache.org/curator-recipes/leader-latch.html";>Leader Latch Recipe</a> of <a class="externalLink" href="http://curator.apache.org";>Apache Curator</a>.</li> +<li>The Active instance is the only one which initializes, modifies or reads state in the backend stores to keep them consistent.</li> +<li>Also, when an instance is elected as Active, it refreshes any cached information from the backend stores to get up to date.</li> +<li>A servlet filter ensures that only the active instance services user requests. If a passive instance receives these requests, it automatically redirects them to the current active instance.</li></ul></div> +<div class="section"> +<h3><a name="Metadata_Store"></a>Metadata Store</h3> +<p>As described above, Atlas uses Titan to store the metadata it manages. By default, Atlas uses a standalone HBase instance as the backing store for Titan. In order to provide HA for the metadata store, we recommend that Atlas be configured to use distributed HBase as the backing store for Titan. Doing this implies that you could benefit from the HA guarantees HBase provides. In order to configure Atlas to use HBase in HA mode, do the following:</p> +<p></p> +<ul> +<li>Choose an existing HBase cluster that is set up in HA mode to configure in Atlas (OR) Set up a new HBase cluster in <a class="externalLink" href="http://hbase.apache.org/book.html#quickstart_fully_distributed";>HA mode</a>. +<ul> +<li>If setting up HBase for Atlas, please following instructions listed for setting up HBase in the <a href="./InstallationSteps.html">Installation Steps</a>.</li></ul></li> +<li>We recommend using more than one HBase masters (at least 2) in the cluster on different physical hosts that use Zookeeper for coordination to provide redundancy and high availability of HBase. +<ul> +<li>Refer to the <a href="./Configuration.html">Configuration page</a> for the options to configure in atlas.properties to setup Atlas with HBase.</li></ul></li></ul></div> +<div class="section"> +<h3><a name="Index_Store"></a>Index Store</h3> +<p>As described above, Atlas indexes metadata through Titan to support full text search queries. In order to provide HA for the index store, we recommend that Atlas be configured to use Solr as the backing index store for Titan. In order to configure Atlas to use Solr in HA mode, do the following:</p> +<p></p> +<ul> +<li>Choose an existing SolrCloud cluster setup in HA mode to configure in Atlas (OR) Set up a new <a class="externalLink" href="https://cwiki.apache.org/confluence/display/solr/SolrCloud";>SolrCloud cluster</a>. +<ul> +<li>Ensure Solr is brought up on at least 2 physical hosts for redundancy, and each host runs a Solr node.</li> +<li>We recommend the number of replicas to be set to at least 2 for redundancy.</li></ul></li> +<li>Create the SolrCloud collections required by Atlas, as described in <a href="./InstallationSteps.html">Installation Steps</a></li> +<li>Refer to the <a href="./Configuration.html">Configuration page</a> for the options to configure in atlas.properties to setup Atlas with Solr.</li></ul></div> +<div class="section"> +<h3><a name="Notification_Server"></a>Notification Server</h3> +<p>Metadata notification events from Hooks are sent to Atlas by writing them to a Kafka topic called <b>ATLAS_HOOK</b>. Similarly, events from Atlas to other integrating components like Ranger, are written to a Kafka topic called <b>ATLAS_ENTITIES</b>. Since Kafka persists these messages, the events will not be lost even if the consumers are down as the events are being sent. In addition, we recommend Kafka is also setup for fault tolerance so that it has higher availability guarantees. In order to configure Atlas to use Kafka in HA mode, do the following:</p> +<p></p> +<ul> +<li>Choose an existing Kafka cluster set up in HA mode to configure in Atlas (OR) Set up a new Kafka cluster.</li> +<li>We recommend that there are more than one Kafka brokers in the cluster on different physical hosts that use Zookeeper for coordination to provide redundancy and high availability of Kafka. +<ul> +<li>Setup at least 2 physical hosts for redundancy, each hosting a Kafka broker.</li></ul></li> +<li>Set up Kafka topics for Atlas usage: +<ul> +<li>The number of partitions for the ATLAS topics should be set to 1 (numPartitions)</li> +<li>Decide number of replicas for Kafka topic: Set this to at least 2 for redundancy.</li> +<li>Run the following commands:</li></ul></li></ul> +<div class="source"> +<pre> + $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper <list of zookeeper host:port entries> --topic ATLAS_HOOK --replication-factor <numReplicas> --partitions 1 + $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper <list of zookeeper host:port entries> --topic ATLAS_ENTITIES --replication-factor <numReplicas> --partitions 1 + Here KAFKA_HOME points to the Kafka installation directory. + +</pre></div> +<p></p> +<ul> +<li>In atlas-application.properties, set the following configuration:</li></ul> +<div class="source"> +<pre> + atlas.notification.embedded=false + atlas.kafka.zookeeper.connect=<comma separated list of servers forming Zookeeper quorum used by Kafka> + atlas.kafka.bootstrap.servers=<comma separated list of Kafka broker endpoints in host:port form> - Give at least 2 for redundancy. + +</pre></div></div> +<div class="section"> +<h3><a name="Known_Issues"></a>Known Issues</h3> +<p></p> +<ul> +<li>If the HBase region servers hosting the Atlas ‘titan’ HTable are down, Atlas would not be able to store or retrieve metadata from HBase until they are brought back online.</li></ul></div> + </div> + </div> + + <hr/> + + <footer> + <div class="container"> + <div class="row span12">Copyright © 2015-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + <p id="poweredBy" class="pull-right"> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </p> + + </div> + </footer> + </body> +</html> http://git-wip-us.apache.org/repos/asf/incubator-atlas-website/blob/60041d8d/0.7.0-incubating/InstallationSteps.html ---------------------------------------------------------------------- diff --git a/0.7.0-incubating/InstallationSteps.html b/0.7.0-incubating/InstallationSteps.html new file mode 100644 index 0000000..63cd759 --- /dev/null +++ b/0.7.0-incubating/InstallationSteps.html @@ -0,0 +1,556 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-07-09 + | Rendered using Apache Maven Fluido Skin 1.3.0 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160709" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache Atlas – Building & Installing Apache Atlas</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + + + <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script> + + + +<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script> + + </head> + <body class="topBarEnabled"> + + + + + + <div id="topbar" class="navbar navbar-fixed-top "> + <div class="navbar-inner"> + <div class="container" style="width: 68%;"><div class="nav-collapse"> + + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Atlas <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="index.html" title="About">About</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="Wiki">Wiki</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="News">News</a> +</li> + + <li> <a href="https://git-wip-us.apache.org/repos/asf/incubator-atlas.git"; title="Git">Git</a> +</li> + + <li> <a href="https://issues.apache.org/jira/browse/ATLAS"; title="Jira">Jira</a> +</li> + + <li> <a href="https://cwiki.apache.org/confluence/display/ATLAS/PoweredBy"; title="Powered by">Powered by</a> +</li> + + <li> <a href="http://blogs.apache.org/atlas/"; title="Blog">Blog</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Information <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="project-info.html" title="Summary">Summary</a> +</li> + + <li> <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="http://webchat.freenode.net?channels=apacheatlas&uio=d4"; title="IRC">IRC</a> +</li> + + <li> <a href="team-list.html" title="Team">Team</a> +</li> + + <li> <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="license.html" title="License">License</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Releases <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/dyn/closer.cgi/incubator/atlas/0.7.0-incubating/"; title="0.7-incubating">0.7-incubating</a> +</li> + + <li> <a href="http://archive.apache.org/dist/incubator/atlas/0.6.0-incubating/"; title="0.6-incubating">0.6-incubating</a> +</li> + + <li> <a href="http://archive.apache.org/dist/incubator/atlas/0.5.0-incubating/"; title="0.5-incubating">0.5-incubating</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="index.html" title="0.7-incubating">0.7-incubating</a> +</li> + + <li> <a href="../0.6.0-incubating/index.html" title="0.6-incubating">0.6-incubating</a> +</li> + + <li> <a href="../0.5.0-incubating/index.html" title="0.5-incubating">0.5-incubating</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/foundation/how-it-works.html"; title="How Apache Works">How Apache Works</a> +</li> + + <li> <a href="http://www.apache.org/foundation/"; title="Foundation">Foundation</a> +</li> + + <li> <a href="http://www.apache.org/foundation/sponsorship.html"; title="Sponsoring Apache">Sponsoring Apache</a> +</li> + + <li> <a href="http://www.apache.org/foundation/thanks.html"; title="Thanks">Thanks</a> +</li> + </ul> + </li> + </ul> + + <form id="search-form" action="http://www.google.com/search"; method="get" class="navbar-search pull-right" > + + <input value="http://atlas.incubator.apache.org"; name="sitesearch" type="hidden"/> + <input class="search-query" name="q" id="query" type="text" /> +</form> +<script type="text/javascript" src="http://www.google.com/coop/cse/brand?form=search-form";></script> + + + + + + <iframe src="http://www.facebook.com/plugins/like.php?href=http://atlas.incubator.apache.org/atlas-docs&send=false&layout=button_count&show-faces=false&action=like&colorscheme=dark"; + scrolling="no" frameborder="0" + style="border:none; width:80px; height:20px; margin-top: 10px;" class="pull-right" ></iframe> + + <script type="text/javascript" src="https://apis.google.com/js/plusone.js";></script> + + <ul class="nav pull-right"><li style="margin-top: 10px;"> + + <div class="g-plusone" data-href="http://atlas.incubator.apache.org/atlas-docs"; data-size="medium" width="60px" align="right" ></div> + + </li></ul> + + + </div> + + </div> + </div> + </div> + + <div class="container"> + <div id="banner"> + <div class="pull-left"> + <a href=".." id="bannerLeft"> + <img src="images/atlas-logo.png" alt="Apache Atlas" width="200px" height="45px"/> + </a> + </div> + <div class="pull-right"> <a href="http://incubator.apache.org"; id="bannerRight"> + <img src="images/apache-incubator-logo.png" alt="Apache Incubator"/> + </a> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="http://www.apache.org"; class="externalLink" title="Apache"> + Apache</a> + </li> + <li class="divider ">/</li> + <li class=""> + <a href="index.html" title="Atlas"> + Atlas</a> + </li> + <li class="divider ">/</li> + <li class="">Building & Installing Apache Atlas</li> + + + + <li id="publishDate" class="pull-right">Last Published: 2016-07-09</li> <li class="divider pull-right">|</li> + <li id="projectVersion" class="pull-right">Version: 0.7-incubating</li> + + </ul> + </div> + + + + <div id="bodyColumn" > + + <div class="section"> +<h3><a name="Building__Installing_Apache_Atlas"></a>Building & Installing Apache Atlas</h3></div> +<div class="section"> +<h4><a name="Building_Atlas"></a>Building Atlas</h4> +<div class="source"> +<pre> +git clone https://git-wip-us.apache.org/repos/asf/incubator-atlas.git atlas + +cd atlas + +export MAVEN_OPTS="-Xmx1536m -XX:MaxPermSize=512m" && mvn clean install + +</pre></div> +<p>Once the build successfully completes, artifacts can be packaged for deployment.</p> +<div class="source"> +<pre> + +mvn clean package -Pdist + + +</pre></div> +<p>To build a distribution that configures Atlas for external HBase and Solr, build with the external-hbase-solr profile.</p> +<div class="source"> +<pre> + +mvn clean package -Pdist,external-hbase-solr + + +</pre></div> +<p>Note that when the external-hbase-solr profile is used the following steps need to be completed to make Atlas functional.</p> +<ul> +<li>Configure atlas.graph.storage.hostname (see "Graph persistence engine - HBase" in the <a href="./Configuration.html">Configuration</a> section).</li> +<li>Configure atlas.graph.index.search.solr.zookeeper-url (see "Graph Search Index - Solr" in the <a href="./Configuration.html">Configuration</a> section).</li> +<li>Set HBASE_CONF_DIR to point to a valid HBase config directory (see "Graph persistence engine - HBase" in the <a href="./Configuration.html">Configuration</a> section).</li> +<li>Create the SOLR indices (see "Graph Search Index - Solr" in the <a href="./Configuration.html">Configuration</a> section).</li></ul> +<p>To build a distribution that packages HBase and Solr, build with the embedded-hbase-solr profile.</p> +<div class="source"> +<pre> + +mvn clean package -Pdist,embedded-hbase-solr + + +</pre></div> +<p>Using the embedded-hbase-solr profile will configure Atlas so that an HBase instance and a Solr instance will be started and stopped along with the Atlas server by default.</p> +<p>Atlas also supports building a distribution that can use BerkeleyDB and Elastic search as the graph and index backends. To build a distribution that is configured for these backends, build with the berkeley-elasticsearch profile.</p> +<div class="source"> +<pre> + +mvn clean package -Pdist,berkeley-elasticsearch + + +</pre></div> +<p>An additional step is required for the binary built using this profile to be used along with the Atlas distribution. Due to licensing requirements, Atlas does not bundle the BerkeleyDB Java Edition in the tarball.</p> +<p>You can download the Berkeley DB jar file from the URL: <verbatim><a class="externalLink" href="http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip";>http://download.oracle.com/otn/berkeley-db/je-5.0.73.zip</a></verbatim> and copy the je-5.0.73.jar to the ${atlas_home}/libext directory.</p> +<p>Tar can be found in atlas/distro/target/apache-atlas-${project.version}-bin.tar.gz</p> +<p>Tar is structured as follows</p> +<div class="source"> +<pre> + +|- bin + |- atlas_start.py + |- atlas_stop.py + |- atlas_config.py + |- quick_start.py + |- cputil.py +|- conf + |- atlas-application.properties + |- atlas-env.sh + |- hbase + |- hbase-site.xml.template + |- log4j.xml + |- solr + |- currency.xml + |- lang + |- stopwords_en.txt + |- protowords.txt + |- schema.xml + |- solrconfig.xml + |- stopwords.txt + |- synonyms.txt +|- docs +|- hbase + |- bin + |- conf + ... +|- server + |- webapp + |- atlas.war +|- solr + |- bin + ... +|- README +|- NOTICE +|- LICENSE +|- DISCLAIMER.txt +|- CHANGES.txt + + +</pre></div> +<p>Note that if the embedded-hbase-solr profile is specified for the build then HBase and Solr are included in the distribution.</p> +<p>In this case, a standalone instance of HBase can be started as the default storage backend for the graph repository. During Atlas installation the conf/hbase/hbase-site.xml.template gets expanded and moved to hbase/conf/hbase-site.xml for the initial standalone HBase configuration. To configure ATLAS graph persistence for a different HBase instance, please see "Graph persistence engine - HBase" in the <a href="./Configuration.html">Configuration</a> section.</p> +<p>Also, a standalone instance of Solr can be started as the default search indexing backend. To configure ATLAS search indexing for a different Solr instance please see "Graph Search Index - Solr" in the <a href="./Configuration.html">Configuration</a> section.</p></div> +<div class="section"> +<h4><a name="Installing__Running_Atlas"></a>Installing & Running Atlas</h4></div> +<div class="section"> +<h5><a name="Installing_Atlas"></a>Installing Atlas</h5> +<div class="source"> +<pre> +tar -xzvf apache-atlas-${project.version}-bin.tar.gz + +cd atlas-${project.version} + +</pre></div></div> +<div class="section"> +<h5><a name="Configuring_Atlas"></a>Configuring Atlas</h5> +<p>By default config directory used by Atlas is {package dir}/conf. To override this set environment variable ATLAS_CONF to the path of the conf dir.</p> +<p>atlas-env.sh has been added to the Atlas conf. This file can be used to set various environment variables that you need for you services. In addition you can set any other environment variables you might need. This file will be sourced by atlas scripts before any commands are executed. The following environment variables are available to set.</p> +<div class="source"> +<pre> +# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path +#export JAVA_HOME= + +# any additional java opts you want to set. This will apply to both client and server operations +#export ATLAS_OPTS= + +# any additional java opts that you want to set for client only +#export ATLAS_CLIENT_OPTS= + +# java heap size we want to set for the client. Default is 1024MB +#export ATLAS_CLIENT_HEAP= + +# any additional opts you want to set for atlas service. +#export ATLAS_SERVER_OPTS= + +# java heap size we want to set for the atlas server. Default is 1024MB +#export ATLAS_SERVER_HEAP= + +# What is is considered as atlas home dir. Default is the base locaion of the installed software +#export ATLAS_HOME_DIR= + +# Where log files are stored. Defatult is logs directory under the base install location +#export ATLAS_LOG_DIR= + +# Where pid files are stored. Defatult is logs directory under the base install location +#export ATLAS_PID_DIR= + +# where the atlas titan db data is stored. Defatult is logs/data directory under the base install location +#export ATLAS_DATA_DIR= + +# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. +#export ATLAS_EXPANDED_WEBAPP_DIR= + +</pre></div> +<p><b>Settings to support large number of metadata objects</b></p> +<p>If you plan to store several tens of thousands of metadata objects, it is recommended that you use values tuned for better GC performance of the JVM.</p> +<p>The following values are common server side options:</p> +<div class="source"> +<pre> +export ATLAS_SERVER_OPTS="-server -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+PrintTenuringDistribution -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dumps/atlas_server.hprof -Xloggc:logs/gc-worker.log -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1m -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCTimeStamps" + +</pre></div> +<p>The <tt>-XX:SoftRefLRUPolicyMSPerMB</tt> option was found to be particularly helpful to regulate GC performance for query heavy workloads with many concurrent users.</p> +<p>The following values are recommended for JDK 7:</p> +<div class="source"> +<pre> +export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=3072m -XX:PermSize=100M -XX:MaxPermSize=512m" + +</pre></div> +<p>The following values are recommended for JDK 8:</p> +<div class="source"> +<pre> +export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=5120m -XX:MetaspaceSize=100M -XX:MaxMetaspaceSize=512m" + +</pre></div> +<p><b>NOTE for Mac OS users</b> If you are using a Mac OS, you will need to configure the ATLAS_SERVER_OPTS (explained above).</p> +<p>In {package dir}/conf/atlas-env.sh uncomment the following line</p> +<div class="source"> +<pre> +#export ATLAS_SERVER_OPTS= + +</pre></div> +<p>and change it to look as below</p> +<div class="source"> +<pre> +export ATLAS_SERVER_OPTS="-Djava.awt.headless=true -Djava.security.krb5.realm= -Djava.security.krb5.kdc=" + +</pre></div> +<p><b>Hbase as the Storage Backend for the Graph Repository</b></p> +<p>By default, Atlas uses Titan as the graph repository and is the only graph repository implementation available currently. The HBase versions currently supported are 1.1.x. For configuring ATLAS graph persistence on HBase, please see "Graph persistence engine - HBase" in the <a href="./Configuration.html">Configuration</a> section for more details.</p> +<p>Pre-requisites for running HBase as a distributed cluster</p> +<ul> +<li>3 or 5 ZooKeeper nodes</li> +<li>Atleast 3 RegionServer nodes. It would be ideal to run the DataNodes on the same hosts as the Region servers for data locality.</li></ul> +<p>HBase tablename in Titan can be set using the following configuration in ATLAS_HOME/conf/atlas-application.properties:</p> +<div class="source"> +<pre> +atlas.graph.storage.hbase.table=apache_atlas_titan +atlas.audit.hbase.tablename=apache_atlas_entity_audit + +</pre></div> +<p><b>Configuring SOLR as the Indexing Backend for the Graph Repository</b></p> +<p>By default, Atlas uses Titan as the graph repository and is the only graph repository implementation available currently. For configuring Titan to work with Solr, please follow the instructions below</p> +<p></p> +<ul> +<li>Install solr if not already running. The version of SOLR supported is 5.2.1. Could be installed from <a class="externalLink" href="http://archive.apache.org/dist/lucene/solr/5.2.1/solr-5.2.1.tgz";>http://archive.apache.org/dist/lucene/solr/5.2.1/solr-5.2.1.tgz</a></li></ul> +<p></p> +<ul> +<li>Start solr in cloud mode.</li></ul>SolrCloud mode uses a ZooKeeper Service as a highly available, central location for cluster management. For a small cluster, running with an existing ZooKeeper quorum should be fine. For larger clusters, you would want to run separate multiple ZooKeeper quorum with atleast 3 servers. Note: Atlas currently supports solr in "cloud" mode only. "http" mode is not supported. For more information, refer solr documentation - <a class="externalLink" href="https://cwiki.apache.org/confluence/display/solr/SolrCloud";>https://cwiki.apache.org/confluence/display/solr/SolrCloud</a> +<p></p> +<ul> +<li>For e.g., to bring up a Solr node listening on port 8983 on a machine, you can use the command:</li></ul> +<div class="source"> +<pre> + $SOLR_HOME/bin/solr start -c -z <zookeeper_host:port> -p 8983 + +</pre></div> +<p></p> +<ul> +<li>Run the following commands from SOLR_BIN (e.g. $SOLR_HOME/bin) directory to create collections in Solr corresponding to the indexes that Atlas uses. In the case that the ATLAS and SOLR instance are on 2 different hosts,</li></ul>first copy the required configuration files from ATLAS_HOME/conf/solr on the ATLAS instance host to the Solr instance host. SOLR_CONF in the below mentioned commands refer to the directory where the solr configuration files have been copied to on Solr host: +<div class="source"> +<pre> + $SOLR_BIN/solr create -c vertex_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor + $SOLR_BIN/solr create -c edge_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor + $SOLR_BIN/solr create -c fulltext_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor + +</pre></div> +<p>Note: If numShards and replicationFactor are not specified, they default to 1 which suffices if you are trying out solr with ATLAS on a single node instance. Otherwise specify numShards according to the number of hosts that are in the Solr cluster and the maxShardsPerNode configuration. The number of shards cannot exceed the total number of Solr nodes in your !SolrCloud cluster.</p> +<p>The number of replicas (replicationFactor) can be set according to the redundancy required.</p> +<p>Also note that solr will automatically be called to create the indexes when the Atlas server is started if the SOLR_BIN and SOLR_CONF environment variables are set and the search indexing backend is set to 'solr5'.</p> +<p></p> +<ul> +<li>Change ATLAS configuration to point to the Solr instance setup. Please make sure the following configurations are set to the below values in ATLAS_HOME/conf/atlas-application.properties</li></ul> +<div class="source"> +<pre> + atlas.graph.index.search.backend=solr5 + atlas.graph.index.search.solr.mode=cloud + atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181 + +</pre></div> +<p></p> +<ul> +<li>Restart Atlas</li></ul> +<p>For more information on Titan solr configuration , please refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/solr.htm";>http://s3.thinkaurelius.com/docs/titan/0.5.4/solr.htm</a></p> +<p>Pre-requisites for running Solr in cloud mode * Memory - Solr is both memory and CPU intensive. Make sure the server running Solr has adequate memory, CPU and disk. Solr works well with 32GB RAM. Plan to provide as much memory as possible to Solr process * Disk - If the number of entities that need to be stored are large, plan to have at least 500 GB free space in the volume where Solr is going to store the index data * SolrCloud has support for replication and sharding. It is highly recommended to use SolrCloud with at least two Solr nodes running on different servers with replication enabled. If using SolrCloud, then you also need ZooKeeper installed and configured with 3 or 5 ZooKeeper nodes</p> +<p><b>Configuring Kafka Topics</b></p> +<p>Atlas uses Kafka to ingest metadata from other components at runtime. This is described in the <a href="./Architecture.html">Architecture page</a> in more detail. Depending on the configuration of Kafka, sometimes you might need to setup the topics explicitly before using Atlas. To do so, Atlas provides a script <tt>bin/atlas_kafka_setup.py</tt> which can be run from the Atlas server. In some environments, the hooks might start getting used first before Atlas server itself is setup. In such cases, the topics can be run on the hosts where hooks are installed using a similar script <tt>hook-bin/atlas_kafka_setup_hook.py</tt>. Both these use configuration in <tt>atlas-application.properties</tt> for setting up the topics. Please refer to the <a href="./Configuration.html">Configuration page</a> for these details.</p></div> +<div class="section"> +<h5><a name="Setting_up_Atlas"></a>Setting up Atlas</h5> +<p>There are a few steps that setup dependencies of Atlas. One such example is setting up the Titan schema in the storage backend of choice. In a simple single server setup, these are automatically setup with default configuration when the server first accesses these dependencies.</p> +<p>However, there are scenarios when we may want to run setup steps explicitly as one time operations. For example, in a multiple server scenario using <a href="./HighAvailability.html">High Availability</a>, it is preferable to run setup steps from one of the server instances the first time, and then start the services.</p> +<p>To run these steps one time, execute the command <tt>bin/atlas_start.py -setup</tt> from a single Atlas server instance.</p> +<p>However, the Atlas server does take care of parallel executions of the setup steps. Also, running the setup steps multiple times is idempotent. Therefore, if one chooses to run the setup steps as part of server startup, for convenience, then they should enable the configuration option <tt>atlas.server.run.setup.on.start</tt> by defining it with the value <tt>true</tt> in the <tt>atlas-application.properties</tt> file.</p></div> +<div class="section"> +<h5><a name="Starting_Atlas_Server"></a>Starting Atlas Server</h5> +<div class="source"> +<pre> +bin/atlas_start.py [-port <port>] + +</pre></div> +<p>By default,</p> +<ul> +<li>To change the port, use -port option.</li> +<li>atlas server starts with conf from {package dir}/conf. To override this (to use the same conf with multiple atlas upgrades), set environment variable ATLAS_CONF to the path of conf dir</li></ul></div> +<div class="section"> +<h4><a name="Using_Atlas"></a>Using Atlas</h4> +<p></p> +<ul> +<li>Quick start model - sample model and data</li></ul> +<div class="source"> +<pre> + bin/quick_start.py [<atlas endpoint>] + +</pre></div> +<p></p> +<ul> +<li>Verify if the server is up and running</li></ul> +<div class="source"> +<pre> + curl -v http://localhost:21000/api/atlas/admin/version + {"Version":"v0.1"} + +</pre></div> +<p></p> +<ul> +<li>List the types in the repository</li></ul> +<div class="source"> +<pre> + curl -v http://localhost:21000/api/atlas/types + {"results":["Process","Infrastructure","DataSet"],"count":3,"requestId":"1867493731@qtp-262860041-0 - 82d43a27-7c34-4573-85d1-a01525705091"} + +</pre></div> +<p></p> +<ul> +<li>List the instances for a given type</li></ul> +<div class="source"> +<pre> + curl -v http://localhost:21000/api/atlas/entities?type=hive_table + {"requestId":"788558007@qtp-44808654-5","list":["cb9b5513-c672-42cb-8477-b8f3e537a162","ec985719-a794-4c98-b98f-0509bd23aac0","48998f81-f1d3-45a2-989a-223af5c1ed6e","a54b386e-c759-4651-8779-a099294244c4"]} + + curl -v http://localhost:21000/api/atlas/entities/list/hive_db + +</pre></div> +<p></p> +<ul> +<li>Search for entities (instances) in the repository</li></ul> +<div class="source"> +<pre> + curl -v http://localhost:21000/api/atlas/discovery/search/dsl?query="from hive_table" + +</pre></div> +<p><b>Dashboard</b></p> +<p>Once atlas is started, you can view the status of atlas entities using the Web-based dashboard. You can open your browser at the corresponding port to use the web UI.</p></div> +<div class="section"> +<h4><a name="Stopping_Atlas_Server"></a>Stopping Atlas Server</h4> +<div class="source"> +<pre> +bin/atlas_stop.py + +</pre></div></div> +<div class="section"> +<h4><a name="Known_Issues"></a>Known Issues</h4></div> +<div class="section"> +<h5><a name="Setup"></a>Setup</h5> +<p>If the setup of Atlas service fails due to any reason, the next run of setup (either by an explicit invocation of <tt>atlas_start.py -setup</tt> or by enabling the configuration option <tt>atlas.server.run.setup.on.start</tt>) will fail with a message such as <tt>A previous setup run may not have completed cleanly.</tt>. In such cases, you would need to manually ensure the setup can run and delete the Zookeeper node at <tt>/apache_atlas/setup_in_progress</tt> before attempting to run setup again.</p> +<p>If the setup failed due to HBase Titan schema setup errors, it may be necessary to repair the HBase schema. If no data has been stored, one can also disable and drop the 'titan' schema in HBase to let setup run again.</p></div> + </div> + </div> + + <hr/> + + <footer> + <div class="container"> + <div class="row span12">Copyright © 2015-2016 + <a href="http://www.apache.org";>Apache Software Foundation</a>. + All Rights Reserved. + + </div> + + + <p id="poweredBy" class="pull-right"> + <a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"> + <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /> + </a> + </p> + + </div> + </footer> + </body> +</html>
