http://git-wip-us.apache.org/repos/asf/atlas-website/blob/e9346ee2/1.0.0-rc0/Security.html ---------------------------------------------------------------------- diff --git a/1.0.0-rc0/Security.html b/1.0.0-rc0/Security.html new file mode 100644 index 0000000..867a04b --- /dev/null +++ b/1.0.0-rc0/Security.html @@ -0,0 +1,394 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/twiki/Security.twiki at 2018-05-26 + | Rendered using Apache Maven Fluido Skin 1.7 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20180526" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache Atlas – Security Features of Apache Atlas</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.7.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + <script type="text/javascript" src="./js/apache-maven-fluido-1.7.min.js"></script> + </head> + <body class="topBarEnabled"> + <div id="topbar" class="navbar navbar-fixed-top "> + <div class="navbar-inner"> + <div class="container" style="width: 68%;"><div class="nav-collapse"> + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Atlas <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="index.html" title="About">About</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="Wiki">Wiki</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="News">News</a></li> + <li><a href="https://git-wip-us.apache.org/repos/asf/atlas.git"; title="Git">Git</a></li> + <li><a href="https://issues.apache.org/jira/browse/ATLAS"; title="Jira">Jira</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/ATLAS/PoweredBy"; title="Powered by">Powered by</a></li> + <li><a href="http://blogs.apache.org/atlas/"; title="Blog">Blog</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Information <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="project-info.html" title="Summary">Summary</a></li> + <li><a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a></li> + <li><a href="http://webchat.freenode.net?channels=apacheatlas&uio=d4"; title="IRC">IRC</a></li> + <li><a href="team-list.html" title="Team">Team</a></li> + <li><a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a></li> + <li><a href="source-repository.html" title="Source Repository">Source Repository</a></li> + <li><a href="license.html" title="License">License</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Releases <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/1.0.0-alpha/"; title="1.0.0-alpha">1.0.0-alpha</a></li> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/1.0.0-rc0/"; title="1.0.0-rc0">1.0.0-rc0</a></li> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/0.8.2/"; title="0.8.2">0.8.2</a></li> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/0.8.1/"; title="0.8.1">0.8.1</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.8.0-incubating/"; title="0.8-incubating">0.8-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.7.1-incubating/"; title="0.7.1-incubating">0.7.1-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.7.0-incubating/"; title="0.7-incubating">0.7-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.6.0-incubating/"; title="0.6-incubating">0.6-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.5.0-incubating/"; title="0.5-incubating">0.5-incubating</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="../index.html" title="latest">latest</a></li> + <li><a href="../1.0.0-alpha/index.html" title="1.0.0-alpha">1.0.0-alpha</a></li> + <li><a href="../1.0.0-rc0/index.html" title="1.0.0-rc0">1.0.0-rc0</a></li> + <li><a href="../0.8.2/index.html" title="0.8.2">0.8.2</a></li> + <li><a href="../0.8.1/index.html" title="0.8.1">0.8.1</a></li> + <li><a href="../0.8.0-incubating/index.html" title="0.8-incubating">0.8-incubating</a></li> + <li><a href="../0.7.1-incubating/index.html" title="0.7.1-incubating">0.7.1-incubating</a></li> + <li><a href="../0.7.0-incubating/index.html" title="0.7-incubating">0.7-incubating</a></li> + <li><a href="../0.6.0-incubating/index.html" title="0.6-incubating">0.6-incubating</a></li> + <li><a href="../0.5.0-incubating/index.html" title="0.5-incubating">0.5-incubating</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/foundation/how-it-works.html"; title="How Apache Works">How Apache Works</a></li> + <li><a href="https://www.apache.org/events/current-event"; title="Events">Events</a></li> + <li><a href="https://www.apache.org/licenses/"; title="License">License</a></li> + <li><a href="http://www.apache.org/foundation/"; title="Foundation">Foundation</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; title="Sponsoring Apache">Sponsoring Apache</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; title="Thanks">Thanks</a></li> + </ul> + </li> + </ul> +<form id="search-form" action="https://www.google.com/search"; method="get" class="navbar-search pull-right" > + <input value="http://atlas.apache.org"; name="sitesearch" type="hidden"/> + <input class="search-query" name="q" id="query" type="text" /> +</form> +<script type="text/javascript">asyncJs( 'https://cse.google.com/brand?form=search-form' )</script> + <iframe src="https://www.facebook.com/plugins/like.php?href=http://atlas.apache.org/atlas-docs&send=false&layout=button_count&show-faces=false&action=like&colorscheme=dark"; + scrolling="no" frameborder="0" + style="border:none; width:100px; height:20px; margin-top: 10px;" class="pull-right" ></iframe> + <script type="text/javascript">asyncJs( 'https://apis.google.com/js/plusone.js' )</script> + <ul class="nav pull-right"><li style="margin-top: 10px;"> + <div class="g-plusone" data-href="http://atlas.apache.org/atlas-docs"; data-size="medium" width="60px" align="right" ></div> + </li></ul> + </div> + </div> + </div> + </div> + <div class="container"> + <div id="banner"> + <div class="pull-left"><a href=".." id="bannerLeft"><img src="images/atlas-logo.png" alt="Apache Atlas" width="200px" height="45px"/></a></div> + <div class="pull-right"></div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="index.html" title="Atlas">Atlas</a><span class="divider">/</span></li> + <li class="active ">Security Features of Apache Atlas</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-05-26</li> + <li id="projectVersion" class="pull-right">Version: 1.0.0-rc0</li> + </ul> + </div> + <div id="bodyColumn" > +<div class="section"> +<h2><a name="Security_Features_of_Apache_Atlas"></a>Security Features of Apache Atlas</h2></div> +<div class="section"> +<h3><a name="Overview"></a>Overview</h3> +<p>The following features are available for enhancing the security of the platform:</p> +<ul> +<li>SSL</li> +<li>Service Authentication</li> +<li>SPNEGO-based HTTP Authentication</li></ul></div> +<div class="section"> +<h4><a name="SSL"></a>SSL</h4> +<p>Both SSL one-way (server authentication) and two-way (server and client authentication) are supported. The following application properties (properties configured in the atlas-application.properties file) are available for configuring SSL:</p> +<p></p> +<ul> +<li><code>atlas.enableTLS</code> (false|true) [default: false] - enable/disable the SSL listener</li> +<li><code>keystore.file</code> - the path to the keystore file leveraged by the server. This file contains the server certificate.</li> +<li><code>truststore.file</code> - the path to the truststore file. This file contains the certificates of other trusted entities (e.g. the certificates for client processes if two-way SSL is enabled). In most instances this can be set to the same value as the keystore.file property (especially if one-way SSL is enabled).</li> +<li><code>client.auth.enabled</code> (false|true) [default: false] - enable/disable client authentication. If enabled, the client will have to authenticate to the server during the transport session key creation process (i.e. two-way SSL is in effect).</li> +<li><code>cert.stores.credential.provider.path</code> - the path to the Credential Provider store file. The passwords for the keystore, truststore, and server certificate are maintained in this secure file. Utilize the cputil script in the 'bin' directoy (see below) to populate this file with the passwords required.</li> +<li><code>atlas.ssl.exclude.cipher.suites</code> - the excluded Cipher Suites list - <b>NULL.</b>,.*RC4.*,.*MD5.*,.*DES.*,.*DSS.* are weak and unsafe Cipher Suites that are excluded by default. If additional Ciphers need to be excluded, set this property with the default Cipher Suites such as atlas.ssl.exclude.cipher.suites=.*NULL.*, .*RC4.*, .*MD5.*, .*DES.*, .*DSS.*, and add the additional Ciper Suites to the list with a comma separator. They can be added with their full name or a regular expression. The Cipher Suites listed in the atlas.ssl.exclude.cipher.suites property will have precedence over the default Cipher Suites. One would keep the default Cipher Suites, and add additional ones to be safe.</li></ul></div> +<div class="section"> +<h5><a name="Credential_Provider_Utility_Script"></a>Credential Provider Utility Script</h5> +<p>In order to prevent the use of clear-text passwords, the Atlas platofrm makes use of the Credential Provider facility for secure password storage (see <a class="externalLink" href="http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CommandsManual.html#credential";>Hadoop Credential Command Reference</a> for more information about this facility). The cputil script in the 'bin' directory can be leveraged to create the password store required.</p> +<p>To create the credential provdier for Atlas:</p> +<p></p> +<ul> +<li>cd to the '<code>bin</code>' directory</li> +<li>type '<code>./cputil.py</code>'</li> +<li>Enter the path for the generated credential provider. The format for the path is: +<ul> +<li><a class="externalLink" href="jceks://file/local/file/path/file.jceks">jceks://file/local/file/path/file.jceks</a> or <a class="externalLink" href="jceks://hdfs@namenodehost:port/path/in/hdfs/to/file.jceks.">jceks://hdfs@namenodehost:port/path/in/hdfs/to/file.jceks.</a> The files generally use the ".jceks" extension (e.g. test.jceks)</li></ul></li> +<li>Enter the passwords for the keystore, truststore, and server key (these passwords need to match the ones utilized for actually creating the associated certificate store files).</li></ul> +<p>The credential provider will be generated and saved to the path provided.</p></div> +<div class="section"> +<h4><a name="Service_Authentication"></a>Service Authentication</h4> +<p>The Atlas platform, upon startup, is associated to an authenticated identity. By default, in an insecure environment, that identity is the same as the OS authenticated user launching the server. However, in a secure cluster leveraging kerberos, it is considered a best practice to configure a keytab and principal in order for the platform to authenticate to the KDC. This allows the service to subsequently interact with other secure cluster services (e.g. HDFS).</p> +<p>The properties for configuring service authentication are:</p> +<p></p> +<ul> +<li><code>atlas.authentication.method</code> (simple|kerberos) [default: simple] - the authentication method to utilize. Simple will leverage the OS authenticated identity and is the default mechanism. 'kerberos' indicates that the service is required to authenticate to the KDC leveraging the configured keytab and principal.</li> +<li><code>atlas.authentication.keytab</code> - the path to the keytab file.</li> +<li><code>atlas.authentication.principal</code> - the principal to use for authenticating to the KDC. The principal is generally of the form "user/host@realm". You may use the '_HOST' token for the hostname and the local hostname will be substituted in by the runtime (e.g. "Atlas/_h...@example.com").</li></ul> +<p>Note that when Atlas is configured with HBase as the storage backend in a secure cluster, the graph db (JanusGraph) needs sufficient user permissions to be able to create and access an HBase table. To grant the appropriate permissions see <a href="./Configuration.html">Graph persistence engine - Hbase</a>.</p></div> +<div class="section"> +<h4><a name="JAAS_configuration"></a>JAAS configuration</h4> +<p>In a secure cluster, some of the components (such as Kafka) that Atlas interacts with, require Atlas to authenticate itself to them using JAAS. The following properties are used to set up appropriate JAAS Configuration.</p> +<p></p> +<ul> +<li><code>atlas.jaas.<code>client-id<code>.loginModuleName<code> - the authentication method used by the component (for example, com.sun.security.auth.module.Krb5LoginModule)</li> +<li><code>atlas.jaas.<code>client-id<code>.loginModuleControlFlag<code> (required|requisite|sufficient|optional) [default: required]</li> +<li><code>atlas.jaas.<code>client-id<code>.option.useKeyTab<code> (true|false)</li> +<li><code>atlas.jaas.<code>client-id<code>.option.storeKey<code> (true | false)</li> +<li><code>atlas.jaas.<code>client-id<code>.option.serviceName<code> - service name of server component</li> +<li><code>atlas.jaas.<code>client-id<code>.option.keyTab<code> = <atlas keytab></li> +<li><code>atlas.jaas.<code>client-id<code>.option.principal<code> = <atlas principal></li></ul> +<p>For example, the following property settings in jaas-application.properties file</p> +<div class="source"><pre class="prettyprint"> + +atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.KafkaClient.loginModuleControlFlag = required +atlas.jaas.KafkaClient.option.useKeyTab = true +atlas.jaas.KafkaClient.option.storeKey = true +atlas.jaas.KafkaClient.option.serviceName = kafka +atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/kafka_client.keytab +atlas.jaas.KafkaClient.option.principal = kafka-clien...@example.com + +atlas.jaas.MyClient.0.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.MyClient.0.loginModuleControlFlag = required +atlas.jaas.MyClient.0.option.useKeyTab = true +atlas.jaas.MyClient.0.option.storeKey = true +atlas.jaas.MyClient.0.option.serviceName = kafka +atlas.jaas.MyClient.0.option.keyTab = /etc/security/keytabs/kafka_client.keytab +atlas.jaas.MyClient.0.option.principal = kafka-clien...@example.com + +atlas.jaas.MyClient.1.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.MyClient.1.loginModuleControlFlag = optional +atlas.jaas.MyClient.1.option.useKeyTab = true +atlas.jaas.MyClient.1.option.storeKey = true +atlas.jaas.MyClient.1.option.serviceName = kafka +atlas.jaas.MyClient.1.option.keyTab = /etc/security/keytabs/kafka_client.keytab +atlas.jaas.MyClient.1.option.principal = kafka-clien...@example.com + + +</pre></div> +<p>will set the JAAS configuration that is equivalent to the following jaas.conf file entries.</p> +<div class="source"><pre class="prettyprint"> + +KafkaClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + serviceName=kafka + keyTab="/etc/security/keytabs/kafka_client.keytab" + principal="kafka-clien...@example.com"; +}; +MyClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + serviceName=kafka keyTab="/etc/security/keytabs/kafka_client.keytab" + principal="kafka-clien...@example.com"; +}; +MyClient { + com.sun.security.auth.module.Krb5LoginModule optional + useKeyTab=true + storeKey=true + serviceName=kafka + keyTab="/etc/security/keytabs/kafka_client.keytab" + principal="kafka-clien...@example.com"; +}; + + +</pre></div></div> +<div class="section"> +<h4><a name="SPNEGO-based_HTTP_Authentication"></a>SPNEGO-based HTTP Authentication</h4> +<p>HTTP access to the Atlas platform can be secured by enabling the platform's SPNEGO support. There are currently two supported authentication mechanisms:</p> +<p></p> +<ul> +<li><code>simple</code> - authentication is performed via a provided user name</li> +<li><code>kerberos</code> - the KDC authenticated identity of the client is leveraged to authenticate to the server</li></ul> +<p>The kerberos support requires the client accessing the server to first authenticate to the KDC (usually this is done via the 'kinit' command). Once authenticated, the user may access the server (the authenticated identity will be related to the server via the SPNEGO negotiation mechanism).</p> +<p>The properties for configuring the SPNEGO support are:</p> +<p></p> +<ul> +<li><code>atlas.http.authentication.enabled</code> (true|false) [default: false] - a property indicating whether to enable HTTP authentication</li> +<li><code>atlas.http.authentication.type</code> (simple|kerberos) [default: simple] - the authentication type</li> +<li><code>atlas.http.authentication.kerberos.principal</code> - the web-application Kerberos principal name. The Kerberos principal name must start with "HTTP/...". For example: "HTTP/localhost@LOCALHOST". There is no default value.</li> +<li><code>atlas.http.authentication.kerberos.keytab</code> - the path to the keytab file containing the credentials for the kerberos principal.</li> +<li><code>atlas.rest.address</code> - <http/https>://<atlas-fqdn>:<atlas port></li></ul> +<p>For a more detailed discussion of the HTTP authentication mechanism refer to <a class="externalLink" href="http://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html";>Hadoop Auth, Java HTTP SPNEGO 2.6.0 - Server Side Configuration</a>. The prefix that document references is "atlas.http.authentication" in the case of the Atlas authentication implementation.</p></div> +<div class="section"> +<h4><a name="Client_security_configuration"></a>Client security configuration</h4> +<p>When leveraging Atlas client code to communicate with an Atlas server configured for SSL transport and/or Kerberos authentication, there is a requirement to provide the Atlas client configuration file that provides the security properties that allow for communication with, or authenticating to, the server. Update the atlas-application.properties file with the appropriate settings (see below) and copy it to the client's classpath or to the directory specified by the "atlas.conf" system property.</p> +<p>The client properties for SSL communication are:</p> +<p></p> +<ul> +<li><code>atlas.enableTLS</code> (false|true) [default: false] - enable/disable the SSL client communication infrastructure.</li> +<li><code>keystore.file</code> - the path to the keystore file leveraged by the client. This file is only required if 2-Way SSL is enabled at the server and contains the client certificate.</li> +<li><code>truststore.file</code> - the path to the truststore file. This file contains the certificates of trusted entities (e.g. the certificates for the server or a shared certification authority). This file is required for both one-way or two-way SSL.</li> +<li><code>cert.stores.credential.provider.path</code> - the path to the Credential Provider store file. The passwords for the keystore, truststore, and client certificate are maintained in this secure file.</li></ul> +<p>The property required for authenticating to the server (if authentication is enabled):</p> +<p></p> +<ul> +<li><code>atlas.http.authentication.type</code> (simple|kerberos) [default: simple] - the authentication type</li></ul></div> +<div class="section"> +<h4><a name="SOLR_Kerberos_configuration"></a>SOLR Kerberos configuration</h4> +<p>If the authentication type specified is 'kerberos', then the kerberos ticket cache will be accessed for authenticating to the server (Therefore the client is required to authenticate to the KDC prior to communication with the server using 'kinit' or a similar mechanism).</p> +<p>See <a class="externalLink" href="https://cwiki.apache.org/confluence/display/RANGER/How+to+configure+Solr+Cloud+with+Kerberos+for+Ranger+0.5";>the Apache SOLR Kerberos configuration</a>.</p> +<p></p> +<ul> +<li>Add principal and generate the keytab file for solr. Create a keytab per host for each host where Solr is going to run and use the principal name with the host (e.g. addprinc -randkey solr/${HOST1}@EXAMPLE.COM. Replace ${HOST1} with the actual host names).</li></ul> +<div class="source"><pre class="prettyprint"> + kadmin.local + kadmin.local: addprinc -randkey solr/<hostname>@EXAMPLE.COM + kadmin.local: xst -k solr.keytab solr/<hostname>@EXAMPLE.COM + kadmin.local: quit + +</pre></div> +<p></p> +<ul> +<li>Add principal and generate the keytab file for authenticating HTTP request. (Note that if Ambari is used to Kerberize the cluster, the keytab /etc/security/keytabs/spnego.service.keytab can be used)</li></ul> +<div class="source"><pre class="prettyprint"> + kadmin.local + kadmin.local: addprinc -randkey HTTP/<hostname>@EXAMPLE.COM + kadmin.local: xst -k HTTP.keytab HTTP/<hostname>@EXAMPLE.COM + kadmin.local: quit + +</pre></div> +<p></p> +<ul> +<li>Copy the keytab file to all the hosts running Solr.</li></ul> +<div class="source"><pre class="prettyprint"> + cp solr.keytab /etc/security/keytabs/ + chmod 400 /etc/security/keytabs/solr.keytab + + cp HTTP.keytab /etc/security/keytabs/ + chmod 400 /etc/security/keytabs/HTTP.keytab + +</pre></div> +<p></p> +<ul> +<li>Create path in Zookeeper for storing the Solr configs and other parameters.</li></ul> +<div class="source"><pre class="prettyprint"> + $SOLR_INSTALL_HOME/server/scripts/cloud-scripts/zkcli.sh -zkhost $ZK_HOST:2181 -cmd makepath solr + +</pre></div> +<p></p> +<ul> +<li>Upload the configuration to Zookeeper.</li></ul> +<div class="source"><pre class="prettyprint"> + $SOLR_INSTALL_HOME/server/scripts/cloud-scripts/zkcli.sh -cmd upconfig -zkhost $ZK_HOST:2181/solr -confname basic_configs -confdir $SOLR_INSTALL_HOME/server/solr/configsets/basic_configs/conf + +</pre></div> +<p></p> +<ul> +<li>Create the JAAS configuration.</li></ul> +<div class="source"><pre class="prettyprint"> + vi /etc/solr/conf/solr_jaas.conf + + Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="/etc/security/keytabs/solr.keytab" + storeKey=true + useTicketCache=true + debug=true + principal="solr/<hostname>@EXAMPLE.COM"; + }; + +</pre></div> +<p></p> +<ul> +<li>Copy /etc/solr/conf/solr_jaas.conf to all hosts running Solr.</li></ul> +<p></p> +<ul> +<li>Edit solr.in.sh in $SOLR_INSTALL_HOME/bin/</li></ul> +<div class="source"><pre class="prettyprint"> + vi $SOLR_INSTALL_HOME/bin/solr.in.sh + + SOLR_JAAS_FILE=/etc/solr/conf/solr_jaas.conf + SOLR_HOST=`hostname -f` + ZK_HOST="$ZK_HOST1:2181,$ZK_HOST2:2181,$ZK_HOST3:2181/solr" + KERBEROS_REALM="EXAMPLE.COM" + SOLR_KEYTAB=/etc/solr/conf/solr.keytab + SOLR_KERB_PRINCIPAL=HTTP@${KERBEROS_REALM} + SOLR_KERB_KEYTAB=/etc/solr/conf/HTTP.keytab + SOLR_AUTHENTICATION_CLIENT_CONFIGURER="org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer" + SOLR_AUTHENTICATION_OPTS=" -DauthenticationPlugin=org.apache.solr.security.KerberosPlugin -Djava.security.auth.login.config=${SOLR_JAAS_FILE} -Dsolr.kerberos.principal=${SOLR_KERB_PRINCIPAL} -Dsolr.kerberos.keytab=${SOLR_KERB_KEYTAB} -Dsolr.kerberos.cookie.domain=${SOLR_HOST} -Dhost=${SOLR_HOST} -Dsolr.kerberos.name.rules=DEFAULT" + +</pre></div> +<p></p> +<ul> +<li>Copy solr.in.sh to all hosts running Solr.</li></ul> +<p></p> +<ul> +<li>Set up Solr to use the Kerberos plugin by uploading the security.json.</li></ul> +<div class="source"><pre class="prettyprint"> + $SOLR_INSTALL_HOME/server/scripts/cloud-scripts/zkcli.sh -zkhost <zk host>:2181 -cmd put /security.json '{"authentication":{"class": "org.apache.solr.security.KerberosPlugin"}}' + +</pre></div> +<p></p> +<ul> +<li>Start Solr.</li></ul> +<div class="source"><pre class="prettyprint"> + $SOLR_INSTALL_HOME/bin/solr start -cloud -z $ZK_HOST1:2181,$ZK_HOST2:2181,$ZK_HOST3:2181 -noprompt + +</pre></div> +<p></p> +<ul> +<li>Test Solr</li></ul> +<div class="source"><pre class="prettyprint"> + kinit -k -t /etc/security/keytabs/HTTP.keytab HTTP/<host>@EXAMPLE.COM + curl --negotiate -u : "http://<host>:8983/solr/" + +</pre></div> +<p></p> +<ul> +<li>Create collections in Solr corresponding to the indexes that Atlas uses and change the Atlas configuration to point to the Solr instance setup as described in the <a href="./InstallationSteps.html">Install Steps</a>.</li></ul></div> + </div> + </div> + <hr/> + <footer> + <div class="container"> + <div class="row"> +<p><a href="https://www.apache.org/foundation/contributing";><img src="https://www.apache.org/images/SupportApache-small.png"; alt="Support the ASF" id="asf-logo" height="20" width="20" /></a>Copyright © 2011-2018 The Apache Software Foundation. Licensed under the <a href="https://www.apache.org/licenses/";>Apache License, Version 2.0</a>.<br/> +Apache Atlas, Atlas, Apache, the Apache feather logo are trademarks of the <a href="https://www.apache.org";>Apache Software Foundation</a>.<br/> +All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + <p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a> +</p> + </div> + </footer> + </body> +</html>
http://git-wip-us.apache.org/repos/asf/atlas-website/blob/e9346ee2/1.0.0-rc0/TypeSystem.html ---------------------------------------------------------------------- diff --git a/1.0.0-rc0/TypeSystem.html b/1.0.0-rc0/TypeSystem.html new file mode 100644 index 0000000..6bff3e0 --- /dev/null +++ b/1.0.0-rc0/TypeSystem.html @@ -0,0 +1,284 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/twiki/TypeSystem.twiki at 2018-05-26 + | Rendered using Apache Maven Fluido Skin 1.7 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20180526" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache Atlas – Type System</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.7.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + <script type="text/javascript" src="./js/apache-maven-fluido-1.7.min.js"></script> + </head> + <body class="topBarEnabled"> + <div id="topbar" class="navbar navbar-fixed-top "> + <div class="navbar-inner"> + <div class="container" style="width: 68%;"><div class="nav-collapse"> + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Atlas <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="index.html" title="About">About</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="Wiki">Wiki</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="News">News</a></li> + <li><a href="https://git-wip-us.apache.org/repos/asf/atlas.git"; title="Git">Git</a></li> + <li><a href="https://issues.apache.org/jira/browse/ATLAS"; title="Jira">Jira</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/ATLAS/PoweredBy"; title="Powered by">Powered by</a></li> + <li><a href="http://blogs.apache.org/atlas/"; title="Blog">Blog</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Information <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="project-info.html" title="Summary">Summary</a></li> + <li><a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a></li> + <li><a href="http://webchat.freenode.net?channels=apacheatlas&uio=d4"; title="IRC">IRC</a></li> + <li><a href="team-list.html" title="Team">Team</a></li> + <li><a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a></li> + <li><a href="source-repository.html" title="Source Repository">Source Repository</a></li> + <li><a href="license.html" title="License">License</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Releases <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/1.0.0-alpha/"; title="1.0.0-alpha">1.0.0-alpha</a></li> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/1.0.0-rc0/"; title="1.0.0-rc0">1.0.0-rc0</a></li> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/0.8.2/"; title="0.8.2">0.8.2</a></li> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/0.8.1/"; title="0.8.1">0.8.1</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.8.0-incubating/"; title="0.8-incubating">0.8-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.7.1-incubating/"; title="0.7.1-incubating">0.7.1-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.7.0-incubating/"; title="0.7-incubating">0.7-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.6.0-incubating/"; title="0.6-incubating">0.6-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.5.0-incubating/"; title="0.5-incubating">0.5-incubating</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="../index.html" title="latest">latest</a></li> + <li><a href="../1.0.0-alpha/index.html" title="1.0.0-alpha">1.0.0-alpha</a></li> + <li><a href="../1.0.0-rc0/index.html" title="1.0.0-rc0">1.0.0-rc0</a></li> + <li><a href="../0.8.2/index.html" title="0.8.2">0.8.2</a></li> + <li><a href="../0.8.1/index.html" title="0.8.1">0.8.1</a></li> + <li><a href="../0.8.0-incubating/index.html" title="0.8-incubating">0.8-incubating</a></li> + <li><a href="../0.7.1-incubating/index.html" title="0.7.1-incubating">0.7.1-incubating</a></li> + <li><a href="../0.7.0-incubating/index.html" title="0.7-incubating">0.7-incubating</a></li> + <li><a href="../0.6.0-incubating/index.html" title="0.6-incubating">0.6-incubating</a></li> + <li><a href="../0.5.0-incubating/index.html" title="0.5-incubating">0.5-incubating</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/foundation/how-it-works.html"; title="How Apache Works">How Apache Works</a></li> + <li><a href="https://www.apache.org/events/current-event"; title="Events">Events</a></li> + <li><a href="https://www.apache.org/licenses/"; title="License">License</a></li> + <li><a href="http://www.apache.org/foundation/"; title="Foundation">Foundation</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; title="Sponsoring Apache">Sponsoring Apache</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; title="Thanks">Thanks</a></li> + </ul> + </li> + </ul> +<form id="search-form" action="https://www.google.com/search"; method="get" class="navbar-search pull-right" > + <input value="http://atlas.apache.org"; name="sitesearch" type="hidden"/> + <input class="search-query" name="q" id="query" type="text" /> +</form> +<script type="text/javascript">asyncJs( 'https://cse.google.com/brand?form=search-form' )</script> + <iframe src="https://www.facebook.com/plugins/like.php?href=http://atlas.apache.org/atlas-docs&send=false&layout=button_count&show-faces=false&action=like&colorscheme=dark"; + scrolling="no" frameborder="0" + style="border:none; width:100px; height:20px; margin-top: 10px;" class="pull-right" ></iframe> + <script type="text/javascript">asyncJs( 'https://apis.google.com/js/plusone.js' )</script> + <ul class="nav pull-right"><li style="margin-top: 10px;"> + <div class="g-plusone" data-href="http://atlas.apache.org/atlas-docs"; data-size="medium" width="60px" align="right" ></div> + </li></ul> + </div> + </div> + </div> + </div> + <div class="container"> + <div id="banner"> + <div class="pull-left"><a href=".." id="bannerLeft"><img src="images/atlas-logo.png" alt="Apache Atlas" width="200px" height="45px"/></a></div> + <div class="pull-right"></div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="index.html" title="Atlas">Atlas</a><span class="divider">/</span></li> + <li class="active ">Type System</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-05-26</li> + <li id="projectVersion" class="pull-right">Version: 1.0.0-rc0</li> + </ul> + </div> + <div id="bodyColumn" > +<div class="section"> +<h2><a name="Type_System"></a>Type System</h2></div> +<div class="section"> +<h3><a name="Overview"></a>Overview</h3> +<p>Atlas allows users to define a model for the metadata objects they want to manage. The model is composed of definitions called ‘types’. Instances of ‘types’ called ‘entities’ represent the actual metadata objects that are managed. The Type System is a component that allows users to define and manage the types and entities. All metadata objects managed by Atlas out of the box (like Hive tables, for e.g.) are modelled using types and represented as entities. To store new types of metadata in Atlas, one needs to understand the concepts of the type system component.</p></div> +<div class="section"> +<h3><a name="Types"></a>Types</h3> +<p>A ‘Type’ in Atlas is a definition of how a particular type of metadata objects are stored and accessed. A type represents one or a collection of attributes that define the properties for the metadata object. Users with a development background will recognize the similarity of a type to a ‘Class’ definition of object oriented programming languages, or a ‘table schema’ of relational databases.</p> +<p>An example of a type that comes natively defined with Atlas is a Hive table. A Hive table is defined with these attributes:</p> +<div class="source"><pre class="prettyprint"> +Name: hive_table +TypeCategory: Entity +SuperTypes: DataSet +Attributes: + name: string + db: hive_db + owner: string + createTime: date + lastAccessTime: date + comment: string + retention: int + sd: hive_storagedesc + partitionKeys: array<hive_column> + aliases: array<string> + columns: array<hive_column> + parameters: map<string,string> + viewOriginalText: string + viewExpandedText: string + tableType: string + temporary: boolean +</pre></div> +<p>The following points can be noted from the above example:</p> +<p></p> +<ul> +<li>A type in Atlas is identified uniquely by a ‘name’</li> +<li>A type has a metatype. Atlas has the following metatypes: +<ul> +<li>Primitive metatypes: boolean, byte, short, int, long, float, double, biginteger, bigdecimal, string, date</li> +<li>Enum metatypes</li> +<li>Collection metatypes: array, map</li> +<li>Composite metatypes: Entity, Struct, Classification, Relationship</li></ul></li> +<li>Entity & Classification types can ‘extend’ from other types, called ‘supertype’ - by virtue of this, it will get to include the attributes that are defined in the supertype as well. This allows modellers to define common attributes across a set of related types etc. This is again similar to the concept of how Object Oriented languages define super classes for a class. It is also possible for a type in Atlas to extend from multiple super types. +<ul> +<li>In this example, every hive table extends from a pre-defined supertype called a ‘DataSet’. More details about this pre-defined types will be provided later.</li></ul></li> +<li>Types which have a metatype of ‘Entity’, ‘Struct’, ‘Classification’ or 'Relationship' can have a collection of attributes. Each attribute has a name (e.g. ‘name’) and some other associated properties. A property can be referred to using an expression type_name.attribute_name. It is also good to note that attributes themselves are defined using Atlas metatypes. +<ul> +<li>In this example, hive_table.name is a String, hive_table.aliases is an array of Strings, hive_table.db refers to an instance of a type called hive_db and so on.</li></ul></li> +<li>Type references in attributes, (like hive_table.db) are particularly interesting. Note that using such an attribute, we can define arbitrary relationships between two types defined in Atlas and thus build rich models. Note that one can also collect a list of references as an attribute type (e.g. hive_table.columns which represents a list of references from hive_table to hive_column type)</li></ul></div> +<div class="section"> +<h3><a name="Entities"></a>Entities</h3> +<p>An ‘entity’ in Atlas is a specific value or instance of an Entity ‘type’ and thus represents a specific metadata object in the real world. Referring back to our analogy of Object Oriented Programming languages, an ‘instance’ is an ‘Object’ of a certain ‘Class’.</p> +<p>An example of an entity will be a specific Hive Table. Say Hive has a table called ‘customers’ in the ‘default’ database. This table will be an ‘entity’ in Atlas of type hive_table. By virtue of being an instance of an entity type, it will have values for every attribute that are a part of the Hive table ‘type’, such as:</p> +<div class="source"><pre class="prettyprint"> +guid: "9ba387dd-fa76-429c-b791-ffc338d3c91f" +typeName: "hive_table" +status: "ACTIVE" +values: + name: “customers” + db: { "guid": "b42c6cfc-c1e7-42fd-a9e6-890e0adf33bc", "typeName": "hive_db" } + owner: “admin” + createTime: 1490761686029 + updateTime: 1516298102877 + comment: null + retention: 0 + sd: { "guid": "ff58025f-6854-4195-9f75-3a3058dd8dcf", "typeName": "hive_storagedesc" } + partitionKeys: null + aliases: null + columns: [ { "guid": ""65e2204f-6a23-4130-934a-9679af6a211f", "typeName": "hive_column" }, { "guid": ""d726de70-faca-46fb-9c99-cf04f6b579a6", "typeName": "hive_column" }, ...] + parameters: { "transient_lastDdlTime": "1466403208"} + viewOriginalText: null + viewExpandedText: null + tableType: “MANAGED_TABLE” + temporary: false +</pre></div> +<p>The following points can be noted from the example above:</p> +<p></p> +<ul> +<li>Every instance ofan entity type is identified by a unique identifier, a GUID. This GUID is generated by the Atlas server when the object is defined, and remains constant for the entire lifetime of the entity. At any point in time, this particular entity can be accessed using its GUID. +<ul> +<li>In this example, the ‘customers’ table in the default database is uniquely identified by the GUID "9ba387dd-fa76-429c-b791-ffc338d3c91f"</li></ul></li> +<li>An entity is of a given type, and the name of the type is provided with the entity definition. +<ul> +<li>In this example, the ‘customers’ table is a ‘hive_table.</li></ul></li> +<li>The values of this entity are a map of all the attribute names and their values for attributes that are defined in the hive_table type definition.</li> +<li>Attribute values will be according to the datatype of the attribute. Entity-type attributes will have value of type <a href="./AtlasObjectId.html">AtlasObjectId</a></li></ul> +<p>With this idea on entities, we can now see the difference between Entity and Struct metatypes. Entities and Structs both compose attributes of other types. However, instances of Entity types have an identity (with a GUID value) and can be referenced from other entities (like a hive_db entity is referenced from a hive_table entity). Instances of Struct types do not have an identity of their own. The value of a Struct type is a collection of attributes that are ‘embedded’ inside the entity itself.</p></div> +<div class="section"> +<h3><a name="Attributes"></a>Attributes</h3> +<p>We already saw that attributes are defined inside metatypes like Entity, Struct, Classification and Relationship. But we implistically referred to attributes as having a name and a metatype value. However, attributes in Atlas have some more properties that define more concepts related to the type system.</p> +<p>An attribute has the following properties:</p> +<div class="source"><pre class="prettyprint"> + name: string, + typeName: string, + isOptional: boolean, + isIndexable: boolean, + isUnique: boolean, + cardinality: enum +</pre></div> +<p>The properties above have the following meanings:</p> +<p></p> +<ul> +<li>name - the name of the attribute</li> +<li>dataTypeName - the metatype name of the attribute (native, collection or composite)</li> +<li>isComposite - +<ul> +<li>This flag indicates an aspect of modelling. If an attribute is defined as composite, it means that it cannot have a lifecycle independent of the entity it is contained in. A good example of this concept is the set of columns that make a part of a hive table. Since the columns do not have meaning outside of the hive table, they are defined as composite attributes.</li> +<li>A composite attribute must be created in Atlas along with the entity it is contained in. i.e. A hive column must be created along with the hive table.</li></ul></li> +<li>isIndexable - +<ul> +<li>This flag indicates whether this property should be indexed on, so that look ups can be performed using the attribute value as a predicate and can be performed efficiently.</li></ul></li> +<li>isUnique - +<ul> +<li>This flag is again related to indexing. If specified to be unique, it means that a special index is created for this attribute in <a href="./JanusGraph.html">JanusGraph</a> that allows for equality based look ups.</li> +<li>Any attribute with a true value for this flag is treated like a primary key to distinguish this entity from other entities. Hence care should be taken ensure that this attribute does model a unique property in real world. +<ul> +<li>For e.g. consider the name attribute of a hive_table. In isolation, a name is not a unique attribute for a hive_table, because tables with the same name can exist in multiple databases. Even a pair of (database name, table name) is not unique if Atlas is storing metadata of hive tables amongst multiple clusters. Only a cluster location, database name and table name can be deemed unique in the physical world.</li></ul></li></ul></li> +<li>multiplicity - indicates whether this attribute is required, optional, or could be multi-valued. If an entity’s definition of the attribute value does not match the multiplicity declaration in the type definition, this would be a constraint violation and the entity addition will fail. This field can therefore be used to define some constraints on the metadata information.</li></ul> +<p>Using the above, let us expand on the attribute definition of one of the attributes of the hive table below. Let us look at the attribute called ‘db’ which represents the database to which the hive table belongs:</p> +<div class="source"><pre class="prettyprint"> +db: + "name": "db", + "typeName": "hive_db", + "isOptional": false, + "isIndexable": true, + "isUnique": false, + "cardinality": "SINGLE" +</pre></div> +<p>Note the “isOptional=true” constraint - a table entity cannot be created without a db reference.</p> +<div class="source"><pre class="prettyprint"> +columns: + "name": "columns", + "typeName": "array<hive_column>", + "isOptional": optional, + "isIndexable": true, + “isUnique": false, + "constraints": [ { "type": "ownedRef" } ] +</pre></div> +<p>Note the “ownedRef” constraint for columns. By doing this, we are indicating that the defined column entities should always be bound to the table entity they are defined with.</p> +<p>From this description and examples, you will be able to realize that attribute definitions can be used to influence specific modelling behavior (constraints, indexing, etc) to be enforced by the Atlas system.</p></div> +<div class="section"> +<h3><a name="System_specific_types_and_their_significance"></a>System specific types and their significance</h3> +<p>Atlas comes with a few pre-defined system types. We saw one example (DataSet) in preceding sections. In this section we will see more of these types and understand their significance.</p> +<p><b>Referenceable</b>: This type represents all entities that can be searched for using a unique attribute called qualifiedName.</p> +<p><b>Asset</b>: This type extends Referenceable and adds attributes like name, description and owner. Name is a required attribute (isOptional=false), the others are optional.</p> +<p>The purpose of Referenceable and Asset is to provide modellers with way to enforce consistency when defining and querying entities of their own types. Having these fixed set of attributes allows applications and user interfaces to make convention based assumptions about what attributes they can expect of types by default.</p> +<p><b>Infrastructure</b>: This type extends Asset and typically can be used to be a common super type for infrastructural metadata objects like clusters, hosts etc.</p> +<p><b>DataSet</b>: This type extends Referenceable. Conceptually, it can be used to represent an type that stores data. In Atlas, hive tables, hbase_tables etc are all types that extend from DataSet. Types that extend DataSet can be expected to have a Schema in the sense that they would have an attribute that defines attributes of that dataset. For e.g. the columns attribute in a hive_table. Also entities of types that extend DataSet participate in data transformation and this transformation can be captured by Atlas via lineage (or provenance) graphs.</p> +<p><b>Process</b>: This type extends Asset. Conceptually, it can be used to represent any data transformation operation. For example, an ETL process that transforms a hive table with raw data to another hive table that stores some aggregate can be a specific type that extends the Process type. A Process type has two specific attributes, inputs and outputs. Both inputs and outputs are arrays of DataSet entities. Thus an instance of a Process type can use these inputs and outputs to capture how the lineage of a DataSet evolves.</p></div> + </div> + </div> + <hr/> + <footer> + <div class="container"> + <div class="row"> +<p><a href="https://www.apache.org/foundation/contributing";><img src="https://www.apache.org/images/SupportApache-small.png"; alt="Support the ASF" id="asf-logo" height="20" width="20" /></a>Copyright © 2011-2018 The Apache Software Foundation. Licensed under the <a href="https://www.apache.org/licenses/";>Apache License, Version 2.0</a>.<br/> +Apache Atlas, Atlas, Apache, the Apache feather logo are trademarks of the <a href="https://www.apache.org";>Apache Software Foundation</a>.<br/> +All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + <p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a> +</p> + </div> + </footer> + </body> +</html> http://git-wip-us.apache.org/repos/asf/atlas-website/blob/e9346ee2/1.0.0-rc0/WhatsNew-1.0.html ---------------------------------------------------------------------- diff --git a/1.0.0-rc0/WhatsNew-1.0.html b/1.0.0-rc0/WhatsNew-1.0.html new file mode 100644 index 0000000..ac99f74 --- /dev/null +++ b/1.0.0-rc0/WhatsNew-1.0.html @@ -0,0 +1,244 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/WhatsNew-1.0.md at 2018-05-26 + | Rendered using Apache Maven Fluido Skin 1.7 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20180526" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache Atlas – Whats new in Apache Atlas 1.0?</title> + <link rel="stylesheet" href="./css/apache-maven-fluido-1.7.min.css" /> + <link rel="stylesheet" href="./css/site.css" /> + <link rel="stylesheet" href="./css/print.css" media="print" /> + <script type="text/javascript" src="./js/apache-maven-fluido-1.7.min.js"></script> + </head> + <body class="topBarEnabled"> + <div id="topbar" class="navbar navbar-fixed-top "> + <div class="navbar-inner"> + <div class="container" style="width: 68%;"><div class="nav-collapse"> + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Atlas <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="index.html" title="About">About</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="Wiki">Wiki</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/ATLAS"; title="News">News</a></li> + <li><a href="https://git-wip-us.apache.org/repos/asf/atlas.git"; title="Git">Git</a></li> + <li><a href="https://issues.apache.org/jira/browse/ATLAS"; title="Jira">Jira</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/ATLAS/PoweredBy"; title="Powered by">Powered by</a></li> + <li><a href="http://blogs.apache.org/atlas/"; title="Blog">Blog</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Information <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="project-info.html" title="Summary">Summary</a></li> + <li><a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a></li> + <li><a href="http://webchat.freenode.net?channels=apacheatlas&uio=d4"; title="IRC">IRC</a></li> + <li><a href="team-list.html" title="Team">Team</a></li> + <li><a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a></li> + <li><a href="source-repository.html" title="Source Repository">Source Repository</a></li> + <li><a href="license.html" title="License">License</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Releases <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/1.0.0-alpha/"; title="1.0.0-alpha">1.0.0-alpha</a></li> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/1.0.0-rc0/"; title="1.0.0-rc0">1.0.0-rc0</a></li> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/0.8.2/"; title="0.8.2">0.8.2</a></li> + <li><a href="http://www.apache.org/dyn/closer.cgi/atlas/0.8.1/"; title="0.8.1">0.8.1</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.8.0-incubating/"; title="0.8-incubating">0.8-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.7.1-incubating/"; title="0.7.1-incubating">0.7.1-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.7.0-incubating/"; title="0.7-incubating">0.7-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.6.0-incubating/"; title="0.6-incubating">0.6-incubating</a></li> + <li><a href="http://archive.apache.org/dist/incubator/atlas/0.5.0-incubating/"; title="0.5-incubating">0.5-incubating</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="../index.html" title="latest">latest</a></li> + <li><a href="../1.0.0-alpha/index.html" title="1.0.0-alpha">1.0.0-alpha</a></li> + <li><a href="../1.0.0-rc0/index.html" title="1.0.0-rc0">1.0.0-rc0</a></li> + <li><a href="../0.8.2/index.html" title="0.8.2">0.8.2</a></li> + <li><a href="../0.8.1/index.html" title="0.8.1">0.8.1</a></li> + <li><a href="../0.8.0-incubating/index.html" title="0.8-incubating">0.8-incubating</a></li> + <li><a href="../0.7.1-incubating/index.html" title="0.7.1-incubating">0.7.1-incubating</a></li> + <li><a href="../0.7.0-incubating/index.html" title="0.7-incubating">0.7-incubating</a></li> + <li><a href="../0.6.0-incubating/index.html" title="0.6-incubating">0.6-incubating</a></li> + <li><a href="../0.5.0-incubating/index.html" title="0.5-incubating">0.5-incubating</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="http://www.apache.org/foundation/how-it-works.html"; title="How Apache Works">How Apache Works</a></li> + <li><a href="https://www.apache.org/events/current-event"; title="Events">Events</a></li> + <li><a href="https://www.apache.org/licenses/"; title="License">License</a></li> + <li><a href="http://www.apache.org/foundation/"; title="Foundation">Foundation</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html"; title="Sponsoring Apache">Sponsoring Apache</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html"; title="Thanks">Thanks</a></li> + </ul> + </li> + </ul> +<form id="search-form" action="https://www.google.com/search"; method="get" class="navbar-search pull-right" > + <input value="http://atlas.apache.org"; name="sitesearch" type="hidden"/> + <input class="search-query" name="q" id="query" type="text" /> +</form> +<script type="text/javascript">asyncJs( 'https://cse.google.com/brand?form=search-form' )</script> + <iframe src="https://www.facebook.com/plugins/like.php?href=http://atlas.apache.org/atlas-docs&send=false&layout=button_count&show-faces=false&action=like&colorscheme=dark"; + scrolling="no" frameborder="0" + style="border:none; width:100px; height:20px; margin-top: 10px;" class="pull-right" ></iframe> + <script type="text/javascript">asyncJs( 'https://apis.google.com/js/plusone.js' )</script> + <ul class="nav pull-right"><li style="margin-top: 10px;"> + <div class="g-plusone" data-href="http://atlas.apache.org/atlas-docs"; data-size="medium" width="60px" align="right" ></div> + </li></ul> + </div> + </div> + </div> + </div> + <div class="container"> + <div id="banner"> + <div class="pull-left"><a href=".." id="bannerLeft"><img src="images/atlas-logo.png" alt="Apache Atlas" width="200px" height="45px"/></a></div> + <div class="pull-right"></div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + <li class=""><a href="http://www.apache.org"; class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li> + <li class=""><a href="index.html" title="Atlas">Atlas</a><span class="divider">/</span></li> + <li class="active ">Whats new in Apache Atlas 1.0?</li> + <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-05-26</li> + <li id="projectVersion" class="pull-right">Version: 1.0.0-rc0</li> + </ul> + </div> + <div id="bodyColumn" > +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<h1>What’s new in Apache Atlas 1.0?</h1> +<div class="section"> +<h2><a name="Features"></a>Features</h2> +<ul> + +<li>Introduction of relationships as a first-class type</li> +<li>Support for propagation of classifications along entity relationships – like lineage</li> +<li>Fine-grained metadata security, which enables access controls up to entity instance level</li> +<li>Introduction of Glossary feature</li> +<li>Introduction of V2 style notifications</li> +<li>Introduction of Atlas hook for HBase</li> +<li>Support for Cassandra and Elasticsearch (tech-preview)</li> +</ul></div> +<div class="section"> +<h2><a name="Updates"></a>Updates</h2> +<ul> + +<li>Graph store has been updated from Titan 0.5.4 to JanusGraph 0.2.0</li> +<li>DSL rewrite, to replace use of Scala based implementation with ANTLR</li> +<li>Performance improvements in Atlas Hooks, by switching to use V2 style notifications</li> +<li>Significant updates in Atlas Web UI</li> +</ul></div> +<div class="section"> +<h2><a name="Changes"></a>Changes</h2> +<div class="section"> +<h3><a name="DSL_search"></a>DSL search</h3> +<p>With DSL rewrite and simplification, some older constructs may not work. Here’s a list of behavior changes from previous releases. More DSL related changes can be found <a href="Search-Advanced.html">here</a>.</p> +<ul> + +<li>When filtering or narrowing results using string attribute, the value <b>MUST</b> be enclosed in double quotes +<ul> + +<li>Table name=“Table1”</li> +<li>Table where name=“Table1”</li> +</ul> +</li> +<li>Join queries are no longer supported e.g. hive_table, hive_db</li> +<li>Select clauses only work with immediate entity attributes or a single referred (entity) type. +<ul> + +<li>Table select name, owner</li> +<li>Table select Columns</li> +<li>Table select name, owner, Columns <i><i>(won’t work)</i></i></li> +</ul> +</li> +<li>OrderBy clause can only be used with a <i><i>single primitive</i></i> attribute.</li> +<li>GroupBy clause can only be used with a <i><i>single primitive</i></i> attribute. +<ul> + +<li>Table groupby name</li> +<li>Table groupby Columns (won’t work)</li> +</ul> +</li> +<li>Typename can’t have multiple aliases +<ul> + +<li>Table as t (OK)</li> +<li>Table as t1, t2 (won’t work)</li> +</ul> +</li> +<li>Has clause only works with primitive attributes. +<ul> + +<li>Table has name</li> +<li>Table has Columns or Table has DB (NOT supported)</li> +</ul> +</li> +<li>Aggregator clause can only be used with a <i><i>single primitive</i></i> attribute. +<ul> + +<li>Table select min(name)</li> +<li>Table select max(name)</li> +<li>Table select sum(createTime)</li> +<li>Table select min(Columns) (won’t work)</li> +<li>Table select max(Columns) (won’t work)</li> +<li>Table select sum(Columns) (won’t work)</li> +</ul> +</li> +<li>Aggregator clause can’t be repeated with different <i><i>primitive attribute</i></i>, the clause appearing last would take preference. +<ul> + +<li>Table select min(name), min(createTime) will ignore <i><i>min(name)</i></i></li> +</ul> +</li> +<li>Limit and offset are not applicable when using aggregator clauses (min, max, sum) +<ul> + +<li>Table select min(name) limit 10 offset 5 - min(name) is computed over <b>ALL</b> entities of type Asset</li> +</ul> +</li> +</ul></div></div> + </div> + </div> + <hr/> + <footer> + <div class="container"> + <div class="row"> +<p><a href="https://www.apache.org/foundation/contributing";><img src="https://www.apache.org/images/SupportApache-small.png"; alt="Support the ASF" id="asf-logo" height="20" width="20" /></a>Copyright © 2011-2018 The Apache Software Foundation. Licensed under the <a href="https://www.apache.org/licenses/";>Apache License, Version 2.0</a>.<br/> +Apache Atlas, Atlas, Apache, the Apache feather logo are trademarks of the <a href="https://www.apache.org";>Apache Software Foundation</a>.<br/> +All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + <p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/"; title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a> +</p> + </div> + </footer> + </body> +</html>
![https://www.apache.org/images/SupportApache-small.png"](https://www.apache.org/images/SupportApache-small.png"){kind=link}