http://git-wip-us.apache.org/repos/asf/accumulo-website/blob/eab65f94/docs/unreleased/getting-started/clients.html ---------------------------------------------------------------------- diff --git a/docs/unreleased/getting-started/clients.html b/docs/unreleased/getting-started/clients.html new file mode 100644 index 0000000..5b9037e --- /dev/null +++ b/docs/unreleased/getting-started/clients.html @@ -0,0 +1,636 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<meta charset="utf-8"> +<meta http-equiv="X-UA-Compatible" content="IE=edge"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<link href="https://maxcdn.bootstrapcdn.com/bootswatch/3.3.7/paper/bootstrap.min.css"; rel="stylesheet" integrity="sha384-awusxf8AUojygHf2+joICySzB780jVvQaVCAt1clU3QsyAitLGul28Qxb2r1e5g+" crossorigin="anonymous"> +<link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet"> +<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.css";> +<link href="/css/accumulo.css" rel="stylesheet" type="text/css"> + +<title>Accumulo Documentation - Accumulo Clients</title> + +<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js";></script> +<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"; integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script> +<script type="text/javascript" src="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.js";></script> +<script> + // show location of canonical site if not currently on the canonical site + $(function() { + var host = window.location.host; + if (typeof host !== 'undefined' && host !== 'accumulo.apache.org') { + $('#non-canonical').show(); + } + }); + + $(function() { + // decorate section headers with anchors + return $("h2, h3, h4, h5, h6").each(function(i, el) { + var $el, icon, id; + $el = $(el); + id = $el.attr('id'); + icon = '<i class="fa fa-link"></i>'; + if (id) { + return $el.append($("<a />").addClass("header-link").attr("href", "#" + id).html(icon)); + } + }); + }); + + // fix sidebar width in documentation + $(function() { + var $affixElement = $('div[data-spy="affix"]'); + $affixElement.width($affixElement.parent().width()); + }); + + // configure Google Analytics + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + if (ga.hasOwnProperty('loaded') && ga.loaded === true) { + ga('create', 'UA-50934829-1', 'apache.org'); + ga('send', 'pageview'); + } +</script> + +</head> +<body style="padding-top: 100px"> + + <nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a href="/"><img id="nav-logo" alt="Apache Accumulo" class="img-responsive" src="/images/accumulo-logo.png" width="200" + /></a> + </div> + <div class="collapse navbar-collapse" id="navbar-items"> + <ul class="nav navbar-nav"> + <li class="nav-link"><a href="/downloads">Download</a></li> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Releases<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/release/accumulo-1.8.1/">1.8.1 (Latest)</a></li> + <li><a href="/release/accumulo-1.7.3/">1.7.3</a></li> + <li><a href="/release/accumulo-1.6.6/">1.6.6</a></li> + <li><a href="/release/">Archive</a></li> + </ul> + </li> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/1.8/accumulo_user_manual.html">User Manual (1.8)</a></li> + <li><a href="/1.8/apidocs">Javadocs (1.8)</a></li> + <li><a href="/1.8/examples">Examples (1.8)</a></li> + <li><a href="/features">Features</a></li> + <li><a href="/glossary">Glossary</a></li> + <li><a href="/external-docs">External Docs</a></li> + <li><a href="/docs-archive/">Archive</a></li> + </ul> + </li> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Community<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/get_involved">Get Involved</a></li> + <li><a href="/mailing_list">Mailing Lists</a></li> + <li><a href="/people">People</a></li> + <li><a href="/related-projects">Related Projects</a></li> + <li><a href="/contributor/">Contributor Guide</a></li> + </ul> + </li> + </ul> + <ul class="nav navbar-nav navbar-right"> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Apache Software Foundation<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="https://www.apache.org";>Apache Homepage <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/licenses/LICENSE-2.0";>License <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/foundation/sponsorship";>Sponsorship <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/security";>Security <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/foundation/thanks";>Thanks <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/foundation/policies/conduct";>Code of Conduct <i class="fa fa-external-link"></i></a></li> + </ul> + </li> + </ul> + </div> + </div> +</nav> + + <div class="container"> + <div class="row"> + <div class="col-md-12"> + + <div id="non-canonical" style="display: none; background-color: #F0E68C; padding-left: 1em;"> + Visit the official site at: <a href="https://accumulo.apache.org";>https://accumulo.apache.org</a> + </div> + <div id="content"> + + <div class="row"> + <div class="col-md-3"> + <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true" data-spy="affix"> + <div class="panel panel-default"> + + + + + + + + + + <div class="panel-heading" role="tab" id="headingOne"> + <h4 class="panel-title"> + <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsegetting-started" aria-expanded="true" aria-controls="collapsegetting-started"> + Getting started + </a> + </h4> + </div> + <div id="collapsegetting-started" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingOne"> + <div class="panel-body"> + + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/design">Accumulo Design</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/quick-install">Quick Installation</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/clients">Accumulo Clients</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/shell">Accumulo Shell</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/table_design">Table Design</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/table_configuration">Table Configuration</a></div> + + </div> + </div> + + + + + + + + + + + + <div class="panel-heading" role="tab" id="headingOne"> + <h4 class="panel-title"> + <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsedevelopment" aria-expanded="false" aria-controls="collapsedevelopment"> + Development + </a> + </h4> + </div> + <div id="collapsedevelopment" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne"> + <div class="panel-body"> + + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/iterators">Iterators</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/mapreduce">MapReduce</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/proxy">Proxy</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/development_tools">Development Tools</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/sampling">Sampling</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/summaries">Summary Statistics</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/security">Security</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/high_speed_ingest">High-Speed Ingest</a></div> + + </div> + </div> + + + + + + + + + + + + <div class="panel-heading" role="tab" id="headingOne"> + <h4 class="panel-title"> + <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapseadministration" aria-expanded="false" aria-controls="collapseadministration"> + Administration + </a> + </h4> + </div> + <div id="collapseadministration" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne"> + <div class="panel-body"> + + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/in-depth-install">In-depth Installation</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/configuration-management">Configuration Management</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/configuration-properties">Configuration Properties</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/monitoring-metrics">Monitoring & Metrics</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/tracing">Tracing</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/fate">FATE</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/multivolume">Multi-Volume Installations</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/ssl">SSL</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/kerberos">Kerberos</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/replication">Replication</a></div> + + </div> + </div> + + + + + + + + + + + + + + + + + + + + + + <div class="panel-heading" role="tab" id="headingOne"> + <h4 class="panel-title"> + <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsetroubleshooting" aria-expanded="false" aria-controls="collapsetroubleshooting"> + Troubleshooting + </a> + </h4> + </div> + <div id="collapsetroubleshooting" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne"> + <div class="panel-body"> + + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/basic">Basic Troubleshooting</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/advanced">Advanced Troubleshooting</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/tools">Troubleshooting Tools</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/system-metadata-tables">System Metadata Tables</a></div> + + </div> + </div> + + + + </div> + </div> + </div> + <div class="col-md-9"> + + <p><a href="/docs/unreleased/">Accumulo unreleased docs</a> >> Getting started >> Accumulo Clients</p> + + + + <div class="alert alert-danger" style="margin-bottom: 0px;" role="alert">This documentation is for a future release of Accumulo! <a href="/1.8/accumulo_user_manual.html">View documentation for the latest release</a>.</div> + + <div class="row"> + <div class="col-md-10"><h1>Accumulo Clients</h1></div> + <div class="col-md-2"><a class="pull-right" style="margin-top: 25px;" href="https://github.com/apache/accumulo-website/edit/master/_docs-unreleased/getting-started/clients.md"; role="button"><i class="glyphicon glyphicon-pencil"></i> <small>Edit this page</small></a></div> + </div> + + <h2 id="running-client-code">Running Client Code</h2> + +<p>There are multiple ways to run Java code that uses Accumulo. Below is a list +of the different ways to execute client code.</p> + +<ul> + <li>using the <code class="highlighter-rouge">java</code> command</li> + <li>using the <code class="highlighter-rouge">accumulo</code> command</li> + <li>using the <code class="highlighter-rouge">accumulo-util hadoop-jar</code> command</li> +</ul> + +<h3 id="using-the-java-command">Using the java command</h3> + +<p>To run Accumulo client code using the <code class="highlighter-rouge">java</code> command, use the <code class="highlighter-rouge">accumulo classpath</code> command +to include all of Accumuloâs dependencies on your classpath:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>java -classpath /path/to/my.jar:/path/to/dep.jar:$(accumulo classpath) com.my.Main arg1 arg2 +</code></pre> +</div> + +<p>If you would like to review which jars are included, the <code class="highlighter-rouge">accumulo classpath</code> command can +output a more human readable format using the <code class="highlighter-rouge">-d</code> option which enables debugging:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>accumulo classpath -d +</code></pre> +</div> + +<h3 id="using-the-accumulo-command">Using the accumulo command</h3> + +<p>Another option for running your code is to use the Accumulo script which can execute a +main class (if it exists on its classpath):</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>accumulo com.foo.Client arg1 arg2 +</code></pre> +</div> + +<p>While the Accumulo script will add all of Accumuloâs dependencies to the classpath, you +will need to add any jars that your create or depend on beyond what Accumulo already +depends on. This can be accomplished by either adding the jars to the <code class="highlighter-rouge">lib/ext</code> directory +of your Accumulo installation or by adding jars to the CLASSPATH variable before calling +the accumulo command.</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>export CLASSPATH=/path/to/my.jar:/path/to/dep.jar; accumulo com.foo.Client arg1 arg2 +</code></pre> +</div> + +<h3 id="using-the-accumulo-util-hadoop-jar-command">Using the âaccumulo-util hadoop-jarâ command</h3> + +<p>If you are writing map reduce job that accesses Accumulo, then you can use +<code class="highlighter-rouge">accumulo-util hadoop-jar</code> to run those jobs. See the <a href="https://github.com/apache/accumulo-examples/blob/master/docs/mapred.md";>MapReduce example</a> +for more information.</p> + +<h2 id="connecting">Connecting</h2> + +<p>All clients must first identify the Accumulo instance to which they will be +communicating. Code to do this is as follows:</p> + +<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">String</span> <span class="n">instanceName</span> <span class="o">=</span> <span class="s">"myinstance"</span><span class="o">;</span> +<span class="n">String</span> <span class="n">zooServers</span> <span class="o">=</span> <span class="s">"zooserver-one,zooserver-two"</span> +<span class="n">Instance</span> <span class="n">inst</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ZooKeeperInstance</span><span class="o">(</span><span class="n">instanceName</span><span class="o">,</span> <span class="n">zooServers</span><span class="o">);</span> + +<span class="n">Connector</span> <span class="n">conn</span> <span class="o">=</span> <span class="n">inst</span><span class="o">.</span><span class="na">getConnector</span><span class="o">(</span><span class="s">"user"</span><span class="o">,</span> <span class="k">new</span> <span class="n">PasswordToken</span><span class="o">(</span><span class="s">"passwd"</span><span class="o">));</span> +</code></pre> +</div> + +<p>The <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/security/tokens/PasswordToken.html";>PasswordToken</a> is the most common implementation of an <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/security/tokens/AuthenticationToken.html";>AuthenticationToken</a>. +This general interface allow authentication as an Accumulo user to come from +a variety of sources or means. The <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/security/tokens/CredentialProviderToken.html";>CredentialProviderToken</a> leverages the Hadoop +CredentialProviders (new in Hadoop 2.6).</p> + +<p>For example, the <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/security/tokens/CredentialProviderToken.html";>CredentialProviderToken</a> can be used in conjunction with a Java +KeyStore to alleviate passwords stored in cleartext. When stored in HDFS, a single +KeyStore can be used across an entire instance. Be aware that KeyStores stored on +the local filesystem must be made available to all nodes in the Accumulo cluster.</p> + +<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">KerberosToken</span> <span class="n">token</span> <span class="o">=</span> <span class="k">new</span> <span class="n">KerberosToken</span><span class="o">();</span> +<span class="n">Connector</span> <span class="n">conn</span> <span class="o">=</span> <span class="n">inst</span><span class="o">.</span><span class="na">getConnector</span><span class="o">(</span><span class="n">token</span><span class="o">.</span><span class="na">getPrincipal</span><span class="o">(),</span> <span class="n">token</span><span class="o">);</span> +</code></pre> +</div> + +<p>The <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/security/tokens/KerberosToken.html";>KerberosToken</a> can be provided to use the authentication provided by Kerberos. +Using Kerberos requires external setup and additional configuration, but provides +a single point of authentication through HDFS, YARN and ZooKeeper and allowing +for password-less authentication with Accumulo.</p> + +<h2 id="writing-data">Writing Data</h2> + +<p>Data are written to Accumulo by creating <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/data/Mutation.html";>Mutation</a> objects that represent all the +changes to the columns of a single row. The changes are made atomically in the +TabletServer. Clients then add Mutations to a <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/BatchWriter.html";>BatchWriter</a> which submits them to +the appropriate TabletServers.</p> + +<p>Mutations can be created thus:</p> + +<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">Text</span> <span class="n">rowID</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Text</span><span class="o">(</span><span class="s">"row1"</span><span class="o">);</span> +<span class="n">Text</span> <span class="n">colFam</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Text</span><span class="o">(</span><span class="s">"myColFam"</span><span class="o">);</span> +<span class="n">Text</span> <span class="n">colQual</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Text</span><span class="o">(</span><span class="s">"myColQual"</span><span class="o">);</span> +<span class="n">ColumnVisibility</span> <span class="n">colVis</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ColumnVisibility</span><span class="o">(</span><span class="s">"public"</span><span class="o">);</span> +<span class="kt">long</span> <span class="n">timestamp</span> <span class="o">=</span> <span class="n">System</span><span class="o">.</span><span class="na">currentTimeMillis</span><span class="o">();</span> + +<span class="n">Value</span> <span class="n">value</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Value</span><span class="o">(</span><span class="s">"myValue"</span><span class="o">.</span><span class="na">getBytes</span><span class="o">());</span> + +<span class="n">Mutation</span> <span class="n">mutation</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Mutation</span><span class="o">(</span><span class="n">rowID</span><span class="o">);</span> +<span class="n">mutation</span><span class="o">.</span><span class="na">put</span><span class="o">(</span><span class="n">colFam</span><span class="o">,</span> <span class="n">colQual</span><span class="o">,</span> <span class="n">colVis</span><span class="o">,</span> <span class="n">timestamp</span><span class="o">,</span> <span class="n">value</span><span class="o">);</span> +</code></pre> +</div> + +<h3 id="batchwriter">BatchWriter</h3> + +<p>The <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/BatchWriter.html";>BatchWriter</a> is highly optimized to send Mutations to multiple TabletServers +and automatically batches Mutations destined for the same TabletServer to +amortize network overhead. Care must be taken to avoid changing the contents of +any Object passed to the BatchWriter since it keeps objects in memory while +batching.</p> + +<p>Mutations are added to a BatchWriter thus:</p> + +<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="c1">// BatchWriterConfig has reasonable defaults</span> +<span class="n">BatchWriterConfig</span> <span class="n">config</span> <span class="o">=</span> <span class="k">new</span> <span class="n">BatchWriterConfig</span><span class="o">();</span> +<span class="n">config</span><span class="o">.</span><span class="na">setMaxMemory</span><span class="o">(</span><span class="mi">10000000L</span><span class="o">);</span> <span class="c1">// bytes available to batchwriter for buffering mutations</span> + +<span class="n">BatchWriter</span> <span class="n">writer</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="na">createBatchWriter</span><span class="o">(</span><span class="s">"table"</span><span class="o">,</span> <span class="n">config</span><span class="o">)</span> +<span class="n">writer</span><span class="o">.</span><span class="na">addMutation</span><span class="o">(</span><span class="n">mutation</span><span class="o">);</span> +<span class="n">writer</span><span class="o">.</span><span class="na">close</span><span class="o">();</span> +</code></pre> +</div> + +<p>For more example code, see the <a href="https://github.com/apache/accumulo-examples/blob/master/docs/batch.md";>batch writing and scanning example</a>.</p> + +<h3 id="conditionalwriter">ConditionalWriter</h3> + +<p>The <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/ConditionalWriter.html";>ConditionalWriter</a> enables efficient, atomic read-modify-write operations on +rows. The ConditionalWriter writes special Mutations which have a list of per +column conditions that must all be met before the mutation is applied. The +conditions are checked in the tablet server while a row lock is +held (Mutations written by the <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/BatchWriter.html";>BatchWriter</a> will not obtain a row +lock). The conditions that can be checked for a column are equality and +absence. For example a conditional mutation can require that column A is +absent inorder to be applied. Iterators can be applied when checking +conditions. Using iterators, many other operations besides equality and +absence can be checked. For example, using an iterator that converts values +less than 5 to 0 and everything else to 1, its possible to only apply a +mutation when a column is less than 5.</p> + +<p>In the case when a tablet server dies after a client sent a conditional +mutation, its not known if the mutation was applied or not. When this happens +the <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/ConditionalWriter.html";>ConditionalWriter</a> reports a status of UNKNOWN for the ConditionalMutation. +In many cases this situation can be dealt with by simply reading the row again +and possibly sending another conditional mutation. If this is not sufficient, +then a higher level of abstraction can be built by storing transactional +information within a row.</p> + +<p>See the <a href="https://github.com/apache/accumulo-examples/blob/master/docs/reservations.md";>reservations example</a> for example code that uses the <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/ConditionalWriter.html";>ConditionalWriter</a>.</p> + +<h3 id="durability">Durability</h3> + +<p>By default, Accumulo writes out any updates to the Write-Ahead Log (WAL). Every change +goes into a file in HDFS and is syncâd to disk for maximum durability. In +the event of a failure, writes held in memory are replayed from the WAL. Like +all files in HDFS, this file is also replicated. Sending updates to the +replicas, and waiting for a permanent sync to disk can significantly write speeds.</p> + +<p>Accumulo allows users to use less tolerant forms of durability when writing. +These levels are:</p> + +<ul> + <li>none: no durability guarantees are made, the WAL is not used</li> + <li>log: the WAL is used, but not flushed; loss of the server probably means recent writes are lost</li> + <li>flush: updates are written to the WAL, and flushed out to replicas; loss of a single server is unlikely to result in data loss.</li> + <li>sync: updates are written to the WAL, and synced to disk on all replicas before the write is acknowledge. Data will not be lost even if the entire cluster suddenly loses power.</li> +</ul> + +<p>The user can set the default durability of a table in the shell. When +writing, the user can configure the BatchWriter or ConditionalWriter to use +a different level of durability for the session. This will override the +default durability setting.</p> + +<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">BatchWriterConfig</span> <span class="n">cfg</span> <span class="o">=</span> <span class="k">new</span> <span class="n">BatchWriterConfig</span><span class="o">();</span> +<span class="c1">// We don't care about data loss with these writes:</span> +<span class="c1">// This is DANGEROUS:</span> +<span class="n">cfg</span><span class="o">.</span><span class="na">setDurability</span><span class="o">(</span><span class="n">Durability</span><span class="o">.</span><span class="na">NONE</span><span class="o">);</span> + +<span class="n">Connection</span> <span class="n">conn</span> <span class="o">=</span> <span class="o">...</span> <span class="o">;</span> +<span class="n">BatchWriter</span> <span class="n">bw</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="na">createBatchWriter</span><span class="o">(</span><span class="n">table</span><span class="o">,</span> <span class="n">cfg</span><span class="o">);</span> +</code></pre> +</div> + +<h2 id="reading-data">Reading Data</h2> + +<p>Accumulo is optimized to quickly retrieve the value associated with a given key, and +to efficiently return ranges of consecutive keys and their associated values.</p> + +<h3 id="scanner">Scanner</h3> + +<p>To retrieve data, Clients use a <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/Scanner.html";>Scanner</a>, which acts like an Iterator over +keys and values. Scanners can be configured to start and stop at particular keys, and +to return a subset of the columns available.</p> + +<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="c1">// specify which visibilities we are allowed to see</span> +<span class="n">Authorizations</span> <span class="n">auths</span> <span class="o">=</span> <span class="k">new</span> <span class="n">Authorizations</span><span class="o">(</span><span class="s">"public"</span><span class="o">);</span> + +<span class="n">Scanner</span> <span class="n">scan</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="na">createScanner</span><span class="o">(</span><span class="s">"table"</span><span class="o">,</span> <span class="n">auths</span><span class="o">);</span> +<span class="n">scan</span><span class="o">.</span><span class="na">setRange</span><span class="o">(</span><span class="k">new</span> <span class="n">Range</span><span class="o">(</span><span class="s">"harry"</span><span class="o">,</span><span class="s">"john"</span><span class="o">));</span> +<span class="n">scan</span><span class="o">.</span><span class="na">fetchColumnFamily</span><span class="o">(</span><span class="k">new</span> <span class="n">Text</span><span class="o">(</span><span class="s">"attributes"</span><span class="o">));</span> + +<span class="k">for</span> <span class="o">(</span><span class="n">Entry</span><span class="o"><</span><span class="n">Key</span><span class="o">,</span><span class="n">Value</span><span class="o">></span> <span class="n">entry</span> <span class="o">:</span> <span class="n">scan</span><span class="o">)</span> <span class="o">{</span> + <span class="n">Text</span> <span class="n">row</span> <span class="o">=</span> <span class="n">entry</span><span class="o">.</span><span class="na">getKey</span><span class="o">().</span><span class="na">getRow</span><span class="o">();</span> + <span class="n">Value</span> <span class="n">value</span> <span class="o">=</span> <span class="n">entry</span><span class="o">.</span><span class="na">getValue</span><span class="o">();</span> +<span class="o">}</span> +</code></pre> +</div> + +<h3 id="isolated-scanner">Isolated Scanner</h3> + +<p>Accumulo supports the ability to present an isolated view of rows when +scanning. There are three possible ways that a row could change in Accumulo :</p> + +<ul> + <li>a mutation applied to a table</li> + <li>iterators executed as part of a minor or major compaction</li> + <li>bulk import of new files</li> +</ul> + +<p>Isolation guarantees that either all or none of the changes made by these +operations on a row are seen. Use the <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/IsolatedScanner.html";>IsolatedScanner</a> to obtain an isolated +view of an Accumulo table. When using the regular scanner it is possible to see +a non isolated view of a row. For example if a mutation modifies three +columns, it is possible that you will only see two of those modifications. +With the isolated scanner either all three of the changes are seen or none.</p> + +<p>The <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/IsolatedScanner.html";>IsolatedScanner</a> buffers rows on the client side so a large row will not +crash a tablet server. By default rows are buffered in memory, but the user +can easily supply their own buffer if they wish to buffer to disk when rows are +large.</p> + +<p>See the <a href="https://github.com/apache/accumulo-examples/blob/master/docs/isolation.md";>isolation example</a> for example code that uses the <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/IsolatedScanner.html";>IsolatedScanner</a>.</p> + +<h3 id="batchscanner">BatchScanner</h3> + +<p>For some types of access, it is more efficient to retrieve several ranges +simultaneously. This arises when accessing a set of rows that are not consecutive +whose IDs have been retrieved from a secondary index, for example.</p> + +<p>The <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/BatchScanner.html";>BatchScanner</a> is configured similarly to the <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/Scanner.html";>Scanner</a>; it can be configured to +retrieve a subset of the columns available, but rather than passing a single <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/data/Range.html";>Range</a>, +BatchScanners accept a set of Ranges. It is important to note that the keys returned +by a <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/BatchScanner.html";>BatchScanner</a> are not in sorted order since the keys streamed are from multiple +TabletServers in parallel.</p> + +<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="n">ArrayList</span><span class="o"><</span><span class="n">Range</span><span class="o">></span> <span class="n">ranges</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o"><</span><span class="n">Range</span><span class="o">>();</span> +<span class="c1">// populate list of ranges ...</span> + +<span class="n">BatchScanner</span> <span class="n">bscan</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="na">createBatchScanner</span><span class="o">(</span><span class="s">"table"</span><span class="o">,</span> <span class="n">auths</span><span class="o">,</span> <span class="mi">10</span><span class="o">);</span> +<span class="n">bscan</span><span class="o">.</span><span class="na">setRanges</span><span class="o">(</span><span class="n">ranges</span><span class="o">);</span> +<span class="n">bscan</span><span class="o">.</span><span class="na">fetchColumnFamily</span><span class="o">(</span><span class="s">"attributes"</span><span class="o">);</span> + +<span class="k">for</span> <span class="o">(</span><span class="n">Entry</span><span class="o"><</span><span class="n">Key</span><span class="o">,</span><span class="n">Value</span><span class="o">></span> <span class="n">entry</span> <span class="o">:</span> <span class="n">bscan</span><span class="o">)</span> <span class="o">{</span> + <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">entry</span><span class="o">.</span><span class="na">getValue</span><span class="o">());</span> +<span class="o">}</span> +</code></pre> +</div> + +<p>For more example code, see the <a href="https://github.com/apache/accumulo-examples/blob/master/docs/batch.md";>batch writing and scanning example</a>.</p> + +<p>At this time, there is no client side isolation support for the <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/client/BatchScanner.html";>BatchScanner</a>. +You may consider using the <a href="https://static.javadoc.io/org.apache.accumulo/accumulo-core/1.8.1/org/apache/accumulo/core/iterators/user/WholeRowIterator.html";>WholeRowIterator</a> with the BatchScanner to achieve +isolation. The drawback of this approach is that entire rows are read into +memory on the server side. If a row is too big, it may crash a tablet server.</p> + +<h2 id="additional-documentation">Additional Documentation</h2> + +<p>This page covers Accumulo client basics. Below are links to additional documentation that may be useful when creating Accumulo clients:</p> + +<ul> + <li><a href="/docs/unreleased/development/iterators">Iterators</a> - Server-side programming mechanism that can modify key/value pairs at various points in data management process</li> + <li><a href="/docs/unreleased/development/proxy">Proxy</a> - Documentation for interacting with Accumulo using non-Java languages through a proxy server</li> + <li><a href="/docs/unreleased/development/mapreduce">MapReduce</a> - Documentation for reading and writing to Accumulo using MapReduce.</li> +</ul> + + + + <div class="row" style="margin-top: 20px;"> + <div class="col-md-10"><strong>Find documentation for all releases in the <a href="/docs-archive">archive</strong></div> + <div class="col-md-2"><a class="pull-right" href="https://github.com/apache/accumulo-website/edit/master/_docs-unreleased/getting-started/clients.md"; role="button"><i class="glyphicon glyphicon-pencil"></i> <small>Edit this page</small></a></div> + </div> + </div> +</div> + + </div> + + +<footer> + + <p><a href="https://www.apache.org/foundation/contributing";><img src="https://www.apache.org/images/SupportApache-small.png"; alt="Support the ASF" id="asf-logo" height="100" /></a></p> + + <p>Copyright © 2011-2017 The Apache Software Foundation. Licensed under the <a href="https://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>.</p> + +</footer> + + + </div> + </div> + </div> +</body> +</html>
http://git-wip-us.apache.org/repos/asf/accumulo-website/blob/eab65f94/docs/unreleased/getting-started/design.html ---------------------------------------------------------------------- diff --git a/docs/unreleased/getting-started/design.html b/docs/unreleased/getting-started/design.html new file mode 100644 index 0000000..99e1f86 --- /dev/null +++ b/docs/unreleased/getting-started/design.html @@ -0,0 +1,515 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<meta charset="utf-8"> +<meta http-equiv="X-UA-Compatible" content="IE=edge"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<link href="https://maxcdn.bootstrapcdn.com/bootswatch/3.3.7/paper/bootstrap.min.css"; rel="stylesheet" integrity="sha384-awusxf8AUojygHf2+joICySzB780jVvQaVCAt1clU3QsyAitLGul28Qxb2r1e5g+" crossorigin="anonymous"> +<link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet"> +<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.css";> +<link href="/css/accumulo.css" rel="stylesheet" type="text/css"> + +<title>Accumulo Documentation - Accumulo Design</title> + +<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.4/jquery.min.js";></script> +<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"; integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script> +<script type="text/javascript" src="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.js";></script> +<script> + // show location of canonical site if not currently on the canonical site + $(function() { + var host = window.location.host; + if (typeof host !== 'undefined' && host !== 'accumulo.apache.org') { + $('#non-canonical').show(); + } + }); + + $(function() { + // decorate section headers with anchors + return $("h2, h3, h4, h5, h6").each(function(i, el) { + var $el, icon, id; + $el = $(el); + id = $el.attr('id'); + icon = '<i class="fa fa-link"></i>'; + if (id) { + return $el.append($("<a />").addClass("header-link").attr("href", "#" + id).html(icon)); + } + }); + }); + + // fix sidebar width in documentation + $(function() { + var $affixElement = $('div[data-spy="affix"]'); + $affixElement.width($affixElement.parent().width()); + }); + + // configure Google Analytics + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + if (ga.hasOwnProperty('loaded') && ga.loaded === true) { + ga('create', 'UA-50934829-1', 'apache.org'); + ga('send', 'pageview'); + } +</script> + +</head> +<body style="padding-top: 100px"> + + <nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a href="/"><img id="nav-logo" alt="Apache Accumulo" class="img-responsive" src="/images/accumulo-logo.png" width="200" + /></a> + </div> + <div class="collapse navbar-collapse" id="navbar-items"> + <ul class="nav navbar-nav"> + <li class="nav-link"><a href="/downloads">Download</a></li> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Releases<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/release/accumulo-1.8.1/">1.8.1 (Latest)</a></li> + <li><a href="/release/accumulo-1.7.3/">1.7.3</a></li> + <li><a href="/release/accumulo-1.6.6/">1.6.6</a></li> + <li><a href="/release/">Archive</a></li> + </ul> + </li> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/1.8/accumulo_user_manual.html">User Manual (1.8)</a></li> + <li><a href="/1.8/apidocs">Javadocs (1.8)</a></li> + <li><a href="/1.8/examples">Examples (1.8)</a></li> + <li><a href="/features">Features</a></li> + <li><a href="/glossary">Glossary</a></li> + <li><a href="/external-docs">External Docs</a></li> + <li><a href="/docs-archive/">Archive</a></li> + </ul> + </li> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Community<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/get_involved">Get Involved</a></li> + <li><a href="/mailing_list">Mailing Lists</a></li> + <li><a href="/people">People</a></li> + <li><a href="/related-projects">Related Projects</a></li> + <li><a href="/contributor/">Contributor Guide</a></li> + </ul> + </li> + </ul> + <ul class="nav navbar-nav navbar-right"> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#">Apache Software Foundation<span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="https://www.apache.org";>Apache Homepage <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/licenses/LICENSE-2.0";>License <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/foundation/sponsorship";>Sponsorship <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/security";>Security <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/foundation/thanks";>Thanks <i class="fa fa-external-link"></i></a></li> + <li><a href="https://www.apache.org/foundation/policies/conduct";>Code of Conduct <i class="fa fa-external-link"></i></a></li> + </ul> + </li> + </ul> + </div> + </div> +</nav> + + <div class="container"> + <div class="row"> + <div class="col-md-12"> + + <div id="non-canonical" style="display: none; background-color: #F0E68C; padding-left: 1em;"> + Visit the official site at: <a href="https://accumulo.apache.org";>https://accumulo.apache.org</a> + </div> + <div id="content"> + + <div class="row"> + <div class="col-md-3"> + <div class="panel-group" id="accordion" role="tablist" aria-multiselectable="true" data-spy="affix"> + <div class="panel panel-default"> + + + + + + + + + + <div class="panel-heading" role="tab" id="headingOne"> + <h4 class="panel-title"> + <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsegetting-started" aria-expanded="true" aria-controls="collapsegetting-started"> + Getting started + </a> + </h4> + </div> + <div id="collapsegetting-started" class="panel-collapse collapse in" role="tabpanel" aria-labelledby="headingOne"> + <div class="panel-body"> + + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/design">Accumulo Design</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/quick-install">Quick Installation</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/clients">Accumulo Clients</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/shell">Accumulo Shell</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/table_design">Table Design</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/getting-started/table_configuration">Table Configuration</a></div> + + </div> + </div> + + + + + + + + + + + + <div class="panel-heading" role="tab" id="headingOne"> + <h4 class="panel-title"> + <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsedevelopment" aria-expanded="false" aria-controls="collapsedevelopment"> + Development + </a> + </h4> + </div> + <div id="collapsedevelopment" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne"> + <div class="panel-body"> + + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/iterators">Iterators</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/mapreduce">MapReduce</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/proxy">Proxy</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/development_tools">Development Tools</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/sampling">Sampling</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/summaries">Summary Statistics</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/security">Security</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/development/high_speed_ingest">High-Speed Ingest</a></div> + + </div> + </div> + + + + + + + + + + + + <div class="panel-heading" role="tab" id="headingOne"> + <h4 class="panel-title"> + <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapseadministration" aria-expanded="false" aria-controls="collapseadministration"> + Administration + </a> + </h4> + </div> + <div id="collapseadministration" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne"> + <div class="panel-body"> + + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/in-depth-install">In-depth Installation</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/configuration-management">Configuration Management</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/configuration-properties">Configuration Properties</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/monitoring-metrics">Monitoring & Metrics</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/tracing">Tracing</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/fate">FATE</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/multivolume">Multi-Volume Installations</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/ssl">SSL</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/kerberos">Kerberos</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/administration/replication">Replication</a></div> + + </div> + </div> + + + + + + + + + + + + + + + + + + + + + + <div class="panel-heading" role="tab" id="headingOne"> + <h4 class="panel-title"> + <a role="button" data-toggle="collapse" data-parent="#accordion" href="#collapsetroubleshooting" aria-expanded="false" aria-controls="collapsetroubleshooting"> + Troubleshooting + </a> + </h4> + </div> + <div id="collapsetroubleshooting" class="panel-collapse collapse" role="tabpanel" aria-labelledby="headingOne"> + <div class="panel-body"> + + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/basic">Basic Troubleshooting</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/advanced">Advanced Troubleshooting</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/tools">Troubleshooting Tools</a></div> + + <div class="row doc-sidebar-link"><a href="/docs/unreleased/troubleshooting/system-metadata-tables">System Metadata Tables</a></div> + + </div> + </div> + + + + </div> + </div> + </div> + <div class="col-md-9"> + + <p><a href="/docs/unreleased/">Accumulo unreleased docs</a> >> Getting started >> Accumulo Design</p> + + + + <div class="alert alert-danger" style="margin-bottom: 0px;" role="alert">This documentation is for a future release of Accumulo! <a href="/1.8/accumulo_user_manual.html">View documentation for the latest release</a>.</div> + + <div class="row"> + <div class="col-md-10"><h1>Accumulo Design</h1></div> + <div class="col-md-2"><a class="pull-right" style="margin-top: 25px;" href="https://github.com/apache/accumulo-website/edit/master/_docs-unreleased/getting-started/design.md"; role="button"><i class="glyphicon glyphicon-pencil"></i> <small>Edit this page</small></a></div> + </div> + + <h2 id="data-model">Data Model</h2> + +<p>Accumulo provides a richer data model than simple key-value stores, but is not a +fully relational database. Data is represented as key-value pairs, where the key and +value are comprised of the following elements:</p> + +<p><img src="https://accumulo.apache.org/images/docs/key_value.png"; alt="key value pair" /></p> + +<p>All elements of the Key and the Value are represented as byte arrays except for +Timestamp, which is a Long. Accumulo sorts keys by element and lexicographically +in ascending order. Timestamps are sorted in descending order so that later +versions of the same Key appear first in a sequential scan. Tables consist of a set of +sorted key-value pairs.</p> + +<h2 id="architecture">Architecture</h2> + +<p>Accumulo is a distributed data storage and retrieval system and as such consists of +several architectural components, some of which run on many individual servers. +Much of the work Accumulo does involves maintaining certain properties of the +data, such as organization, availability, and integrity, across many commodity-class +machines.</p> + +<h2 id="components">Components</h2> + +<p>An instance of Accumulo includes many TabletServers, one Garbage Collector process, +one Master server and many Clients.</p> + +<h3 id="tablet-server">Tablet Server</h3> + +<p>The TabletServer manages some subset of all the tablets (partitions of tables). This includes receiving writes from clients, persisting writes to a +write-ahead log, sorting new key-value pairs in memory, periodically +flushing sorted key-value pairs to new files in HDFS, and responding +to reads from clients, forming a merge-sorted view of all keys and +values from all the files it has created and the sorted in-memory +store.</p> + +<p>TabletServers also perform recovery of a tablet +that was previously on a server that failed, reapplying any writes +found in the write-ahead log to the tablet.</p> + +<h3 id="garbage-collector">Garbage Collector</h3> + +<p>Accumulo processes will share files stored in HDFS. Periodically, the Garbage +Collector will identify files that are no longer needed by any process, and +delete them. Multiple garbage collectors can be run to provide hot-standby support. +They will perform leader election among themselves to choose a single active instance.</p> + +<h3 id="master">Master</h3> + +<p>The Accumulo Master is responsible for detecting and responding to TabletServer +failure. It tries to balance the load across TabletServer by assigning tablets carefully +and instructing TabletServers to unload tablets when necessary. The Master ensures all +tablets are assigned to one TabletServer each, and handles table creation, alteration, +and deletion requests from clients. The Master also coordinates startup, graceful +shutdown and recovery of changes in write-ahead logs when Tablet servers fail.</p> + +<p>Multiple masters may be run. The masters will choose among themselves a single master, +and the others will become backups if the master should fail.</p> + +<h3 id="tracer">Tracer</h3> + +<p>The Accumulo Tracer process supports the distributed timing API provided by Accumulo. +One to many of these processes can be run on a cluster which will write the timing +information to a given Accumulo table for future reference. See the +<a href="/docs/unreleased/administration/tracing">tracing documentation</a> for more information.</p> + +<h3 id="monitor">Monitor</h3> + +<p>The Accumulo Monitor is a web application that provides a wealth of information about +the state of an instance. The Monitor shows graphs and tables which contain information +about read/write rates, cache hit/miss rates, and Accumulo table information such as scan +rate and active/queued compactions. Additionally, the Monitor should always be the first +point of entry when attempting to debug an Accumulo problem as it will show high-level problems +in addition to aggregated errors from all nodes in the cluster. See the <a href="/docs/unreleased/administration/monitoring-metrics#monitoring">Accumulo monitor documentation</a> +for more information.</p> + +<p>Multiple Monitors can be run to provide hot-standby support in the face of failure. Due to the +forwarding of logs from remote hosts to the Monitor, only one Monitor process should be active +at one time. Leader election will be performed internally to choose the active Monitor.</p> + +<h3 id="client">Client</h3> + +<p>Accumulo has a client library that can be used to write applications that write and read +data to/from Accumulo. See the <a href="/docs/unreleased/getting-started/clients">Accumulo clients documentation</a> for more information.</p> + +<h2 id="data-management">Data Management</h2> + +<p>Accumulo stores data in tables, which are partitioned into tablets. Tablets are +partitioned on row boundaries so that all of the columns and values for a particular +row are found together within the same tablet. The Master assigns Tablets to one +TabletServer at a time. This enables row-level transactions to take place without +using distributed locking or some other complicated synchronization mechanism. As +clients insert and query data, and as machines are added and removed from the +cluster, the Master migrates tablets to ensure they remain available and that the +ingest and query load is balanced across the cluster.</p> + +<p><img src="https://accumulo.apache.org/images/docs/data_distribution.png"; alt="data distribution" /></p> + +<h2 id="tablet-service">Tablet Service</h2> + +<p>When a write arrives at a TabletServer it is written to a Write-Ahead Log and +then inserted into a sorted data structure in memory called a MemTable. When the +MemTable reaches a certain size, the TabletServer writes out the sorted +key-value pairs to a file in HDFS called a Relative Key File (RFile), which is a +kind of Indexed Sequential Access Method (ISAM) file. This process is called a +minor compaction. A new MemTable is then created and the fact of the compaction +is recorded in the Write-Ahead Log.</p> + +<p>When a request to read data arrives at a TabletServer, the TabletServer does a +binary search across the MemTable as well as the in-memory indexes associated +with each RFile to find the relevant values. If clients are performing a scan, +several key-value pairs are returned to the client in order from the MemTable +and the set of RFiles by performing a merge-sort as they are read.</p> + +<h2 id="compactions">Compactions</h2> + +<p>In order to manage the number of files per tablet, periodically the TabletServer +performs Major Compactions of files within a tablet, in which some set of RFiles +are combined into one file. The previous files will eventually be removed by the +Garbage Collector. This also provides an opportunity to permanently remove +deleted key-value pairs by omitting key-value pairs suppressed by a delete entry +when the new file is created. See the <a href="/docs/unreleased/getting-started/table_configuration#compaction">compaction documentation</a> +for more information.</p> + +<h2 id="splitting">Splitting</h2> + +<p>When a table is created it has one tablet. As the table grows its initial +tablet eventually splits into two tablets. Its likely that one of these +tablets will migrate to another tablet server. As the table continues to grow, +its tablets will continue to split and be migrated. The decision to +automatically split a tablet is based on the size of a tablets files. The +size threshold at which a tablet splits is configurable per table. In addition +to automatic splitting, a user can manually add split points to a table to +create new tablets. Manually splitting a new table can parallelize reads and +writes giving better initial performance without waiting for automatic +splitting.</p> + +<p>As data is deleted from a table, tablets may shrink. Over time this can lead +to small or empty tablets. To deal with this, the <a href="/docs/unreleased/getting-started/table_configuration#merging-tablets">merging of tablets</a> +was introduced in Accumulo 1.4.</p> + +<h2 id="fault-tolerance">Fault-Tolerance</h2> + +<p>If a TabletServer fails, the Master detects it and automatically reassigns the tablets +assigned from the failed server to other servers. Any key-value pairs that were in +memory at the time the TabletServer fails are automatically reapplied from the Write-Ahead +Log(WAL) to prevent any loss of data.</p> + +<p>Tablet servers write their WALs directly to HDFS so the logs are available to all tablet +servers for recovery. To make the recovery process efficient, the updates within a log are +grouped by tablet. TabletServers can quickly apply the mutations from the sorted logs +that are destined for the tablets they have now been assigned.</p> + +<p>TabletServer failures are noted on the Masterâs monitor page, accessible via +<code class="highlighter-rouge">http://master-address:9995/monitor</code>.</p> + +<p><img src="https://accumulo.apache.org/images/docs/failure_handling.png"; alt="failure handling" /></p> + + + + <div class="row" style="margin-top: 20px;"> + <div class="col-md-10"><strong>Find documentation for all releases in the <a href="/docs-archive">archive</strong></div> + <div class="col-md-2"><a class="pull-right" href="https://github.com/apache/accumulo-website/edit/master/_docs-unreleased/getting-started/design.md"; role="button"><i class="glyphicon glyphicon-pencil"></i> <small>Edit this page</small></a></div> + </div> + </div> +</div> + + </div> + + +<footer> + + <p><a href="https://www.apache.org/foundation/contributing";><img src="https://www.apache.org/images/SupportApache-small.png"; alt="Support the ASF" id="asf-logo" height="100" /></a></p> + + <p>Copyright © 2011-2017 The Apache Software Foundation. Licensed under the <a href="https://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>.</p> + +</footer> + + + </div> + </div> + </div> +</body> +</html>
![https://www.apache.org/images/SupportApache-small.png"](https://www.apache.org/images/SupportApache-small.png"){kind=link}
![https://accumulo.apache.org/images/docs/key_value.png"](https://accumulo.apache.org/images/docs/key_value.png"){kind=link}
![https://accumulo.apache.org/images/docs/data_distribution.png"](https://accumulo.apache.org/images/docs/data_distribution.png"){kind=link}
![https://accumulo.apache.org/images/docs/failure_handling.png"](https://accumulo.apache.org/images/docs/failure_handling.png"){kind=link}