http://git-wip-us.apache.org/repos/asf/hbase-site/blob/86fde03b/apidocs/src-html/org/apache/hadoop/hbase/util/ByteBufferUtils.html ---------------------------------------------------------------------- diff --git a/apidocs/src-html/org/apache/hadoop/hbase/util/ByteBufferUtils.html b/apidocs/src-html/org/apache/hadoop/hbase/util/ByteBufferUtils.html index 3776d0f..77ccd85 100644 --- a/apidocs/src-html/org/apache/hadoop/hbase/util/ByteBufferUtils.html +++ b/apidocs/src-html/org/apache/hadoop/hbase/util/ByteBufferUtils.html @@ -37,7 +37,7 @@ <span class="sourceLineNo">029</span><a name="line.29"></a> <span class="sourceLineNo">030</span>import org.apache.hadoop.hbase.classification.InterfaceAudience;<a name="line.30"></a> <span class="sourceLineNo">031</span>import org.apache.hadoop.hbase.classification.InterfaceStability;<a name="line.31"></a> -<span class="sourceLineNo">032</span>import org.apache.hadoop.hbase.io.ByteBufferSupportOutputStream;<a name="line.32"></a> +<span class="sourceLineNo">032</span>import org.apache.hadoop.hbase.io.ByteBufferWriter;<a name="line.32"></a> <span class="sourceLineNo">033</span>import org.apache.hadoop.hbase.io.util.StreamUtils;<a name="line.33"></a> <span class="sourceLineNo">034</span>import org.apache.hadoop.io.IOUtils;<a name="line.34"></a> <span class="sourceLineNo">035</span>import org.apache.hadoop.io.WritableUtils;<a name="line.35"></a> @@ -152,8 +152,8 @@ <span class="sourceLineNo">144</span> // We have writeInt in ByteBufferOutputStream so that it can directly write<a name="line.144"></a> <span class="sourceLineNo">145</span> // int to underlying<a name="line.145"></a> <span class="sourceLineNo">146</span> // ByteBuffer in one step.<a name="line.146"></a> -<span class="sourceLineNo">147</span> if (out instanceof ByteBufferSupportOutputStream) {<a name="line.147"></a> -<span class="sourceLineNo">148</span> ((ByteBufferSupportOutputStream) out).writeInt(value);<a name="line.148"></a> +<span class="sourceLineNo">147</span> if (out instanceof ByteBufferWriter) {<a name="line.147"></a> +<span class="sourceLineNo">148</span> ((ByteBufferWriter) out).writeInt(value);<a name="line.148"></a> <span class="sourceLineNo">149</span> } else {<a name="line.149"></a> <span class="sourceLineNo">150</span> StreamUtils.writeInt(out, value);<a name="line.150"></a> <span class="sourceLineNo">151</span> }<a name="line.151"></a> @@ -190,8 +190,8 @@ <span class="sourceLineNo">182</span> */<a name="line.182"></a> <span class="sourceLineNo">183</span> public static void copyBufferToStream(OutputStream out, ByteBuffer in,<a name="line.183"></a> <span class="sourceLineNo">184</span> int offset, int length) throws IOException {<a name="line.184"></a> -<span class="sourceLineNo">185</span> if (out instanceof ByteBufferSupportOutputStream) {<a name="line.185"></a> -<span class="sourceLineNo">186</span> ((ByteBufferSupportOutputStream) out).write(in, offset, length);<a name="line.186"></a> +<span class="sourceLineNo">185</span> if (out instanceof ByteBufferWriter) {<a name="line.185"></a> +<span class="sourceLineNo">186</span> ((ByteBufferWriter) out).write(in, offset, length);<a name="line.186"></a> <span class="sourceLineNo">187</span> } else if (in.hasArray()) {<a name="line.187"></a> <span class="sourceLineNo">188</span> out.write(in.array(), in.arrayOffset() + offset, length);<a name="line.188"></a> <span class="sourceLineNo">189</span> } else {<a name="line.189"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/86fde03b/book.html ---------------------------------------------------------------------- diff --git a/book.html b/book.html index 3062a3f..5854552 100644 --- a/book.html +++ b/book.html @@ -20864,26 +20864,32 @@ See <a href="#block.cache">Block Cache</a></p> <p>See <a href="#recommended_configurations">Recommended Configurations</a>.</p> </div> <div class="sect2"> -<h3 id="perf.compactions.and.splits"><a class="anchor" href="#perf.compactions.and.splits"></a>96.1. Managing Compactions</h3> +<h3 id="perf.99th.percentile"><a class="anchor" href="#perf.99th.percentile"></a>96.1. Improving the 99th Percentile</h3> +<div class="paragraph"> +<p>Try link:[hedged_reads].</p> +</div> +</div> +<div class="sect2"> +<h3 id="perf.compactions.and.splits"><a class="anchor" href="#perf.compactions.and.splits"></a>96.2. Managing Compactions</h3> <div class="paragraph"> <p>For larger systems, managing link:[compactions and splits] may be something you want to consider.</p> </div> </div> <div class="sect2"> -<h3 id="perf.handlers"><a class="anchor" href="#perf.handlers"></a>96.2. <code>hbase.regionserver.handler.count</code></h3> +<h3 id="perf.handlers"><a class="anchor" href="#perf.handlers"></a>96.3. <code>hbase.regionserver.handler.count</code></h3> <div class="paragraph"> <p>See <a href="#hbase.regionserver.handler.count">[hbase.regionserver.handler.count]</a>.</p> </div> </div> <div class="sect2"> -<h3 id="perf.hfile.block.cache.size"><a class="anchor" href="#perf.hfile.block.cache.size"></a>96.3. <code>hfile.block.cache.size</code></h3> +<h3 id="perf.hfile.block.cache.size"><a class="anchor" href="#perf.hfile.block.cache.size"></a>96.4. <code>hfile.block.cache.size</code></h3> <div class="paragraph"> <p>See <a href="#hfile.block.cache.size">[hfile.block.cache.size]</a>. A memory setting for the RegionServer process.</p> </div> </div> <div class="sect2"> -<h3 id="blockcache.prefetch"><a class="anchor" href="#blockcache.prefetch"></a>96.4. Prefetch Option for Blockcache</h3> +<h3 id="blockcache.prefetch"><a class="anchor" href="#blockcache.prefetch"></a>96.5. Prefetch Option for Blockcache</h3> <div class="paragraph"> <p><a href="https://issues.apache.org/jira/browse/HBASE-9857";>HBASE-9857</a> adds a new option to prefetch HFile contents when opening the BlockCache, if a Column family or RegionServer property is set. This option is available for HBase 0.98.3 and later. @@ -20930,35 +20936,35 @@ or on <code>org.apache.hadoop.hbase.io.hfile.HFileReaderV2</code> in earlier ver </div> </div> <div class="sect2"> -<h3 id="perf.rs.memstore.size"><a class="anchor" href="#perf.rs.memstore.size"></a>96.5. <code>hbase.regionserver.global.memstore.size</code></h3> +<h3 id="perf.rs.memstore.size"><a class="anchor" href="#perf.rs.memstore.size"></a>96.6. <code>hbase.regionserver.global.memstore.size</code></h3> <div class="paragraph"> <p>See <a href="#hbase.regionserver.global.memstore.size">[hbase.regionserver.global.memstore.size]</a>. This memory setting is often adjusted for the RegionServer process depending on needs.</p> </div> </div> <div class="sect2"> -<h3 id="perf.rs.memstore.size.lower.limit"><a class="anchor" href="#perf.rs.memstore.size.lower.limit"></a>96.6. <code>hbase.regionserver.global.memstore.size.lower.limit</code></h3> +<h3 id="perf.rs.memstore.size.lower.limit"><a class="anchor" href="#perf.rs.memstore.size.lower.limit"></a>96.7. <code>hbase.regionserver.global.memstore.size.lower.limit</code></h3> <div class="paragraph"> <p>See <a href="#hbase.regionserver.global.memstore.size.lower.limit">[hbase.regionserver.global.memstore.size.lower.limit]</a>. This memory setting is often adjusted for the RegionServer process depending on needs.</p> </div> </div> <div class="sect2"> -<h3 id="perf.hstore.blockingstorefiles"><a class="anchor" href="#perf.hstore.blockingstorefiles"></a>96.7. <code>hbase.hstore.blockingStoreFiles</code></h3> +<h3 id="perf.hstore.blockingstorefiles"><a class="anchor" href="#perf.hstore.blockingstorefiles"></a>96.8. <code>hbase.hstore.blockingStoreFiles</code></h3> <div class="paragraph"> <p>See <a href="#hbase.hstore.blockingStoreFiles">[hbase.hstore.blockingStoreFiles]</a>. If there is blocking in the RegionServer logs, increasing this can help.</p> </div> </div> <div class="sect2"> -<h3 id="perf.hregion.memstore.block.multiplier"><a class="anchor" href="#perf.hregion.memstore.block.multiplier"></a>96.8. <code>hbase.hregion.memstore.block.multiplier</code></h3> +<h3 id="perf.hregion.memstore.block.multiplier"><a class="anchor" href="#perf.hregion.memstore.block.multiplier"></a>96.9. <code>hbase.hregion.memstore.block.multiplier</code></h3> <div class="paragraph"> <p>See <a href="#hbase.hregion.memstore.block.multiplier">[hbase.hregion.memstore.block.multiplier]</a>. If there is enough RAM, increasing this can help.</p> </div> </div> <div class="sect2"> -<h3 id="hbase.regionserver.checksum.verify.performance"><a class="anchor" href="#hbase.regionserver.checksum.verify.performance"></a>96.9. <code>hbase.regionserver.checksum.verify</code></h3> +<h3 id="hbase.regionserver.checksum.verify.performance"><a class="anchor" href="#hbase.regionserver.checksum.verify.performance"></a>96.10. <code>hbase.regionserver.checksum.verify</code></h3> <div class="paragraph"> <p>Have HBase write the checksum into the datablock and save having to do the checksum seek whenever you read.</p> </div> @@ -20967,7 +20973,7 @@ If there is enough RAM, increasing this can help.</p> </div> </div> <div class="sect2"> -<h3 id="_tuning_code_callqueue_code_options"><a class="anchor" href="#_tuning_code_callqueue_code_options"></a>96.10. Tuning <code>callQueue</code> Options</h3> +<h3 id="_tuning_code_callqueue_code_options"><a class="anchor" href="#_tuning_code_callqueue_code_options"></a>96.11. Tuning <code>callQueue</code> Options</h3> <div class="paragraph"> <p><a href="https://issues.apache.org/jira/browse/HBASE-11355";>HBASE-11355</a> introduces several callQueue tuning mechanisms which can increase performance. See the JIRA for some benchmarking information.</p> @@ -21627,16 +21633,45 @@ See the <em>Development Process</em> section of the document <a href="https://is </div> </div> <div class="sect2"> -<h3 id="_hedged_reads"><a class="anchor" href="#_hedged_reads"></a>101.10. Hedged Reads</h3> +<h3 id="hedged.reads"><a class="anchor" href="#hedged.reads"></a>101.10. Hedged Reads</h3> <div class="paragraph"> -<p>Hedged reads are a feature of HDFS, introduced in <a href="https://issues.apache.org/jira/browse/HDFS-5776";>HDFS-5776</a>. +<p>Hedged reads are a feature of HDFS, introduced in Hadoop 2.4.0 with <a href="https://issues.apache.org/jira/browse/HDFS-5776";>HDFS-5776</a>. Normally, a single thread is spawned for each read request. -However, if hedged reads are enabled, the client waits some configurable amount of time, and if the read does not return, the client spawns a second read request, against a different block replica of the same data. -Whichever read returns first is used, and the other read request is discarded. -Hedged reads can be helpful for times where a rare slow read is caused by a transient error such as a failing disk or flaky network connection.</p> +However, if hedged reads are enabled, the client waits some +configurable amount of time, and if the read does not return, +the client spawns a second read request, against a different +block replica of the same data. Whichever read returns first is +used, and the other read request is discarded.</p> +</div> +<div class="paragraph"> +<p>Hedged reads are "…​very good at eliminating outlier datanodes, which +in turn makes them very good choice for latency sensitive setups. +But, if you are looking for maximizing throughput, hedged reads tend to +create load amplification as things get slower in general. In short, +the thing to watch out for is the non-graceful performance degradation +when you are running close a certain throughput threshold." (Quote from Ashu Pachauri in HBASE-17083).</p> +</div> +<div class="paragraph"> +<p>Other concerns to keep in mind while running with hedged reads enabled +include:</p> +</div> +<div class="ulist"> +<ul> +<li> +<p>They may lead to network congestion. See <a href="https://issues.apache.org/jira/browse/HBASE-17083";>HBASE-17083</a></p> +</li> +<li> +<p>Make sure you set the thread pool large enough so as blocking on the pool does not become a bottleneck (Again see <a href="https://issues.apache.org/jira/browse/HBASE-17083";>HBASE-17083</a>)</p> +</li> +</ul> +</div> +<div class="paragraph"> +<p>(From Yu Li up in HBASE-17083)</p> </div> <div class="paragraph"> -<p>Because an HBase RegionServer is a HDFS client, you can enable hedged reads in HBase, by adding the following properties to the RegionServer’s hbase-site.xml and tuning the values to suit your environment.</p> +<p>Because an HBase RegionServer is a HDFS client, you can enable hedged +reads in HBase, by adding the following properties to the RegionServer’s +hbase-site.xml and tuning the values to suit your environment.</p> </div> <div class="ulist"> <div class="title">Configuration for Hedged Reads</div> @@ -30271,9 +30306,11 @@ Starting the mini-cluster takes about 20-30 seconds, but that should be appropri </div> </div> <h1 id="protobuf" class="sect0"><a class="anchor" href="#protobuf"></a>Protobuf in HBase</h1> -<div class="openblock partintro"> -<div class="content"> -HBase uses Google’s <a href="http://protobuf.protobufs";>protobufs</a> wherever +<div class="sect1"> +<h2 id="_protobuf"><a class="anchor" href="#_protobuf"></a>156. Protobuf</h2> +<div class="sectionbody"> +<div class="paragraph"> +<p>HBase uses Google’s <a href="http://protobuf.protobufs";>protobufs</a> wherever it persists metadata — in the tail of hfiles or Cells written by HBase into the system hbase;meta table or when HBase writes znodes to zookeeper, etc. — and when it passes objects over the wire making @@ -30281,15 +30318,13 @@ to zookeeper, etc. — and when it passes objects over the wire Interfaces (Services) we expose to clients, for example the <code>Admin</code> and <code>Client</code> Interfaces that the RegionServer fields, or specifying the arbitrary extensions added by developers via our -<a href="#cp">Coprocessor Endpoint</a> mechanism. -In this chapter we go into detail for developers who are looking to -understand better how it all works. This chapter is of particular -use to those who would amend or extend HBase functionality. +<a href="#cp">Coprocessor Endpoint</a> mechanism.</p> </div> +<div class="paragraph"> +<p>In this chapter we go into detail for developers who are looking to +understand better how it all works. This chapter is of particular +use to those who would amend or extend HBase functionality.</p> </div> -<div class="sect1"> -<h2 id="_protobuf"><a class="anchor" href="#_protobuf"></a>156. Protobuf</h2> -<div class="sectionbody"> <div class="paragraph"> <p>With protobuf, you describe serializations and services in a <code>.protos</code> file. You then feed these descriptors to a protobuf tool, the <code>protoc</code> binary, http://git-wip-us.apache.org/repos/asf/hbase-site/blob/86fde03b/bulk-loads.html ---------------------------------------------------------------------- diff --git a/bulk-loads.html b/bulk-loads.html index 11ac63a..5d0e7dc 100644 --- a/bulk-loads.html +++ b/bulk-loads.html @@ -7,7 +7,7 @@ <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20161114" /> + <meta name="Date-Revision-yyyymmdd" content="20161115" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache HBase – Bulk Loads in Apache HBase (TM) @@ -191,6 +191,18 @@ </li> <li class="dropdown-submenu"> + <a href="" title="1.2 Documentation">1.2 Documentation</a> + <ul class="dropdown-menu"> + <li> <a href="1.2/apidocs/index.html" target="_blank" title="API">API</a> +</li> + <li> <a href="1.2/xref/index.html" target="_blank" title="X-Ref">X-Ref</a> +</li> + <li> <a href="1.2/book.html" target="_blank" title="Ref Guide (single-page)">Ref Guide (single-page)</a> +</li> + </ul> + </li> + + <li class="dropdown-submenu"> <a href="" title="1.1 Documentation">1.1 Documentation</a> <ul class="dropdown-menu"> <li> <a href="1.1/apidocs/index.html" target="_blank" title="API">API</a> @@ -317,7 +329,7 @@ under the License. --> <a href="http://www.apache.org/";>The Apache Software Foundation</a>. All rights reserved. - <li id="publishDate" class="pull-right">Last Published: 2016-11-14</li> + <li id="publishDate" class="pull-right">Last Published: 2016-11-15</li> </p> </div>
