http://git-wip-us.apache.org/repos/asf/hbase-site/blob/5f066b2e/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.html ---------------------------------------------------------------------- diff --git a/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.html b/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.html new file mode 100644 index 0000000..39ebcf3 --- /dev/null +++ b/apidocs/src-html/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.html @@ -0,0 +1,174 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html lang="en"> +<head> +<title>Source code</title> +<link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style"> +</head> +<body> +<div class="sourceContainer"> +<pre><span class="sourceLineNo">001</span>/**<a name="line.1"></a> +<span class="sourceLineNo">002</span> * Licensed to the Apache Software Foundation (ASF) under one or more contributor license<a name="line.2"></a> +<span class="sourceLineNo">003</span> * agreements. See the NOTICE file distributed with this work for additional information regarding<a name="line.3"></a> +<span class="sourceLineNo">004</span> * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the<a name="line.4"></a> +<span class="sourceLineNo">005</span> * "License"); you may not use this file except in compliance with the License. You may obtain a<a name="line.5"></a> +<span class="sourceLineNo">006</span> * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable<a name="line.6"></a> +<span class="sourceLineNo">007</span> * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"<a name="line.7"></a> +<span class="sourceLineNo">008</span> * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License<a name="line.8"></a> +<span class="sourceLineNo">009</span> * for the specific language governing permissions and limitations under the License.<a name="line.9"></a> +<span class="sourceLineNo">010</span> */<a name="line.10"></a> +<span class="sourceLineNo">011</span>package org.apache.hadoop.hbase.mapreduce;<a name="line.11"></a> +<span class="sourceLineNo">012</span><a name="line.12"></a> +<span class="sourceLineNo">013</span>import java.io.IOException;<a name="line.13"></a> +<span class="sourceLineNo">014</span>import java.util.HashMap;<a name="line.14"></a> +<span class="sourceLineNo">015</span>import java.util.Map;<a name="line.15"></a> +<span class="sourceLineNo">016</span>import org.apache.commons.logging.Log;<a name="line.16"></a> +<span class="sourceLineNo">017</span>import org.apache.commons.logging.LogFactory;<a name="line.17"></a> +<span class="sourceLineNo">018</span>import org.apache.hadoop.hbase.classification.InterfaceAudience;<a name="line.18"></a> +<span class="sourceLineNo">019</span>import org.apache.hadoop.hbase.classification.InterfaceStability;<a name="line.19"></a> +<span class="sourceLineNo">020</span>import org.apache.hadoop.conf.Configuration;<a name="line.20"></a> +<span class="sourceLineNo">021</span>import org.apache.hadoop.fs.FileSystem;<a name="line.21"></a> +<span class="sourceLineNo">022</span>import org.apache.hadoop.fs.Path;<a name="line.22"></a> +<span class="sourceLineNo">023</span>import org.apache.hadoop.hbase.Cell;<a name="line.23"></a> +<span class="sourceLineNo">024</span>import org.apache.hadoop.hbase.io.ImmutableBytesWritable;<a name="line.24"></a> +<span class="sourceLineNo">025</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.25"></a> +<span class="sourceLineNo">026</span>import org.apache.hadoop.mapreduce.RecordWriter;<a name="line.26"></a> +<span class="sourceLineNo">027</span>import org.apache.hadoop.mapreduce.TaskAttemptContext;<a name="line.27"></a> +<span class="sourceLineNo">028</span>import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;<a name="line.28"></a> +<span class="sourceLineNo">029</span>import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;<a name="line.29"></a> +<span class="sourceLineNo">030</span>import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;<a name="line.30"></a> +<span class="sourceLineNo">031</span><a name="line.31"></a> +<span class="sourceLineNo">032</span>import com.google.common.annotations.VisibleForTesting;<a name="line.32"></a> +<span class="sourceLineNo">033</span>/**<a name="line.33"></a> +<span class="sourceLineNo">034</span> * Create 3 level tree directory, first level is using table name as parent directory and then use<a name="line.34"></a> +<span class="sourceLineNo">035</span> * family name as child directory, and all related HFiles for one family are under child directory<a name="line.35"></a> +<span class="sourceLineNo">036</span> * -tableName1<a name="line.36"></a> +<span class="sourceLineNo">037</span> * -columnFamilyName1<a name="line.37"></a> +<span class="sourceLineNo">038</span> * -columnFamilyName2<a name="line.38"></a> +<span class="sourceLineNo">039</span> * -HFiles<a name="line.39"></a> +<span class="sourceLineNo">040</span> * -tableName2<a name="line.40"></a> +<span class="sourceLineNo">041</span> * -columnFamilyName1<a name="line.41"></a> +<span class="sourceLineNo">042</span> * -HFiles<a name="line.42"></a> +<span class="sourceLineNo">043</span> * -columnFamilyName2<a name="line.43"></a> +<span class="sourceLineNo">044</span> * <p><a name="line.44"></a> +<span class="sourceLineNo">045</span> */<a name="line.45"></a> +<span class="sourceLineNo">046</span>@InterfaceAudience.Public<a name="line.46"></a> +<span class="sourceLineNo">047</span>@InterfaceStability.Evolving<a name="line.47"></a> +<span class="sourceLineNo">048</span>@VisibleForTesting<a name="line.48"></a> +<span class="sourceLineNo">049</span>public class MultiHFileOutputFormat extends FileOutputFormat<ImmutableBytesWritable, Cell> {<a name="line.49"></a> +<span class="sourceLineNo">050</span> private static final Log LOG = LogFactory.getLog(MultiHFileOutputFormat.class);<a name="line.50"></a> +<span class="sourceLineNo">051</span><a name="line.51"></a> +<span class="sourceLineNo">052</span> @Override<a name="line.52"></a> +<span class="sourceLineNo">053</span> public RecordWriter<ImmutableBytesWritable, Cell><a name="line.53"></a> +<span class="sourceLineNo">054</span> getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException {<a name="line.54"></a> +<span class="sourceLineNo">055</span> return createMultiHFileRecordWriter(context);<a name="line.55"></a> +<span class="sourceLineNo">056</span> }<a name="line.56"></a> +<span class="sourceLineNo">057</span><a name="line.57"></a> +<span class="sourceLineNo">058</span> static <V extends Cell> RecordWriter<ImmutableBytesWritable, V><a name="line.58"></a> +<span class="sourceLineNo">059</span> createMultiHFileRecordWriter(final TaskAttemptContext context) throws IOException {<a name="line.59"></a> +<span class="sourceLineNo">060</span><a name="line.60"></a> +<span class="sourceLineNo">061</span> // Get the path of the output directory<a name="line.61"></a> +<span class="sourceLineNo">062</span> final Path outputPath = FileOutputFormat.getOutputPath(context);<a name="line.62"></a> +<span class="sourceLineNo">063</span> final Path outputDir = new FileOutputCommitter(outputPath, context).getWorkPath();<a name="line.63"></a> +<span class="sourceLineNo">064</span> final Configuration conf = context.getConfiguration();<a name="line.64"></a> +<span class="sourceLineNo">065</span> final FileSystem fs = outputDir.getFileSystem(conf);<a name="line.65"></a> +<span class="sourceLineNo">066</span><a name="line.66"></a> +<span class="sourceLineNo">067</span> // Map of tables to writers<a name="line.67"></a> +<span class="sourceLineNo">068</span> final Map<ImmutableBytesWritable, RecordWriter<ImmutableBytesWritable, V>> tableWriters =<a name="line.68"></a> +<span class="sourceLineNo">069</span> new HashMap<ImmutableBytesWritable, RecordWriter<ImmutableBytesWritable, V>>();<a name="line.69"></a> +<span class="sourceLineNo">070</span><a name="line.70"></a> +<span class="sourceLineNo">071</span> return new RecordWriter<ImmutableBytesWritable, V>() {<a name="line.71"></a> +<span class="sourceLineNo">072</span> @Override<a name="line.72"></a> +<span class="sourceLineNo">073</span> public void write(ImmutableBytesWritable tableName, V cell)<a name="line.73"></a> +<span class="sourceLineNo">074</span> throws IOException, InterruptedException {<a name="line.74"></a> +<span class="sourceLineNo">075</span> RecordWriter<ImmutableBytesWritable, V> tableWriter = tableWriters.get(tableName);<a name="line.75"></a> +<span class="sourceLineNo">076</span> // if there is new table, verify that table directory exists<a name="line.76"></a> +<span class="sourceLineNo">077</span> if (tableWriter == null) {<a name="line.77"></a> +<span class="sourceLineNo">078</span> // using table name as directory name<a name="line.78"></a> +<span class="sourceLineNo">079</span> final Path tableOutputDir = new Path(outputDir, Bytes.toString(tableName.copyBytes()));<a name="line.79"></a> +<span class="sourceLineNo">080</span> fs.mkdirs(tableOutputDir);<a name="line.80"></a> +<span class="sourceLineNo">081</span> LOG.info("Writing Table '" + tableName.toString() + "' data into following directory"<a name="line.81"></a> +<span class="sourceLineNo">082</span> + tableOutputDir.toString());<a name="line.82"></a> +<span class="sourceLineNo">083</span><a name="line.83"></a> +<span class="sourceLineNo">084</span> // Create writer for one specific table<a name="line.84"></a> +<span class="sourceLineNo">085</span> tableWriter = new HFileOutputFormat2.HFileRecordWriter<V>(context, tableOutputDir);<a name="line.85"></a> +<span class="sourceLineNo">086</span> // Put table into map<a name="line.86"></a> +<span class="sourceLineNo">087</span> tableWriters.put(tableName, tableWriter);<a name="line.87"></a> +<span class="sourceLineNo">088</span> }<a name="line.88"></a> +<span class="sourceLineNo">089</span> // Write <Row, Cell> into tableWriter<a name="line.89"></a> +<span class="sourceLineNo">090</span> // in the original code, it does not use Row<a name="line.90"></a> +<span class="sourceLineNo">091</span> tableWriter.write(null, cell);<a name="line.91"></a> +<span class="sourceLineNo">092</span> }<a name="line.92"></a> +<span class="sourceLineNo">093</span><a name="line.93"></a> +<span class="sourceLineNo">094</span> @Override<a name="line.94"></a> +<span class="sourceLineNo">095</span> public void close(TaskAttemptContext c) throws IOException, InterruptedException {<a name="line.95"></a> +<span class="sourceLineNo">096</span> for (RecordWriter<ImmutableBytesWritable, V> writer : tableWriters.values()) {<a name="line.96"></a> +<span class="sourceLineNo">097</span> writer.close(c);<a name="line.97"></a> +<span class="sourceLineNo">098</span> }<a name="line.98"></a> +<span class="sourceLineNo">099</span> }<a name="line.99"></a> +<span class="sourceLineNo">100</span> };<a name="line.100"></a> +<span class="sourceLineNo">101</span> }<a name="line.101"></a> +<span class="sourceLineNo">102</span>}<a name="line.102"></a> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +</pre> +</div> +</body> +</html>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/5f066b2e/book.html ---------------------------------------------------------------------- diff --git a/book.html b/book.html index 2dee923..9b4d569 100644 --- a/book.html +++ b/book.html @@ -5292,15 +5292,6 @@ See <a href="https://issues.apache.org/jira/browse/HBASE-6389";>HBASE-6389 Modify starting region assignments</a> for more detail.</p> </div> </div> -<div class="sect3"> -<h4 id="backup.master.fail.fast"><a class="anchor" href="#backup.master.fail.fast"></a>9.1.2. If a backup Master exists, make the primary Master fail fast</h4> -<div class="paragraph"> -<p>If the primary Master loses its connection with ZooKeeper, it will fall into a loop where it keeps trying to reconnect. -Disable this functionality if you are running more than one Master: i.e. a backup Master. -Failing to do so, the dying Master may continue to receive RPCs though another Master has assumed the role of primary. -See the configuration <a href="#fail.fast.expired.active.master">fail.fast.expired.active.master</a>.</p> -</div> -</div> </div> <div class="sect2"> <h3 id="recommended_configurations"><a class="anchor" href="#recommended_configurations"></a>9.2. Recommended Configurations</h3> @@ -13986,9 +13977,9 @@ For background, see <a href="https://issues.apache.org/jira/browse/HBASE-2643";>H <div class="sect4"> <h5 id="_performance_improvements_during_log_splitting"><a class="anchor" href="#_performance_improvements_during_log_splitting"></a>Performance Improvements during Log Splitting</h5> <div class="paragraph"> -<p>WAL log splitting and recovery can be resource intensive and take a long time, depending on the number of RegionServers involved in the crash and the size of the regions. <a href="#distributed.log.splitting">[distributed.log.splitting]</a> was developed to improve performance during log splitting.</p> +<p>WAL log splitting and recovery can be resource intensive and take a long time, depending on the number of RegionServers involved in the crash and the size of the regions. <a href="#distributed.log.splitting">Enabling or Disabling Distributed Log Splitting</a> was developed to improve performance during log splitting.</p> </div> -<div class="paragraph"> +<div id="distributed.log.splitting" class="paragraph"> <div class="title">Enabling or Disabling Distributed Log Splitting</div> <p>Distributed log processing is enabled by default since HBase 0.92. The setting is controlled by the <code>hbase.master.distributed.log.splitting</code> property, which can be set to <code>true</code> or <code>false</code>, but defaults to <code>true</code>.</p> @@ -14213,7 +14204,7 @@ A split log worker directly replays edits from the WAL of the failed RegionServe When a region is in "recovering" state, it can accept writes but no reads (including Append and Increment), region splits or merges.</p> </div> <div class="paragraph"> -<p>Distributed Log Replay extends the <a href="#distributed.log.splitting">[distributed.log.splitting]</a> framework. +<p>Distributed Log Replay extends the <a href="#distributed.log.splitting">Enabling or Disabling Distributed Log Splitting</a> framework. It works by directly replaying WAL edits to another RegionServer instead of creating <em>recovered.edits</em> files. It provides the following advantages over distributed log splitting alone:</p> </div> @@ -19862,9 +19853,10 @@ an easier way to load a coprocessor dynamically.</p> <div class="listingblock"> <div class="content"> <pre class="CodeRay highlight"><code data-lang="java">TableName tableName = TableName.valueOf(<span class="string"><span class="delimiter">"</span><span class="content">users</span><span class="delimiter">"</span></span>); -<span class="predefined-type">String</span> path = <span class="string"><span class="delimiter">"</span><span class="content">hdfs://<namenode>:<port>/user/<hadoop-user>/coprocessor.jar</span><span class="delimiter">"</span></span>; +Path path = <span class="keyword">new</span> Path(<span class="string"><span class="delimiter">"</span><span class="content">hdfs://<namenode>:<port>/user/<hadoop-user>/coprocessor.jar</span><span class="delimiter">"</span></span>); <span class="predefined-type">Configuration</span> conf = HBaseConfiguration.create(); -HBaseAdmin admin = <span class="keyword">new</span> HBaseAdmin(conf); +<span class="predefined-type">Connection</span> connection = ConnectionFactory.createConnection(conf); +Admin admin = connection.getAdmin(); admin.disableTable(tableName); HTableDescriptor hTableDescriptor = <span class="keyword">new</span> HTableDescriptor(tableName); HColumnDescriptor columnFamily1 = <span class="keyword">new</span> HColumnDescriptor(<span class="string"><span class="delimiter">"</span><span class="content">personalDet</span><span class="delimiter">"</span></span>); @@ -20081,23 +20073,15 @@ empty result. Otherwise, process the request as normal.</p> <span class="directive">private</span> <span class="directive">static</span> <span class="directive">final</span> <span class="type">byte</span><span class="type">[]</span> VALUE = Bytes.toBytes(<span class="string"><span class="delimiter">"</span><span class="content">You can't see Admin details</span><span class="delimiter">"</span></span>); <span class="annotation">@Override</span> - <span class="directive">public</span> <span class="type">void</span> preGetOp(<span class="directive">final</span> ObserverContext e, <span class="directive">final</span> Get get, <span class="directive">final</span> <span class="predefined-type">List</span> results) + <span class="directive">public</span> <span class="type">void</span> preGetOp(<span class="directive">final</span> ObserverContext<RegionCoprocessorEnvironment> e, <span class="directive">final</span> Get get, <span class="directive">final</span> <span class="predefined-type">List</span><Cell> results) <span class="directive">throws</span> <span class="exception">IOException</span> { <span class="keyword">if</span> (Bytes.equals(get.getRow(),ADMIN)) { - Cell c = CellUtil.createCell(get.getRow(),COLUMN _FAMILY, COLUMN, + Cell c = CellUtil.createCell(get.getRow(),COLUMN_FAMILY, COLUMN, <span class="predefined-type">System</span>.currentTimeMillis(), (<span class="type">byte</span>)<span class="integer">4</span>, VALUE); results.add(c); e.bypass(); } - - <span class="predefined-type">List</span> kvs = <span class="keyword">new</span> <span class="predefined-type">ArrayList</span>(results.size()); - <span class="keyword">for</span> (Cell c : results) { - kvs.add(KeyValueUtil.ensureKeyValue(c)); - } - preGet(e, get, kvs); - results.clear(); - results.addAll(kvs); } }</code></pre> </div> @@ -20109,7 +20093,7 @@ the <code>preScannerOpen()</code> method to filter the <code>admin</code> row fr <div class="listingblock"> <div class="content"> <pre class="CodeRay highlight"><code data-lang="java"><span class="annotation">@Override</span> -<span class="directive">public</span> RegionScanner preScannerOpen(<span class="directive">final</span> ObserverContext e, <span class="directive">final</span> Scan scan, +<span class="directive">public</span> RegionScanner preScannerOpen(<span class="directive">final</span> ObserverContext<RegionCoprocessorEnvironment> e, <span class="directive">final</span> Scan scan, <span class="directive">final</span> RegionScanner s) <span class="directive">throws</span> <span class="exception">IOException</span> { <span class="predefined-type">Filter</span> filter = <span class="keyword">new</span> RowFilter(CompareOp.NOT_EQUAL, <span class="keyword">new</span> BinaryComparator(ADMIN)); @@ -20126,10 +20110,10 @@ remove any <code>admin</code> results from the scan:</p> <div class="listingblock"> <div class="content"> <pre class="CodeRay highlight"><code data-lang="java"><span class="annotation">@Override</span> -<span class="directive">public</span> <span class="type">boolean</span> postScannerNext(<span class="directive">final</span> ObserverContext e, <span class="directive">final</span> InternalScanner s, -<span class="directive">final</span> <span class="predefined-type">List</span> results, <span class="directive">final</span> <span class="type">int</span> limit, <span class="directive">final</span> <span class="type">boolean</span> hasMore) <span class="directive">throws</span> <span class="exception">IOException</span> { +<span class="directive">public</span> <span class="type">boolean</span> postScannerNext(<span class="directive">final</span> ObserverContext<RegionCoprocessorEnvironment> e, <span class="directive">final</span> InternalScanner s, +<span class="directive">final</span> <span class="predefined-type">List</span><<span class="predefined-type">Result</span>> results, <span class="directive">final</span> <span class="type">int</span> limit, <span class="directive">final</span> <span class="type">boolean</span> hasMore) <span class="directive">throws</span> <span class="exception">IOException</span> { <span class="predefined-type">Result</span> result = <span class="predefined-constant">null</span>; - <span class="predefined-type">Iterator</span> iterator = results.iterator(); + <span class="predefined-type">Iterator</span><<span class="predefined-type">Result</span>> iterator = results.iterator(); <span class="keyword">while</span> (iterator.hasNext()) { result = iterator.next(); <span class="keyword">if</span> (Bytes.equals(result.getRow(), ROWKEY)) { @@ -34075,7 +34059,7 @@ The server will return cellblocks compressed using this same compressor as long <div id="footer"> <div id="footer-text"> Version 2.0.0-SNAPSHOT<br> -Last updated 2016-06-15 16:11:32 UTC +Last updated 2016-04-10 14:30:44 UTC </div> </div> </body> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/5f066b2e/bulk-loads.html ---------------------------------------------------------------------- diff --git a/bulk-loads.html b/bulk-loads.html index b4c1ea7..bfe3494 100644 --- a/bulk-loads.html +++ b/bulk-loads.html @@ -7,7 +7,7 @@ <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20160714" /> + <meta name="Date-Revision-yyyymmdd" content="20160717" /> <meta http-equiv="Content-Language" content="en" /> <title>Apache HBase – Bulk Loads in Apache HBase (TM) @@ -305,7 +305,7 @@ under the License. --> <a href="http://www.apache.org/";>The Apache Software Foundation</a>. All rights reserved. - <li id="publishDate" class="pull-right">Last Published: 2016-07-14</li> + <li id="publishDate" class="pull-right">Last Published: 2016-07-17</li> </p> </div>
