Fixed white space issue in the bash section.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a4840274 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a4840274 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a4840274 Branch: refs/heads/master Commit: a4840274d8845dd7264f9f660077316d200e7e89 Parents: 0dcc076 Author: Aaron McCurry <[email protected]> Authored: Sat Aug 24 14:15:30 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Sat Aug 24 14:15:30 2013 -0400 ---------------------------------------------------------------------- .../java/org/apache/blur/shell/GenerateDoc.java | 1 - docs/using-blur.html | 368 +++++++++---------- 2 files changed, 184 insertions(+), 185 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a4840274/blur-shell/src/main/java/org/apache/blur/shell/GenerateDoc.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/GenerateDoc.java b/blur-shell/src/main/java/org/apache/blur/shell/GenerateDoc.java index 0065ae9..42d2fb2 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/GenerateDoc.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/GenerateDoc.java @@ -52,7 +52,6 @@ public class GenerateDoc { BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(source))); String line; while ((line = reader.readLine()) != null) { - line = line.trim(); if (line.equals("|||Shell-Menu|||")) { writer.println(menuReplacementText); } else if (line.equals("|||Shell-Body|||")) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a4840274/docs/using-blur.html ---------------------------------------------------------------------- diff --git a/docs/using-blur.html b/docs/using-blur.html index 192f2c0..cfe0668 100644 --- a/docs/using-blur.html +++ b/docs/using-blur.html @@ -1,66 +1,66 @@ <!DOCTYPE html> -<!-- -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. --> <html> -<head> -<title>Using Blur - Apache Blur (Incubator) Documentation</title> -<meta name="viewport" content="width=device-width, initial-scale=1.0"> -<!-- Bootstrap --> -<link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen"> -<link href="resources/css/bs-docs.css" rel="stylesheet" media="screen"> -</head> -<body> -<div class="navbar navbar-inverse navbar-fixed-top"> -<div class="container"> -<div class="navbar-header"> -<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> -<span class="icon-bar"></span> -<span class="icon-bar"></span> -<span class="icon-bar"></span> -</button> -<a class="navbar-brand" href="index.html">Apache Blur (Incubator)</a> -</div> -<div class="collapse navbar-collapse"> -<ul class="nav navbar-nav"> -<li><a href="getting-started.html">Getting Started</a></li> -<li><a href="data-model.html">Data Model</a></li> -<li><a href="cluster-setup.html">Cluster Setup</a></li> -<li class="active"><a href="using-blur.html">Using Blur</a></li> -<li><a href="Blur.html">Blur API</a></li> -</ul> -</div> -</div> -</div> -<div class="container bs-docs-container"> -<div class="row"> -<div class="col-md-3"> -<div class="bs-sidebar hidden-print affix" role="complementary"> -<ul class="nav bs-sidenav"> -<li><a href="#thrift-client">Thrift Client</a> -<ul class="nav"> -<li><a href="#simple_query_example">Query Example</a></li> -<li><a href="#simple_query_example_data">Query Example with Data</a></li> -<li><a href="#simple_fetch_data">Fetch Data</a></li> -<li><a href="#simple_mutate_example">Mutate Example</a></li> -<li><a href="#simple_shortened_mutate_example">Shortened Mutate Example</a></li> -</ul> -</li> -<li><a href="#shell">Shell</a> -<ul class="nav"> + <head> + <title>Using Blur - Apache Blur (Incubator) Documentation</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <!-- Bootstrap --> + <link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen"> + <link href="resources/css/bs-docs.css" rel="stylesheet" media="screen"> + </head> + <body> + <div class="navbar navbar-inverse navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="index.html">Apache Blur (Incubator)</a> + </div> + <div class="collapse navbar-collapse"> + <ul class="nav navbar-nav"> + <li><a href="getting-started.html">Getting Started</a></li> + <li><a href="data-model.html">Data Model</a></li> + <li><a href="cluster-setup.html">Cluster Setup</a></li> + <li class="active"><a href="using-blur.html">Using Blur</a></li> + <li><a href="Blur.html">Blur API</a></li> + </ul> + </div> + </div> + </div> + <div class="container bs-docs-container"> + <div class="row"> + <div class="col-md-3"> + <div class="bs-sidebar hidden-print affix" role="complementary"> + <ul class="nav bs-sidenav"> + <li><a href="#thrift-client">Thrift Client</a> + <ul class="nav"> + <li><a href="#simple_query_example">Query Example</a></li> + <li><a href="#simple_query_example_data">Query Example with Data</a></li> + <li><a href="#simple_fetch_data">Fetch Data</a></li> + <li><a href="#simple_mutate_example">Mutate Example</a></li> + <li><a href="#simple_shortened_mutate_example">Shortened Mutate Example</a></li> + </ul> + </li> + <li><a href="#shell">Shell</a> + <ul class="nav"> <li><a href="#shell_table_commands">Table Commands</a> <ul class="nav"> <li><a href="#shell_command_create"> create</a></li> @@ -103,19 +103,19 @@ limitations under the License. <li><a href="#shell_command_reset"> reset</a></li> </ul></li> -</ul> -</li> -<li><a href="#map-reduce">Map Reduce</a></li> -<li><a href="#csv-loader">CSV Loader</a></li> -<li><a href="#jdbc">JDBC</a></li> -</ul> -</div> -</div> -<div class="col-md-9" role="main"> -<section> -<div class="page-header"> -<h1 id="thrift-client">Thrift Client</h1> -</div> + </ul> + </li> + <li><a href="#map-reduce">Map Reduce</a></li> + <li><a href="#csv-loader">CSV Loader</a></li> + <li><a href="#jdbc">JDBC</a></li> + </ul> + </div> + </div> + <div class="col-md-9" role="main"> + <section> + <div class="page-header"> + <h1 id="thrift-client">Thrift Client</h1> + </div> <h3 id="simple_query_example">Query Example</h3> <p> This is a simple example of how to run a query via the Thrift API and get back search results. By default @@ -138,7 +138,7 @@ for (BlurResult result : results.getResults()) { <h3 id="simple_query_example_data">Query Example with Data</h3> <p> This is an example of how to run a query via the Thrift API and get back search results with data. All -the columns in the "fam0" family are returned for each Record in the Row. +the columns in the "fam0" family are returned for each Record in the Row. </p> <pre><code class="java">Iface client = BlurClient.getClient("controller1:40010,controller2:40010"); @@ -182,7 +182,7 @@ FetchResult fetchRow = client.fetchRow("table1", selector); FetchRowResult rowResult = fetchRow.getRowResult(); Row row = rowResult.getRow(); for (Record record : row.getRecords()) { -System.out.println(record); + System.out.println(record); } </code></pre> @@ -198,25 +198,25 @@ record1.setRecordId("recordid1"); record1.setFamily("fam0"); record1.addToColumns(new Column("col0", "val0")); record1.addToColumns(new Column("col1", "val1")); - + Record record2 = new Record(); record2.setRecordId("recordid2"); record2.setFamily("fam1"); record2.addToColumns(new Column("col4", "val4")); record2.addToColumns(new Column("col5", "val5")); - + List<RecordMutation> recordMutations = new ArrayList<RecordMutation>(); - + recordMutations.add(new RecordMutation(RecordMutationType.REPLACE_ENTIRE_RECORD, record1)); recordMutations.add(new RecordMutation(RecordMutationType.REPLACE_ENTIRE_RECORD, record2)); // This will replace the exiting Row of "rowid1" (if one exists) in table "table1". It will -// write the mutate to the write ahead log (WAL) and it will not block waiting for the -// mutate to become visible. +// write the mutate to the write ahead log (WAL) and it will not block waiting for the +// mutate to become visible. RowMutation mutation = new RowMutation("table1", "rowid1", true, RowMutationType.REPLACE_ROW, -recordMutations, false); + recordMutations, false); mutation.setRecordMutations(recordMutations); - + client.mutate(mutation); </code></pre> @@ -229,19 +229,19 @@ This is the same example as above but is shorted with a help class. Iface client = BlurClient.getClient("controller1:40010,controller2:40010"); // This will replace the exiting Row of "rowid1" (if one exists) in table "table1". It will -// write the mutate to the write ahead log (WAL) and it will not block waiting for the -// mutate to become visible. +// write the mutate to the write ahead log (WAL) and it will not block waiting for the +// mutate to become visible. RowMutation mutation = newRowMutation("table1", "rowid1", -newRecordMutation("fam0", "recordid1", newColumn("col0", "val0"), newColumn("col1", "val2")), -newRecordMutation("fam1", "recordid2", newColumn("col4", "val4"), newColumn("col5", "val4"))); + newRecordMutation("fam0", "recordid1", newColumn("col0", "val0"), newColumn("col1", "val2")), + newRecordMutation("fam1", "recordid2", newColumn("col4", "val4"), newColumn("col5", "val4"))); client.mutate(mutation); </code></pre> -</section> -<section> -<div class="page-header"> -<h1 id="shell">Shell</h1> -</div> + </section> + <section> + <div class="page-header"> + <h1 id="shell">Shell</h1> + </div> <p> The shell can be invoked by running: <pre><code class="bash">$BLUR_HOME/bin/blur shell</code></pre> @@ -350,24 +350,24 @@ The following rules are used when interacting with the shell: <p>Description: Resets the terminal window.<br/> <pre><code class="bash">reset </code></pre></p> -</section> -<section> -<div class="page-header"> -<h1 id="map-reduce">Map Reduce</h1> -</div> -<p>Here is an example of the typical usage of the BlurOutputFormat. The Blur table has to be created before the MapReduce job is started. The setupJob method configures the following:</p> -<ul> -<li>The reducer class to be DefaultBlurReducer</li> -<li>The number of reducers to be equal to the number of shards in the table.</li> -<li>The output key class to a standard Text writable from the Hadoop library</li> -<li>The output value class is a BlurMutate writable from the Blur library</li> -<li>The output format to be BlurOutputFormat</li> -<li>Sets the TableDescriptor in the Configuration</li> -<li>Sets the output path to the TableDescriptor.getTableUri() value</li> -<li>Also the job will use the BlurOutputCommitter class to commit or rollback the MapReduce job</li> -</ul> -<h3>Example Usage</h3> -<pre><code class="java">Iface client = BlurClient.getClient("controller1:40010"); + </section> + <section> + <div class="page-header"> + <h1 id="map-reduce">Map Reduce</h1> + </div> + <p>Here is an example of the typical usage of the BlurOutputFormat. The Blur table has to be created before the MapReduce job is started. The setupJob method configures the following:</p> + <ul> + <li>The reducer class to be DefaultBlurReducer</li> + <li>The number of reducers to be equal to the number of shards in the table.</li> + <li>The output key class to a standard Text writable from the Hadoop library</li> + <li>The output value class is a BlurMutate writable from the Blur library</li> + <li>The output format to be BlurOutputFormat</li> + <li>Sets the TableDescriptor in the Configuration</li> + <li>Sets the output path to the TableDescriptor.getTableUri() value</li> + <li>Also the job will use the BlurOutputCommitter class to commit or rollback the MapReduce job</li> + </ul> + <h3>Example Usage</h3> + <pre><code class="java">Iface client = BlurClient.getClient("controller1:40010"); TableDescriptor tableDescriptor = client.describe(tableName); @@ -384,33 +384,33 @@ BlurOutputFormat.setIndexLocally(job, true); BlurOutputFormat.setOptimizeInFlight(job, true); job.waitForCompletion(true);</code></pre> -<h3>Options</h3> -<ul> -<li> -BlurOutputFormat.setIndexLocally(Job,boolean) -<ul><li>Enabled by default, this will enable local indexing on the machine where the task is running. Then when the RecordWriter closes the index is copied to the remote destination in HDFS.</li></ul> -</li> -<li> -BlurOutputFormat.setMaxDocumentBufferSize(Job,int) -<ul><li>Sets the maximum number of documents that the buffer will hold in memory before overflowing to disk. By default this is 1000 which will probably be very low for most systems.</li></ul> -</li> -<li> -BlurOutputFormat.setOptimizeInFlight(Job,boolean) -<ul><li>Enabled by default, this will optimize the index while copying from the local index to the remote destination in HDFS. Used in conjunction with the setIndexLocally.</li></ul> -</li> -<li> -BlurOutputFormat.setReducerMultiplier(Job,int) -<ul><li>This will multiple the number of reducers for this job. For example if the table has 256 shards the normal number of reducers is 256. However if the reducer multiplier is set to 4 then the number of reducers will be 1024 and each shard will get 4 new segments instead of the normal 1.</li></ul> -</li> -</ul> -</section> -<section> -<div class="page-header"> -<h1 id="csv-loader">CSV Loader</h1> -</div> + <h3>Options</h3> + <ul> + <li> + BlurOutputFormat.setIndexLocally(Job,boolean) + <ul><li>Enabled by default, this will enable local indexing on the machine where the task is running. Then when the RecordWriter closes the index is copied to the remote destination in HDFS.</li></ul> + </li> + <li> + BlurOutputFormat.setMaxDocumentBufferSize(Job,int) + <ul><li>Sets the maximum number of documents that the buffer will hold in memory before overflowing to disk. By default this is 1000 which will probably be very low for most systems.</li></ul> + </li> + <li> + BlurOutputFormat.setOptimizeInFlight(Job,boolean) + <ul><li>Enabled by default, this will optimize the index while copying from the local index to the remote destination in HDFS. Used in conjunction with the setIndexLocally.</li></ul> + </li> + <li> + BlurOutputFormat.setReducerMultiplier(Job,int) + <ul><li>This will multiple the number of reducers for this job. For example if the table has 256 shards the normal number of reducers is 256. However if the reducer multiplier is set to 4 then the number of reducers will be 1024 and each shard will get 4 new segments instead of the normal 1.</li></ul> + </li> + </ul> + </section> + <section> + <div class="page-header"> + <h1 id="csv-loader">CSV Loader</h1> + </div> <p> The CSV Loader program can be invoked by running:<pre><code class="bash">$BLUR_HOME/bin/blur csvloader</code></pre> -<div class="bs-callout bs-callout-warning"><h4>Caution</h4>Also the machine that will execute this command will need to have Hadoop installed and configured locally, +<div class="bs-callout bs-callout-warning"><h4>Caution</h4>Also the machine that will execute this command will need to have Hadoop installed and configured locally, otherwise the scripts will not work correctly.</div> <pre><code class="bash">usage: csvloader The "csvloader" command is used to load delimited into a Blur table. @@ -420,56 +420,56 @@ recordid can be generated based on the data in the record via the "-A" and "-a" can assigned based on the path via the "-I" option. The column name order can be mapped via the "-d" option. Also you can set the input format to either sequence files vie the "-S" option or leave the default text files. --A No Row Ids - Automatically generate row ids for each record based on a MD5 -has of the data within the record. --a No Record Ids - Automatically generate record ids for each record based on a -MD5 has of the data within the record. --b <size> The maximum number of Lucene documents to buffer in the reducer for a single -row before spilling over to disk. (default 1000) --c <controller*> * Thrift controller connection string. (host1:40010 host2:40010 ...) --C <minimum maximum> Enables a combine file input to help deal with many small files as the -input. Provide the minimum and maximum size per mapper. For a minimum of -1GB and a maximum of 2.5GB: (1000000000 2500000000) --d <family column*> * Define the mapping of fields in the CSV file to column names. (family col1 -col2 col3 ...) --I <family path*> The directory to index with a family name, the family name is assumed to NOT -be present in the file contents. (family hdfs://namenode/input/in1) --i <path*> The directory to index, the family name is assumed to BE present in the file -contents. (hdfs://namenode/input/in1) --l Disable the use storage local on the server that is running the reducing -task and copy to Blur table once complete. (enabled by default) --o Disable optimize indexes during copy, this has very little overhead. -(enabled by default) --p <codec> Sets the compression codec for the map compress output setting. -(SNAPPY,GZIP,BZIP,DEFAULT, or classname) --r <multiplier> The reducer multipler allows for an increase in the number of reducers per -shard in the given table. For example if the table has 128 shards and the -reducer multiplier is 4 the total number of reducers will be 512, 4 reducers -per shard. (default 1) --s <delimiter> The file delimiter to be used. (default value ',') NOTE: For special -charactors like the default hadoop separator of ASCII value 1, you can use -standard java escaping (\u0001) --S The input files are sequence files. --t <tablename> * Blur table name.</code></pre> + -A No Row Ids - Automatically generate row ids for each record based on a MD5 + has of the data within the record. + -a No Record Ids - Automatically generate record ids for each record based on a + MD5 has of the data within the record. + -b <size> The maximum number of Lucene documents to buffer in the reducer for a single + row before spilling over to disk. (default 1000) + -c <controller*> * Thrift controller connection string. (host1:40010 host2:40010 ...) + -C <minimum maximum> Enables a combine file input to help deal with many small files as the + input. Provide the minimum and maximum size per mapper. For a minimum of + 1GB and a maximum of 2.5GB: (1000000000 2500000000) + -d <family column*> * Define the mapping of fields in the CSV file to column names. (family col1 + col2 col3 ...) + -I <family path*> The directory to index with a family name, the family name is assumed to NOT + be present in the file contents. (family hdfs://namenode/input/in1) + -i <path*> The directory to index, the family name is assumed to BE present in the file + contents. (hdfs://namenode/input/in1) + -l Disable the use storage local on the server that is running the reducing + task and copy to Blur table once complete. (enabled by default) + -o Disable optimize indexes during copy, this has very little overhead. + (enabled by default) + -p <codec> Sets the compression codec for the map compress output setting. + (SNAPPY,GZIP,BZIP,DEFAULT, or classname) + -r <multiplier> The reducer multipler allows for an increase in the number of reducers per + shard in the given table. For example if the table has 128 shards and the + reducer multiplier is 4 the total number of reducers will be 512, 4 reducers + per shard. (default 1) + -s <delimiter> The file delimiter to be used. (default value ',') NOTE: For special + charactors like the default hadoop separator of ASCII value 1, you can use + standard java escaping (\u0001) + -S The input files are sequence files. + -t <tablename> * Blur table name.</code></pre> </p> -</section> -<section> -<div class="page-header"> -<h1 id="jdbc">JDBC</h1> -</div> -<p>TODO</p> -</section> -</div> -</div> -</div> - -<!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> -<script src="resources/js/jquery-2.0.3.min.js"></script> -<!-- Include all compiled plugins (below), or include individual files as needed --> -<script src="resources/js/bootstrap.min.js"></script> -<!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) --> -<script src="resources/js/respond.min.js"></script> -<script src="resources/js/docs.js"></script> -</body> + </section> + <section> + <div class="page-header"> + <h1 id="jdbc">JDBC</h1> + </div> + <p>TODO</p> + </section> + </div> + </div> + </div> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <script src="resources/js/jquery-2.0.3.min.js"></script> + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="resources/js/bootstrap.min.js"></script> + <!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) --> + <script src="resources/js/respond.min.js"></script> + <script src="resources/js/docs.js"></script> + </body> </html>
