Added: incubator/blur/site/trunk/content/blur/docs/0.2.4/data-model.html URL: http://svn.apache.org/viewvc/incubator/blur/site/trunk/content/blur/docs/0.2.4/data-model.html?rev=1648906&view=auto ============================================================================== --- incubator/blur/site/trunk/content/blur/docs/0.2.4/data-model.html (added) +++ incubator/blur/site/trunk/content/blur/docs/0.2.4/data-model.html Thu Jan 1 17:21:48 2015 @@ -0,0 +1,646 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html> + <head> + <title>Data Model - Apache Blur (Incubator) Documentation</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <!-- Bootstrap --> + <link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen"> + <link href="resources/css/bs-docs.css" rel="stylesheet" media="screen"> + </head> + <body> + <div class="navbar navbar-inverse navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://incubator.apache.org/blur";>Apache Blur (Incubator)</a> + </div> + <div class="collapse navbar-collapse"> + <ul class="nav navbar-nav"> + <li><a href="index.html">Main</a></li> + <li><a href="getting-started.html">Getting Started</a></li> + <li><a href="platform.html">Platform</a></li> + <li class="active"><a href="data-model.html">Data Model</a></li> + <li><a href="cluster-setup.html">Cluster Setup</a></li> + <li><a href="using-blur.html">Using Blur</a></li> + <li><a href="Blur.html">Blur API</a></li> + <li><a href="console.html">Console</a></li> + </ul> + </div> + </div> + </div> + <div class="container bs-docs-container"> + <div class="row"> + <div class="col-md-3"> + <div class="bs-sidebar hidden-print affix" role="complementary"> + <ul class="nav bs-sidenav"> + <li> + <a href="#structure">Structure</a> + <ul class="nav"> + <li><a href="#columns">Columns</a></li> + <li><a href="#records">Records</a></li> + <li><a href="#rows">Rows</a></li> + </ul> + </li> + <li> + <a href="#querying">Querying</a> + <ul class="nav"> + <li><a href="#row_query">Row Query</a></li> + <li><a href="#text_query">Text</a></li> + <li><a href="#string_query">String</a></li> + <li><a href="#numeric_query">Numeric</a></li> + <li><a href="#date_query">Date</a></li> + <li><a href="#spatial_query">Spatial</a></li> + </ul> + </li> + <li> + <a href="#types">Types</a> + <ul class="nav"> + <li><a href="#text_type">Text</a></li> + <li><a href="#string_type">String</a></li> + <li><a href="#long_type">Long</a></li> + <li><a href="#int_type">Integer</a></li> + <li><a href="#float_type">Float</a></li> + <li><a href="#double_type">Double</a></li> + <li><a href="#date_type">Date</a></li> + <li><a href="#stored_type">Stored</a></li> + <li><a href="#spatial_type">Spatial</a> + <ul class="nav"> + <li><a href="#spatial_pointvector"> Point Vector</a></li> + <li><a href="#spatial_termprefix"> Term Prefix</a></li> + <li><a href="#spatial_recursiveprefix"> Recursive Prefix</a></li> + </ul> + </li> + </ul> + </li> + <li> + <a href="#custom_types">Custom Types</a> + <ul class="nav"> + <li><a href="#custom_types_creating">Creating</a></li> + <li><a href="#custom_types_distributing">Distributing</a></li> + <li><a href="#custom_types_using">Using</a> + <ul class="nav"> + <li><a href="#custom_types_using_cluster"> Cluster Wide</a></li> + <li><a href="#custom_types_using_table"> Single Table</a></li> + </ul> + </li> + </ul> + </li> + </ul> + </div> + </div> + <div class="col-md-9" role="main"> + <section> + <div class="page-header"> + <h1 id="structure">Data Structure</h1> + </div> + <p class="lead"> + Blur is a table based query system. So within a single shard cluster there can be many different tables, each with a different schema, shard size, analyzers, etc. Each table contains Rows. A Row contains a row id (Lucene StringField internally) and many Records. A record has a record id (Lucene StringField internally), a family (Lucene StringField internally), and many Columns. A column contains a name and value, both are Strings in the API but the value can be interpreted as different types. All base Lucene Field types are supported, Text, String, Long, Int, Double, and Float. + </p> + <p>Starting with the most basic structure and building on it.</p> + <h3 id="columns">Columns</h3> + <p> + Columns contain a name and value, both are strings in the API but can be interpreted as an Integer, Float, Long, Double, String, or Text. All Column types default to Text and will be analyzed during the indexing process. + </p> + <pre><code class="json">Column {"name" => "value"}</code></pre> + <h3 id="records">Records</h3> + <p> + Record contains a Record Id, Family, and one or more Columns + </p> + <pre><code class="json">Record { + "recordId" => "1234", + "family" => "family1", + "columns" => [ + Column {"column1" => "value1"}, + Column {"column2" => "value2"}, + Column {"column2" => "value3"}, + Column {"column3" => "value4"} + ] +}</code></pre> + <div class="bs-callout bs-callout-info"><h4>Quick Tip!</h4><p>The column names do not have to be unique within the Record. So you can treat multiple Columns with the same name as an array of values. Also the order of the values will be maintained.</p></div> + <h3 id="rows">Rows</h3> + <p> + Rows contain a row id and a list of Records. + </p> + <pre><code class="json">Row { + "id" => "r-5678", + "records" => [ + Record { + "recordId" => "1234", + "family" => "family1", + "columns" => [ + Column {"column1" => "value1"}, + Column {"column2" => "value2"}, + Column {"column2" => "value3"}, + Column {"column3" => "value4"} + ] + }, + Record { + "recordId" => "9012", + "family" => "family1", + "columns" => [ + Column {"column1" => "value1"} + ] + }, + Record { + "recordId" => "4321", + "family" => "family2", + "columns" => [ + Column {"column16" => "value1"} + ] + } + ] +}</code></pre> + </section> + <section> + <div class="page-header"> + <h1 id="querying">Querying</h1> + <p> + All queries follow the basic Lucene query syntax see (<a href="http://lucene.apache.org/core/4_4_0/queryparser/org/apache/lucene/queryparser/classic/package-summary.html";>http://lucene.apache.org/core/4_4_0/queryparser/org/apache/lucene/queryparser/classic/package-summary.html</a>) + for an extensive explanation on the syntax. + </p> + <p> + All queries can have boolean logic like: + <pre><code class="json">+docs.body:hadoop +docs.author:jon</code></pre> + </p><p> + Which is the same as: + <pre><code class="json">docs.body:hadoop AND docs.author:jon</code></pre> + </p> + <h3 id="row_query">Row Queries</h3> + +<p> +Row queries allow you to execute queries across Records within the same Row. Row queries are a similar idea to an inner join. +Let's say you want to find all the Rows that contain a Record with the family "author" and has a "name" Column that has that contains a term "Jon" and another Record with the family "docs" and has a "body" Column with a term of "Hadoop". +<pre><code class="json">+<author.name:Jon> +<docs.body:Hadoop></code></pre> +</p> + +<p> +The meaning of the modifier on the outside of the < indicates how you want Rows with matching Records +handled. In the case above, we've asked for Blur to include all Rows that match the criteria. However, +we could also have negated Rows with matching Records as well. Suppose, we're interested in books that +contain Hadoop but we happen to already own the one written by Jon, so we'd like to exclude them: +<pre><code class="json">-<author.name:Jon> +<docs.body:Hadoop></code></pre> +</p> + +<p> +So far, we've included only simple term queries inside the < > operators, but they support full +query expressions too. Suppose our previous query excluded to many good books because of our author criteria. +Well, we happen to know that the book we have was written by Jon in Atlanta, so let's further limit that part +of our criteria: +<pre><code class="json">-<+author.name:Jon +author.city:Atlanta> +<docs.body:Hadoop></code></pre> + +Seeing that positive (+) criteria inside the negated clause can be confusing, so let's "read" what we're asking +for from the inside out. First, the record criteria - find any Row that contains a Record with an author named +Jon AND in the city of Atlanta. Now, the negation (-) outside the < says that we want to take all Rows +that match that criteria and exclude them from our results. +</p> + <h3 id="text_query">Text</h3> +<p>Text fields are analyzed with Lucene's standard analyzers, which mean that the string is broken down into terms +and the terms capitalization is removed as well as any special punctuation. See Lucene's documentation for further explanation. +</p> +<h4>Examples:</h4> +<p> +To run a query to find all the rows that contain a column with a term of "hadoop" where the family is "docs" and the column is "body". +<pre><code class="json">docs.body:hadoop</code></pre> +</p> +<p> +To run a query to find all the rows that contain a column with a term of "hadoop" and "awesome" where the family is "docs" and the column is "body". +<pre><code class="json">docs.body:(+hadoop +awesome)</code></pre> +</p> +<p> +To run a query to find all the rows that contain a column with a phrase of "hadoop is awesome" where the family is "docs" and the column is "body". +<pre><code class="json">docs.body:"hadoop is awesome"</code></pre> +</p> +<p> +To run a query to find all the rows that contain a column with a word of "hadoop" and we want to deal with misspellings (or a Fuzzy query) where the family is "docs" and the column is "body". +<pre><code class="json">docs.body:hadoop~</code></pre> +</p> +<p> +To run a query to find all the rows that contain a column with a word that matches a wildcard pattern of "h*d?op" where the family is "docs" and the column is "body". +<pre><code class="json">docs.body:h*d?op</code></pre> +</p> + + <h3 id="string_query">String</h3> +<p>String fields are indexed Columns that are not analyzed they are indexed as is. Do not use String fields for +large amount of text, this will increase the size of your index and probably not give you the desired behavior. +So given the string "Hadoop" and "hadoop", these will be indexed as two different term because the String field +is case sensitive. Also if the string contains "The cow jumps over the moon." the single term that will be placed +into the index is "The cow jumps over the moon." as a single string. This field type is normally used for id or +type lookups. +</p> +<h4>Examples:</h4> +<p> +To run a query to find all the rows that contain a column with a term of "Hadoop" where the family is "docs" and the column is "type". +<pre><code class="json">docs.type:Hadoop</code></pre> +</p> + <h3 id="numeric_query">Numeric</h3> +<p>The numerics types are: +<ul> +<li>int</li> +<li>long</li> +<li>float</li> +<li>double</li> +</ul> +All numerics types can perform two types of queries: +<ul> +<li>Exact Match</li> +<li>Range</li> +</ul> +</p> + +<h4>Examples:</h4> +<p> +To run a query to find all the rows that contain a column with a value of "12345" where the family is "docs" and the column is "id". +<pre><code class="json">docs.id:12345</code></pre> +</p> +<p> +To run a query to find all the rows that contain a column with a starting value of "12345" and an ending values of "54321" where the family is "docs" and the column is "id". +<pre><code class="json">docs.id:[12345 TO 54321]</code></pre> +</p> + +<p> +To run a query to find all the rows that contain a column with a value less than "12345" where the family is "docs" and the column is "id". +<pre><code class="json">docs.id:[MIN TO 12345}</code></pre> +</p> + +<p> +To run a query to find all the rows that contain a column with a value less than or equal to "12345" where the family is "docs" and the column is "id". +<pre><code class="json">docs.id:[MIN TO 12345]</code></pre> +</p> + +<p> +To run a query to find all the rows that contain a column with a value great than to "12345" where the family is "docs" and the column is "id". +<pre><code class="json">docs.id:{12345 TO MAX]</code></pre> +</p> + +<p> +To run a query to find all the rows that contain a column with a value great than or equal to "12345" where the family is "docs" and the column is "id". +<pre><code class="json">docs.id:[12345 TO MAX]</code></pre> +</p> + +<h3 id="date_query">Date</h3> +<p>Date types are basically a long field type with a date parser built-in. + +The date type can perform two types of queries: +<ul> +<li>Exact Match</li> +<li>Range</li> +</ul> +</p> + +<h4>Examples:</h4> +<p> +To run a query to find all the rows that contain a column with a value of "2012-09-11" where the family is "docs" and the column is "published_date". +<pre><code class="json">docs.published_date:2013-09-11</code></pre> +</p> +<p> +To run a query to find all the rows that contain a column with a starting value of "2012-09-11" and an ending values of "2013-09-11" where the family is "docs" and the column is "published_date". +<pre><code class="json">docs.published_date:[2012-09-11 TO 2013-09-11]</code></pre> +</p> + + <h3 id="spatial_query">Spatial</h3> +<p> +Spatial queries are supported through the Lucene spatial module. There are currently three +different types of strategies: +<ul> +<li>Point Vector</li> +<li>Term Query Prefix Tree</li> +<li>Recursive Prefix Tree</li> +</ul> +Currently all of the built in spatial types are using a GEO spatial context, meaning the assumed +coordinates are on the planet Earth. Each of these strategies support different indexing features +such as the shapes they can index, the shapes they can query with, and the operations they +support during the query. See the type section below to get a list of supported operations and +shapes per strategy. +</p> +<h4>Examples:</h4> +<p> +To run a query to find all the rows that contain a location within 10 km (0.089932 10 km in degrees) of gis coordinate "33.0, -88.0" where the family is "docs" and the column is "location". +<pre><code class="json">docs.location:"Intersects(Circle(33.0, -88.0 d=0.089932))"</code></pre> +</p> +<p> +To run a query to find all the rows that contain a location within 10 km of gis coordinate "33.0, -88.0" where the family is "docs" and the column is "location". +<pre><code class="json">docs.location:"Intersects(Circle(33.0, -88.0 d=10.0km))"</code></pre> +</p> +<p> +To run a query to find all the rows that contain a location within 10 miles of gis coordinate "33.0, -88.0" where the family is "docs" and the column is "location". +<pre><code class="json">docs.location:"Intersects(Circle(33.0, -88.0 d=10.0m))"</code></pre> +</p> + </div> + </section> + <section> + <div class="page-header"> + <h1 id="types">Types</h1> + <h3 id="text_type">Text</h3> + <p> + The Text Type has the type name of: +<pre><code class="json">text</code></pre> + <h5>Property Options:</h5> + <ul> + <li>"stopWordPath" -Optional- default value is no stop words. This should be a HDFS path. + <br/>This will load stop words into the StandardAnalyzer for this field, one term per line.</li> + + <li>"analyzerClass" -Optional- default value is a standard analyzer with no stop words.<br/>This could be any Analyzer class that has a default constructor or one that takes a Lucene Version enum.</li> + </ul> + </p> + <h3 id="string_type">String</h3> + <p> + The String Type has the type name of: +<pre><code class="json">string</code></pre> + <h5>Property Options:</h5> + <ul> + <li>None</li> + </ul> + </p> + <h3 id="long_type">Long</h3> + <p> + The Long Type has the type name of: +<pre><code class="json">long</code></pre> + <h5>Property Options:</h5> + <ul> + <li>"numericPrecisionStep" -Optional- default value is "4"</li> + </ul> + </p> + <h3 id="int_type">Integer</h3> + <p> + The Integer Type has the type name of: +<pre><code class="json">int</code></pre> + <h5>Property Options:</h5> + <ul> + <li>"numericPrecisionStep" -Optional- default value is "4"</li> + </ul> + </p> + <h3 id="float_type">Float</h3> + <p> + The Float Type has the type name of: +<pre><code class="json">float</code></pre> + <h5>Property Options:</h5> + <ul> + <li>"numericPrecisionStep" -Optional- default value is "4"</li> + </ul> + </p> + <h3 id="double_type">Double</h3> + <p> + The Double Type has the type name of: +<pre><code class="json">double</code></pre> + <h5>Property Options:</h5> + <ul> + <li>"numericPrecisionStep" -Optional- default value is "4"</li> + </ul> + </p> + <h3 id="date_type">Date</h3> + <p> + The Date Type has the type name of: + <pre><code class="json">date</code></pre> + <h5>Property Options:</h5> + <ul> + <li>"dateFormat" -Required- Examples: "yyyy-MM-dd", "MM/dd/yyyy" or anything that SimpleDateFormat can parse.</li> + <li>"timeUnit" -Optional- Default is SECONDS. Other options (DAYS, HOURS, MINUTES, SECONDS, MILLISECONDS)</li> + <li>"numericPrecisionStep" -Optional- default value is "4"</li> + </ul> + </p> + + <h3 id="stored_type">Stored</h3> + <p> + The Stored Type has the type name of: +<pre><code class="json">stored</code></pre> + <h5>Property Options:</h5> + <ul> + <li>None</li> + </ul> + </p> + <h3 id="spatial_type">Spatial</h3> + <p></p> + <h4 id="spatial_pointvector">Point Vector</h4> + <p> + The Point Vector Spatial Type has the type name of: +<pre><code class="json">geo-pointvector</code></pre> + <h5>Property Options:</h5> + <ul> + <li>None</li> + </ul> + <h5>Supported Indexing Shapes:</h5> + <ul> + <li>Point</li> + </ul> + <h5>Supported Querying Shapes:</h5> + <ul> + <li>Circle</li> + <li>Rectangle</li> + </ul> + <h5>Supported Querying Operations:</h5> + <ul> + <li>Intersects</li> + </ul> + </p> + <h4 id="spatial_termprefix">Term Prefix</h4> + <p> + The Term Prefix Spatial Type has the type name of: +<pre><code class="json">geo-termprefix</code></pre> + <h5>Property Options:</h5> + <ul> + <li>"spatialPrefixTree" can either equal to "GeohashPrefixTree" or "QuadPrefixTree"</li> + <li>"maxLevels" -Optional- default value is "11"</li> + </ul> + <h5>Supported Indexing Shapes:</h5> + <ul> + <li>Point</li> + </ul> + <h5>Supported Querying Shapes:</h5> + <ul> + <li>Circle</li> + <li>Rectangle</li> + </ul> + <h5>Supported Querying Operations:</h5> + <ul> + <li>Intersects</li> + </ul> + </p> + <h4 id="spatial_recursiveprefix">Recursive Prefix</h4> + <p> + The Term Prefix Spatial Type has the type name of: +<pre><code class="json">geo-recursiveprefix</code></pre> + <h5>Property Options:</h5> + <ul> + <li>"spatialPrefixTree" can either equal to "GeohashPrefixTree" or "QuadPrefixTree"</li> + <li>"maxLevels" -Optional- default value is "11"</li> + </ul> + <h5>Supported Indexing Shapes:</h5> + <ul> + <li>Point</li> + <li>Circle</li> + <li>Rectangle</li> + </ul> + <h5>Supported Querying Shapes:</h5> + <ul> + <li>Circle</li> + <li>Rectangle</li> + </ul> + <h5>Supported Querying Operations:</h5> + <ul> + <li>IsDisjointTo</li> + <li>Intersects</li> + <li>IsWithin</li> + <li>Contains</li> + </ul> + </p> + </div> + </section> + <section> + <div class="page-header"> + <h1 id="custom_types">Custom Types</h1> + </div> + <p class="lead">Custom types in Blur allow you to create your own types in Lucene + as well as plugging into the query parser so that you can use your custom type.</p> + <h3 id="custom_types_creating">Creating</h3> + <p> +You will need to extend the "org.apache.blur.analysis.FieldTypeDefinition" class found in the blur-query module. If you need to use a different Analyzer than the StandardAnalyzer used in the "text" type just extend the "org.apache.blur.analysis.type.TextFieldTypeDefinition" and make the appropriate changes. + </p> +<p> +For types that require custom query parsing or custom "org.apache.lucene.index.IndexableField" manipulation without the use of an Analyzer. Please extend "org.apache.blur.analysis.type.CustomFieldTypeDefinition". +</p> +<p> +<h4>Example</h4> +Below is a simple type that is basically the same as a "string" type, however it's implemented by extending "org.apache.blur.analysis.type.CustomFieldTypeDefinition".<br/><br/> +<pre><code class="java">public class ExampleType extends CustomFieldTypeDefinition { + + private String _fieldNameForThisInstance; + + /** + * Get the name of the type. + * + * @return the name. + */ + @Override + public String getName() { + return "example"; + } + + /** + * Configures this instance for the type. + * + * @param fieldNameForThisInstance + * the field name for this instance. + * @param properties + * the properties passed into this type definition from the + * {@link Blur.Iface#addColumnDefinition(String, ColumnDefinition)} + * method. + */ + @Override + public void configure(String fieldNameForThisInstance, Map<String, String> properties, + Configuration configuration) { + _fieldNameForThisInstance = fieldNameForThisInstance; + } + + /** + * Create {@link Field}s for the index as well as for storing the original + * data for retrieval. + * + * @param family + * the family name. + * @param column + * the column that holds the name and value. + * + * @return the {@link Iterable} of {@link Field}s. + */ + @Override + public Iterable<? extends Field> getFieldsForColumn(String family, Column column) { + String name = family + "." + column.getName(); + String value = column.getValue(); + return makeIterable(new StringField(name, value, Store.YES)); + } + + /** + * Create {@link Field}s for the index do NOT store the data because the is a + * sub column. + * + * @param family + * the family name. + * @param column + * the column that holds the name and value. + * @param subName + * the sub column name. + * + * @return the {@link Iterable} of {@link Field}s. + */ + @Override + public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, + String subName) { + String name = family + "." + column.getName() + "." + subName; + String value = column.getValue(); + return makeIterable(new StringField(name, value, Store.NO)); + } + + /** + * Gets the query from the text provided by the query parser. + * + * @param text + * the text provided by the query parser. + * @return the {@link Query}. + */ + @Override + public Query getCustomQuery(String text) { + return new TermQuery(new Term(_fieldNameForThisInstance, text)); + } + +} +</code></pre> + +</p> + + <h3 id="custom_types_distributing">Distributing</h3> + <p>Once you have created and tested your custom type you will need to copy the jar file containing your custom type to all the servers in the cluster. The jar file will need to be located within the $BLUR_HOME/lib directory. Once there all the servers will need to be restarted to have the jar file be picked up in the classpath.<br/><br/>In a later version of Blur we hope to have this be a dynamic operation that can be performed without restarting the cluster.</p> + <h3 id="custom_types_using">Using</h3> + <p>You can either add your custom type to the entire cluster or per table.</p> + <h4 id="custom_types_using_cluster">Cluster Wide</h4> + <p>For cluster wide configuration you will need to add the new field types into the blur-site.properties file on each server. +<pre><code class="bash">blur.fieldtype.customtype1=org.apache.blur.analysis.type.ExampleType1 +blur.fieldtype.customtype2=org.apache.blur.analysis.type.ExampleType2 +...</code></pre> + +Please note that the prefix of "blur.fieldtype." is all that is used from the property name because the type gets it's name from the internal method of "getName". However the property names will need to be unique within the file. +</p> + <h4 id="custom_types_using_table">Single Table</h4> + <p>For a single table configuration you will need to add the new field types into the tableProperties map in the TableDescriptor as you define the table. +<pre><code class="java">tableDescriptor.putToTableProperties("blur.fieldtype.customtype1", + "org.apache.blur.analysis.type.ExampleType1"); +tableDescriptor.putToTableProperties("blur.fieldtype.customtype2", + "org.apache.blur.analysis.type.ExampleType2"); +...</code></pre> + +Please note that the prefix of "blur.fieldtype." is all that is used from the property name because the type gets it's name from the internal method of "getName". However the property names will need to be unique within the map.</p> + </section> + </div> + </div> + </div> + +<br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <script src="resources/js/jquery-2.0.3.min.js"></script> + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="resources/js/bootstrap.min.js"></script> + <!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) --> + <script src="resources/js/respond.min.js"></script> + <script src="resources/js/docs.js"></script> + </body> +</html>
Added: incubator/blur/site/trunk/content/blur/docs/0.2.4/extra.html URL: http://svn.apache.org/viewvc/incubator/blur/site/trunk/content/blur/docs/0.2.4/extra.html?rev=1648906&view=auto ============================================================================== --- incubator/blur/site/trunk/content/blur/docs/0.2.4/extra.html (added) +++ incubator/blur/site/trunk/content/blur/docs/0.2.4/extra.html Thu Jan 1 17:21:48 2015 @@ -0,0 +1,72 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html> + <head> + <title>Extra - Apache Blur (Incubator) Documentation</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <!-- Bootstrap --> + <link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen"> + <link href="resources/css/bs-docs.css" rel="stylesheet" media="screen"> + </head> + <body> + <div class="navbar navbar-inverse navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://incubator.apache.org/blur";>Apache Blur (Incubator)</a> + </div> + <div class="collapse navbar-collapse"> + <ul class="nav navbar-nav"> + <li><a href="index.html">Main</a></li> + <li><a href="getting-started.html">Getting Started</a></li> + <li><a href="data-model.html">Data Model</a></li> + <li><a href="cluster-setup.html">Cluster Setup</a></li> + <li><a href="using-blur.html">Using Blur</a></li> + <li class="active"><a href="extra.html">Extra</a></li> + </ul> + </div> + </div> + </div> + <div class="container bs-docs-container"> + <div class="row"> + <div class="col-md-3"> + <div class="bs-sidebar hidden-print affix" role="complementary"> + <ul class="nav bs-sidenav"> + + </ul> + </div> + </div> + <div class="col-md-9" role="main"> + + </div> + </div> + </div> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <script src="resources/js/jquery-2.0.3.min.js"></script> + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="resources/js/bootstrap.min.js"></script> + <!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) --> + <script src="resources/js/respond.min.js"></script> + <script src="resources/js/docs.js"></script> + </body> +</html> Added: incubator/blur/site/trunk/content/blur/docs/0.2.4/getting-started.html URL: http://svn.apache.org/viewvc/incubator/blur/site/trunk/content/blur/docs/0.2.4/getting-started.html?rev=1648906&view=auto ============================================================================== --- incubator/blur/site/trunk/content/blur/docs/0.2.4/getting-started.html (added) +++ incubator/blur/site/trunk/content/blur/docs/0.2.4/getting-started.html Thu Jan 1 17:21:48 2015 @@ -0,0 +1,276 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html> + <head> + <title>Getting started - Apache Blur (Incubator) Documentation</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <!-- Bootstrap --> + <link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen"> + <link href="resources/css/bs-docs.css" rel="stylesheet" media="screen"> + </head> + <body> + <div class="navbar navbar-inverse navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://incubator.apache.org/blur";>Apache Blur (Incubator)</a> + </div> + <div class="collapse navbar-collapse"> + <ul class="nav navbar-nav"> + <li><a href="index.html">Main</a></li> + <li class="active"><a href="getting-started.html">Getting Started</a></li> + <li><a href="platform.html">Platform</a></li> + <li><a href="data-model.html">Data Model</a></li> + <li><a href="cluster-setup.html">Cluster Setup</a></li> + <li><a href="using-blur.html">Using Blur</a></li> + <li><a href="Blur.html">Blur API</a></li> + <li><a href="console.html">Console</a></li> + </ul> + </div> + </div> + </div> + <div class="container bs-docs-container"> + <div class="row"> + <div class="col-md-3"> + <div class="bs-sidebar hidden-print affix" role="complementary"> + <ul class="nav bs-sidenav"> + <li><a href="#prerequisites">Prerequisites</a></li> + <li><a href="#download">Download</a></li> + <li><a href="#compile">Compile</a></li> + <li><a href="#install">Install</a></li> + <li><a href="#min-config">Minimum Configuration</a></li> + <li><a href="#start">Starting Apache Blur</a></li> + <li><a href="#shell">Shell</a></li> + <li><a href="#shell-example">Simple Shell Example</a> + <ul class="nav"> + <li><a href="#shell-create">Create Table</a></li> + <li><a href="#shell-mutate">Mutate</a></li> + <li><a href="#shell-query">Query</a></li> + <li><a href="#shell-enable-highlighting">Enable Highlighting</a></li> + <li><a href="#shell-query-highlight">Query with Highlights</a></li> + </ul> + </li> + </ul> + </div> + </div> + <div class="col-md-9" role="main"> + <section> + <div class="page-header"> + <h1 id="prerequisites">Prerequisites</h1> + </div> + <p class="lead"> + You will at a minimum need the following: + </p> + <ul> + <li>Java 6 or Java 7 (Java 7 is recommended)</li> + </ul> + <h3>Setup passphraseless ssh</h3> + <p>These instructions are taken from the Hadoop Quick Start Guide.</p> + <p>Now check that you can ssh to the localhost without a passphrase:</p> + <pre><code class="bash">ssh localhost</code></pre> + <p>If you cannot ssh to localhost without a passphrase, execute the following commands:</p> + <pre><code class="bash">ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa +cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys</code></pre> + <div class="bs-callout bs-callout-info"><h4>Heads Up!</h4><p>Also you will need to know the location of the JAVA_HOME directory.</p></div> + </section> + <section> + <div class="page-header"> + <h1 id="download">Download</h1> + </div> + <h3>Download Source and Binary Artifacts</h3> + <p> + Both the source and binary artifacts are provided via mirrors here:<br/><br/> + <a href="http://www.apache.org/dyn/closer.cgi/incubator/blur/0.2.4-incubating-SNAPSHOT/apache-blur-0.2.4-incubating-src.tar.gz"; class="btn btn-large btn-primary">Apache Blur 0.2.4 Source</a> + <a href="http://www.apache.org/dyn/closer.cgi/incubator/blur/0.2.4-incubating-SNAPSHOT/apache-blur-0.2.4-incubating-hadoop1-bin.tar.gz"; class="btn btn-large btn-primary">Apache Blur 0.2.4 Hadoop1 Binary</a> + <a href="http://www.apache.org/dyn/closer.cgi/incubator/blur/0.2.4-incubating-SNAPSHOT/apache-blur-0.2.4-incubating-hadoop2-bin.tar.gz"; class="btn btn-large btn-primary">Apache Blur 0.2.4 Hadoop2 Binary</a> + </p> + </section> + <section> + <div class="page-header"> + <h1 id="compile">Compile Hadoop</h1> + </div> + <p class="lead"> + If building from source, the distribution needs to be compiled before use + </p> + <p>Clone master</p> + <pre><code class="bash">git clone https://git-wip-us.apache.org/repos/asf/incubator-blur.git</code></pre> + <h4>Hadoop 1</h4> + <p>Build the artifacts for Hadoop 1 (if you want to run the tests remove the "-DskipTests")</p> + <pre><code class="bash">cd incubator-blur/ +mvn install -DskipTests -Dhadoop1</code></pre> + <p>The binary artifact is located <code>distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin.tar.gz</code>.</p> + <h4>Hadoop 2</h4> + <div class="bs-callout bs-callout-info"><h4>Heads Up!</h4><p>While all the tests pass on Hadoop 2, Blur has not be tested at scale on Hadoop 2 and bin/blur-config.sh script will likely require modification to include the correct Hadoop 2 libraries.</p></div> + <p>Build the artifacts for Hadoop 2 (if you want to run the tests remove the "-DskipTests")</p> + <pre><code class="bash">cd incubator-blur/ +mvn install -DskipTests -Dhadoop2</code></pre> + <p>The binary artifact is located <code>distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop2-bin.tar.gz</code>.</p> + </section> + <section> + <div class="page-header"> + <h1 id="install">Install</h1> + </div> + <p class="lead"> + Once a distribution is available, follow the simple steps to install. + </p> + <p>Extract the contents of the distribution</p> + <pre><code class="bash">tar -xzvf apache-blur-*-bin.tar.gz</code></pre> + <div class="bs-callout bs-callout-info">While it's not required it is a good idea to set BLUR_HOME in your environment variables.</div> + <p>For bash edit .bash_profile and add:</p> + <pre><code class="bash">export BLUR_HOME=<directory where Blur was extracted></code></pre> + </section> + <section> + <div class="page-header"> + <h1 id="min-config">Minimum Configuration</h1> + </div> + <p class="lead"> + There are a few things at a minimum that will need to be configured to start Apache Blur + </p> + <p>Edit $BLUR_HOME/conf/blur-env.sh and set JAVA_HOME:</p> + <pre><code class="bash">export JAVA_HOME=<Java Home Directory></code></pre> + <div class="bs-callout bs-callout-warning"><h4>Caution</h4>If this variable is not set, then the script will attempt to locate JAVA_HOME by using the location of the "java" command.</div> + </section> + <section> + <div class="page-header"> + <h1 id="start">Starting Apache Blur</h1> + </div> + <p class="lead"> + Starting Apache blur is a simple one command step + </p> + <p>To start Apache Blur run the following command:</p> + <pre><code class="bash">$BLUR_HOME/bin/start-all.sh</code></pre> + <p> + This will start a single Controller server and a single Shard server on your localhost. + </p> + <p> + You should see: + </p> + <pre><code class="bash">blur@blurvm:~$ apache-blur-0.2.4-incubating-SNAPSHOT/bin/start-all.sh +localhost: ZooKeeper starting as process 6650. +localhost: Shard [0] starting as process 6783. +localhost: Controller [0] starting as process 6933.</code></pre> + <p>If you run the start command again you should see:</p> + <pre><code class="bash">blur@blurvm:~$ apache-blur-0.2.4-incubating-SNAPSHOT/bin/stop-all.sh +localhost: Stopping Controller [0] server with pid [6933]. +localhost: Stopping Shard [0] server with pid [6783]. +localhost: Stopping ZooKeeper with pid [6650].</code></pre> + <p>If you see it starting the servers again, then there is likely some issue with startup. Look in the $BLUR_HOME/logs directory for log and out files.</p> + </section> + <section> + <div class="page-header"> + <h1 id="shell">Shell</h1> + </div> + <p class="lead"> + Once the servers have been started, you can use the shell to interact with Blur. + </p> + <p>The shell command can be found in the bin directory</p> + <p>Auto detect the controller servers from the $BLUR_HOME/conf/controllers file</p> + <pre><code class="bash">$BLUR_HOME/bin/blur shell</code></pre> + <p>You can also explicitly call out the controller servers.</p> + <pre><code class="bash">$BLUR_HOME/bin/blur shell controller1:40010,controller2:40010</code></pre> + <p>Once in the shell, tables can be created, enabled, disabled, and removed. Type help to get a list of the commands.</p> + </section> + <section> + <div class="page-header"> + <h1 id="shell-example">Shell Example</h1> + </div> + <p class="lead"> + The below example creates a table and stores the contents of the table in a local directory of /data/testTableName which will only work if you are running blur in a single instance. Normally if you are running a hadoop cluster this will be a hdfs URI for example hdfs://host:port/blur/tables/testTableName. + </p> +<h3 id="shell-create">Create Table</h3> +<p> + <pre><code class="bash">blur> #Creates a table called testtable in the hdfs directory of /data/testtable with 11 shards +blur> create -t testtable -c 11 -l hdfs://namenode/data/testtable +</code></pre> + <div class="bs-callout bs-callout-info"><h4>Note</h4><p>The local directory can be used however the integrity of the data may be compromised.</p></div> + <pre><code class="bash">blur> #Creates a table called testtable in the local directory of /data/testtable with 11 shards +blur> create -t testtable -c 11 -l file:///data/testtable +</code></pre> +</p> +<h3 id="shell-mutate">Mutate</h3> +<p> +<pre><code class="bash">blur> #Adds a row to testtable +blur> mutate testtable rowid1 recordid1 fam0 col1:value1 +</code></pre> +</p> +<h3 id="shell-query">Query</h3> +<p> +<pre><code class="bash">blur> #Runs a query on testtable +blur> query testtable fam0.col1:value1 + - Results Summary - + total : 1 + time : 7.874 ms +----------------------------------------------------------------------------------------------------- + hit : 0 + score : 1.4142135381698608 + id : rowid1 + recordId : recordid1 + family : fam0 + col1 : value1 +----------------------------------------------------------------------------------------------------- + - Results Summary - + total : 1 + time : 7.874 ms +</code></pre> +</p> +<h3 id="shell-enable-highlighting">Enable Highlighting</h3> +<p> +<pre><code class="bash">blur> #Turns highlighting on +blur> highlight +highlight of query command is now on +</code></pre> +</p> +<h3 id="shell-query-highlight">Query with Highlights</h3> +<p> +<pre><code class="bash">blur> #Runs a query on testtable with highlighting on, notice <<<value1>>> is highlighted +blur> query testtable2 fam0.col1:value1 + - Results Summary - + total : 1 + time : 13.395 ms +----------------------------------------------------------------------------------------------------- + hit : 0 + score : 1.4142135381698608 + id : rowid1 + recordId : recordid1 + family : fam0 + col1 : <<<value1>>> +----------------------------------------------------------------------------------------------------- + - Results Summary - + total : 1 + time : 13.395 ms +blur></code></pre> + </section> + </div> + </div> + </div> + +<br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <script src="resources/js/jquery-2.0.3.min.js"></script> + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="resources/js/bootstrap.min.js"></script> + <!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) --> + <script src="resources/js/respond.min.js"></script> + <script src="resources/js/docs.js"></script> + </body> +</html> Added: incubator/blur/site/trunk/content/blur/docs/0.2.4/index.html URL: http://svn.apache.org/viewvc/incubator/blur/site/trunk/content/blur/docs/0.2.4/index.html?rev=1648906&view=auto ============================================================================== --- incubator/blur/site/trunk/content/blur/docs/0.2.4/index.html (added) +++ incubator/blur/site/trunk/content/blur/docs/0.2.4/index.html Thu Jan 1 17:21:48 2015 @@ -0,0 +1,69 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html> + <head> + <title>Apache Blur (Incubator) Documentation</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <!-- Bootstrap --> + <link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen"> + <link href="resources/css/bs-docs.css" rel="stylesheet" media="screen"> + </head> + <body> + <div class="navbar navbar-inverse navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://incubator.apache.org/blur";>Apache Blur (Incubator)</a> + </div> + <div class="collapse navbar-collapse"> + <ul class="nav navbar-nav"> + <li class="active"><a href="index.html">Main</a></li> + <li><a href="getting-started.html">Getting Started</a></li> + <li><a href="platform.html">Platform</a></li> + <li><a href="data-model.html">Data Model</a></li> + <li><a href="cluster-setup.html">Cluster Setup</a></li> + <li><a href="using-blur.html">Using Blur</a></li> + <li><a href="Blur.html">Blur API</a></li> + <li><a href="console.html">Console</a></li> + </ul> + </div> + </div> + </div> + <div class="container"> + <div class="jumbotron"> + <h1>Apache Blur (Incubator)</h1> + <p> + Blur is an open source search platform capable of querying massive amounts of data at incredible speeds. Blur is built on top of Lucene, Hadoop, Thrift, and ZooKeeper. Tables consist of a series of shards (Lucene indexes) that are distributed across a cluster of commodity servers. All index information is stored in HDFS, cluster coordination is handled by Apache ZooKeeper, Apache Thrift is used for the RPC, and Lucene handles all of the indexing and data storage. + </p> + <a href="getting-started.html" class="btn btn-primary btn-large">Get Started »</a> + <a href="release-notes.html" class="btn btn-primary btn-large">View Release Notes »</a> + </div> + </div> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <script src="resources/js/jquery-2.0.3.min.js"></script> + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="resources/js/bootstrap.min.js"></script> + <!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) --> + <script src="resources/js/respond.min.js"></script> + </body> +</html> Added: incubator/blur/site/trunk/content/blur/docs/0.2.4/platform.html URL: http://svn.apache.org/viewvc/incubator/blur/site/trunk/content/blur/docs/0.2.4/platform.html?rev=1648906&view=auto ============================================================================== --- incubator/blur/site/trunk/content/blur/docs/0.2.4/platform.html (added) +++ incubator/blur/site/trunk/content/blur/docs/0.2.4/platform.html Thu Jan 1 17:21:48 2015 @@ -0,0 +1,278 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html> + <head> + <title>Platform - Apache Blur (Incubator) Documentation</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <!-- Bootstrap --> + <link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen"> + <link href="resources/css/bs-docs.css" rel="stylesheet" media="screen"> + </head> + <body> + <div class="navbar navbar-inverse navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://incubator.apache.org/blur";>Apache Blur (Incubator)</a> + </div> + <div class="collapse navbar-collapse"> + <ul class="nav navbar-nav"> + <li><a href="index.html">Main</a></li> + <li><a href="getting-started.html">Getting Started</a></li> + <li class="active"><a href="platform.html">Platform</a></li> + <li><a href="data-model.html">Data Model</a></li> + <li><a href="cluster-setup.html">Cluster Setup</a></li> + <li><a href="using-blur.html">Using Blur</a></li> + <li><a href="Blur.html">Blur API</a></li> + <li><a href="console.html">Console</a></li> + </ul> + </div> + </div> + </div> + <div class="container bs-docs-container"> + <div class="row"> + <div class="col-md-3"> + <div class="bs-sidebar hidden-print affix" role="complementary"> + <ul class="nav bs-sidenav"> + <li><a href="#intro">Introduction</a></li> + <li><a href="#motivation">Motivation</a></li> + <li><a href="#arch">Blur Architecture Review</a></li> + <li><a href="#affordances">Platform Affordances</a></li> + <li><a href="#commands">Command Overview</a></li> + <li><a href="#arguments">Arguments</a></li> + <li><a href="#installation">Installation</a></li> + <li><a href="#docs">Documentation</a></li> + <li><a href="#cli">CLI</a></li> + </ul> + </div> + </div> + <div class="col-md-9" role="main"> + <!-- In some places, we need to describe both where the system is heading and + describe its current limitations. For ease of documentation maintenance, let + us try to write in a way that disclaimers are in their own paragraphs that can + easily be stripped out when they no longer apply, thus not requiring a bunch + of re-writing. + --> + <section> + <div class="page-header"> + <h1 id="intro">Introduction</h1> + </div> + <p class="lead"> + While many users of Blur will find the search system sufficient for their + needs out of the box, the Blur platform exposes a simple set of lower-level + primitives that allow the user to easily and quickly introduce new system behavior. + </p> + <!-- Disclaimer para for 0.2.4 --> + <p class="lead"> + With this release, we expose the initial read-only constructs for the platform. + Future releases, will allow introduce more rich read-write constructs. + </p> + <!-- Disclaimer para for 0.2.4 --> + <p class="alert"> + <strong>NOTE:</strong> In 0.2.4, the platform capability described here exists, + but existing functionality of Blur has not yet been ported to use it. + </p> + </section> + <section> + <div class="page-header"> + <h2 id="motivation">Motivation</h2> + </div> + <p> + In modern open source search platforms, we find Lucene at the very core and + a monolithic application stack implemented on top of it handling the distributed + indexing, searching, failures, features, etc. Indeed, this was true of Blur + as well. + </p> + <p>We wanted more flexibility. We wanted to rapidly be able to introduce brand new + features into the system. So, we supposed it would be helpful if an + intermediate abstraction could be introduced providing the primitives for a + distributed Lucene server on which specific search applications could be built. + </p> + <p>Some specific goals we had in mind:</p> + <ul> + <li>To allow for indexing/searching based on other/new data models (e.g. + more than just the Row/Record constructs).</li> + <li>Allow implementations to build whole new APIs given direct access to the Lucene primitives.</li> + <li>Allow flexibility to build totally custom applications.</li> + <li>Remove the complexities of threading, networking and concurrency from + new feature creation.</li> + </ul> + </section> + <section> + <div class="page-header"> + <h2 id="arch">Blur Architecture Review</h2> + </div> + <p> + The Blur platform provides a set of <code>Command</code> classes that can + be implemented to achieve new functionality. A basic understanding of how Blur + works will greatly help in understanding how to implement commands. So let's + take a moment to review. + </p> + <!-- + @TODO: Does this content exist somewhere we can just point to? + @TODO: If the answer is no, we should beef up this quick-n-dirty explanation. + --> + <p>In Blur, we refer to a logical Lucene index as a table. Tables are typically + very large, and so we divide them up into 'shards'. Now, each shard is exposed + through a Shard Server, which is sort of a container of shards. The Shard Server(s) + are organized into a cluster that work together to make all the shards of the table(s) + available. For scalability, we've divided up the logical table into shards spread + across the Shard Server(s). We then put another type of server, called a Controller, + in front of the cluster to present all the shards as a single logical table. + </p> + <!-- + @TODO: Find a graphic of the architecture so the bevy of words above can be simplified. + --> + <p>For the controller to present all the shards as a single index, it needs to accept + a request, then scatter the request to all the shard servers, combine the results in + some meaningful way, and send them back to the client. + </p> + </section> + <section> + <div class="page-header"> + <h2 id="affordances">Platform Affordances</h2> + </div> + <p> + <code>@TODO</code> + </p> + + </section> + <section> + <div class="page-header"> + <h2 id="commands">Command Overview</h2> + </div> + <p> + As we've gathered from above, the heart of a distributed search system is the ability + to execute some function across a set of indices and combine the results in a logical + way to be returned to the user. Not surprisingly, this is also at the heart of the + Blur Platform. As an introduction, we'll explore how to take a look at finding the number + of documents that contain a particular term across all shards in a table. + </p> + <p>Our first step will be to find the answer for a single shard/index. Lucene's + <code>IndexReader</code>, to which we'll have access in our command, conveniently + gives us that. Getting the answer for a single index requires implementing an <code>execute</code> + method. + </p> + <pre>@Override +public Long execute(IndexContext context) throws IOException { + return (long) context.getIndexReader().numDocs(); +}</pre> + <p>We'll learn where the field name and term are defined later in the Arguments + section. Inside of the <code>execute</code> method, we're focused on finding the answer for + a single shard/index. To find our answer, we're given an <code>IndexContext</code> which + provides us access to the underlying Lucene index, so for our trivial command we can simply + return the answer directly from the IndexReader. + </p> + <p>Now we need to let Blur know how to combine the results from the individual shards + into a single logical response. We do this by implementing the <code>combine</code> method. + </p> + <pre>@Override +public Long combine(CombiningContext context, Map<? extends Location<?>, Long> results) throws IOException { + long total = 0; + for (Long l : results.values()) { + total += l; + } + return total; +}</pre> + <p> + Again, we're given some execution context (which we don't need for our sample command) and we're + given an <code>Map<? extends Location<?>, Long></code> of result values. + </p> + + </section> + <section> + <div class="page-header"> + <h2 id="arguments">Arguments</h2> + </div> + <p> + Recall from above that in the execute method we were able to use some member variables that + were treated like arguments to the command. Now, let's take a closer look how they were provided. + </p> + <p> + We've kept it very simple for you to declare arguments and for your users to provide arguments. We provide + two simple annotations that you can place right on your member field declarations indicating whether + they are required or optional. You can [and are encouraged to] provide some helpful documentation + on the intent of the argument. As an example, by extending the <code>TableReadCommand</code> you get + the <code>table</code> argument as required for free. Let's look at how it's declared: + </p> + <pre> +@RequiredArgument("The name of the table.") +private String table; + </pre> + <p>Naturally, we can also declare optional arguments as well:</p> + <pre> +@OptionalArgument("The number of results to be returned. default=10") +private short size = 10; + </pre> + <p> + By annotating your parameters, the Blur Platform is able to do the basic requirement checking for you + allowing you to keep the inside of your execute/combine clean of argument validation. + </p> + </section> + <section> + <div class="page-header"> + <h2 id="installation">Installation</h2> + </div> + <p> + <code>@TODO</code> + </p> + + </section> + <section> + <div class="page-header"> + <h2 id="docs">Documentation</h2> + </div> + <p> + Commands should be self-documenting starting with a good name. But good naming + is not sufficient, so Blur offers a <code>@Description</code> annotation to provide + a nice way to better express what your command does. It's simply used like so: + </p> + <pre> +@Description("Returns the number of documents containing the term in the given field.") +public class DocFreqCommand extends TableReadCommand<Long> { + ... +} + </pre> + + </section> + <section> + <div class="page-header"> + <h2 id="cli">CLI</h2> + </div> + <p> +See Using Blur -> Shell -> <a href="using-blur.html#shell_platform_commands">Platform Commands</a>. + </p> + + </section> + </div> + </div> + </div> + + <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> + <script src="resources/js/jquery-2.0.3.min.js"></script> + <!-- Include all compiled plugins (below), or include individual files as needed --> + <script src="resources/js/bootstrap.min.js"></script> + <!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) --> + <script src="resources/js/respond.min.js"></script> + <script src="resources/js/docs.js"></script> + </body> +</html> Added: incubator/blur/site/trunk/content/blur/docs/0.2.4/release-notes.html URL: http://svn.apache.org/viewvc/incubator/blur/site/trunk/content/blur/docs/0.2.4/release-notes.html?rev=1648906&view=auto ============================================================================== --- incubator/blur/site/trunk/content/blur/docs/0.2.4/release-notes.html (added) +++ incubator/blur/site/trunk/content/blur/docs/0.2.4/release-notes.html Thu Jan 1 17:21:48 2015 @@ -0,0 +1,159 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- Header for Release Notes --> +<html> + <head> + <title>Apache Blur (Incubator) Documentation</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <!-- Bootstrap --> + <link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen"> + <link href="resources/css/bs-docs.css" rel="stylesheet" media="screen"> + </head> + <body> + <div class="navbar navbar-inverse navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://incubator.apache.org/blur";>Apache Blur (Incubator)</a> + </div> + <div class="collapse navbar-collapse"> + <ul class="nav navbar-nav"> + <li><a href="index.html">Main</a></li> + <li><a href="getting-started.html">Getting Started</a></li> + <li><a href="platform.html">Platform</a></li> + <li><a href="data-model.html">Data Model</a></li> + <li><a href="cluster-setup.html">Cluster Setup</a></li> + <li><a href="using-blur.html">Using Blur</a></li> + <li><a href="Blur.html">Blur API</a></li> + <li><a href="console.html">Console</a></li> + <li><a href="site/index.html" target="_blank">Maven Site</a></li> + <li><a href="site/apidocs/index.html" target="_blank">Javadocs</a></li> + </ul> + </div> + </div> + </div> + <div class="container"> + <div class="jumbotron"> + <h1>Apache Blur (Incubator)</h1> + <p> +<!-- Header for Release Notes --> + + Release Notes - Apache Blur - Version 0.2.3<br/><br/> + +<b>** Apache Blur 0.2.3 is a re-release of 0.2.2 with a few bug fixes and a new version of Apache Blur Console.</b><br/> +<br/> + + + + Release Notes - Apache Blur - Version 0.2.2 + +<h2> Bug +</h2> +<ul> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-297'>BLUR-297</a>] - The ShardServerEventHandler class is log noisy when clients disconnect +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-298'>BLUR-298</a>] - ZooKeeperClusterStatus has several methods that are not working +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-299'>BLUR-299</a>] - Custom types at the system/cluster level aren't registered when the OutputFormat creates the table. +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-300'>BLUR-300</a>] - System types need to be registered before the tablecontext is loaded +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-301'>BLUR-301</a>] - List command should be sorted +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-303'>BLUR-303</a>] - Change class sig for DefaultBlurReducer +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-305'>BLUR-305</a>] - Blur CombineInputFormat doesn't work at all +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-306'>BLUR-306</a>] - Tables with the same name errors +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-307'>BLUR-307</a>] - CsvBlurMapper not does work with combined input format +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-308'>BLUR-308</a>] - The MR jobs tmpjars contains more jar than needed +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-309'>BLUR-309</a>] - CsvBlurMapper has an extra comma in the error message +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-310'>BLUR-310</a>] - Remove the index size gathering code +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-318'>BLUR-318</a>] - Bug in TypeDefinitions while having separate analyzers for index and query +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-319'>BLUR-319</a>] - Custom TypeDef's can get Blur into a state where one cannot make new tables. +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-324'>BLUR-324</a>] - SpatialPointVectorStrategyFieldTypeDefinition is throwing an exception during indexing +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-325'>BLUR-325</a>] - The MasterBasedDistributedLayout is not removing the layout when a table is deleted +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-326'>BLUR-326</a>] - The thrift call shardServerList is failing +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-328'>BLUR-328</a>] - Index curruption during mutateBatch calls +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-329'>BLUR-329</a>] - Shard server query cache bug +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-330'>BLUR-330</a>] - Remove the ZooKeeper trace store +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-331'>BLUR-331</a>] - Large Row Paging issue +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-332'>BLUR-332</a>] - FileNotFoundException during stats call +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-337'>BLUR-337</a>] - Shard server should fail hard on a bad misconfiguration +</li> +</ul> + +<h2> Improvement +</h2> +<ul> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-302'>BLUR-302</a>] - start-all script sometimes causes errors +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-317'>BLUR-317</a>] - Enable Changing of Compression alg and chunk size in Codec +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-323'>BLUR-323</a>] - Improve error message on CustomFieldTypeDefinition +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-327'>BLUR-327</a>] - Schedule a daily optimize of each shard +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-333'>BLUR-333</a>] - Remove polling from the index importer +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-334'>BLUR-334</a>] - Add sorting +</li> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-335'>BLUR-335</a>] - Field existence query +</li> +</ul> + +<h2> New Feature +</h2> +<ul> +<li>[<a href='https://issues.apache.org/jira/browse/BLUR-314'>BLUR-314</a>] - Allow configurable Thrift frame size +</li> +</ul> + + +<!-- Footer for Release Notes --> +</p> +</div> +</div> +<!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> +<script src="resources/js/jquery-2.0.3.min.js"></script> +<!-- Include all compiled plugins (below), or include individual files as needed --> +<script src="resources/js/bootstrap.min.js"></script> +<!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) --> +<script src="resources/js/respond.min.js"></script> +</body> +</html> +<!-- Footer for Release Notes --> Added: incubator/blur/site/trunk/content/blur/docs/0.2.4/release-notes.template.html URL: http://svn.apache.org/viewvc/incubator/blur/site/trunk/content/blur/docs/0.2.4/release-notes.template.html?rev=1648906&view=auto ============================================================================== --- incubator/blur/site/trunk/content/blur/docs/0.2.4/release-notes.template.html (added) +++ incubator/blur/site/trunk/content/blur/docs/0.2.4/release-notes.template.html Thu Jan 1 17:21:48 2015 @@ -0,0 +1,74 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- Header for Release Notes --> +<html> + <head> + <title>Apache Blur (Incubator) Documentation</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <!-- Bootstrap --> + <link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen"> + <link href="resources/css/bs-docs.css" rel="stylesheet" media="screen"> + </head> + <body> + <div class="navbar navbar-inverse navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://incubator.apache.org/blur";>Apache Blur (Incubator)</a> + </div> + <div class="collapse navbar-collapse"> + <ul class="nav navbar-nav"> + <li><a href="index.html">Main</a></li> + <li><a href="getting-started.html">Getting Started</a></li> + <li><a href="platform.html">Platform</a></li> + <li><a href="data-model.html">Data Model</a></li> + <li><a href="cluster-setup.html">Cluster Setup</a></li> + <li><a href="using-blur.html">Using Blur</a></li> + <li><a href="Blur.html">Blur API</a></li> + <li><a href="console.html">Console</a></li> + <li><a href="site/index.html" target="_blank">Maven Site</a></li> + <li><a href="site/apidocs/index.html" target="_blank">Javadocs</a></li> + </ul> + </div> + </div> + </div> + <div class="container"> + <div class="jumbotron"> + <h1>Apache Blur (Incubator)</h1> + <p> +<!-- Header for Release Notes --> + +PUT HTML RELEASE NOTES HERE + +<!-- Footer for Release Notes --> +</p> +</div> +</div> +<!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> +<script src="resources/js/jquery-2.0.3.min.js"></script> +<!-- Include all compiled plugins (below), or include individual files as needed --> +<script src="resources/js/bootstrap.min.js"></script> +<!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) --> +<script src="resources/js/respond.min.js"></script> +</body> +</html> +<!-- Footer for Release Notes -->
