Updated Branches: refs/heads/master a10c12597 -> 9db2f8411
Cleanup removing old files, documenting the blur-default file. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2716101a Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2716101a Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2716101a Branch: refs/heads/master Commit: 2716101a42ca50bc1105a71907a6d45134bd338b Parents: a10c125 Author: Aaron McCurry <[email protected]> Authored: Sun Aug 18 16:21:11 2013 -0400 Committer: Aaron McCurry <[email protected]> Committed: Sun Aug 18 16:21:11 2013 -0400 ---------------------------------------------------------------------- .../src/main/resources/blur-default.properties | 112 +++++++++++++++-- .../main/resources/hadoop-metrics.properties | 43 ------- .../src/main/scripts/conf/blur-site.properties | 14 ++- .../main/scripts/conf/hadoop-metrics.properties | 28 ----- docs/cluster-setup.html | 124 +++++++++++++++++-- docs/data-model.html | 27 +++- 6 files changed, 250 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index ba5c838..098cc1a 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -13,58 +13,152 @@ # See the License for the specific language governing permissions and # limitations under the License. +# The zookeeper session timeout +blur.zookeeper.timeout=90000 + +# The maximum number of results that can be fetched in a single request +blur.query.max.results.fetch=1000 + +# The maximum number of rows that can be fetched in a single request +blur.query.max.row.fetch=100 + +# The maximum number of records that can be fetched in a single request +blur.query.max.record.fetch=1000 + +# Setup metric reporter +blur.metrics.reporters= + +# Shard Properties + +# The hostname for the shard, if blank the hostname is automatically detected blur.shard.hostname= + +# The binding address of the shard blur.shard.bind.address=0.0.0.0 + +# The default binding port of the shard server blur.shard.bind.port=40020 + +# The number of fetcher threads blur.shard.data.fetch.thread.count=8 + +# The number of the thrift threads blur.shard.server.thrift.thread.count=8 + +# The number of threads that are used for opening indexes blur.shard.opener.thread.count=8 + +# The number of cached queries blur.shard.cache.max.querycache.elements=128 + +# The time to live for the cache query blur.shard.cache.max.timetolive=60000 + +# Default implementation of the blur cache filter, which is a pass through filter that does nothing blur.shard.filter.cache.class=org.apache.blur.manager.DefaultBlurFilterCache + +# Default Blur index warmup class that warms the fields provided in the table descriptor blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup + +# Throttles the warmup to 30MB/s across all the warmup threads blur.shard.index.warmup.throttle=30000000 + +# By default the block cache using off heap memory blur.shard.blockcache.direct.memory.allocation=true + +# The slabs in the blockcache are automatically configured by default (-1) otherwise 1 slab equals 128MB +# The auto config is detected through the MaxDirectoryMemorySize provided to the JVM blur.shard.blockcache.slab.count=-1 + +# The number of 1K byte buffers blur.shard.buffercache.1024=8192 + +# The number of 8K byte buffers blur.shard.buffercache.8192=8192 + +# The number of milliseconds to wait for the cluster to settle once changes have ceased blur.shard.safemodedelay=5000 + +# The default time between index commits blur.shard.time.between.commits=30000 + +# The default time between index refreshs blur.shard.time.between.refreshs=3000 + +# The maximum number of clauses in a BooleanQuery blur.max.clause.count=1024 + +# The number of thread used for parallel searching in the index manager blur.indexmanager.search.thread.count=8 + +# The number of threads used for parallel searching in the index searchers blur.shard.internal.search.thread.count=8 + +# Number of threads used for warming up the index blur.shard.warmup.thread.count=8 + +# The fetch count per Lucene search, this fetches pointers to hits blur.shard.fetchcount=100 + +# Heap limit on row fetch, once this limit has been reached the request will return blur.max.heap.per.row.fetch=10000000 + +# The maximum number of records in a single row fetch blur.max.records.per.row.fetch.request=1000 +# The http status page port for the shard server +blur.gui.shard.port=40090 + + +# Controller Properties + +# Sets the hostname for the controller, if blank the hostname is automatically detected blur.controller.hostname= + +# The binding address of the controller blur.controller.bind.address=0.0.0.0 + +# The default binding port of the controller server blur.controller.bind.port=40010 + +# The number of threads used for thrift requests blur.controller.server.thrift.thread.count=32 + +# The number of threads used for remote thrift requests to the shards server. This should be a large number. blur.controller.server.remote.thread.count=64 + +# The number of hits to fetch per request to the shard servers blur.controller.remote.fetch.count=100 +# The max number of retries to the shard server when there is an error during fetch blur.controller.retry.max.fetch.retries=3 + +# The max number of retries to the shard server when there is an error during mutate blur.controller.retry.max.mutate.retries=3 + +# The max number of retries to the shard server when there is an error during all other request blur.controller.retry.max.default.retries=3 + +# The starting backoff delay for the first retry for a fetch errors blur.controller.retry.fetch.delay=500 + +# The starting backoff delay for the first retry for a mutate errors blur.controller.retry.mutate.delay=500 + +# The starting backoff delay for the first retry for a all other request errors blur.controller.retry.default.delay=500 + +# The ending backoff delay for the last retry for a fetch errors blur.controller.retry.max.fetch.delay=2000 -blur.controller.retry.max.mutate.delay=2000 -blur.controller.retry.max.default.delay=2000 -blur.query.max.results.fetch=1000 -blur.query.max.row.fetch=100 -blur.query.max.record.fetch=1000 +# The ending backoff delay for the last retry for a mutate errors +blur.controller.retry.max.mutate.delay=2000 -blur.zookeeper.system.time.tolerance=3000 -blur.zookeeper.timeout=90000 +# The ending backoff delay for the last retry for a all other request errors +blur.controller.retry.max.default.delay=2000 +# The http status page port for the controller server blur.gui.controller.port=40080 -blur.gui.shard.port=40090 -blur.metrics.reporters= + http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/blur-util/src/main/resources/hadoop-metrics.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/hadoop-metrics.properties b/blur-util/src/main/resources/hadoop-metrics.properties deleted file mode 100644 index 5f4bb56..0000000 --- a/blur-util/src/main/resources/hadoop-metrics.properties +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Configuration of the "blur" context for null -blur.class=org.apache.hadoop.metrics.spi.NullContext - -# Configuration of the "blur" context for file -# blur.class=org.apache.hadoop.metrics.file.FileContext -# blur.period=10 -# blur.fileName=/tmp/blurmetrics.log - -# Configuration of the "blur" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# blur.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# blur.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# blur.period=10 -# blur.servers=localhost:8649 - - -# Configuration of the "jvm" context for null -jvm.class=org.apache.hadoop.metrics.spi.NullContext - -# Configuration of the "jvm" context for file -# jvm.class=org.apache.hadoop.metrics.file.FileContext -# jvm.period=10 -# jvm.fileName=/tmp/jvmmetrics.log - -# Configuration of the "jvm" context for ganglia -# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# jvm.period=10 -# jvm.servers=localhost:8649 http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/distribution/src/main/scripts/conf/blur-site.properties ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/conf/blur-site.properties b/distribution/src/main/scripts/conf/blur-site.properties index a5b33ad..3064f67 100644 --- a/distribution/src/main/scripts/conf/blur-site.properties +++ b/distribution/src/main/scripts/conf/blur-site.properties @@ -13,9 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -#Zookeeper connection string -blur.zookeeper.connection=localhost - -#The cluster name +blur.zookeeper.connection=127.0.0.1 blur.cluster.name=default +# +# Sets the default table location in hdfs. If left null the table uri property in +# the table descriptor will be required for all tables. +# +# blur.cluster.<cluster name here>.table.uri=<hdfs uri> +# +# Example: +# blur.cluster.default.table.uri=hdfs://namenode/blur/tables +# http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/distribution/src/main/scripts/conf/hadoop-metrics.properties ---------------------------------------------------------------------- diff --git a/distribution/src/main/scripts/conf/hadoop-metrics.properties b/distribution/src/main/scripts/conf/hadoop-metrics.properties deleted file mode 100644 index dba5756..0000000 --- a/distribution/src/main/scripts/conf/hadoop-metrics.properties +++ /dev/null @@ -1,28 +0,0 @@ -# Configuration of the "blur" context for null -blur.class=org.apache.hadoop.metrics.spi.NullContext - -# Configuration of the "blur" context for file -# blur.class=org.apache.hadoop.metrics.file.FileContext -# blur.period=10 -# blur.fileName=/tmp/blurmetrics.log - -# Configuration of the "blur" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# blur.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# blur.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# blur.period=10 -# blur.servers=localhost:8649 - - -# Configuration of the "jvm" context for null -jvm.class=org.apache.hadoop.metrics.spi.NullContext - -# Configuration of the "jvm" context for file -# jvm.class=org.apache.hadoop.metrics.file.FileContext -# jvm.period=10 -# jvm.fileName=/tmp/jvmmetrics.log - -# Configuration of the "jvm" context for ganglia -# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# jvm.period=10 -# jvm.servers=localhost:8649 http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/docs/cluster-setup.html ---------------------------------------------------------------------- diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html index 9168581..39854a9 100644 --- a/docs/cluster-setup.html +++ b/docs/cluster-setup.html @@ -84,30 +84,67 @@ <p> These are the default settings for the shard server that can be overridden in the blur-site.properties file. Consider increasing the various thread pool counts (*.thread.count). The blur.controller.server.remote.thread.count is very important to increase for larger clusters, basically one thread is used per shard server per query. Some production cluster have used set this thread pool to 2000 or more threads. </p> - <pre><code class="bash">blur.controller.hostname= +<pre> +<code class="bash"> +# Sets the hostname for the controller, if blank the hostname is automatically detected +blur.controller.hostname= + +# The binding address of the controller blur.controller.bind.address=0.0.0.0 + +# The default binding port of the controller server blur.controller.bind.port=40010 + +# The number of threads used for thrift requests blur.controller.server.thrift.thread.count=32 + +# The number of threads used for remote thrift requests to +# the shards server. This should be a large number. blur.controller.server.remote.thread.count=64 + +# The number of hits to fetch per request to the shard servers blur.controller.remote.fetch.count=100 +# The max number of retries to the shard server when there +# is an error during fetch blur.controller.retry.max.fetch.retries=3 + +# The max number of retries to the shard server when there +# is an error during mutate blur.controller.retry.max.mutate.retries=3 + +# The max number of retries to the shard server when there +# is an error during all other request blur.controller.retry.max.default.retries=3 + +# The starting backoff delay for the first retry for a +# fetch errors blur.controller.retry.fetch.delay=500 + +# The starting backoff delay for the first retry for a +# mutate errors blur.controller.retry.mutate.delay=500 + +# The starting backoff delay for the first retry for a +# all other request errors blur.controller.retry.default.delay=500 + +# The ending backoff delay for the last retry for a +# fetch errors blur.controller.retry.max.fetch.delay=2000 + +# The ending backoff delay for the last retry for a +# mutate errors blur.controller.retry.max.mutate.delay=2000 -blur.controller.retry.max.default.delay=2000 -blur.query.max.results.fetch=1000 -blur.query.max.row.fetch=100 -blur.query.max.record.fetch=1000 +# The ending backoff delay for the last retry for a +# all other request errors +blur.controller.retry.max.default.delay=2000 +# The http status page port for the controller server blur.gui.controller.port=40080 - -blur.metrics.reporters=</code></pre> +</code> +</pre> <h3 id="controller-blur-env">blur-env.sh</h3> <pre><code class="bash"># JAVA JVM OPTIONS for the controller servers, jvm tuning parameters are placed here. # Consider adding the -XX:OnOutOfMemoryError="kill -9 %p" option to kill jvms that are failing due to memory issues. @@ -127,33 +164,94 @@ export BLUR_NUMBER_OF_CONTROLLER_SERVER_INSTANCES_PER_MACHINE=1</code></pre> <p> These are the default settings for the shard server that can be overridden in the blur-site.properties file. Consider increasing the various thread pool counts (*.thread.count). Also the blur.max.clause.count sets the BooleanQuery max clause count for Lucene queries. </p> - <pre><code class="bash">blur.shard.hostname= +<pre> +<code class="bash"> +# The hostname for the shard, if blank the hostname is automatically detected +blur.shard.hostname= + +# The binding address of the shard blur.shard.bind.address=0.0.0.0 + +# The default binding port of the shard server blur.shard.bind.port=40020 + +# The number of fetcher threads blur.shard.data.fetch.thread.count=8 + +# The number of the thrift threads blur.shard.server.thrift.thread.count=8 + +# The number of threads that are used for opening indexes blur.shard.opener.thread.count=8 + +# The number of cached queries blur.shard.cache.max.querycache.elements=128 + +# The time to live for the cache query blur.shard.cache.max.timetolive=60000 + +# Default implementation of the blur cache filter, which is +# a pass through filter that does nothing blur.shard.filter.cache.class=org.apache.blur.manager.DefaultBlurFilterCache + +# Default Blur index warmup class that warms the fields provided +# in the table descriptor blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup + +# Throttles the warmup to 30MB/s across all the warmup threads +blur.shard.index.warmup.throttle=30000000 + +# By default the block cache using off heap memory blur.shard.blockcache.direct.memory.allocation=true + +# The slabs in the blockcache are automatically configured by +# default (-1) otherwise 1 slab equals 128MB. The auto config +# is detected through the MaxDirectoryMemorySize provided to +# the JVM blur.shard.blockcache.slab.count=-1 + +# The number of 1K byte buffers blur.shard.buffercache.1024=8192 + +# The number of 8K byte buffers blur.shard.buffercache.8192=8192 + +# The number of milliseconds to wait for the cluster to settle +# once changes have ceased blur.shard.safemodedelay=5000 + +# The default time between index commits blur.shard.time.between.commits=30000 + +# The default time between index refreshs blur.shard.time.between.refreshs=3000 + +# The maximum number of clauses in a BooleanQuery blur.max.clause.count=1024 + +# The number of thread used for parallel searching in the index manager blur.indexmanager.search.thread.count=8 -blur.query.max.results.fetch=1000 -blur.query.max.row.fetch=100 -blur.query.max.record.fetch=1000 +# The number of threads used for parallel searching in the index searchers +blur.shard.internal.search.thread.count=8 -blur.gui.shard.port=40090 +# Number of threads used for warming up the index +blur.shard.warmup.thread.count=8 -blur.metrics.reporters=</code></pre> +# The fetch count per Lucene search, this fetches pointers to hits +blur.shard.fetchcount=100 + +# Heap limit on row fetch, once this limit has been reached the +# request will return +blur.max.heap.per.row.fetch=10000000 + +# The maximum number of records in a single row fetch +blur.max.records.per.row.fetch.request=1000 + +# The http status page port for the shard server +blur.gui.shard.port=40090 +</code> +</pre> <h3 id="shard-blur-env">blur-env.sh</h3> <pre><code class="bash"># JAVA JVM OPTIONS for the shard servers, jvm tuning parameters are placed here. export BLUR_SHARD_JVM_OPTIONS="-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m " http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/docs/data-model.html ---------------------------------------------------------------------- diff --git a/docs/data-model.html b/docs/data-model.html index 1528ed5..6eeca72 100644 --- a/docs/data-model.html +++ b/docs/data-model.html @@ -58,7 +58,32 @@ <li><a href="#rows">Rows</a></li> </ul> </li> - <li><a href="#querying">Querying</a></li> + <li> + <a href="#querying">Querying</a> + <ul class="nav"> + <li><a href="#string_query">String / Text</a></li> + <li><a href="#numeric_query">Numeric</a></li> + <li><a href="#spatial_query">Spatial</a></li> + </ul> + </li> + <li> + <a href="#types">Types</a> + <ul class="nav"> + <li><a href="#text_type">Text</a></li> + <li><a href="#string_type">String</a></li> + <li><a href="#long_type">Long</a></li> + <li><a href="#int_type">Integer</a></li> + <li><a href="#float_type">Float</a></li> + <li><a href="#double_type">Double</a></li> + <li><a href="#stored_type">Stored</a></li> + <a href="#spatial_type">Spatial</a> + <ul class="nav"> + <li><a href="#spatial_pointvector">Point Vector</a></li> + <li><a href="#spatial_termprefix">Term Prefix</a></li> + <li><a href="#spatial_recursiveprefix">Recursive Prefix</a></li> + </ul> + </ul> + </li> </ul> </div> </div>
