Hi All,
I am trying to use Hbase as a key-value store, where the key is stored as
row-key and the value as a column family.
The desc of the table is as follows:
{NAME => 'cookies', FAMILIES => [{NAME => 'cf', DATA_BLOCK_ENCODING =>
'NONE', BLOOMFILTER => 'ROWCOL', true
REPLICATION_SCOPE => '0', VERSIONS => '1', COMPRESSION => 'GZ',
MIN_VERSIONS => '0', TTL => '214748364
7', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '8192', IN_MEMORY =>
'false', ENCODE_ON_DISK => 'true',
BLOCKCACHE => 'true'}]}
I am accessing the values using rest API.
curl -H "Accept:application/octet-stream"
http://10.4.41.133:10000/cookies/ed36b1d218315_ziv/cf:data
This is bound to return a single result, since it used as key-value pair.
Now the strange part is sometimes my request takes 40 millisecs to answer.
I am firing some 50000 request per second. Is there something I can do to
optimize. Basically I am looking at returning the results in 5 millisecs.
The hbase configuration is as:
1 node--> running hbase master and hadoop-namenode/secondary node and hbase
rest client.
+
2 nodes--> running data-nodes and regionservers.
The 2 nodes that are running region-servers/datanodes are having 4 ssd's
each and the data is spread across all the four ssd's.
Each node has a 12 GB RAM.
Also I find this very strange, there is negligible disk I/O, CPU usages on
datanodes.
I have tried various combinations, but none of them have given any results.
I am attaching the hbase-site.xml. If required I will also attach the
hadoop configuration files also.
-Vibhav
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://10.4.41.133:54310/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>10.4.41.133</value>
</property>
<property>
<name>hbase.regionserver.codecs</name>
<value>gz</value>
</property>
<property>
<name>hbase.client.scanner.caching</name>
<value>1</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>3200</value>
</property>
<property>
<name>hbase.hregion.max.filesize</name>
<value>107374182400</value>
</property>
<property>
<name>hbase.hregion.memstore.mslab.enabled</name>
<value>true</value>
</property>
<property>
<name>hfile.min.blocksize.size</name>
<value>16384</value>
</property>
<property>
<name>hbase.hregion.memstore.block.multiplier</name>
<value>4</value>
</property>
<property>
<name>hbase.zookeeper.property.maxClientCnxns</name>
<value>100</value>
</property>
<property>
<name>hbase.ipc.warn.response.time</name>
<value>4000</value>
</property>
<property>
<name>hbase.hregion.memstore.flush.size</name>
<value>536870912</value>
</property>
<property>
<name>hbase.hstore.blockingStoreFiles</name>
<value>200</value>
</property>
<property>
<name>hbase.regionserver.optionallogflushinterval</name>
<value>60000</value>
</property>
<property>
<name>hbase.hregion.majorcompaction</name>
<value>0</value>
</property>
<property>
<name>hbase.hstore.compaction.max</name>
<value>100</value>
</property>
<property>
<name>hbase.hstore.compactionThreshold</name>
<value>100</value>
</property>
<property>
<name>hbase.ipc.client.tcpnodelay</name>
<value>true</value>
</property>
<property>
<name>ipc.server.tcpnodelay</name>
<value>true</value>
</property>
<property>
<name>hbase.regions.slop</name>
<value>0.02</value>
</property>
<property>
<name>hbase.regionserver.global.memstore.upperLimit</name>
<value>0.10</value>
</property>
<property>
<name>hbase.regionserver.global.memstore.lowerLimit</name>
<value>0.05</value>
</property>
</configuration>
