I was running some similar tests and came across a surprising finding. I
compared reads and write on ConcurrentSkipListMap ( which the memstore uses)
and synchronized TreeMap ( Which was literally treemap synchronized).
Executed concurrent reads, writes and deletes on both of them.
Surprisingly synchronized treeMap performed better, though just slightly
better, than ConcurrentSkipListMap which KeyValueSkipListSet uses.
Here are the output of a few runs
Sometimes the difference was considerable
Using HBaseMap it took 20438ms
Using TreeMap it took 11613ms
Time Difference:8825ms
And sometimes the difference was negligible
Using HBaseMap it took 13370ms
Using TreeMap it took 9482ms
Time Difference:3888ms
I've attaching the test java file which I wrote to test it.
This might be a very minor differece but still surprising considering the
fact that ConcurrentSkipListMap uses fancy 2 level indexes which they say
improves the deletion performance.
And here are the details about the test run.
100 Threads each fetching 1,000,000 records
100 threads each adding 1,000,000 records.
100 threads each deletin 1,000,000 records
( Reads, Writes and deletes simultaneously )
Cheers,
Akash A
On Sun, Oct 23, 2011 at 3:25 AM, Stack <st...@duboce.net> wrote:
> On Sat, Oct 22, 2011 at 2:41 PM, N Keywal <nkey...@gmail.com> wrote:
> > I would think that the bottleneck for insert is the wal part?
> > It would be possible to do a kind of memory list preparation during the
> wal
> > insertion, and if the wal insertion is confirmed, do the insertion in the
> > memory list. But it's strange to have the insertion in memory important
> vs.
> > the insertion on disk...
> >
>
> Yes, WAL is the long pole writing. But MemStore has issues too;
> Dhruba says cpu above. Reading and writing it is also 'slow'.
> St.Ack
>
package org.apache.hadoop.hbase.regionserver;
import java.util.Collections;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.util.Bytes;
public class TestKV {
static final int NUM_RUNS = 1;
static final int NUM_THREADS = 100;
static final int NUM_RECORDS = 1000000;
static final int NUM_ITERATIONS = NUM_RECORDS/NUM_THREADS;
boolean useHBaseMap;
ConcurrentSkipListMap<KeyValue, KeyValue> hbaseMap =
new ConcurrentSkipListMap<KeyValue, KeyValue>(KeyValue.COMPARATOR);
SortedMap<KeyValue, KeyValue> myMap = Collections.synchronizedSortedMap((
SortedMap<KeyValue, KeyValue>)new TreeMap<KeyValue, KeyValue>(
KeyValue.COMPARATOR));
AtomicLong totalTime = new AtomicLong();
TestKV(boolean useHBaseMap){
this.useHBaseMap = useHBaseMap;
}
public static void main(String[] args) throws InterruptedException {
if(NUM_RUNS > 0) {
long timeUsingHBaseMap = runMemStoreTest(true);
System.out.println("Using HBaseMap it took "+timeUsingHBaseMap+"ms");
long timeUsingMyMap = runMemStoreTest(false);
System.out.println("Using TreeMap it took "+timeUsingMyMap+"ms");
System.out.println("Time Difference:"+ (timeUsingHBaseMap - timeUsingMyMap) + "ms");
}
}
public static long runMemStoreTest(boolean useHBaseMap) throws InterruptedException {
int totalTime=0;
for(int j=0;j<NUM_RUNS;j++) {
TestKV kv = new TestKV(useHBaseMap);
KeyValueWriter writer = new KeyValueWriter(kv);
KeyValueReader reader = new KeyValueReader(kv);
KeyValueDeleter deleter = new KeyValueDeleter(kv);
long start = System.currentTimeMillis();
ExecutorService e= Executors.newFixedThreadPool(NUM_THREADS*3);
for(int i=0;i<NUM_THREADS;i++) {
e.submit(writer);
e.submit(reader);
e.submit(deleter);
}
e.shutdown();
e.awaitTermination(10, TimeUnit.MINUTES);
long end = System.currentTimeMillis()-start;
totalTime+=end;
}
return totalTime/NUM_RUNS;
}
public static class KeyValueReader implements Runnable {
ConcurrentSkipListMap<KeyValue,KeyValue> hbaseMap;
SortedMap<KeyValue, KeyValue> myMap;
AtomicLong totalTime = new AtomicLong();
boolean useHBaseMap;
KeyValueReader(TestKV kv) {
hbaseMap = kv.hbaseMap;
myMap = kv.myMap;
totalTime = kv.totalTime;
useHBaseMap = kv.useHBaseMap;
}
@Override
public void run() {
for(long i=0;i<NUM_ITERATIONS;i++) {
byte[] random = Bytes.toBytes(Math.random());
KeyValue keyValue = new KeyValue(random, random, random, random);
if(useHBaseMap)
hbaseMap.get(keyValue);
else
myMap.get(keyValue);
}
}
}
public static class KeyValueWriter implements Runnable {
ConcurrentSkipListMap<KeyValue, KeyValue> hbaseMap;
SortedMap<KeyValue, KeyValue> myMap;
AtomicLong totalTime = new AtomicLong();
boolean useHBaseMap;
KeyValueWriter(TestKV kv) {
hbaseMap = kv.hbaseMap;
myMap = kv.myMap;
totalTime = kv.totalTime;
useHBaseMap = kv.useHBaseMap;
}
@Override
public void run() {
long start = System.currentTimeMillis();
for(long i=0;i<NUM_ITERATIONS;i++) {
byte[] random = Bytes.toBytes(Math.random());
KeyValue keyValue = new KeyValue(random, random, random, random);
if(useHBaseMap)
hbaseMap.put(keyValue,keyValue);
else {
myMap.put(keyValue,keyValue);
}
}
long timeElapsed=System.currentTimeMillis()-start;
totalTime.addAndGet(timeElapsed);
}
}
public static class KeyValueDeleter implements Runnable {
ConcurrentSkipListMap<KeyValue, KeyValue> hbaseMap;
SortedMap<KeyValue, KeyValue> myMap;
AtomicLong totalTime = new AtomicLong();
boolean useHBaseMap;
KeyValueDeleter(TestKV kv) {
hbaseMap = kv.hbaseMap;
myMap = kv.myMap;
totalTime = kv.totalTime;
useHBaseMap = kv.useHBaseMap;
}
@Override
public void run() {
long start = System.currentTimeMillis();
for(long i=0;i<NUM_ITERATIONS;i++) {
byte[] random = Bytes.toBytes(Math.random());
KeyValue keyValue = new KeyValue(random, random, random, random);
if(useHBaseMap)
hbaseMap.remove(keyValue);
else {
myMap.remove(keyValue);
}
}
long timeElapsed=System.currentTimeMillis()-start;
totalTime.addAndGet(timeElapsed);
}
}
}
