[
https://issues.apache.org/jira/browse/HBASE-20188?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16423279#comment-16423279
]
huaxiang sun commented on HBASE-20188:
--------------------------------------
Hi [~stack], I was looking at tree.txt. The following concerned me, it seems
that getMin() was taking lots of cpu time. 9.1/81.8 = 11 percent of cpu. It
makes me wondering the fix I did for HBASE-12148 is causing trouble for write
performance. Let me prepare a patch for undoing the change in HBASE-12148 and
we can rerun to see if this improves, thanks.
{code:java}
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.RSRpcServices::doNonAtomicRegionMutation(org.apache.hadoop.hbase.regionserver.HRegion,org.apache.hadoop.hbase.quotas.OperationQuota,org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$RegionAction,org.apache.hadoop.hbase.CellScanner,org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$RegionActionResult$Builder,java.util.List,long,org.apache.hadoop.hbase.regionserver.RSRpcServices$RegionScannersCloseCallBack,org.apache.hadoop.hbase.ipc.RpcCallContext,org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.RSRpcServices::doNonAtomicBatchOp(org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$RegionActionResult$Builder,org.apache.hadoop.hbase.regionserver.HRegion,org.apache.hadoop.hbase.quotas.OperationQuota,java.util.List,org.apache.hadoop.hbase.CellScanner,org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.RSRpcServices::doBatchOp(org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$RegionActionResult$Builder,org.apache.hadoop.hbase.regionserver.HRegion,org.apache.hadoop.hbase.quotas.OperationQuota,java.util.List,org.apache.hadoop.hbase.CellScanner,org.apache.hadoop.hbase.quotas.ActivePolicyEnforcement,boolean)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion::batchMutate(org.apache.hadoop.hbase.client.Mutation[],boolean,long,long)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion::batchMutate(org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion::doMiniBatchMutate(org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation::writeMiniBatchOperationsToMemStore(org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress,org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl$WriteEntry)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation::writeMiniBatchOperationsToMemStore(org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress,long)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation::visitBatchOperations(boolean,int,org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation$Visitor)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation$$Lambda$247.826973093::visit(int)
(t 81.8,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation::lambda$writeMiniBatchOperationsToMemStore$0(long,org.apache.hadoop.hbase.regionserver.MemStoreSizing,int)
(t 81.8,s 9.1)
org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation::applyFamilyMapToMemStore(java.util.Map,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 72.7,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion::access$600(org.apache.hadoop.hbase.regionserver.HRegion,org.apache.hadoop.hbase.regionserver.HStore,java.util.List,boolean,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 72.7,s 0.0)
org.apache.hadoop.hbase.regionserver.HRegion::applyToMemStore(org.apache.hadoop.hbase.regionserver.HStore,java.util.List,boolean,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 72.7,s 0.0)
org.apache.hadoop.hbase.regionserver.HStore::add(java.lang.Iterable,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 72.7,s 0.0)
org.apache.hadoop.hbase.regionserver.AbstractMemStore::add(java.lang.Iterable,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 72.7,s 0.0)
org.apache.hadoop.hbase.regionserver.AbstractMemStore::add(org.apache.hadoop.hbase.Cell,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 72.7,s 0.0)
org.apache.hadoop.hbase.regionserver.AbstractMemStore::internalAdd(org.apache.hadoop.hbase.Cell,boolean,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 72.7,s 0.0)
org.apache.hadoop.hbase.regionserver.MutableSegment::add(org.apache.hadoop.hbase.Cell,boolean,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 72.7,s 0.0)
org.apache.hadoop.hbase.regionserver.Segment::internalAdd(org.apache.hadoop.hbase.Cell,boolean,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 63.6,s 0.0)
org.apache.hadoop.hbase.regionserver.CellSet::add(org.apache.hadoop.hbase.Cell)
(t 63.6,s 0.0)
java.util.concurrent.ConcurrentSkipListMap::put(java.lang.Object,java.lang.Object)
(t 63.6,s 0.0)
java.util.concurrent.ConcurrentSkipListMap::doPut(java.lang.Object,java.lang.Object,boolean)
(t 36.4,s 0.0)
java.util.concurrent.ConcurrentSkipListMap::findPredecessor(java.lang.Object,java.util.Comparator)
(t 36.4,s 0.0)
java.util.concurrent.ConcurrentSkipListMap::cpr(java.util.Comparator,java.lang.Object,java.lang.Object)
(t 36.4,s 0.0)
org.apache.hadoop.hbase.CellComparatorImpl::compare(java.lang.Object,java.lang.Object)
(t 36.4,s 0.0)
org.apache.hadoop.hbase.CellComparatorImpl::compare(org.apache.hadoop.hbase.Cell,org.apache.hadoop.hbase.Cell)
(t 36.4,s 0.0)
org.apache.hadoop.hbase.CellComparatorImpl::compare(org.apache.hadoop.hbase.Cell,org.apache.hadoop.hbase.Cell,boolean)
(t 36.4,s 36.4)
org.apache.hadoop.hbase.CellComparatorImpl::compareRows(org.apache.hadoop.hbase.Cell,org.apache.hadoop.hbase.Cell)
(t 27.3,s 0.0)
java.util.concurrent.ConcurrentSkipListMap::cpr(java.util.Comparator,java.lang.Object,java.lang.Object)
(t 27.3,s 0.0)
org.apache.hadoop.hbase.CellComparatorImpl::compare(java.lang.Object,java.lang.Object)
(t 27.3,s 0.0)
org.apache.hadoop.hbase.CellComparatorImpl::compare(org.apache.hadoop.hbase.Cell,org.apache.hadoop.hbase.Cell)
(t 27.3,s 0.0)
org.apache.hadoop.hbase.CellComparatorImpl::compare(org.apache.hadoop.hbase.Cell,org.apache.hadoop.hbase.Cell,boolean)
(t 27.3,s 0.0)
org.apache.hadoop.hbase.CellComparatorImpl::compareRows(org.apache.hadoop.hbase.Cell,org.apache.hadoop.hbase.Cell)
(t 18.2,s 18.2)
org.apache.hadoop.hbase.util.ByteBufferUtils::compareTo(java.nio.ByteBuffer,int,int,java.nio.ByteBuffer,int,int)
(t 9.1,s 0.0)
org.apache.hadoop.hbase.ByteBufferKeyValue::getRowLength()
(t 9.1,s 0.0)
org.apache.hadoop.hbase.ByteBufferKeyValue::getRowLen()
(t 9.1,s 0.0)
org.apache.hadoop.hbase.util.ByteBufferUtils::toShort(java.nio.ByteBuffer,int)
(t 9.1,s 0.0)
org.apache.hadoop.hbase.util.UnsafeAccess::toShort(java.nio.ByteBuffer,int)
(t 9.1,s 9.1)
org.apache.hadoop.hbase.util.UnsafeAccess::getAsShort(java.nio.ByteBuffer,int)
(t 9.1,s 0.0)
org.apache.hadoop.hbase.regionserver.Segment::updateMetaInfo(org.apache.hadoop.hbase.Cell,boolean,boolean,org.apache.hadoop.hbase.regionserver.MemStoreSizing)
(t 9.1,s 0.0)
org.apache.hadoop.hbase.regionserver.TimeRangeTracker::includeTimestamp(org.apache.hadoop.hbase.Cell)
(t 9.1,s 0.0)
org.apache.hadoop.hbase.regionserver.TimeRangeTracker::includeTimestamp(long)
(t 9.1,s 9.1)
org.apache.hadoop.hbase.regionserver.TimeRangeTracker$SyncTimeRangeTracker::getMin(){code}
> [TESTING] Performance
> ---------------------
>
> Key: HBASE-20188
> URL: https://issues.apache.org/jira/browse/HBASE-20188
> Project: HBase
> Issue Type: Umbrella
> Components: Performance
> Reporter: stack
> Assignee: stack
> Priority: Blocker
> Fix For: 2.0.0
>
> Attachments: CAM-CONFIG-V01.patch, ITBLL2.5B_1.2.7vs2.0.0_cpu.png,
> ITBLL2.5B_1.2.7vs2.0.0_gctime.png, ITBLL2.5B_1.2.7vs2.0.0_iops.png,
> ITBLL2.5B_1.2.7vs2.0.0_load.png, ITBLL2.5B_1.2.7vs2.0.0_memheap.png,
> ITBLL2.5B_1.2.7vs2.0.0_memstore.png, ITBLL2.5B_1.2.7vs2.0.0_ops.png,
> ITBLL2.5B_1.2.7vs2.0.0_ops_NOT_summing_regions.png, YCSB_CPU.png,
> YCSB_GC_TIME.png, YCSB_IN_MEMORY_COMPACTION=NONE.ops.png, YCSB_MEMSTORE.png,
> YCSB_OPs.png, YCSB_in-memory-compaction=NONE.ops.png, YCSB_load.png,
> flamegraph-1072.1.svg, flamegraph-1072.2.svg, tree.txt
>
>
> How does 2.0.0 compare to old versions? Is it faster, slower? There is rumor
> that it is much slower, that the problem is the asyncwal writing. Does
> in-memory compaction slow us down or speed us up? What happens when you
> enable offheaping?
> Keep notes here in this umbrella issue. Need to be able to say something
> about perf when 2.0.0 ships.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
