I am working with kafka streamer , and the write speed is too low, 1k-2k rec
/sec, which further reduces after 12million records dump to 200 rec/sec
below are my logs
Metrics for local node (to disable set 'metricsLogFrequency' to 0)
^-- Node [id=44f7cdbc, uptime=00:21:00.103]
^-- H/N/C [hosts=4, nodes=7, CPUs=64]
^-- CPU [cur=7.93%, avg=8.56%, GC=0.03%]
^-- PageMemory [pages=0]
^-- Heap [used=960MB, free=93.15%, comm=6042MB]
^-- Off-heap [used=0MB, free=100%, comm=10576MB]
^-- sysMemPlc region [used=0MB, free=100%, comm=40MB]
^-- default region [used=0MB, free=100%, comm=256MB]
^-- 500MB_Region region [used=0MB, free=100%, comm=10240MB]
^-- TxLog region [used=0MB, free=100%, comm=40MB]
^-- Outbound messages queue [size=0]
^-- Public thread pool [active=0, idle=0, qSize=0]
^-- System thread pool [active=0, idle=0, qSize=0]
IgniteConfiguration [igniteInstanceName=null, pubPoolSize=16,
svcPoolSize=16, callbackPoolSize=16, stripedPoolSize=16, sysPoolSize=16,
mgmtPoolSize=4, igfsPoolSize=16, dataStreamerPoolSize=16,
utilityCachePoolSize=16, utilityCacheKeepAliveTime=60000, p2pPoolSize=2,
qryPoolSize=16, igniteHome=null, igniteWorkDir=/home/centos/ignite/work,
mbeanSrv=com.sun.jmx.mbeanserver.JmxMBeanServer@6ee52dcd,
nodeId=44f7cdbc-81ae-40fd-8f39-a701b5076369, marsh=BinaryMarshaller [],
marshLocJobs=false, daemon=false, p2pEnabled=true, netTimeout=5000,
sndRetryDelay=1000, sndRetryCnt=3, metricsHistSize=10000,
metricsUpdateFreq=2000, metricsExpTime=9223372036854775807,
discoSpi=TcpDiscoverySpi [addrRslvr=null, sockTimeout=0, ackTimeout=0,
marsh=null, reconCnt=10, reconDelay=2000, maxAckTimeout=600000,
forceSrvMode=false, clientReconnectDisabled=false, internalLsnr=null],
segPlc=STOP, segResolveAttempts=2, waitForSegOnStart=true,
allResolversPassReq=true, segChkFreq=10000, commSpi=TcpCommunicationSpi
[connectGate=null,
connPlc=org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$FirstConnectionPolicy@55fe41ea,
enableForcibleNodeKill=false, enableTroubleshootingLog=false, locAddr=null,
locHost=null, locPort=47100, locPortRange=100, shmemPort=-1, directBuf=true,
directSndBuf=false, idleConnTimeout=600000, connTimeout=5000,
maxConnTimeout=600000, reconCnt=10, sockSndBuf=32768, sockRcvBuf=32768,
msgQueueLimit=0, slowClientQueueLimit=0, nioSrvr=null, shmemSrv=null,
usePairedConnections=false, connectionsPerNode=1, tcpNoDelay=true,
filterReachableAddresses=false, ackSndThreshold=32, unackedMsgsBufSize=0,
sockWriteTimeout=2000, boundTcpPort=-1, boundTcpShmemPort=-1,
selectorsCnt=8, selectorSpins=0, addrRslvr=null,
ctxInitLatch=java.util.concurrent.CountDownLatch@fbd1f6[Count = 1],
stopping=false],
evtSpi=org.apache.ignite.spi.eventstorage.NoopEventStorageSpi@1ce24091,
colSpi=NoopCollisionSpi [], deploySpi=LocalDeploymentSpi [],
indexingSpi=org.apache.ignite.spi.indexing.noop.NoopIndexingSpi@24aed80c,
addrRslvr=null,
encryptionSpi=org.apache.ignite.spi.encryption.noop.NoopEncryptionSpi@3a52dba3,
clientMode=true, rebalanceThreadPoolSize=1, txCfg=TransactionConfiguration
[txSerEnabled=false, dfltIsolation=REPEATABLE_READ,
dfltConcurrency=PESSIMISTIC, dfltTxTimeout=0,
txTimeoutOnPartitionMapExchange=0, pessimisticTxLogSize=0,
pessimisticTxLogLinger=10000, tmLookupClsName=null, txManagerFactory=null,
useJtaSync=false], cacheSanityCheckEnabled=true, discoStartupDelay=60000,
deployMode=SHARED, p2pMissedCacheSize=100, locHost=null,
timeSrvPortBase=31100, timeSrvPortRange=100, failureDetectionTimeout=10000,
sysWorkerBlockedTimeout=null, clientFailureDetectionTimeout=30000,
metricsLogFreq=60000, hadoopCfg=null, connectorCfg=ConnectorConfiguration
[jettyPath=null, host=null, port=11211, noDelay=true, directBuf=false,
sndBufSize=32768, rcvBufSize=32768, idleQryCurTimeout=600000,
idleQryCurCheckFreq=60000, sndQueueLimit=0, selectorCnt=4, idleTimeout=7000,
sslEnabled=false, sslClientAuth=false, sslCtxFactory=null, sslFactory=null,
portRange=100, threadPoolSize=16, msgInterceptor=null], odbcCfg=null,
warmupClos=null, atomicCfg=AtomicConfiguration [seqReserveSize=1000,
cacheMode=PARTITIONED, backups=1, aff=null, grpName=null], classLdr=null,
sslCtxFactory=null, platformCfg=null, binaryCfg=null, memCfg=null,
pstCfg=null, dsCfg=DataStorageConfiguration [sysRegionInitSize=41943040,
sysRegionMaxSize=104857600, pageSize=0, concLvl=0,
dfltDataRegConf=DataRegionConfiguration [name=default, maxSize=13239418880,
initSize=268435456, swapPath=null, pageEvictionMode=DISABLED,
evictionThreshold=0.9, emptyPagesPoolSize=100, metricsEnabled=false,
metricsSubIntervalCount=5, metricsRateTimeInterval=60000,
persistenceEnabled=false, checkpointPageBufSize=0],
dataRegions=[DataRegionConfiguration [name=500MB_Region,
maxSize=53687091200, initSize=10737418240, swapPath=null,
pageEvictionMode=DISABLED, evictionThreshold=0.9, emptyPagesPoolSize=100,
metricsEnabled=false, metricsSubIntervalCount=5,
metricsRateTimeInterval=60000, persistenceEnabled=false,
checkpointPageBufSize=0]], storagePath=null, checkpointFreq=180000,
lockWaitTime=10000, checkpointThreads=4, checkpointWriteOrder=SEQUENTIAL,
walHistSize=20, maxWalArchiveSize=1073741824, walSegments=10,
walSegmentSize=67108864, walPath=db/wal, walArchivePath=db/wal/archive,
metricsEnabled=false, walMode=LOG_ONLY, walTlbSize=131072, walBuffSize=0,
walFlushFreq=2000, walFsyncDelay=1000, walRecordIterBuffSize=67108864,
alwaysWriteFullPages=false,
fileIOFactory=org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory@65e98b1c,
metricsSubIntervalCnt=5, metricsRateTimeInterval=60000,
walAutoArchiveAfterInactivity=-1, writeThrottlingEnabled=false,
walCompactionEnabled=false, walCompactionLevel=1,
checkpointReadLockTimeout=null], activeOnStart=true, autoActivation=true,
longQryWarnTimeout=3000, sqlConnCfg=null,
cliConnCfg=ClientConnectorConfiguration [host=null, port=10800,
portRange=100, sockSndBufSize=0, sockRcvBufSize=0, tcpNoDelay=true,
maxOpenCursorsPerConn=128, threadPoolSize=16, idleTimeout=0,
jdbcEnabled=true, odbcEnabled=true, thinCliEnabled=true, sslEnabled=false,
useIgniteSslCtxFactory=true, sslClientAuth=false, sslCtxFactory=null],
mvccVacuumThreadCnt=2, mvccVacuumFreq=5000, authEnabled=false,
failureHnd=null, commFailureRslvr=null]
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
