[ 
https://issues.apache.org/jira/browse/IGNITE-13093?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Tomasz Grygo updated IGNITE-13093:
----------------------------------
     Attachment: ignite.xml
                 thread_dump.txt
    Environment: 
Java 1.8.0_231
Apache Ignite 2.8.1
Windows 10, 64G memory


Java settings

-Xms1024m -Xmx50g -Xss1024m
-Xverify:none
-server
-DIGNITE_QUIET=true
-XX:+UseG1GC
-XX:+DisableExplicitGC
-Djava.net.preferIPv4Stack=true
-XX:+AlwaysPreTouch
-XX:+ScavengeBeforeFullGC
-XX:+AggressiveOpts


  was:
Java 1.8.0_231
Apache Ignite 2.8.1
Windows 10, 64G memory

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans";
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
       xsi:schemaLocation="
        http://www.springframework.org/schema/beans
        http://www.springframework.org/schema/beans/spring-beans.xsd";>

    <bean class="org.apache.ignite.configuration.IgniteConfiguration">
        <property name="gridLogger">
            <bean class="org.apache.ignite.logger.log4j2.Log4J2Logger">
                <constructor-arg type="java.lang.String" value="log4j2.xml"/>
            </bean>
        </property>

        <property name="communicationSpi">
            <bean 
class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
              <!-- Override local port. -->
              <property name="localPort" value="47400"/>
            </bean>
        </property>

        <!-- Configure internal thread pool. 64-->
        <property name="publicThreadPoolSize" value="16"/>

        <!-- Configure system thread pool. 32-->
        <property name="systemThreadPoolSize" value="8"/>

        <property name="systemWorkerBlockedTimeout" value="#{5 * 60 * 1000}"/>

        <property name="failureHandler">
            <bean class="org.apache.ignite.failure.StopNodeFailureHandler">

                <!-- uncomment to enable this handler to 
                process critical workers' hung-ups -->
                <property name="ignoredFailureTypes">
                    <list>
                    </list>
                </property>

            </bean>
        </property>
        <!-- Set to true to enable distributed class loading for examples, 
default is false. -->
        <property name="peerClassLoadingEnabled" value="false"/>

        <property name="dataStorageConfiguration">
            <bean 
class="org.apache.ignite.configuration.DataStorageConfiguration">

                <!--
                     Sets a path to the root directory where data and indexes 
are
                     to be persisted. It's assumed the directory is on a 
separated SSD.
                -->
                <property name="storagePath" value="persistence"/>

                <!--
                     Sets a path to the directory where WAL is stored.
                     It's assumed the directory is on a separated HDD.
                -->
                <property name="walPath" value="wal"/>

                <!--
                    Sets a path to the directory where WAL archive is stored.
                    The directory is on the same HDD as the WAL.
                -->
                <property name="walArchivePath" value="wal/archive"/>

                <!-- Changing WAL Mode. --> 
                <property name="walMode" value="NONE"/>

                <!-- Set the page size to 4 KB, default -->
                <!-- limit 1 KB - 16 KB -->
                <property name="pageSize" value="#{4 * 1024}"/>

                <!-- Enable write throttling. -->
                <property name="writeThrottlingEnabled" value="false"/>

                <property name="checkpointFrequency" value="500"/>
                <property name="lockWaitTime" value="2000"/>
                <property name="checkpointThreads" value="1"/>
                <property name="checkpointWriteOrder" value="RANDOM"/>

                <!--
                    Default memory region that grows endlessly. A cache is 
bound to this memory region
                    unless it sets another one in its CacheConfiguration.
                -->
                <property name="defaultDataRegionConfiguration">
                    <bean 
class="org.apache.ignite.configuration.DataRegionConfiguration">
                        <!--property name="name" value="Default_Region"/-->
                        <property name="name" value="default"/>
                        <!-- 100 MB memory region with disabled eviction -->
                        <property name="initialSize" value="#{100L * 1024 * 
1024}"/>
                        <!-- maxSize 20 MB is too little -->
                        <!-- sum of all maxSize values has to be less than 
total memory of the system -->
                        <!-- limits size in memory, not on disk -->
                        <!-- default value 1.2GB -->
                        <property name="maxSize"     value="#{2L * 1024 * 1024 
* 1024}"/>
                        <property name="persistenceEnabled" value="true"/>
                        <!-- Increasing the buffer size to 1 GB. -->
                        <property name="checkpointPageBufferSize" value="#{1L * 
1024 * 1024 * 1024}"/>
                    </bean>
                </property>

                <!-- Defining several data regions for different memory regions 
-->
                <property name="dataRegionConfigurations">
                    <list>
                    </list>
                </property>
            </bean>
        </property>

        <property name="cacheConfiguration">
            <list>
                <!--bean 
class="org.apache.ignite.configuration.CacheConfiguration">
                    <property name="dataRegionName" value="default"/>
                    <property name="name" value=".ShardDetectorStorage"/>
                    <property name="onheapCacheEnabled" value="true"/>
                </bean>
                <bean 
class="org.apache.ignite.configuration.CacheConfiguration">
                    <property name="dataRegionName" value="default"/>
                    <property name="name" value=".ChildrenStore"/>
                    <property name="onheapCacheEnabled" value="true"/>
                </bean>
                <bean 
class="org.apache.ignite.configuration.CacheConfiguration">
                    <property name="dataRegionName" value="default"/>
                    <property name="name" value=".ChildrenStore.listsize"/>
                    <property name="onheapCacheEnabled" value="true"/>
                </bean>
                <bean 
class="org.apache.ignite.configuration.CacheConfiguration">
                    <property name="dataRegionName" value="default"/>
                    <property name="name" value=".RootStorage"/>
                    <property name="onheapCacheEnabled" value="true"/>
                </bean>
                <bean 
class="org.apache.ignite.configuration.CacheConfiguration">
                    <property name="dataRegionName" value="default"/>
                    <property name="name" value=".QualifierStorage"/>
                    <property name="onheapCacheEnabled" value="true"/>
                </bean-->
            </list>
        </property>
    </bean>

</beans>

Java settings

-Xms1024m -Xmx50g -Xss1024m
-Xverify:none
-server
-DIGNITE_QUIET=true
-XX:+UseG1GC
-XX:+DisableExplicitGC
-Djava.net.preferIPv4Stack=true
-XX:+AlwaysPreTouch
-XX:+ScavengeBeforeFullGC
-XX:+AggressiveOpts


partial thread dump during slowdown

"db-checkpoint-thread-#54" #99 prio=5 os_prio=0 tid=0x0000000070344800 
nid=0x2d54 runnable [0x0000001c5df3e000]
   java.lang.Thread.State: RUNNABLE
        at 
org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl$Segment.removePageForReplacement(PageMemoryImpl.java:2398)
        at 
org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl$Segment.access$900(PageMemoryImpl.java:2093)
        at 
org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:773)
        at 
org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:701)
        at 
org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.saveStoreMetadata(GridCacheOffheapManager.java:342)
        at 
org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.syncMetadata(GridCacheOffheapManager.java:268)
        at 
org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.syncMetadata(GridCacheOffheapManager.java:254)
        at 
org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager.beforeCheckpointBegin(GridCacheOffheapManager.java:226)
        at 
org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$Checkpointer.markCheckpointBegin(GridCacheDatabaseSharedManager.java:4125)
        at 
org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$Checkpointer.doCheckpoint(GridCacheDatabaseSharedManager.java:3738)
        at 
org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$Checkpointer.body(GridCacheDatabaseSharedManager.java:3623)
        at 
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
        at java.lang.Thread.run(Thread.java:748)

   Locked ownable synchronizers:
        - <0x000000009b71a250> (a 
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)




> Unidentified Apache Ignite worker blocked when inserting large amount of 
> records to the persistent storage
> ----------------------------------------------------------------------------------------------------------
>
>                 Key: IGNITE-13093
>                 URL: https://issues.apache.org/jira/browse/IGNITE-13093
>             Project: Ignite
>          Issue Type: Bug
>          Components: cache
>    Affects Versions: 2.8.1
>         Environment: Java 1.8.0_231
> Apache Ignite 2.8.1
> Windows 10, 64G memory
> Java settings
> -Xms1024m -Xmx50g -Xss1024m
> -Xverify:none
> -server
> -DIGNITE_QUIET=true
> -XX:+UseG1GC
> -XX:+DisableExplicitGC
> -Djava.net.preferIPv4Stack=true
> -XX:+AlwaysPreTouch
> -XX:+ScavengeBeforeFullGC
> -XX:+AggressiveOpts
>            Reporter: Tomasz Grygo
>            Priority: Blocker
>         Attachments: ignite.xml, thread_dump.txt
>
>
> I'm looking at Apache Ignite to use as a fast database. Performance is very 
> important, I need to build it as fast as possible with resources available. 
> First I copy all (450M) records from my original test database to Ignite 
> caches through IgniteDataStreams using PK as a key. Database does not fit in 
> memory so I have disk persistence enabled and eviction disabled. Data is 
> inserted in parallel using 8 threads. I have only one but fairly powerful 
> Windows PC doing all the work, no separate Ignite cluster. I'm not interested 
> in cache recovery so WAL is disabled. Everything goes well until I hit around 
> 310 million entries (2 hours of work). At this point Ignite starts to choke, 
> inserts slow down and then stop with exceptions. Exception is triggered by 
> systemWorkerBlockedTimeout setting set to 5 minutes. Extending this time does 
> not help at all. Based on heap dump I tried adding 
> -DIGNITE_PAGES_LIST_DISABLE_ONHEAP_CACHING=true and it failed slightly later 
> but still could not finish the job. I read the performance guides and I tried 
> tweaking other Ignite settings too but didn't see any impact. How can if find 
> which worker is being blocked and why?
> 2020-05-27 21:54:26,176 [Storage2 ] [ERROR] - DTR_0030 worker Storage2 had 
> error: FATAL ERROR java.lang.IllegalStateException: Data streamer has been 
> closed.
> java.lang.IllegalStateException: Data streamer has been closed.
>     at 
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.closedException(DataStreamerImpl.java:1095)
>     at 
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.lock(DataStreamerImpl.java:446)
>     at 
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.addDataInternal(DataStreamerImpl.java:646)
>     at 
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.addDataInternal(DataStreamerImpl.java:631)
>     at 
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.addData(DataStreamerImpl.java:753)
>     at 
> com.sc.extr.cache.PureIgniteDynamicRowStorage.putIfAbsent(PureIgniteDynamicRowStorage.java:83)
>     at 
> com.sc.extr.cache.PureIgniteDynamicRowStorage.addRowOnKey(PureIgniteDynamicRowStorage.java:160)
>     at 
> com.sc.extr.tree.MultiCacheTreeBuilder.addRootRowToCache(MultiCacheTreeBuilder.java:409)
>     at 
> com.sc.extr.tree.MultiCacheTreeBuilder.parentRev1to1(MultiCacheTreeBuilder.java:237)
>     at 
> com.sc.extr.tree.MultiCacheTreeBuilder.addRowToCache(MultiCacheTreeBuilder.java:333)
>     at 
> com.sc.extr.tree.MultiCacheTreeBuilder.parentRev(MultiCacheTreeBuilder.java:274)
>     at 
> com.sc.extr.tree.MultiCacheTreeBuilder.addRow(MultiCacheTreeBuilder.java:379)
>     at 
> com.sc.extr.tree.MultiCacheTreeBuilder.process(MultiCacheTreeBuilder.java:206)
>     at com.sc.bi.workflow.WorkTransformer.processOne(WorkTransformer.java:84)
>     at com.sc.bi.workflow.WorkTransformer.doWork(WorkTransformer.java:145)
>     at 
> com.sc.bi.workflow.WorkTransformer.processQueue(WorkTransformer.java:210)
>     at com.sc.bi.workflow.WorkTransformer.run(WorkTransformer.java:169)
> Caused by: class org.apache.ignite.IgniteCheckedException: Data streamer has 
> been cancelled: DataStreamerImpl [bufLdrSzPerThread=4096, 
> rcvr=org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl$IsolatedUpdater@381b03ed,
>  ioPlcRslvr=null, cacheName=PERSON.PTINTN, bufSize=512, parallelOps=0, 
> timeout=-1, autoFlushFreq=0, bufMappings=ConcurrentHashMap 
> {03e74462-12ec-4140-b9fb-a975572ac3bb=Buffer [node=TcpDiscoveryNode 
> [id=03e74462-12ec-4140-b9fb-a975572ac3bb, 
> consistentId=b01eb38b-7728-4e43-a697-0bc52f872e44, addrs=ArrayList 
> [127.0.0.1, 172.27.179.112], sockAddrs=HashSet 
> [SOFTBI-DEV.sc.com/172.27.179.112:47500, /127.0.0.1:47500], discPort=47500, 
> order=1, intOrder=1, lastExchangeTime=1590614830815, loc=true, 
> ver=2.8.1#20200521-sha1:86422096, isClient=false], isLocNode=true, idGen=0, 
> sem=java.util.concurrent.Semaphore@2a869d9[Permits = 64], 
> perNodeParallelOps=64, entriesCnt=2048, locFutsSize=0, reqsSize=0]}, 
> cacheObjProc=GridProcessorAdapter [], 
> cacheObjCtx=org.apache.ignite.internal.processors.cache.CacheObjectContext@2a5313b0,
>  cancelled=true, cancellationReason=null, failCntr=0, 
> activeFuts=GridConcurrentHashSet [GridFutureAdapter [ignoreInterrupts=false, 
> state=INIT, res=null, hash=2102798044], GridFutureAdapter 
> [ignoreInterrupts=false, state=INIT, res=null, hash=1195632760], 
> GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, 
> hash=370791970], GridFutureAdapter [ignoreInterrupts=false, state=INIT, 
> res=null, hash=420732031], GridFutureAdapter [ignoreInterrupts=false, 
> state=INIT, res=null, hash=1453517070]], jobPda=null, depCls=null, 
> fut=DataStreamerFuture [super=GridFutureAdapter [ignoreInterrupts=false, 
> state=INIT, res=null, hash=1165180540]], publicFut=IgniteFuture 
> [orig=DataStreamerFuture [super=GridFutureAdapter [ignoreInterrupts=false, 
> state=INIT, res=null, hash=1165180540]]], disconnectErr=null, closed=true, 
> lastFlushTime=1590629894701, skipStore=false, keepBinary=false, 
> maxRemapCnt=32, remapSem=java.util.concurrent.Semaphore@6e6f060b[Permits = 
> 2147483647], remapOwning=false]
>     at 
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.closeEx(DataStreamerImpl.java:1347)
>     at 
> org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl.closeEx(DataStreamerImpl.java:1318)
>     at 
> org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.onKernalStop(DataStreamProcessor.java:155)
>     at org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2551)
>     at org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2499)
>     at 
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2650)
>     at 
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2613)
>     at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:339)
>     at 
> org.apache.ignite.failure.StopNodeFailureHandler$1.run(StopNodeFailureHandler.java:36)
>     at java.lang.Thread.run(Thread.java:748)



--
This message was sent by Atlassian Jira
(v8.3.4#803005)

Reply via email to