[
https://issues.apache.org/jira/browse/ASTERIXDB-2306?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16397435#comment-16397435
]
Taewoo Kim commented on ASTERIXDB-2306:
---------------------------------------
This is continuously happening and breaking the index structure.
09:46:39.018 [Executor-1340:1] WARN
org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness - Failure scheduling
replication or destroying merged component
java.lang.IllegalStateException: Page 3263:3 is pinned and file is being
closed. Pincount is: 1 Page is confiscated: false
at
org.apache.hyracks.storage.common.buffercache.BufferCache.invalidateIfFileIdMatch(BufferCache.java:930)
~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.common.buffercache.BufferCache.sweepAndFlush(BufferCache.java:896)
~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.common.buffercache.BufferCache.deleteFile(BufferCache.java:997)
~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.common.buffercache.BufferCache.deleteFile(BufferCache.java:983)
~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex.destroy(OnDiskInvertedIndex.java:170)
~[hyracks-storage-am-lsm-invertedindex-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent.deactivateAndDestroy(AbstractLSMDiskComponent.java:166)
~[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent.deactivateAndDestroy(AbstractLSMWithBloomFilterDiskComponent.java:63)
~[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyDiskComponent.deactivateAndDestroy(AbstractLSMWithBuddyDiskComponent.java:56)
~[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness.doExitComponents(LSMHarness.java:324)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness.exitComponents(LSMHarness.java:411)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness.merge(LSMHarness.java:639)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexAccessor.merge(LSMInvertedIndexAccessor.java:125)
[hyracks-storage-am-lsm-invertedindex-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation.call(MergeOperation.java:45)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation.call(MergeOperation.java:30)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0]
at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
13:40:41.022 [Executor-1908:1] WARN
org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness - Failure scheduling
replication or destroying merged component
java.lang.IllegalStateException: Page 3595:0 is pinned and file is being
closed. Pincount is: 1 Page is confiscated: false
at
org.apache.hyracks.storage.common.buffercache.BufferCache.invalidateIfFileIdMatch(BufferCache.java:930)
~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.common.buffercache.BufferCache.sweepAndFlush(BufferCache.java:904)
~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.common.buffercache.BufferCache.deleteFile(BufferCache.java:997)
~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.common.buffercache.BufferCache.deleteFile(BufferCache.java:983)
~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex.destroy(OnDiskInvertedIndex.java:170)
~[hyracks-storage-am-lsm-invertedindex-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent.deactivateAndDestroy(AbstractLSMDiskComponent.java:166)
~[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent.deactivateAndDestroy(AbstractLSMWithBloomFilterDiskComponent.java:63)
~[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyDiskComponent.deactivateAndDestroy(AbstractLSMWithBuddyDiskComponent.java:56)
~[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness.doExitComponents(LSMHarness.java:324)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness.exitComponents(LSMHarness.java:411)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness.merge(LSMHarness.java:639)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexAccessor.merge(LSMInvertedIndexAccessor.java:125)
[hyracks-storage-am-lsm-invertedindex-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation.call(MergeOperation.java:45)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at
org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation.call(MergeOperation.java:30)
[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[?:1.8.0]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[?:1.8.0]
at java.lang.Thread.run(Thread.java:744) [?:1.8.0]
> Inverted list file is not deleted after an inverted index component merge
> operation
> -----------------------------------------------------------------------------------
>
> Key: ASTERIXDB-2306
> URL: https://issues.apache.org/jira/browse/ASTERIXDB-2306
> Project: Apache AsterixDB
> Issue Type: Bug
> Components: IDX - Indexes, STO - Storage
> Reporter: Taewoo Kim
> Assignee: Taewoo Kim
> Priority: Critical
> Labels: triaged
>
> After the following exception, an inverted list file of an old component was
> not deleted. This was happened during an ingestion of 1 Billion tweets on a
> Cloudberry cluster with five nodes.
> {code:java}
> 23:15:15.269 [Executor-6082:5] WARN
> org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness - Failure
> scheduling replication or destroying merged component
> java.lang.IllegalStateException: Page 20629:2 is pinned and file is being
> closed. Pincount is: 1 Page is confiscated: false
> at
> org.apache.hyracks.storage.common.buffercache.BufferCache.invalidateIfFileIdMatch(BufferCache.java:930)
> ~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.common.buffercache.BufferCache.sweepAndFlush(BufferCache.java:904)
> ~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.common.buffercache.BufferCache.deleteFile(BufferCache.java:997)
> ~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.common.buffercache.BufferCache.deleteFile(BufferCache.java:983)
> ~[hyracks-storage-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex.destroy(OnDiskInvertedIndex.java:170)
> ~[hyracks-storage-am-lsm-invertedindex-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent.deactivateAndDestroy(AbstractLSMDiskComponent.java:166)
> ~[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent.deactivateAndDestroy(AbstractLSMWithBloomFilterDiskComponent.java:63)
> ~[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyDiskComponent.deactivateAndDestroy(AbstractLSMWithBuddyDiskComponent.java:56)
> ~[hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness.doExitComponents(LSMHarness.java:324)
> [hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness.exitComponents(LSMHarness.java:411)
> [hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.common.impls.LSMHarness.merge(LSMHarness.java:639)
> [hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexAccessor.merge(LSMInvertedIndexAccessor.java:125)
> [hyracks-storage-am-lsm-invertedindex-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation.call(MergeOperation.java:45)
> [hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at
> org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation.call(MergeOperation.java:30)
> [hyracks-storage-am-lsm-common-0.3.4-SNAPSHOT.jar:0.3.4-SNAPSHOT]
> at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> [?:1.8.0]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> [?:1.8.0]
> at java.lang.Thread.run(Thread.java:744) [?:1.8.0]{code}
>
> The directory:
> {code:java}
> -rw-r--r-- 1 waans11 waans11 262160 Feb 27 13:13
> 2018-02-27-13-12-52-923_2018-02-27-12-45-21-400_d
> -rw-r--r-- 1 waans11 waans11 0 Feb 27 13:12
> 2018-02-27-13-12-52-923_2018-02-27-12-45-21-400_f
> -rw-r--r-- 1 waans11 waans11 76288560 Feb 27 13:13
> 2018-02-27-13-12-52-923_2018-02-27-12-45-21-400_i
> -rw-r--r-- 1 waans11 waans11 262160 Feb 27 13:15
> 2018-02-27-13-15-21-671_2018-02-27-13-15-21-671_d
> -rw-r--r-- 1 waans11 waans11 0 Feb 27 13:15
> 2018-02-27-13-15-21-671_2018-02-27-13-15-21-671_f
> -rw-r--r-- 1 waans11 waans11 2097280 Feb 27 13:15
> 2018-02-27-13-15-21-671_2018-02-27-13-15-21-671_i
> -rw-r--r-- 1 waans11 waans11 43256400 Feb 27 23:15
> 2018-02-27-23-14-41-965_2018-02-27-12-45-21-400_b
> -rw-r--r-- 1 waans11 waans11 262160 Feb 27 23:15
> 2018-02-27-23-14-41-965_2018-02-27-12-45-21-400_d
> -rw-r--r-- 1 waans11 waans11 0 Feb 27 23:15
> 2018-02-27-23-14-41-965_2018-02-27-12-45-21-400_f
> -rw-r--r-- 1 waans11 waans11 148120400 Feb 27 23:15
> 2018-02-27-23-14-41-965_2018-02-27-12-45-21-400_i{code}
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
