[
https://issues.apache.org/jira/browse/LUCENE-2729?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=12926858#action_12926858
]
Nico Krijnen commented on LUCENE-2729:
--------------------------------------
{code}
jteb:assetIndex jteb$ ls -la
total 41550832
drwxr-xr-x 2 jteb jteb 4862 1 nov 08:52 .
drwxr-xr-x 4 jteb jteb 238 29 okt 14:10 ..
-rw-r--r--@ 1 jteb jteb 21508 1 nov 08:52 .DS_Store
-rw-r--r-- 1 jteb jteb 969134416 18 okt 16:41 _2q.fdt
-rw-r--r-- 1 jteb jteb 36652 18 okt 16:41 _2q.fdx
-rw-r--r-- 1 jteb jteb 276 18 okt 16:41 _2q.fnm
-rw-r--r-- 1 jteb jteb 4685726 18 okt 16:41 _2q.frq
-rw-r--r-- 1 jteb jteb 9166 18 okt 16:41 _2q.nrm
-rw-r--r-- 1 jteb jteb 393230403 18 okt 16:42 _2q.prx
-rw-r--r-- 1 jteb jteb 7447 18 okt 16:42 _2q.tii
-rw-r--r-- 1 jteb jteb 746299 18 okt 16:42 _2q.tis
-rw-r--r-- 1 jteb jteb 8394 18 okt 16:42 _2q.tvd
-rw-r--r-- 1 jteb jteb 599185081 18 okt 16:42 _2q.tvf
-rw-r--r-- 1 jteb jteb 73300 18 okt 16:42 _2q.tvx
-rw-r--r-- 1 jteb jteb 1595882722 18 okt 16:45 _3u.fdt
-rw-r--r-- 1 jteb jteb 63692 18 okt 16:45 _3u.fdx
-rw-r--r-- 1 jteb jteb 330 18 okt 16:45 _3u.fnm
-rw-r--r-- 1 jteb jteb 8001869 18 okt 16:45 _3u.frq
-rw-r--r-- 1 jteb jteb 15926 18 okt 16:45 _3u.nrm
-rw-r--r-- 1 jteb jteb 647374863 18 okt 16:45 _3u.prx
-rw-r--r-- 1 jteb jteb 11319 18 okt 16:45 _3u.tii
-rw-r--r-- 1 jteb jteb 1168399 18 okt 16:45 _3u.tis
-rw-r--r-- 1 jteb jteb 14209 18 okt 16:45 _3u.tvd
-rw-r--r-- 1 jteb jteb 986370136 18 okt 16:46 _3u.tvf
-rw-r--r-- 1 jteb jteb 127380 18 okt 16:46 _3u.tvx
-rw-r--r-- 1 jteb jteb 2691565961 18 okt 16:49 _4c.fdt
-rw-r--r-- 1 jteb jteb 39572 18 okt 16:49 _4c.fdx
-rw-r--r-- 1 jteb jteb 276 18 okt 16:49 _4c.fnm
-rw-r--r-- 1 jteb jteb 18724620 18 okt 16:49 _4c.frq
-rw-r--r-- 1 jteb jteb 9896 18 okt 16:49 _4c.nrm
-rw-r--r-- 1 jteb jteb 590255960 18 okt 16:50 _4c.prx
-rw-r--r-- 1 jteb jteb 141243 18 okt 16:50 _4c.tii
-rw-r--r-- 1 jteb jteb 12185869 18 okt 16:50 _4c.tis
-rw-r--r-- 1 jteb jteb 9894 18 okt 16:50 _4c.tvd
-rw-r--r-- 1 jteb jteb 932649779 18 okt 16:51 _4c.tvf
-rw-r--r-- 1 jteb jteb 79140 18 okt 16:51 _4c.tvx
-rw-r--r-- 1 jteb jteb 2398908136 18 okt 16:52 _4d.fdt
-rw-r--r-- 1 jteb jteb 548 18 okt 16:52 _4d.fdx
-rw-r--r-- 1 jteb jteb 354 18 okt 16:52 _4d.fnm
-rw-r--r-- 1 jteb jteb 24581614 18 okt 16:52 _4d.frq
-rw-r--r-- 1 jteb jteb 140 18 okt 16:52 _4d.nrm
-rw-r--r-- 1 jteb jteb 158243133 18 okt 16:52 _4d.prx
-rw-r--r-- 1 jteb jteb 141948 18 okt 16:52 _4d.tii
-rw-r--r-- 1 jteb jteb 12259425 18 okt 16:52 _4d.tis
-rw-r--r-- 1 jteb jteb 140 18 okt 16:52 _4d.tvd
-rw-r--r-- 1 jteb jteb 303769970 18 okt 16:53 _4d.tvf
-rw-r--r-- 1 jteb jteb 1092 18 okt 16:53 _4d.tvx
-rw-r--r-- 1 jteb jteb 4118409126 29 okt 16:26 _6g.fdt
-rw-r--r-- 1 jteb jteb 1484 29 okt 16:26 _6g.fdx
-rw-r--r-- 1 jteb jteb 384 29 okt 16:17 _6g.fnm
-rw-r--r-- 1 jteb jteb 35294399 29 okt 16:27 _6g.frq
-rw-r--r-- 1 jteb jteb 374 29 okt 16:27 _6g.nrm
-rw-r--r-- 1 jteb jteb 230791431 29 okt 16:27 _6g.prx
-rw-r--r-- 1 jteb jteb 143860 29 okt 16:27 _6g.tii
-rw-r--r-- 1 jteb jteb 12491845 29 okt 16:27 _6g.tis
-rw-r--r-- 1 jteb jteb 295 29 okt 16:28 _6g.tvd
-rw-r--r-- 1 jteb jteb 444939185 29 okt 16:28 _6g.tvf
-rw-r--r-- 1 jteb jteb 2964 29 okt 16:28 _6g.tvx
-rw-r--r-- 1 jteb jteb 2758122671 29 okt 16:31 _6h.fdt
-rw-r--r-- 1 jteb jteb 96388 29 okt 16:31 _6h.fdx
-rw-r--r-- 1 jteb jteb 723 29 okt 16:29 _6h.fnm
-rw-r--r-- 1 jteb jteb 51142700 29 okt 16:31 _6h.frq
-rw-r--r-- 1 jteb jteb 24100 29 okt 16:31 _6h.nrm
-rw-r--r-- 1 jteb jteb 189178767 29 okt 16:31 _6h.prx
-rw-r--r-- 1 jteb jteb 270472 29 okt 16:31 _6h.tii
-rw-r--r-- 1 jteb jteb 21710405 29 okt 16:31 _6h.tis
-rw-r--r-- 1 jteb jteb 23873 29 okt 16:31 _6h.tvd
-rw-r--r-- 1 jteb jteb 394088075 29 okt 16:31 _6h.tvf
-rw-r--r-- 1 jteb jteb 192772 29 okt 16:31 _6h.tvx
-rw-r--r-- 1 jteb jteb 0 29 okt 20:22 _8b.fnm
-rw-r--r-- 1 jteb jteb 0 29 okt 20:26 _8b.tvd
-rw-r--r-- 1 jteb jteb 0 29 okt 20:26 _8b.tvf
-rw-r--r-- 1 jteb jteb 0 29 okt 20:22 _8c.fdt
-rw-r--r-- 1 jteb jteb 0 29 okt 20:22 _8c.fdx
-rw-r--r-- 1 jteb jteb 0 29 okt 20:26 _8c.frq
-rw-r--r-- 1 jteb jteb 0 29 okt 20:24 _8c.tii
-rw-r--r-- 1 jteb jteb 0 29 okt 20:24 _8c.tis
-rw-r--r-- 1 jteb jteb 0 29 okt 20:28 _8c.tvf
-rw-r--r-- 1 jteb jteb 0 29 okt 20:30 _8c.tvx
-rw-r--r-- 1 jteb jteb 0 29 okt 20:24 _8d.fdt
-rw-r--r-- 1 jteb jteb 0 29 okt 20:25 _8d.fdx
-rw-r--r-- 1 jteb jteb 0 29 okt 20:24 _8d.frq
-rw-r--r-- 1 jteb jteb 0 29 okt 20:26 _8d.nrm
-rw-r--r-- 1 jteb jteb 0 29 okt 20:26 _8d.prx
-rw-r--r-- 1 jteb jteb 0 29 okt 20:26 _8d.tii
-rw-r--r-- 1 jteb jteb 0 29 okt 20:26 _8d.tis
-rw-r--r-- 1 jteb jteb 0 29 okt 20:30 _8d.tvd
-rw-r--r-- 1 jteb jteb 0 29 okt 20:22 _8e.fdt
-rw-r--r-- 1 jteb jteb 0 29 okt 20:22 _8e.fdx
-rw-r--r-- 1 jteb jteb 0 29 okt 20:26 _8e.frq
-rw-r--r-- 1 jteb jteb 0 29 okt 20:24 _8e.nrm
-rw-r--r-- 1 jteb jteb 0 29 okt 20:07 _8e.tis
-rw-r--r-- 1 jteb jteb 0 29 okt 21:09 _8e.tvd
-rw-r--r-- 1 jteb jteb 0 29 okt 21:09 _8e.tvf
-rw-r--r-- 1 jteb jteb 0 29 okt 21:04 _8e.tvx
-rw-r--r-- 1 jteb jteb 331348323 29 okt 19:49 _9a.fdt
-rw-r--r-- 1 jteb jteb 97020 29 okt 19:49 _9a.fdx
-rw-r--r-- 1 jteb jteb 1480 29 okt 19:49 _9a.fnm
-rw-r--r-- 1 jteb jteb 38868223 29 okt 19:50 _9a.frq
-rw-r--r-- 1 jteb jteb 24258 29 okt 19:50 _9a.nrm
-rw-r--r-- 1 jteb jteb 88673922 29 okt 19:50 _9a.prx
-rw-r--r-- 1 jteb jteb 147173 29 okt 19:50 _9a.tii
-rw-r--r-- 1 jteb jteb 11155683 29 okt 19:50 _9a.tis
-rw-r--r-- 1 jteb jteb 23837 29 okt 19:50 _9a.tvd
-rw-r--r-- 1 jteb jteb 194739895 29 okt 19:50 _9a.tvf
-rw-r--r-- 1 jteb jteb 194036 29 okt 19:50 _9a.tvx
-rw-r--r-- 1 jteb jteb 27 29 okt 19:54 _9a_1.del
-rw-r--r-- 1 jteb jteb 562391 29 okt 19:59 _9e.fdt
-rw-r--r-- 1 jteb jteb 9868 29 okt 19:59 _9e.fdx
-rw-r--r-- 1 jteb jteb 1271 29 okt 19:59 _9e.fnm
-rw-r--r-- 1 jteb jteb 325115 29 okt 19:59 _9e.frq
-rw-r--r-- 1 jteb jteb 1237 29 okt 19:59 _9e.nrm
-rw-r--r-- 1 jteb jteb 29378 29 okt 19:59 _9e.prx
-rw-r--r-- 1 jteb jteb 3367 29 okt 19:59 _9e.tii
-rw-r--r-- 1 jteb jteb 308103 29 okt 19:59 _9e.tis
-rw-r--r-- 1 jteb jteb 117116 29 okt 20:04 _9f.fdt
-rw-r--r-- 1 jteb jteb 2124 29 okt 20:04 _9f.fdx
-rw-r--r-- 1 jteb jteb 416 29 okt 20:04 _9f.fnm
-rw-r--r-- 1 jteb jteb 67935 29 okt 20:04 _9f.frq
-rw-r--r-- 1 jteb jteb 269 29 okt 20:04 _9f.nrm
-rw-r--r-- 1 jteb jteb 6165 29 okt 20:04 _9f.prx
-rw-r--r-- 1 jteb jteb 903 29 okt 20:04 _9f.tii
-rw-r--r-- 1 jteb jteb 75060 29 okt 20:04 _9f.tis
-rw-r--r-- 1 jteb jteb 120877 29 okt 20:04 _9g.fdt
-rw-r--r-- 1 jteb jteb 2188 29 okt 20:04 _9g.fdx
-rw-r--r-- 1 jteb jteb 416 29 okt 20:04 _9g.fnm
-rw-r--r-- 1 jteb jteb 70021 29 okt 20:04 _9g.frq
-rw-r--r-- 1 jteb jteb 277 29 okt 20:04 _9g.nrm
-rw-r--r-- 1 jteb jteb 6315 29 okt 20:04 _9g.prx
-rw-r--r-- 1 jteb jteb 842 29 okt 20:04 _9g.tii
-rw-r--r-- 1 jteb jteb 72269 29 okt 20:04 _9g.tis
-rw-r--r-- 1 jteb jteb 11704 29 okt 20:04 _9h.fdt
-rw-r--r-- 1 jteb jteb 212 29 okt 20:04 _9h.fdx
-rw-r--r-- 1 jteb jteb 416 29 okt 20:04 _9h.fnm
-rw-r--r-- 1 jteb jteb 6063 29 okt 20:04 _9h.frq
-rw-r--r-- 1 jteb jteb 30 29 okt 20:04 _9h.nrm
-rw-r--r-- 1 jteb jteb 627 29 okt 20:04 _9h.prx
-rw-r--r-- 1 jteb jteb 146 29 okt 20:04 _9h.tii
-rw-r--r-- 1 jteb jteb 9576 29 okt 20:04 _9h.tis
-rw-r--r-- 1 jteb jteb 5 29 okt 20:04 index.directory
-rw-r--r-- 1 jteb jteb 20 29 okt 20:04 segments.gen
-rw-r--r-- 1 jteb jteb 0 29 okt 20:07 segments_44
-rw-r--r-- 1 jteb jteb 2641 29 okt 20:04 segments_4m
{code}
> Index corruption after 'read past EOF' under heavy update load and snapshot
> export
> ----------------------------------------------------------------------------------
>
> Key: LUCENE-2729
> URL: https://issues.apache.org/jira/browse/LUCENE-2729
> Project: Lucene - Java
> Issue Type: Bug
> Components: Index
> Affects Versions: 3.0.1, 3.0.2
> Environment: Happens on both OS X 10.6 and Windows 2008 Server.
> Integrated with zoie (using a zoie snapshot from 2010-08-06:
> zoie-2.0.0-snapshot-20100806.jar).
> Reporter: Nico Krijnen
>
> We have a system running lucene and zoie. We use lucene as a content store
> for a CMS/DAM system. We use the hot-backup feature of zoie to make scheduled
> backups of the index. This works fine for small indexes and when there are
> not a lot of changes to the index when the backup is made.
> On large indexes (about 5 GB to 19 GB), when a backup is made while the index
> is being changed a lot (lots of document additions and/or deletions), we
> almost always get a 'read past EOF' at some point, followed by lots of 'Lock
> obtain timed out'.
> At that point we get lots of 0 kb files in the index, data gets lots, and the
> index is unusable.
> When we stop our server, remove the 0kb files and restart our server, the
> index is operational again, but data has been lost.
> I'm not sure if this is a zoie or a lucene issue, so i'm posting it to both.
> Hopefully someone has some ideas where to look to fix this.
> Some more details...
> Stack trace of the read past EOF and following Lock obtain timed out:
> {code}
> 78307 [proj.zoie.impl.indexing.internal.realtimeindexdataloa...@31ca5085]
> ERROR proj.zoie.impl.indexing.internal.BaseSearchIndex - read past EOF
> java.io.IOException: read past EOF
> at
> org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:154)
> at
> org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:39)
> at
> org.apache.lucene.store.ChecksumIndexInput.readByte(ChecksumIndexInput.java:37)
> at org.apache.lucene.store.IndexInput.readInt(IndexInput.java:69)
> at org.apache.lucene.index.SegmentInfos.read(SegmentInfos.java:245)
> at
> org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:166)
> at
> org.apache.lucene.index.DirectoryReader.doCommit(DirectoryReader.java:725)
> at org.apache.lucene.index.IndexReader.commit(IndexReader.java:987)
> at org.apache.lucene.index.IndexReader.commit(IndexReader.java:973)
> at org.apache.lucene.index.IndexReader.decRef(IndexReader.java:162)
> at org.apache.lucene.index.IndexReader.close(IndexReader.java:1003)
> at
> proj.zoie.impl.indexing.internal.BaseSearchIndex.deleteDocs(BaseSearchIndex.java:203)
> at
> proj.zoie.impl.indexing.internal.BaseSearchIndex.loadFromIndex(BaseSearchIndex.java:223)
> at
> proj.zoie.impl.indexing.internal.LuceneIndexDataLoader.loadFromIndex(LuceneIndexDataLoader.java:153)
> at
> proj.zoie.impl.indexing.internal.DiskLuceneIndexDataLoader.loadFromIndex(DiskLuceneIndexDataLoader.java:134)
> at
> proj.zoie.impl.indexing.internal.RealtimeIndexDataLoader.processBatch(RealtimeIndexDataLoader.java:171)
> at
> proj.zoie.impl.indexing.internal.BatchedIndexDataLoader$LoaderThread.run(BatchedIndexDataLoader.java:373)
> 579336 [proj.zoie.impl.indexing.internal.realtimeindexdataloa...@31ca5085]
> ERROR proj.zoie.impl.indexing.internal.LuceneIndexDataLoader - Problem
> copying segments: Lock obtain timed out:
> org.apache.lucene.store.singleinstancel...@5ad0b895: write.lock
> org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out:
> org.apache.lucene.store.singleinstancel...@5ad0b895: write.lock
> at org.apache.lucene.store.Lock.obtain(Lock.java:84)
> at org.apache.lucene.index.IndexWriter.init(IndexWriter.java:1060)
> at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:957)
> at
> proj.zoie.impl.indexing.internal.DiskSearchIndex.openIndexWriter(DiskSearchIndex.java:176)
> at
> proj.zoie.impl.indexing.internal.BaseSearchIndex.loadFromIndex(BaseSearchIndex.java:228)
> at
> proj.zoie.impl.indexing.internal.LuceneIndexDataLoader.loadFromIndex(LuceneIndexDataLoader.java:153)
> at
> proj.zoie.impl.indexing.internal.DiskLuceneIndexDataLoader.loadFromIndex(DiskLuceneIndexDataLoader.java:134)
> at
> proj.zoie.impl.indexing.internal.RealtimeIndexDataLoader.processBatch(RealtimeIndexDataLoader.java:171)
> at
> proj.zoie.impl.indexing.internal.BatchedIndexDataLoader$LoaderThread.run(BatchedIndexDataLoader.java:373)
> {code}
> We get exactly the same behavour on both OS X and on Windows. On both zoie is
> using a SimpleFSDirectory.
> We also use a SingleInstanceLockFactory (since our process is the only one
> working with the index), but we get the same behaviour with a NativeFSLock.
> The snapshot backup is being made by calling:
> *proj.zoie.impl.indexing.ZoieSystem.exportSnapshot(WritableByteChannel)*
> Same issue in zoie JIRA:
> http://snaprojects.jira.com/browse/ZOIE-51
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
