Can this be put back? It currently causes a test failure.
- // LUCENE-6299: check we are in bounds
- if (totalDocs > IndexWriter.getActualMaxDocs()) {
- throw new CorruptIndexException("Too many documents: an index
cannot exceed " + IndexWriter.getActualMaxDocs() + " but readers have
total maxDoc=" + totalDocs, input);
- }
On Tue, Apr 28, 2015 at 7:46 PM, <[email protected]> wrote:
> Author: rjernst
> Date: Tue Apr 28 23:46:20 2015
> New Revision: 1676648
>
> URL: http://svn.apache.org/r1676648
> Log:
> LUCENE-6213: Add useful exception message when commit contains segments from
> legacy codecs (forward port from branch_5x)
>
> Added:
>
> lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.5x-with-4x-segments-cfs.zip
> (with props)
>
> lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.5x-with-4x-segments-nocfs.zip
> (with props)
> Modified:
> lucene/dev/trunk/ (props changed)
> lucene/dev/trunk/lucene/CHANGES.txt
>
> lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
>
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
>
> Modified: lucene/dev/trunk/lucene/CHANGES.txt
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1676648&r1=1676647&r2=1676648&view=diff
> ==============================================================================
> --- lucene/dev/trunk/lucene/CHANGES.txt (original)
> +++ lucene/dev/trunk/lucene/CHANGES.txt Tue Apr 28 23:46:20 2015
> @@ -928,6 +928,9 @@ Other
>
> * LUCENE-5915: Remove Pulsing postings format. (Robert Muir)
>
> +* LUCENE-6213: Add useful exception message when commit contains segments
> from legacy codecs.
> + (Ryan Ernst)
> +
> ======================= Lucene 4.10.4 ======================
>
> Bug fixes
>
> Modified:
> lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1676648&r1=1676647&r2=1676648&view=diff
> ==============================================================================
> ---
> lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
> (original)
> +++
> lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
> Tue Apr 28 23:46:20 2015
> @@ -321,7 +321,9 @@ public class TestBackwardsCompatibility
> "4.10.3-cfs",
> "4.10.3-nocfs",
> "4.10.4-cfs",
> - "4.10.4-nocfs"
> + "4.10.4-nocfs",
> + "5x-with-4x-segments-cfs",
> + "5x-with-4x-segments-nocfs"
> };
>
> final static String[] oldSingleSegmentNames = {
> @@ -845,7 +847,7 @@ public class TestBackwardsCompatibility
> mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
> // TODO: remove randomness
> IndexWriterConfig conf = new IndexWriterConfig(new
> MockAnalyzer(random()))
> - .setMaxBufferedDocs(10).setMergePolicy(mp);
> + .setMaxBufferedDocs(10).setMergePolicy(NoMergePolicy.INSTANCE);
> IndexWriter writer = new IndexWriter(dir, conf);
>
> for(int i=0;i<35;i++) {
> @@ -863,7 +865,7 @@ public class TestBackwardsCompatibility
> mp.setNoCFSRatio(doCFS ? 1.0 : 0.0);
> // TODO: remove randomness
> conf = new IndexWriterConfig(new MockAnalyzer(random()))
> - .setMaxBufferedDocs(10).setMergePolicy(mp);
> + .setMaxBufferedDocs(10).setMergePolicy(NoMergePolicy.INSTANCE);
> writer = new IndexWriter(dir, conf);
> addNoProxDoc(writer);
> writer.close();
>
> Added:
> lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.5x-with-4x-segments-cfs.zip
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.5x-with-4x-segments-cfs.zip?rev=1676648&view=auto
> ==============================================================================
> Binary file - no diff available.
>
> Added:
> lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.5x-with-4x-segments-nocfs.zip
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/unsupported.5x-with-4x-segments-nocfs.zip?rev=1676648&view=auto
> ==============================================================================
> Binary file - no diff available.
>
> Modified:
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
> URL:
> http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1676648&r1=1676647&r2=1676648&view=diff
> ==============================================================================
> ---
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
> (original)
> +++
> lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
> Tue Apr 28 23:46:20 2015
> @@ -37,6 +37,7 @@ import org.apache.lucene.codecs.DocValue
> import org.apache.lucene.codecs.FieldInfosFormat;
> import org.apache.lucene.codecs.LiveDocsFormat;
> import org.apache.lucene.store.ChecksumIndexInput;
> +import org.apache.lucene.store.DataInput;
> import org.apache.lucene.store.DataOutput;
> import org.apache.lucene.store.Directory;
> import org.apache.lucene.store.IOContext;
> @@ -276,7 +277,7 @@ public final class SegmentInfos implemen
> byte id[] = new byte[StringHelper.ID_LENGTH];
> input.readBytes(id, 0, id.length);
> CodecUtil.checkIndexHeaderSuffix(input, Long.toString(generation,
> Character.MAX_RADIX));
> -
> +
> SegmentInfos infos = new SegmentInfos();
> infos.id = id;
> infos.generation = generation;
> @@ -295,10 +296,12 @@ public final class SegmentInfos implemen
> if (hasID == 1) {
> segmentID = new byte[StringHelper.ID_LENGTH];
> input.readBytes(segmentID, 0, segmentID.length);
> + } else if (hasID == 0) {
> + throw new IndexFormatTooOldException(input, "Segment is from
> Lucene 4.x");
> } else {
> throw new CorruptIndexException("invalid hasID byte, got: " +
> hasID, input);
> }
> - Codec codec = Codec.forName(input.readString());
> + Codec codec = readCodec(input);
> SegmentInfo info = codec.segmentInfoFormat().read(directory,
> segName, segmentID, IOContext.READ);
> info.setCodec(codec);
> totalDocs += info.maxDoc();
> @@ -341,15 +344,33 @@ public final class SegmentInfos implemen
>
> CodecUtil.checkFooter(input);
>
> - // LUCENE-6299: check we are in bounds
> - if (totalDocs > IndexWriter.getActualMaxDocs()) {
> - throw new CorruptIndexException("Too many documents: an index cannot
> exceed " + IndexWriter.getActualMaxDocs() + " but readers have total maxDoc="
> + totalDocs, input);
> - }
> -
> return infos;
> }
> }
>
> + private static final List<String> unsupportedCodecs = Arrays.asList(
> + "Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene45",
> "Lucene46", "Lucene49", "Lucene410"
> + );
> +
> + private static Codec readCodec(DataInput input) throws IOException {
> + final String name = input.readString();
> + try {
> + return Codec.forName(name);
> + } catch (IllegalArgumentException e) {
> + // give better error messages if we can, first check if this is a
> legacy codec
> + if (unsupportedCodecs.contains(name)) {
> + IOException newExc = new IndexFormatTooOldException(input, "Codec '"
> + name + "' is too old");
> + newExc.initCause(e);
> + throw newExc;
> + }
> + // or maybe it's an old default codec that moved
> + if (name.startsWith("Lucene")) {
> + throw new IllegalArgumentException("Could not load codec '" + name +
> "'. Did you forget to add lucene-backward-codecs.jar?", e);
> + }
> + throw e;
> + }
> + }
> +
> /** Find the latest commit ({@code segments_N file}) and
> * load all {@link SegmentCommitInfo}s. */
> public static final SegmentInfos readLatestCommit(Directory directory)
> throws IOException {
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
