Revision: 16667
http://sourceforge.net/p/gate/code/16667
Author: valyt
Date: 2013-04-29 16:35:35 +0000 (Mon, 29 Apr 2013)
Log Message:
-----------
It seems the upgrade gods are smiling on us today: mimir-core upgraded to MG4J
5.2.1 (latest). It can create new indexes and read them back. It can also read
pre-existing indexes.
N.B. Because of MG4J deps, M?\195?\173mir 5.0 will require Java 7.
N.B. The other sub-projects have not yet been ported to this new version of
core, which means that the current trunk *does not compile*! Bear with us,
we're working as fast as we can...
Modified Paths:
--------------
mimir/trunk/mimir-core/.classpath
mimir/trunk/mimir-core/build/ivy.xml
mimir/trunk/mimir-core/src/gate/mimir/IndexConfig.java
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/GATEDocument.java
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/GATEDocumentFactory.java
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/MimirDirectIndexBuilder.java
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/MimirIndexBuilder.java
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/RelocatableZipDocumentCollectionHolder.java
mimir/trunk/mimir-core/src/gate/mimir/search/IndexReaderPool.java
mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractIntersectionQueryExecutor.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractOverlapQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractQueryExecutor.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/ConstQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/GapQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/MinusQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/OrQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/QueryExecutor.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/RepeatsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/query/TermQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/score/BindingScorer.java
mimir/trunk/mimir-core/src/gate/mimir/search/score/DelegatingScoringQueryExecutor.java
mimir/trunk/mimir-core/src/gate/mimir/search/score/MimirScorer.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractIndexTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsAndTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsOrTermsQuery.java
mimir/trunk/mimir-core/src/gate/mimir/util/MG4JTools.java
mimir/trunk/mimir-core/src/gate/mimir/util/NormalizingTermProcessor.java
mimir/trunk/mimir-test/src/gate/mimir/test/Scratch.java
mimir/trunk/mimir-test/src/gate/mimir/test/TestUtils.java
Modified: mimir/trunk/mimir-core/.classpath
===================================================================
--- mimir/trunk/mimir-core/.classpath 2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/.classpath 2013-04-29 16:35:35 UTC (rev 16667)
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
- <classpathentry kind="con"
path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry exported="true" kind="con"
path="org.apache.ivyde.eclipse.cpcontainer.IVYDE_CONTAINER/?project=mimir-core&ivyXmlPath=build%2Fivy.xml&confs=*"/>
+ <classpathentry kind="con"
path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
<classpathentry kind="output" path="classes"/>
</classpath>
Modified: mimir/trunk/mimir-core/build/ivy.xml
===================================================================
--- mimir/trunk/mimir-core/build/ivy.xml 2013-04-29 14:16:07 UTC (rev
16666)
+++ mimir/trunk/mimir-core/build/ivy.xml 2013-04-29 16:35:35 UTC (rev
16667)
@@ -15,15 +15,14 @@
<!-- GATE -->
<dependency name="gate-core" org="uk.ac.gate" rev="7.0">
<artifact name="gate-core" type="jar" />
- </dependency>
+ </dependency>
<!-- MG4J -->
- <dependency org="it.unimi.dsi" name="mg4j-big" rev="4.0.4" >
+ <dependency org="it.unimi.di" name="mg4j-big" rev="5.2.1" >
<artifact name="mg4j-big" type="jar" />
</dependency>
- <!-- explicit versions for DSI tools -->
- <dependency org="it.unimi.dsi" name="webgraph" rev="3.0.7" >
- <artifact name="webgraph" type="jar" />
- </dependency>
+ <!-- Force Ivy to use the correct version -->
+ <dependency org="commons-configuration" name="commons-configuration"
rev="1.8"/>
+
<!-- Exclude xmlbeans as it includes forbidden classes
(a copy excluding the offending classes is included in lib/ -->
Modified: mimir/trunk/mimir-core/src/gate/mimir/IndexConfig.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/IndexConfig.java 2013-04-29
14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/IndexConfig.java 2013-04-29
16:35:35 UTC (rev 16667)
@@ -16,8 +16,8 @@
import gate.Gate;
import gate.mimir.index.IndexException;
-import it.unimi.dsi.big.mg4j.index.NullTermProcessor;
-import it.unimi.dsi.big.mg4j.index.TermProcessor;
+import it.unimi.di.big.mg4j.index.NullTermProcessor;
+import it.unimi.di.big.mg4j.index.TermProcessor;
import java.io.BufferedInputStream;
import java.io.File;
@@ -207,7 +207,7 @@
* The current format version for the XML files containing serialisations of
* IndexConfig instances.
*/
- private static final int FORMAT_VERSION = 5;
+ private static final int FORMAT_VERSION = 6;
/**
* The default feature name for obtaining document URIs (provided as features
@@ -396,7 +396,10 @@
xs.alias("indexConfig", IndexConfig.class);
xs.alias("tokenIndexer", TokenIndexerConfig.class);
xs.alias("semanticIndexer", SemanticIndexerConfig.class);
- xs.aliasPackage("it.unimi.dsi.mg4j", "it.unimi.dsi.big.mg4j");
+ // when loading old indexes, add the '.big.'
+ xs.aliasPackage("it.unimi.dsi.mg4j", "it.unimi.di.big.mg4j");
+ // when loading pre-5.0 indexes, replace the package name
+ xs.aliasPackage("it.unimi.dsi.big.mg4j", "it.unimi.di.big.mg4j");
return xs;
}
Modified: mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/GATEDocument.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/GATEDocument.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/GATEDocument.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -20,7 +20,7 @@
import gate.util.OffsetComparator;
import it.unimi.dsi.io.WordReader;
import it.unimi.dsi.lang.MutableString;
-import it.unimi.dsi.big.mg4j.document.Document;
+import it.unimi.di.big.mg4j.document.Document;
import java.io.*;
import java.util.Arrays;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/GATEDocumentFactory.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/GATEDocumentFactory.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/GATEDocumentFactory.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -16,8 +16,8 @@
import gate.mimir.IndexConfig;
import it.unimi.dsi.fastutil.objects.Reference2ObjectMap;
-import it.unimi.dsi.big.mg4j.document.Document;
-import it.unimi.dsi.big.mg4j.document.DocumentFactory;
+import it.unimi.di.big.mg4j.document.Document;
+import it.unimi.di.big.mg4j.document.DocumentFactory;
import java.io.IOException;
import java.io.InputStream;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/MimirDirectIndexBuilder.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/MimirDirectIndexBuilder.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/MimirDirectIndexBuilder.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -21,10 +21,10 @@
import gate.mimir.util.MG4JTools;
import gate.util.GateRuntimeException;
import it.unimi.dsi.Util;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.index.IndexIterator;
-import it.unimi.dsi.big.mg4j.index.IndexReader;
-import it.unimi.dsi.big.mg4j.tool.Scan.Completeness;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.index.IndexIterator;
+import it.unimi.di.big.mg4j.index.IndexReader;
+import it.unimi.di.big.mg4j.tool.Scan.Completeness;
import it.unimi.dsi.logging.ProgressLogger;
import java.io.File;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/MimirIndexBuilder.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/MimirIndexBuilder.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/MimirIndexBuilder.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -36,27 +36,29 @@
import it.unimi.dsi.io.OutputBitStream;
import it.unimi.dsi.lang.MutableString;
import it.unimi.dsi.logging.ProgressLogger;
-import it.unimi.dsi.big.mg4j.index.BitStreamIndex;
-import it.unimi.dsi.big.mg4j.index.BitStreamIndexWriter;
-import it.unimi.dsi.big.mg4j.index.CompressionFlags;
-import it.unimi.dsi.big.mg4j.index.CompressionFlags.Coding;
-import it.unimi.dsi.big.mg4j.index.CompressionFlags.Component;
-import it.unimi.dsi.big.mg4j.index.DiskBasedIndex;
-import it.unimi.dsi.big.mg4j.index.FileIndex;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.index.IndexWriter;
-import it.unimi.dsi.big.mg4j.index.NullTermProcessor;
-import it.unimi.dsi.big.mg4j.index.SkipBitStreamIndexWriter;
-import it.unimi.dsi.big.mg4j.index.TermProcessor;
-import it.unimi.dsi.big.mg4j.index.cluster.ContiguousDocumentalStrategy;
-import it.unimi.dsi.big.mg4j.index.cluster.DocumentalCluster;
-import it.unimi.dsi.big.mg4j.index.cluster.DocumentalConcatenatedCluster;
-import it.unimi.dsi.big.mg4j.index.cluster.IndexCluster;
-import it.unimi.dsi.big.mg4j.io.ByteArrayPostingList;
-import it.unimi.dsi.big.mg4j.tool.Combine;
-import it.unimi.dsi.big.mg4j.tool.Concatenate;
-import it.unimi.dsi.big.mg4j.tool.Scan;
-import it.unimi.dsi.big.mg4j.tool.Scan.Completeness;
+import it.unimi.di.big.mg4j.index.BitStreamIndex;
+import it.unimi.di.big.mg4j.index.BitStreamIndexWriter;
+import it.unimi.di.big.mg4j.index.CompressionFlags;
+import it.unimi.di.big.mg4j.index.CompressionFlags.Coding;
+import it.unimi.di.big.mg4j.index.CompressionFlags.Component;
+import it.unimi.di.big.mg4j.index.DiskBasedIndex;
+import it.unimi.di.big.mg4j.index.FileIndex;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.index.IndexWriter;
+import it.unimi.di.big.mg4j.index.NullTermProcessor;
+import it.unimi.di.big.mg4j.index.SkipBitStreamIndexWriter;
+import it.unimi.di.big.mg4j.index.TermProcessor;
+import it.unimi.di.big.mg4j.index.cluster.ContiguousDocumentalStrategy;
+import it.unimi.di.big.mg4j.index.cluster.DocumentalCluster;
+import it.unimi.di.big.mg4j.index.cluster.DocumentalConcatenatedCluster;
+import it.unimi.di.big.mg4j.index.cluster.IndexCluster;
+import it.unimi.di.big.mg4j.io.ByteArrayPostingList;
+import it.unimi.di.big.mg4j.io.IOFactory;
+import it.unimi.di.big.mg4j.tool.Combine;
+import it.unimi.di.big.mg4j.tool.Combine.IndexType;
+import it.unimi.di.big.mg4j.tool.Concatenate;
+import it.unimi.di.big.mg4j.tool.Scan;
+import it.unimi.di.big.mg4j.tool.Scan.Completeness;
import it.unimi.dsi.sux4j.mph.LcpMonotoneMinimalPerfectHashFunction;
import it.unimi.dsi.util.Properties;
@@ -661,18 +663,23 @@
new OutputBitStream(
getBatchFile(DiskBasedIndex.FREQUENCIES_EXTENSION));
final OutputBitStream globCountsStream = new OutputBitStream(
- getBatchFile(DiskBasedIndex.GLOBCOUNTS_EXTENSION));
+ getBatchFile(DiskBasedIndex.OCCURRENCIES_EXTENSION));
final OutputBitStream indexStream = new OutputBitStream(
getBatchFile(DiskBasedIndex.INDEX_EXTENSION));
final OutputBitStream offsetsStream = new OutputBitStream(
getBatchFile(DiskBasedIndex.OFFSETS_EXTENSION));
+
final OutputBitStream posLengthsStream = savePositions ?
new OutputBitStream(
getBatchFile(DiskBasedIndex.POSITIONS_NUMBER_OF_BITS_EXTENSION)) :
null;
-
+
+ final OutputBitStream sumsMaxPosStream = savePositions ?
+ new OutputBitStream(
+ getBatchFile(DiskBasedIndex.SUMS_MAX_POSITION_EXTENSION)) :
+ null;
ByteArrayPostingList postingsList;
int maxCount = 0;
@@ -702,9 +709,12 @@
} else indexStream.write(postingsList.buffer, bitLength );
frequenciesStream.writeLongGamma( frequency );
- globCountsStream.writeLongGamma( postingsList.globCount );
+ globCountsStream.writeLongGamma(postingsList.occurrency);
offsetsStream.writeLongGamma( indexStream.writtenBits() - prevOffset );
- if(savePositions) posLengthsStream.writeLongGamma(
postingsList.posNumBits );
+ if(savePositions){
+ posLengthsStream.writeLongGamma( postingsList.posNumBits );
+ sumsMaxPosStream.writeLongDelta( postingsList.sumMaxPos );
+ }
prevOffset = indexStream.writtenBits();
}
@@ -736,7 +746,10 @@
indexStream.close();
offsetsStream.close();
globCountsStream.close();
- if(savePositions) posLengthsStream.close();
+ if(savePositions){
+ posLengthsStream.close();
+ sumsMaxPosStream.close();
+ }
frequenciesStream.close();
termMap.clear();
termMap.trim( INITIAL_TERM_MAP_SIZE );
@@ -837,11 +850,12 @@
logger.info( "Generating empty index " + getBatchFile(""));
makeEmpty(getBatchFile(DiskBasedIndex.TERMS_EXTENSION));
makeEmpty(getBatchFile(DiskBasedIndex.FREQUENCIES_EXTENSION));
- makeEmpty(getBatchFile(DiskBasedIndex.GLOBCOUNTS_EXTENSION));
+ makeEmpty(getBatchFile(DiskBasedIndex.OCCURRENCIES_EXTENSION));
makeEmpty(getBatchFile(DiskBasedIndex.SIZES_EXTENSION));
final IndexWriter indexWriter = new BitStreamIndexWriter(
- getBatchFile("").getAbsolutePath(), totalDocuments, true,
flags );
+ IOFactory.FILESYSTEM_FACTORY,
+ getBatchFile("").getAbsolutePath(), totalDocuments, true, flags );
indexWriter.close();
final Properties properties = indexWriter.properties();
properties.setProperty( Index.PropertyKeys.TERMPROCESSOR,
@@ -1135,23 +1149,26 @@
Map<Component,Coding> codingFlags;
if(savePositions) {
- codingFlags = CompressionFlags.DEFAULT_STANDARD_INDEX;
+ codingFlags = CompressionFlags.DEFAULT_QUASI_SUCCINCT_INDEX;
} else {
- codingFlags = new EnumMap<Component,
Coding>(CompressionFlags.DEFAULT_STANDARD_INDEX);
+ codingFlags = new EnumMap<Component,
Coding>(CompressionFlags.DEFAULT_QUASI_SUCCINCT_INDEX);
codingFlags.remove(Component.POSITIONS);
}
- new Concatenate(outputBaseName,
- inputBasenames, false,
- Combine.DEFAULT_BUFFER_SIZE,
- codingFlags,
- false, true,
- // BitStreamIndex.DEFAULT_QUANTUM,
- // replaced with optimised automatic calculation
- -5,
- BitStreamIndex.DEFAULT_HEIGHT,
- SkipBitStreamIndexWriter.DEFAULT_TEMP_BUFFER_SIZE,
- ProgressLogger.DEFAULT_LOG_INTERVAL).run();
+ new Concatenate(
+ IOFactory.FILESYSTEM_FACTORY,
+ outputBaseName,
+ inputBasenames, false,
+ Combine.DEFAULT_BUFFER_SIZE,
+ codingFlags,
+ IndexType.QUASI_SUCCINCT,
+ true,
+ // BitStreamIndex.DEFAULT_QUANTUM,
+ // replaced with optimised automatic calculation
+ -5,
+ BitStreamIndex.DEFAULT_HEIGHT,
+ SkipBitStreamIndexWriter.DEFAULT_TEMP_BUFFER_SIZE,
+ ProgressLogger.DEFAULT_LOG_INTERVAL).run();
//clean up the used batch files
for(String batchBasename : inputBasenames){
cleanBatchFiles(batchBasename);
@@ -1169,16 +1186,19 @@
*/
protected void cleanBatchFiles(String basename){
new File( basename + DiskBasedIndex.FREQUENCIES_EXTENSION ).delete();
- new File( basename + DiskBasedIndex.GLOBCOUNTS_EXTENSION ).delete();
- new File( basename +
DiskBasedIndex.POSITIONS_NUMBER_OF_BITS_EXTENSION).delete();
+ new File( basename + DiskBasedIndex.OCCURRENCIES_EXTENSION ).delete();
new File( basename + DiskBasedIndex.INDEX_EXTENSION ).delete();
new File( basename + DiskBasedIndex.OFFSETS_EXTENSION ).delete();
new File( basename + DiskBasedIndex.SIZES_EXTENSION ).delete();
new File( basename + DiskBasedIndex.STATS_EXTENSION ).delete();
new File( basename + DiskBasedIndex.PROPERTIES_EXTENSION ).delete();
+ new File( basename +
DiskBasedIndex.POSITIONS_NUMBER_OF_BITS_EXTENSION).delete();
new File( basename + DiskBasedIndex.POSITIONS_EXTENSION ).delete();
new File( basename + DiskBasedIndex.TERMS_EXTENSION ).delete();
new File( basename + DiskBasedIndex.UNSORTED_TERMS_EXTENSION ).delete();
+ new File( basename + DiskBasedIndex.SUMS_MAX_POSITION_EXTENSION ).delete();
+
+
}
}
Modified:
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/RelocatableZipDocumentCollectionHolder.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/RelocatableZipDocumentCollectionHolder.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/index/mg4j/RelocatableZipDocumentCollectionHolder.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -18,8 +18,8 @@
import java.io.IOException;
import java.io.Serializable;
-import it.unimi.dsi.big.mg4j.document.DocumentFactory;
-import it.unimi.dsi.big.mg4j.document.ZipDocumentCollection;
+import it.unimi.di.big.mg4j.document.DocumentFactory;
+import it.unimi.di.big.mg4j.document.ZipDocumentCollection;
/**
* Holder class allowing us to store a zipped document collection with
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/IndexReaderPool.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/IndexReaderPool.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/IndexReaderPool.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -15,9 +15,9 @@
package gate.mimir.search;
import it.unimi.dsi.big.io.FileLinesCollection;
-import it.unimi.dsi.big.mg4j.index.DiskBasedIndex;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.index.IndexReader;
+import it.unimi.di.big.mg4j.index.DiskBasedIndex;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.index.IndexReader;
import it.unimi.dsi.big.util.ImmutableExternalPrefixMap;
import it.unimi.dsi.big.util.SemiExternalGammaBigList;
import it.unimi.dsi.fastutil.longs.LongBigList;
@@ -58,6 +58,11 @@
private static final int DEFAULT_CAPACITY = 100000;
/**
+ * Old (i.e. pre MG4J-5) extension used for the global counts file.
+ */
+ public static final String GLOBCOUNTS_EXTENSION = ".globcounts";
+
+ /**
* How many readers are normally kept. The actual size can be much larger
than
* this, as this pool will never refuse to create new readers. However, upon
* being returned, all excess readers get destroyed instead of being
@@ -183,12 +188,11 @@
public LongBigList getTermOccurenceCounts() throws IOException {
if(termOccurrenceCounts == null) {
File countsFile = new File(URI.create(indexURI.toString() +
- DiskBasedIndex.GLOBCOUNTS_EXTENSION));
+ GLOBCOUNTS_EXTENSION));
if(!countsFile.exists()) {
- // when we change to MG4J 5+ indexes, this will have to change to use
- // the new .occurrences extension
- throw new IOException("Could not find occurrences file at " +
- countsFile.getAbsolutePath() + "!");
+ // is this a mg4j-5+ index?
+ countsFile = new File(URI.create(indexURI.toString() +
+ DiskBasedIndex.OCCURRENCIES_EXTENSION));
}
termOccurrenceCounts = new SemiExternalGammaBigList(
new InputBitStream(countsFile), -1, getDictionary().size64());
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/QueryEngine.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -39,7 +39,7 @@
import it.unimi.dsi.fastutil.ints.IntBigList;
import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
-import it.unimi.dsi.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.index.Index;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractIntersectionQueryExecutor.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractIntersectionQueryExecutor.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractIntersectionQueryExecutor.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -19,8 +19,8 @@
import it.unimi.dsi.fastutil.objects.ReferenceArraySet;
import it.unimi.dsi.fastutil.objects.ReferenceSet;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor;
import java.io.IOException;
import java.util.Arrays;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractOverlapQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractOverlapQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractOverlapQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -19,8 +19,8 @@
import it.unimi.dsi.fastutil.objects.ReferenceArraySet;
import it.unimi.dsi.fastutil.objects.ReferenceSet;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor;
import java.io.IOException;
import java.util.*;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractQueryExecutor.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractQueryExecutor.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/query/AbstractQueryExecutor.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -17,9 +17,9 @@
import gate.mimir.search.QueryEngine;
import it.unimi.dsi.fastutil.objects.Reference2ReferenceMap;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.DocumentIterator;
-import it.unimi.dsi.big.mg4j.search.IntervalIterator;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.DocumentIterator;
+import it.unimi.di.big.mg4j.search.IntervalIterator;
import java.io.IOException;
import java.util.NoSuchElementException;
@@ -143,11 +143,6 @@
}
@Override
- public IntervalIterator iterator() {
- throw new UnsupportedOperationException("This method is not implemented.");
- }
-
- @Override
public boolean mayHaveNext() {
throw new UnsupportedOperationException("This method is not implemented.");
}
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/AnnotationQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -21,8 +21,8 @@
import gate.mimir.search.QueryEngine;
import gate.mimir.search.terms.AnnotationTermsQuery;
import gate.mimir.search.terms.TermsResultSet;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor;
import it.unimi.dsi.fastutil.objects.ReferenceSet;
import it.unimi.dsi.fastutil.objects.ReferenceSets;
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/query/ConstQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/ConstQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/ConstQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -16,8 +16,8 @@
import gate.mimir.search.QueryEngine;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor;
import it.unimi.dsi.fastutil.objects.ReferenceArraySet;
import it.unimi.dsi.fastutil.objects.ReferenceSet;
@@ -88,7 +88,7 @@
/**
* Returns an empty set of indices.
- * @see it.unimi.dsi.big.mg4j.search.DocumentIterator#indices()
+ * @see it.unimi.di.big.mg4j.search.DocumentIterator#indices()
*/
@Override
public ReferenceSet<Index> indices() {
@@ -100,7 +100,7 @@
/**
* Always returns null.
- * @see
it.unimi.dsi.big.mg4j.search.DocumentIterator#accept(it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor)
+ * @see
it.unimi.di.big.mg4j.search.DocumentIterator#accept(it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor)
*/
@Override
public <T> T accept(DocumentIteratorVisitor<T> visitor) throws IOException
{
@@ -109,7 +109,7 @@
/**
* Always returns null
- * @see
it.unimi.dsi.big.mg4j.search.DocumentIterator#acceptOnTruePaths(it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor)
+ * @see
it.unimi.di.big.mg4j.search.DocumentIterator#acceptOnTruePaths(it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor)
*/
@Override
public <T> T acceptOnTruePaths(DocumentIteratorVisitor<T> visitor)
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/query/GapQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/GapQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/GapQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -18,8 +18,8 @@
import gate.mimir.search.QueryEngine;
import it.unimi.dsi.fastutil.objects.ReferenceSet;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor;
import java.io.IOException;
import java.util.*;
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/query/MinusQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/MinusQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/MinusQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -14,8 +14,8 @@
import gate.mimir.search.QueryEngine;
import gate.mimir.search.query.AbstractOverlapQuery.SubQuery;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor;
import it.unimi.dsi.fastutil.objects.ReferenceArraySet;
import it.unimi.dsi.fastutil.objects.ReferenceSet;
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/query/OrQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/OrQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/OrQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -25,8 +25,8 @@
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import it.unimi.dsi.fastutil.objects.ReferenceArraySet;
import it.unimi.dsi.fastutil.objects.ReferenceSet;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor;
import java.io.IOException;
import java.util.*;
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/query/QueryExecutor.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/QueryExecutor.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/QueryExecutor.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -15,7 +15,7 @@
package gate.mimir.search.query;
import gate.mimir.search.QueryEngine;
-import it.unimi.dsi.big.mg4j.search.DocumentIterator;
+import it.unimi.di.big.mg4j.search.DocumentIterator;
import java.io.IOException;
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/query/RepeatsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/RepeatsQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/RepeatsQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -18,8 +18,8 @@
import gate.mimir.search.QueryEngine;
import it.unimi.dsi.fastutil.objects.ReferenceSet;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor;
import java.io.IOException;
import java.util.*;
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/query/TermQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/query/TermQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/query/TermQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -22,13 +22,14 @@
import it.unimi.dsi.fastutil.objects.Reference2ReferenceMap;
import it.unimi.dsi.fastutil.objects.ReferenceSet;
import it.unimi.dsi.lang.MutableString;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.index.IndexIterator;
-import it.unimi.dsi.big.mg4j.index.IndexReader;
-import it.unimi.dsi.big.mg4j.index.payload.Payload;
-import it.unimi.dsi.big.mg4j.search.DocumentIterator;
-import it.unimi.dsi.big.mg4j.search.IntervalIterator;
-import it.unimi.dsi.big.mg4j.search.visitor.DocumentIteratorVisitor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.index.IndexIterator;
+import it.unimi.di.big.mg4j.index.IndexIterators;
+import it.unimi.di.big.mg4j.index.IndexReader;
+import it.unimi.di.big.mg4j.index.payload.Payload;
+import it.unimi.di.big.mg4j.search.DocumentIterator;
+import it.unimi.di.big.mg4j.search.IntervalIterator;
+import it.unimi.di.big.mg4j.search.visitor.DocumentIteratorVisitor;
import java.io.IOException;
@@ -145,17 +146,16 @@
if (from >= latestDocument){
//we do need to skip
latestDocument = indexIterator.skipTo(from + 1);
- if(latestDocument == DocumentIterator.END_OF_LIST){
- //no more documents available
- latestDocument = -1;
- }
}else{
//from is lower than latest document,
//so we just return the next document
latestDocument = indexIterator.nextDocument();
}
- if(latestDocument != -1){
- positionsIterator = indexIterator.positions();
+ if(latestDocument == DocumentIterator.END_OF_LIST){
+ //no more documents available
+ latestDocument = -1;
+ } else {
+ positionsIterator = IndexIterators.positionIterator(indexIterator);
}
return latestDocument;
}
@@ -165,7 +165,8 @@
*/
public Binding nextHit() throws IOException{
if(closed) return null;
- if(positionsIterator == null) positionsIterator =
indexIterator.positions();
+ if(positionsIterator == null) positionsIterator =
+ IndexIterators.positionIterator(indexIterator);
if(latestDocument >= 0 && positionsIterator.hasNext()){
int position = positionsIterator.nextInt();
return new Binding(query, latestDocument, position, query.length,
null);
@@ -186,6 +187,15 @@
}
+ /* (non-Javadoc)
+ * @see it.unimi.di.big.mg4j.index.IndexIterator#nextPosition()
+ */
+ @Override
+ public int nextPosition() throws IOException {
+ // TODO Auto-generated method stub
+ throw new UnsupportedOperationException("Method not implemented!");
+ }
+
public boolean hasNext() {
throw new UnsupportedOperationException("Method not implemented!");
}
@@ -222,23 +232,11 @@
return indexIterator.count();
}
- public IntIterator positions() throws IOException {
- return indexIterator.positions();
- }
-
- public int positions(int[] positions) throws IOException {
- return indexIterator.positions(positions);
- }
-
public Reference2ReferenceMap<Index, IntervalIterator> intervalIterators()
throws IOException {
return indexIterator.intervalIterators();
}
- public int[] positionArray() throws IOException {
- return indexIterator.positionArray();
- }
-
public ReferenceSet<Index> indices() {
return indexIterator.indices();
}
@@ -272,10 +270,6 @@
indexIterator.dispose();
}
- public IntervalIterator iterator() {
- return indexIterator.iterator();
- }
-
public long termNumber() {
return indexIterator.termNumber();
}
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/score/BindingScorer.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/score/BindingScorer.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/score/BindingScorer.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -16,9 +16,9 @@
import gate.mimir.search.query.Binding;
import gate.mimir.search.query.QueryExecutor;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.DocumentIterator;
-import it.unimi.dsi.big.mg4j.search.score.AbstractWeightedScorer;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.DocumentIterator;
+import it.unimi.di.big.mg4j.search.score.AbstractWeightedScorer;
import java.io.IOException;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/score/DelegatingScoringQueryExecutor.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/score/DelegatingScoringQueryExecutor.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/score/DelegatingScoringQueryExecutor.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -17,9 +17,9 @@
import gate.mimir.search.query.Binding;
import gate.mimir.search.query.QueryExecutor;
import it.unimi.dsi.fastutil.objects.Reference2DoubleMap;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.search.DocumentIterator;
-import it.unimi.dsi.big.mg4j.search.score.DelegatingScorer;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.search.DocumentIterator;
+import it.unimi.di.big.mg4j.search.score.DelegatingScorer;
import java.io.IOException;
Modified: mimir/trunk/mimir-core/src/gate/mimir/search/score/MimirScorer.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/score/MimirScorer.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/score/MimirScorer.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -16,8 +16,8 @@
import gate.mimir.search.query.Binding;
import gate.mimir.search.query.QueryExecutor;
-import it.unimi.dsi.big.mg4j.search.DocumentIterator;
-import it.unimi.dsi.big.mg4j.search.score.DelegatingScorer;
+import it.unimi.di.big.mg4j.search.DocumentIterator;
+import it.unimi.di.big.mg4j.search.score.DelegatingScorer;
import java.io.IOException;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractIndexTermsQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractIndexTermsQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AbstractIndexTermsQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -18,10 +18,10 @@
import gate.mimir.search.IndexReaderPool;
import gate.mimir.search.QueryEngine;
import gate.mimir.search.QueryEngine.IndexType;
-import it.unimi.dsi.big.mg4j.search.DocumentIterator;
-import it.unimi.dsi.big.mg4j.search.visitor.CounterCollectionVisitor;
-import it.unimi.dsi.big.mg4j.search.visitor.CounterSetupVisitor;
-import it.unimi.dsi.big.mg4j.search.visitor.TermCollectionVisitor;
+import it.unimi.di.big.mg4j.search.DocumentIterator;
+import it.unimi.di.big.mg4j.search.visitor.CounterCollectionVisitor;
+import it.unimi.di.big.mg4j.search.visitor.CounterSetupVisitor;
+import it.unimi.di.big.mg4j.search.visitor.TermCollectionVisitor;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermsQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermsQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/terms/AnnotationTermsQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -20,8 +20,8 @@
import gate.mimir.search.QueryEngine;
import gate.mimir.search.query.AnnotationQuery;
-import it.unimi.dsi.big.mg4j.index.IndexIterator;
-import it.unimi.dsi.big.mg4j.index.IndexReader;
+import it.unimi.di.big.mg4j.index.IndexIterator;
+import it.unimi.di.big.mg4j.index.IndexReader;
import java.io.IOException;
import java.util.List;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentTermsQuery.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentTermsQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentTermsQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -18,7 +18,7 @@
import gate.mimir.index.mg4j.MimirDirectIndexBuilder;
import gate.mimir.search.QueryEngine;
import gate.mimir.search.QueryEngine.IndexType;
-import it.unimi.dsi.big.mg4j.index.IndexReader;
+import it.unimi.di.big.mg4j.index.IndexReader;
import java.io.IOException;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsAndTermsQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsAndTermsQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsAndTermsQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -18,9 +18,9 @@
import gate.mimir.index.mg4j.MimirDirectIndexBuilder;
import gate.mimir.search.QueryEngine;
import gate.mimir.search.QueryEngine.IndexType;
-import it.unimi.dsi.big.mg4j.index.IndexIterator;
-import it.unimi.dsi.big.mg4j.index.IndexReader;
-import it.unimi.dsi.big.mg4j.search.AndDocumentIterator;
+import it.unimi.di.big.mg4j.index.IndexIterator;
+import it.unimi.di.big.mg4j.index.IndexReader;
+import it.unimi.di.big.mg4j.search.AndDocumentIterator;
import java.io.IOException;
Modified:
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsOrTermsQuery.java
===================================================================
---
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsOrTermsQuery.java
2013-04-29 14:16:07 UTC (rev 16666)
+++
mimir/trunk/mimir-core/src/gate/mimir/search/terms/DocumentsOrTermsQuery.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -18,9 +18,9 @@
import gate.mimir.index.mg4j.MimirDirectIndexBuilder;
import gate.mimir.search.QueryEngine;
import gate.mimir.search.QueryEngine.IndexType;
-import it.unimi.dsi.big.mg4j.index.IndexIterator;
-import it.unimi.dsi.big.mg4j.index.IndexReader;
-import it.unimi.dsi.big.mg4j.search.OrDocumentIterator;
+import it.unimi.di.big.mg4j.index.IndexIterator;
+import it.unimi.di.big.mg4j.index.IndexReader;
+import it.unimi.di.big.mg4j.search.OrDocumentIterator;
import java.io.IOException;
Modified: mimir/trunk/mimir-core/src/gate/mimir/util/MG4JTools.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/util/MG4JTools.java 2013-04-29
14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/util/MG4JTools.java 2013-04-29
16:35:35 UTC (rev 16667)
@@ -16,9 +16,9 @@
import gate.mimir.index.mg4j.MimirIndexBuilder;
import gate.mimir.search.QueryEngine;
-import it.unimi.dsi.big.mg4j.index.DiskBasedIndex;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.index.Index.UriKeys;
+import it.unimi.di.big.mg4j.index.DiskBasedIndex;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.index.Index.UriKeys;
import it.unimi.dsi.fastutil.io.BinIO;
import it.unimi.dsi.util.Properties;
@@ -142,10 +142,19 @@
if(aFile.exists()) {
size += aFile.length();
} else {
- // no index file!
- throw new IllegalArgumentException(
- "Could not locate the index file at " + aFile.getAbsolutePath()
- + "!");
+ // mg4j-5+ may use quasi-succinct indexes
+ File anotherFile =
+ new File(URI.create(basename +
DiskBasedIndex.POINTERS_EXTENSIONS));
+ if(anotherFile.exists()) {
+ size += anotherFile.length();
+ } else {
+ // mg4j-5+ may use quasi-succinct indexes
+ // no index file!
+ throw new IllegalArgumentException(
+ "Could not locate the index file at " +
+ aFile.getAbsolutePath() + ", nor at " +
+ anotherFile.getAbsolutePath() + "!");
+ }
}
aFile =
new File(URI.create(basename + DiskBasedIndex.POSITIONS_EXTENSION));
Modified:
mimir/trunk/mimir-core/src/gate/mimir/util/NormalizingTermProcessor.java
===================================================================
--- mimir/trunk/mimir-core/src/gate/mimir/util/NormalizingTermProcessor.java
2013-04-29 14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-core/src/gate/mimir/util/NormalizingTermProcessor.java
2013-04-29 16:35:35 UTC (rev 16667)
@@ -15,8 +15,8 @@
package gate.mimir.util;
import it.unimi.dsi.lang.MutableString;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.index.TermProcessor;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.index.TermProcessor;
import java.io.InputStream;
import java.lang.reflect.Method;
Modified: mimir/trunk/mimir-test/src/gate/mimir/test/Scratch.java
===================================================================
--- mimir/trunk/mimir-test/src/gate/mimir/test/Scratch.java 2013-04-29
14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-test/src/gate/mimir/test/Scratch.java 2013-04-29
16:35:35 UTC (rev 16667)
@@ -7,15 +7,15 @@
import it.unimi.dsi.fastutil.ints.IntHeapPriorityQueue;
import it.unimi.dsi.fastutil.ints.IntIndirectPriorityQueue;
import it.unimi.dsi.fastutil.ints.IntPriorityQueue;
-import it.unimi.dsi.big.mg4j.index.Index;
-import it.unimi.dsi.big.mg4j.index.IndexIterator;
-import it.unimi.dsi.big.mg4j.index.IndexReader;
-import it.unimi.dsi.big.mg4j.search.score.BM25FScorer;
-import it.unimi.dsi.big.mg4j.search.score.BM25Scorer;
-import it.unimi.dsi.big.mg4j.search.score.CountScorer;
-import it.unimi.dsi.big.mg4j.search.score.DelegatingScorer;
-import it.unimi.dsi.big.mg4j.search.score.Scorer;
-import it.unimi.dsi.big.mg4j.search.score.TfIdfScorer;
+import it.unimi.di.big.mg4j.index.Index;
+import it.unimi.di.big.mg4j.index.IndexIterator;
+import it.unimi.di.big.mg4j.index.IndexReader;
+import it.unimi.di.big.mg4j.search.score.BM25FScorer;
+import it.unimi.di.big.mg4j.search.score.BM25Scorer;
+import it.unimi.di.big.mg4j.search.score.CountScorer;
+import it.unimi.di.big.mg4j.search.score.DelegatingScorer;
+import it.unimi.di.big.mg4j.search.score.Scorer;
+import it.unimi.di.big.mg4j.search.score.TfIdfScorer;
import java.io.BufferedReader;
import java.io.File;
@@ -61,11 +61,11 @@
public class Scratch {
public static void main (String[] args) throws Exception {
-// mainSimple(args);
+ mainSimple(args);
// mainDirectIndexes(args);
// mainBuildDirectIndex(args);
// mainQueryIndex(args);
- mainRemote(args);
+// mainRemote(args);
}
/**
@@ -87,7 +87,7 @@
IndexReaderPool termSource = null;
// open the term supplying index
- URI termsIndexUri = new
File("/home/data/mimir-indexes/index-fastvac-1M.work/mg4j/mimir-token-2").toURI();
+ URI termsIndexUri = new
File("/data/mimir-indexes/index-fastvac-1M.mimir/mg4j/mimir-token-2").toURI();
Index termsIndex = MG4JTools.openMg4jIndex(termsIndexUri);
termSource = new IndexReaderPool(termsIndex, termsIndexUri);
Modified: mimir/trunk/mimir-test/src/gate/mimir/test/TestUtils.java
===================================================================
--- mimir/trunk/mimir-test/src/gate/mimir/test/TestUtils.java 2013-04-29
14:16:07 UTC (rev 16666)
+++ mimir/trunk/mimir-test/src/gate/mimir/test/TestUtils.java 2013-04-29
16:35:35 UTC (rev 16667)
@@ -25,8 +25,8 @@
import gate.mimir.search.QueryEngine;
import gate.mimir.search.query.*;
-import it.unimi.dsi.big.mg4j.index.DowncaseTermProcessor;
-import it.unimi.dsi.big.mg4j.index.NullTermProcessor;
+import it.unimi.di.big.mg4j.index.DowncaseTermProcessor;
+import it.unimi.di.big.mg4j.index.NullTermProcessor;
import java.io.*;
import java.lang.reflect.Constructor;
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Try New Relic Now & We'll Send You this Cool Shirt
New Relic is the only SaaS-based application performance monitoring service
that delivers powerful full stack analytics. Optimize and monitor your
browser, app, & servers with just a few lines of code. Try New Relic
and get this awesome Nerd Life shirt! http://p.sf.net/sfu/newrelic_d2d_apr
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
