path. Text will
be analyzed with a. If create is true, then a
new, empty index will be created in path, replacing the index
already there, if any. */
public IndexWriter2(String path, Analyzer a, boolean create)
throws IOException {
this(FSDirectory.getDirectory(path, create), a, create);
}
/** Constructs an IndexWriter for the index in path. Text will
be analyzed with a. If create is true, then a
new, empty index will be created in path, replacing the index
already there, if any. */
public IndexWriter2(File path, Analyzer a, boolean create)
throws IOException {
this(FSDirectory.getDirectory(path, create), a, create);
}
/** Constructs an IndexWriter for the index in d. Text will be
analyzed with a. If create is true, then a new,
empty index will be created in d, replacing the index already
there, if any. */
public IndexWriter2(Directory d, Analyzer a, final boolean create)
throws IOException {
directory = d;
analyzer = a;
ramDocWriter = new DocumentWriter(ramDirectory, analyzer, maxFieldLength);
Lock writeLock = directory.makeLock("write.lock");
if (!writeLock.obtain()) // obtain write lock
throw new IOException("Index locked for write: " + writeLock);
this.writeLock = writeLock; // save it
synchronized (directory) { // in- & inter-process sync
new Lock.With(directory.makeLock("commit.lock")) {
public Object doBody() throws IOException {
if (create)
segmentInfos.write(directory);
else
segmentInfos.read(directory);
return null;
}
}.run();
}
}
/** Flushes all changes to an index, closes all associated files, and closes
the directory that the index is stored in. */
public final synchronized void close() throws IOException {
flushRamSegments();
ramDirectory.close();
writeLock.release(); // release write lock
writeLock = null;
directory.close();
}
/** Release the write lock, if needed. */
protected final void finalize() throws IOException {
if (writeLock != null) {
writeLock.release(); // release write lock
writeLock = null;
}
}
/** Returns the number of documents currently in this index. */
public final synchronized int docCount() {
int count = 0;
for (int i = 0; i < segmentInfos.size(); i++) {
SegmentInfo si = segmentInfos.info(i);
count += si.docCount;
}
for (int i = 0; i < ramSegmentInfos.size(); i++) {
SegmentInfo si = segmentInfos.info(i);
count += si.docCount;
}
return count;
}
public int maxDocsInRam = 2000;
/** Adds a document to this index.*/
public final void addDocument(Document doc) throws IOException {
String segmentName = newSegmentName(ramSegmentInfos);
ramDocWriter.addDocument(segmentName, doc);
synchronized (this) {
ramSegmentInfos.addElement(new SegmentInfo(segmentName, 1, ramDirectory));
if(ramSegmentInfos.size()>=maxDocsInRam)
flushRamSegments();
}
}
private final synchronized String newSegmentName(SegmentInfos sis) {
return "_" + Integer.toString(sis.counter++, Character.MAX_RADIX);
}
/** If non-null, information about merges will be printed to this. */
public PrintStream infoStream = null;
public int mergeFactor = 10;
/** Merges all segments together into a single segment, optimizing an index
for search. */
public final synchronized void optimize() throws IOException {
flushRamSegments();
while (segmentInfos.size() > 1 || (segmentInfos.size() == 1 && (SegmentReader.hasDeletions(segmentInfos.info(0)) || segmentInfos.info(0).dir != directory))) {
int minSegment = segmentInfos.size() - mergeFactor;
mergeSegments(segmentInfos, (minSegment < 0 ? 0 : minSegment), segmentInfos, directory);
}
}
/** Merges all segments from an array of indexes into this index.
*
* This may be used to parallelize batch indexing. A large document * collection can be broken into sub-collections. Each sub-collection can be * indexed in parallel, on a different thread, process or machine. The * complete index can then be created by merging sub-collection indexes * with this method. * *
After this completes, the index is optimized. */ public final synchronized void addIndexes(Directory[] dirs) throws IOException { optimize(); // start with zero or 1 seg for (int i = 0; i < dirs.length; i++) { SegmentInfos sis = new SegmentInfos(); // read infos from dir sis.read(dirs[i]); for (int j = 0; j < sis.size(); j++) { segmentInfos.addElement(sis.info(j)); // add each info } } optimize(); // final cleanup } /** Merges all RAM-resident segments into one RAM segment and then writes * it into the directory (where the index is being built). **/ private final void flushRamSegments() throws IOException { if(ramSegmentInfos.size()==0) return; //nothing to do. if(ramSegmentInfos.size()>1) //make one segment in the ramDirectory mergeSegments(ramSegmentInfos, 0, ramSegmentInfos, ramDirectory); //merge the one ram segment into the directory: mergeSegments(ramSegmentInfos, 0, segmentInfos, directory); } /** Pops segments off of sourceSegmentInfos stack down to minSegment, merges them, and pushes the merged index onto the top of the segmentInfos stack. */ private final void mergeSegments(SegmentInfos sourceSegmentInfos, int minSegment, final SegmentInfos targetSegmentInfos, final Directory targetDirectory) throws IOException { String mergedName = newSegmentName(targetSegmentInfos); int mergedDocCount = 0; if (infoStream != null) infoStream.print("merging segments"); SegmentMerger merger = new SegmentMerger(targetDirectory, mergedName); final HashMap segmentsToDelete = new HashMap(); //maps (SegmentInfo) segmentInfo -> (Vector) names of files, comprising this segment for (int i = minSegment; i < sourceSegmentInfos.size(); i++) { SegmentInfo si = sourceSegmentInfos.info(i); if (infoStream != null) infoStream.print(" " + si.name + " (" + si.docCount + " docs)"); SegmentReader reader = new SegmentReader(si); merger.add(reader); segmentsToDelete.put(si, reader.files()); mergedDocCount += si.docCount; } if (infoStream != null) infoStream.println("\n into "+mergedName+" ("+mergedDocCount+" docs)"); merger.merge(); //it will close the opened SegmentReader-s at the end. sourceSegmentInfos.setSize(minSegment); // pop old infos & add new targetSegmentInfos.addElement(new SegmentInfo(mergedName, mergedDocCount, targetDirectory)); synchronized (targetDirectory) { // in- & inter-process sync new Lock.With(directory.makeLock("commit.lock")) { public Object doBody() throws IOException { targetSegmentInfos.write(targetDirectory); // commit before deleting Vector deletable = new Vector(); //files, which can't be deleted right now will be stored here and saved to a file, named 'deletable' for(java.util.Iterator iter=segmentsToDelete.keySet().iterator(); iter.hasNext(); ) { SegmentInfo si = (SegmentInfo) iter.next(); Vector filesToDelete = (Vector) segmentsToDelete.get(si); deleteFiles(filesToDelete, deletable, si.dir); // delete now-unused segments } if (infoStream != null) infoStream.println("The following files could not be deleted : "+deletable); //deleteFiles(readDeleteableFiles(), deletable, directory); // try to delete the files. listed in file 'deleteable' //writeDeleteableFiles(deletable); // note files we can't delete return null; } }.run(); } } private final void deleteFiles(Vector files, Vector deletable, Directory d) throws IOException { for (int i = 0; i < files.size(); i++) { String file = (String)files.elementAt(i); try { d.deleteFile(file); // try to delete each file } catch (IOException e) { // if delete fails if (directory.fileExists(file)) { deletable.addElement(file); // add to deletable } } } } private final Vector readDeleteableFiles() throws IOException { Vector result = new Vector(); if (!directory.fileExists("deletable")) return result; InputStream input = directory.openFile("deletable"); try { for (int i = input.readInt(); i > 0; i--) // read file names result.addElement(input.readString()); } finally { input.close(); } return result; } private final void writeDeleteableFiles(Vector files) throws IOException { OutputStream output = directory.createFile("deleteable.new"); try { output.writeInt(files.size()); for (int i = 0; i < files.size(); i++) output.writeString((String)files.elementAt(i)); } finally { output.close(); } directory.renameFile("deleteable.new", "deletable"); } }