goller 2004/08/11 10:37:52
Modified: src/java/org/apache/lucene/index SegmentMerger.java
SegmentReader.java IndexWriter.java
Log:
building of compound file taken out of commit lock
Revision Changes Path
1.13 +6 -10
jakarta-lucene/src/java/org/apache/lucene/index/SegmentMerger.java
Index: SegmentMerger.java
===================================================================
RCS file:
/home/cvs/jakarta-lucene/src/java/org/apache/lucene/index/SegmentMerger.java,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -r1.12 -r1.13
--- SegmentMerger.java 8 Aug 2004 13:03:59 -0000 1.12
+++ SegmentMerger.java 11 Aug 2004 17:37:52 -0000 1.13
@@ -108,13 +108,13 @@
}
}
- final void createCompoundFile()
+ final Vector createCompoundFile(String fileName)
throws IOException {
CompoundFileWriter cfsWriter =
- new CompoundFileWriter(directory, segment + ".cfs");
+ new CompoundFileWriter(directory, fileName);
- ArrayList files =
- new ArrayList(COMPOUND_EXTENSIONS.length + fieldInfos.size());
+ Vector files =
+ new Vector(COMPOUND_EXTENSIONS.length + fieldInfos.size());
// Basic files
for (int i = 0; i < COMPOUND_EXTENSIONS.length; i++) {
@@ -144,12 +144,8 @@
// Perform the merge
cfsWriter.close();
-
- // Now delete the source files
- it = files.iterator();
- while (it.hasNext()) {
- directory.deleteFile((String) it.next());
- }
+
+ return files;
}
/**
1.25 +25 -8
jakarta-lucene/src/java/org/apache/lucene/index/SegmentReader.java
Index: SegmentReader.java
===================================================================
RCS file:
/home/cvs/jakarta-lucene/src/java/org/apache/lucene/index/SegmentReader.java,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -r1.24 -r1.25
--- SegmentReader.java 6 Aug 2004 20:50:29 -0000 1.24
+++ SegmentReader.java 11 Aug 2004 17:37:52 -0000 1.25
@@ -53,7 +53,7 @@
InputStream proxStream;
// Compound File Reader when based on a compound file segment
- CompoundFileReader cfsReader;
+ CompoundFileReader cfsReader = null;
private class Norm {
public Norm(InputStream in, int number)
@@ -75,7 +75,13 @@
} finally {
out.close();
}
- String fileName = segment + ".f" + number;
+ String fileName;
+ if(cfsReader == null)
+ fileName = segment + ".f" + number;
+ else{
+ // use a different file name if we have compound format
+ fileName = segment + ".s" + number;
+ }
directory().renameFile(segment + ".tmp", fileName);
this.dirty = false;
}
@@ -216,8 +222,15 @@
for (int i = 0; i < fieldInfos.size(); i++) {
FieldInfo fi = fieldInfos.fieldInfo(i);
- if (fi.isIndexed)
- files.addElement(segment + ".f" + i);
+ if (fi.isIndexed){
+ String name;
+ if(cfsReader == null)
+ name = segment + ".f" + i;
+ else
+ name = segment + ".s" + i;
+ if (directory().fileExists(name))
+ files.addElement(name);
+ }
}
return files;
}
@@ -363,9 +376,13 @@
for (int i = 0; i < fieldInfos.size(); i++) {
FieldInfo fi = fieldInfos.fieldInfo(i);
if (fi.isIndexed) {
- String fileName = segment + ".f" + fi.number;
- // look first for re-written file, then in compound format
- Directory d = directory().fileExists(fileName) ? directory() : cfsDir;
+ // look first if there are separate norms in compound format
+ String fileName = segment + ".s" + fi.number;
+ Directory d = directory();
+ if(!d.fileExists(fileName)){
+ fileName = segment + ".f" + fi.number;
+ d = cfsDir;
+ }
norms.put(fi.name, new Norm(d.openFile(fileName), fi.number));
}
}
1.36 +49 -15 jakarta-lucene/src/java/org/apache/lucene/index/IndexWriter.java
Index: IndexWriter.java
===================================================================
RCS file:
/home/cvs/jakarta-lucene/src/java/org/apache/lucene/index/IndexWriter.java,v
retrieving revision 1.35
retrieving revision 1.36
diff -u -r1.35 -r1.36
--- IndexWriter.java 8 Aug 2004 13:03:59 -0000 1.35
+++ IndexWriter.java 11 Aug 2004 17:37:52 -0000 1.36
@@ -399,8 +399,8 @@
optimize(); // start with zero or 1 seg
- String mergedName = newSegmentName();
- final SegmentMerger merger = new SegmentMerger(directory, mergedName);
+ final String mergedName = newSegmentName();
+ SegmentMerger merger = new SegmentMerger(directory, mergedName);
final Vector segmentsToDelete = new Vector();
IndexReader sReader = null;
@@ -422,16 +422,29 @@
sReader.close();
synchronized (directory) { // in- & inter-process sync
- new Lock.With(directory.makeLock("commit.lock"), COMMIT_LOCK_TIMEOUT) {
+ new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException {
segmentInfos.write(directory); // commit changes
deleteSegments(segmentsToDelete); // delete now-unused segments
- if(useCompoundFile)
- merger.createCompoundFile();
return null;
}
}.run();
}
+
+ if (useCompoundFile) {
+ final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp");
+ synchronized (directory) { // in- & inter-process sync
+ new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
+ public Object doBody() throws IOException {
+ // make compound file visible for SegmentReaders
+ directory.renameFile(mergedName + ".tmp", mergedName + ".cfs");
+ // delete now unused files of segment
+ deleteFiles(filesToDelete);
+ return null;
+ }
+ }.run();
+ }
+ }
}
/** Merges all RAM-resident segments. */
@@ -479,9 +492,9 @@
and pushes the merged index onto the top of the segmentInfos stack. */
private final void mergeSegments(int minSegment)
throws IOException {
- String mergedName = newSegmentName();
+ final String mergedName = newSegmentName();
if (infoStream != null) infoStream.print("merging segments");
- final SegmentMerger merger =
+ SegmentMerger merger =
new SegmentMerger(directory, mergedName);
final Vector segmentsToDelete = new Vector();
@@ -510,23 +523,37 @@
merger.closeReaders();
synchronized (directory) { // in- & inter-process sync
- new Lock.With(directory.makeLock(IndexWriter.COMMIT_LOCK_NAME),
COMMIT_LOCK_TIMEOUT) {
+ new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
public Object doBody() throws IOException {
segmentInfos.write(directory); // commit before deleting
deleteSegments(segmentsToDelete); // delete now-unused segments
- if(useCompoundFile)
- merger.createCompoundFile();
return null;
}
}.run();
}
-
+
+ if (useCompoundFile) {
+ final Vector filesToDelete = merger.createCompoundFile(mergedName + ".tmp");
+ synchronized (directory) { // in- & inter-process sync
+ new Lock.With(directory.makeLock(COMMIT_LOCK_NAME), COMMIT_LOCK_TIMEOUT) {
+ public Object doBody() throws IOException {
+ // make compound file visible for SegmentReaders
+ directory.renameFile(mergedName + ".tmp", mergedName + ".cfs");
+ // delete now unused files of segment
+ deleteFiles(filesToDelete);
+ return null;
+ }
+ }.run();
+ }
+ }
}
- /* Some operating systems (e.g. Windows) don't permit a file to be deleted
- while it is opened for read (e.g. by another process or thread). So we
- assume that when a delete fails it is because the file is open in another
- process, and queue the file for subsequent deletion. */
+ /*
+ * Some operating systems (e.g. Windows) don't permit a file to be deleted
+ * while it is opened for read (e.g. by another process or thread). So we
+ * assume that when a delete fails it is because the file is open in another
+ * process, and queue the file for subsequent deletion.
+ */
private final void deleteSegments(Vector segments) throws IOException {
Vector deletable = new Vector();
@@ -542,6 +569,13 @@
}
writeDeleteableFiles(deletable); // note files we can't delete
+ }
+
+ private final void deleteFiles(Vector files) throws IOException {
+ Vector deletable = new Vector();
+ deleteFiles(readDeleteableFiles(), deletable); // try to delete deleteable
+ deleteFiles(files, deletable); // try to delete our files
+ writeDeleteableFiles(deletable); // note files we can't delete
}
private final void deleteFiles(Vector files, Directory directory)
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
