Hello.
I am struck in a problem and have been continously getting exception of
/*Stream Closed and LockObtainFailedException..*/
I am trying to read the complete document line by line and once I have
read 100000 lines I will add the doc to the writer and then again read
the next 100000 lines and repeat the same process unless the complete
file is read.
I cannot read the entire document line by line in 1 go due to memory
constraints.
Can anyone kindly let me know what might be wrong with this code:
The */StreamCLOSED/ *exception occurs when I try to add doc to writer
for iterations>1..In first iteration everything goes fine. /*Lock
exception */follows streamclosed exception..
Code Snippet:
package com.iorg;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LiveIndexWriterConfig;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergePolicy.OneMerge;
import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import com.rancore.CustomAnalyzerForCaseSensitive;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.util.Date;
import java.util.Iterator;
public class MainClass2 {
public static void main(String[] args) {
String indexPath = args[0]; //Place where indexes will be created
String docsPath=args[1]; //Place where the files are kept.
final File docDir = new File(docsPath);
if (!docDir.exists() || !docDir.canRead()) {
System.out.println("Document directory '"
+docDir.getAbsolutePath()+ "' does not exist or is not readable, please
check the path");
System.exit(1);
}
Date start = new Date();
try {
System.out.println("Indexing to directory ONLY '" + indexPath +
"'..."+docsPath);
Directory dir = FSDirectory.open(new File(indexPath));
Analyzer analyzer=new
CustomAnalyzerForCaseSensitive(Version.LUCENE_44);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_44,
analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
if(args[2].trim().equalsIgnoreCase("OverAll")){
System.out.println("IN");
indexDocs(docDir,true,dir,iwc);
}
Date end = new Date();
System.out.println(end.getTime() - start.getTime() + " total
milliseconds");
} catch (IOException e) {
System.out.println(" caught a " + e.getClass() +
"\n with message: " + e.getMessage());
}
catch(Exception e)
{
e.printStackTrace();
}
}
//Over All
static void indexDocs(File file,boolean flag,Directory
dir,IndexWriterConfig iwc)
throws IOException {
FileInputStream fis = null;
if (file.canRead()) {
if (file.isDirectory())
{
String[] files = file.list();
System.out.println("size of list is " + files.length);
if (files != null) {
for (int i = 0; i < files.length; i++) {
System.out.println("Invoked for " + i + "and " +
files[i]);
indexDocs(new File(file, files[i]),flag,dir,iwc);
}
}
}
else {
boolean flags=true;
try {
fis = new FileInputStream(file);
} catch (FileNotFoundException fnfe) {
fnfe.printStackTrace();
}
try {
Document doc = new Document();
LineNumberReader lnr=new LineNumberReader(new FileReader(file));
Field pathField = new StringField("path", file.getPath(),
Field.Store.YES);
doc.add(pathField);
String line=null;
System.out.println("INITIALIZING");
int i=0;
doc.add(new StringField("TT",file.getName(),Field.Store.YES));
BufferedReader br=new BufferedReader(new
InputStreamReader(fis));
doc.add(new TextField("DD", br));
while(flags)
{
System.out.println("Looping");
IndexWriter iwcTemp1=new IndexWriter(dir,iwc);
while( null != (line = lnr.readLine()) ){
i++;
StringField sf=new
StringField("EEE",line.trim(),Field.Store.YES);
doc.add(sf);
if(i%10000==0)
{
System.out.println("Breaking" + i);
lnr.mark(i);
break;
}
sf=null;
}
if(line==null)
{
System.out.println("FALSE");
flags=false;
}
System.out.println("Total value is " + i);
if (iwcTemp1.getConfig().getOpenMode() ==
OpenMode.CREATE_OR_APPEND) {
try
{
iwcTemp1.addDocument(doc);
iwcTemp1.commit();
iwcTemp1.close();
}catch(Throwable t)
{
lnr.close();
br.close();
fis.close();
t.printStackTrace();
}
} else {
try
{
System.out.println("updating " + file);
iwcTemp1.updateDocument(new Term("path", file.getPath()), doc);
}catch(Exception e)
{
e.printStackTrace();
}
}
System.out.println("END OF WHILE ");
lnr.reset();
}//end of While
}catch (Exception e) {
e.printStackTrace();
}finally {
fis.close();
}
}
}
}
}
Kindly guide as to where the possible problem lies. Trying to figure out
but to no avail..
--
Regards
Ankit Murarka
"What lies behind us and what lies before us are tiny matters compared with what
lies within us"
