I really recommend you restructure your program, it's a hard to follow. For instance, you open a new IndexWriter every time through the while (flags) loop. You only close it in the if (iwcTemp1.getConfig().getOpenMode() == OpenMode.CREATE_OR_APPEND) { case. That may be the root of your problem right there.
I'd probably open the indexWriter once in the main method, probably make into a member variable that's opened in an init method and closed in a finish method or some such. I'd separate the loop that traverses the directory structure away from the code that indexes the data. FWIW, Erick On Sun, Sep 1, 2013 at 7:28 AM, Ankit Murarka <ankit.mura...@rancoretech.com > wrote: > Hello. > I am struck in a problem and have been continously getting exception of > /*Stream Closed and LockObtainFailedException..*/ > > I am trying to read the complete document line by line and once I have > read 100000 lines I will add the doc to the writer and then again read the > next 100000 lines and repeat the same process unless the complete file is > read. > > I cannot read the entire document line by line in 1 go due to memory > constraints. > > Can anyone kindly let me know what might be wrong with this code: > > The */StreamCLOSED/ *exception occurs when I try to add doc to writer for > iterations>1..In first iteration everything goes fine. /*Lock exception > */follows streamclosed exception.. > > Code Snippet: > > package com.iorg; > > import org.apache.lucene.analysis.**Analyzer; > import org.apache.lucene.document.**Document; > import org.apache.lucene.document.**Field; > import org.apache.lucene.document.**LongField; > import org.apache.lucene.document.**StringField; > import org.apache.lucene.document.**TextField; > import org.apache.lucene.index.**IndexCommit; > import org.apache.lucene.index.**IndexWriter; > import org.apache.lucene.index.**IndexWriterConfig.OpenMode; > import org.apache.lucene.index.**IndexWriterConfig; > import org.apache.lucene.index.**IndexableField; > import org.apache.lucene.index.**LiveIndexWriterConfig; > import org.apache.lucene.index.**LogDocMergePolicy; > import org.apache.lucene.index.**LogMergePolicy; > import org.apache.lucene.index.**MergePolicy; > import org.apache.lucene.index.**MergePolicy.OneMerge; > import org.apache.lucene.index.**MergeScheduler; > import org.apache.lucene.index.Term; > import org.apache.lucene.store.**Directory; > import org.apache.lucene.store.**FSDirectory; > import org.apache.lucene.util.**Version; > > import com.rancore.**CustomAnalyzerForCaseSensitive**; > > import java.io.BufferedReader; > import java.io.File; > import java.io.FileInputStream; > import java.io.FileNotFoundException; > import java.io.FileReader; > import java.io.IOException; > import java.io.InputStreamReader; > import java.io.LineNumberReader; > import java.util.Date; > import java.util.Iterator; > > public class MainClass2 { > public static void main(String[] args) { > > String indexPath = args[0]; //Place where indexes will be created > String docsPath=args[1]; //Place where the files are kept. > > final File docDir = new File(docsPath); > if (!docDir.exists() || !docDir.canRead()) { > System.out.println("Document directory '" +docDir.getAbsolutePath()+ > "' does not exist or is not readable, please check the path"); > System.exit(1); > } > Date start = new Date(); > try { > System.out.println("Indexing to directory ONLY '" + indexPath + > "'..."+docsPath); > Directory dir = FSDirectory.open(new File(indexPath)); > Analyzer analyzer=new CustomAnalyzerForCaseSensitive** > (Version.LUCENE_44); > IndexWriterConfig iwc = new IndexWriterConfig(Version.**LUCENE_44, > analyzer); > iwc.setOpenMode(OpenMode.**CREATE_OR_APPEND); > > if(args[2].trim().**equalsIgnoreCase("OverAll")){ > System.out.println("IN"); > indexDocs(docDir,true,dir,iwc)**; > } > Date end = new Date(); > System.out.println(end.**getTime() - start.getTime() + " total > milliseconds"); > > } catch (IOException e) { > System.out.println(" caught a " + e.getClass() + > "\n with message: " + e.getMessage()); > } > catch(Exception e) > { > e.printStackTrace(); > } > } > > //Over All > static void indexDocs(File file,boolean flag,Directory > dir,IndexWriterConfig iwc) > throws IOException { > FileInputStream fis = null; > if (file.canRead()) { > if (file.isDirectory()) > { > String[] files = file.list(); > System.out.println("size of list is " + files.length); > if (files != null) { > for (int i = 0; i < files.length; i++) { > System.out.println("Invoked for " + i + "and " + > files[i]); > indexDocs(new File(file, files[i]),flag,dir,iwc); > } > } > } > else { > boolean flags=true; > try { > fis = new FileInputStream(file); > } catch (FileNotFoundException fnfe) { > fnfe.printStackTrace(); > } > try { > Document doc = new Document(); > LineNumberReader lnr=new LineNumberReader(new FileReader(file)); > Field pathField = new StringField("path", file.getPath(), > Field.Store.YES); > doc.add(pathField); > String line=null; > System.out.println("**INITIALIZING"); > int i=0; > doc.add(new StringField("TT",file.getName(**),Field.Store.YES)); > BufferedReader br=new BufferedReader(new InputStreamReader(fis)); > doc.add(new TextField("DD", br)); > while(flags) > { > System.out.println("Looping"); > IndexWriter iwcTemp1=new IndexWriter(dir,iwc); > while( null != (line = lnr.readLine()) ){ > i++; > StringField sf=new StringField("EEE",line.trim(),** > Field.Store.YES); > doc.add(sf); > if(i%10000==0) > { > System.out.println("Breaking" + i); > lnr.mark(i); > break; > } > sf=null; > } > if(line==null) > { > System.out.println("FALSE"); > flags=false; > } > System.out.println("Total value is " + i); > if (iwcTemp1.getConfig().**getOpenMode() == > OpenMode.CREATE_OR_APPEND) { > try > { > iwcTemp1.addDocument(doc); > iwcTemp1.commit(); > iwcTemp1.close(); > }catch(Throwable t) > { > lnr.close(); > br.close(); > fis.close(); > t.printStackTrace(); > } > > } else { > try > { > > System.out.println("updating " + file); > iwcTemp1.updateDocument(new Term("path", file.getPath()), doc); > }catch(Exception e) > { > e.printStackTrace(); > } > } > System.out.println("END OF WHILE "); > lnr.reset(); > }//end of While > }catch (Exception e) { > e.printStackTrace(); > }finally { > fis.close(); > } > } > } > } > } > > > Kindly guide as to where the possible problem lies. Trying to figure out > but to no avail.. > > -- > Regards > > Ankit Murarka > > "What lies behind us and what lies before us are tiny matters compared > with what lies within us" > >