lucene 6.1.0 upgrade
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/999e68dd Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/999e68dd Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/999e68dd Branch: refs/heads/master Commit: 999e68ddbed773aaeffe4a912288915a9eb4ab9c Parents: 3939d0a Author: Tom Barber <[email protected]> Authored: Fri Jul 8 21:46:35 2016 +0100 Committer: Lewis John McGibbney <[email protected]> Committed: Thu Mar 9 21:34:33 2017 -0800 ---------------------------------------------------------------------- core/pom.xml | 14 +- filemgr/pom.xml | 11 + .../oodt/cas/filemgr/catalog/LuceneCatalog.java | 373 +++++++++++-------- .../filemgr/catalog/LuceneCatalogFactory.java | 18 +- .../cli/action/LuceneQueryCliAction.java | 25 +- .../oodt/cas/filemgr/tools/CASAnalyzer.java | 36 +- .../oodt/cas/filemgr/tools/CatalogSearch.java | 26 +- .../filemgr/tools/OptimizeLuceneCatalog.java | 24 +- .../oodt/cas/filemgr/tools/QueryTool.java | 23 +- .../cas/filemgr/tools/RangeQueryTester.java | 50 +-- 10 files changed, 374 insertions(+), 226 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index ee8cb7e..e3ff15a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -280,9 +280,21 @@ the License. <dependency> <groupId>org.apache.lucene</groupId> <artifactId>lucene-core</artifactId> - <version>2.0.0</version> + <version>6.1.0</version> </dependency> <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-queryparser</artifactId> + <version>6.1.0</version> + </dependency> + + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + <version>6.1.0</version> + </dependency> + + <dependency> <groupId>org.apache.oodt</groupId> <artifactId>cas-cli</artifactId> <version>${project.parent.version}</version> http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/pom.xml ---------------------------------------------------------------------- diff --git a/filemgr/pom.xml b/filemgr/pom.xml index 6e102dc..54b4e31 100644 --- a/filemgr/pom.xml +++ b/filemgr/pom.xml @@ -125,6 +125,15 @@ <artifactId>lucene-core</artifactId> </dependency> <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-queryparser</artifactId> + </dependency> + + <dependency> <groupId>org.apache.oodt</groupId> <artifactId>cas-cli</artifactId> </dependency> @@ -275,6 +284,8 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> + <forkMode>never</forkMode> + <!--forkMode>pertest</forkMode--> <forkMode>pertest</forkMode> <!-- so we can DEBUG --> <argLine>-enableassertions</argLine> http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java index 26bb983..c8936bf 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java @@ -17,44 +17,29 @@ package org.apache.oodt.cas.filemgr.catalog; -//JDK imports - import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.RangeQuery; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.WildcardQuery; -import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria; -import org.apache.oodt.cas.filemgr.structs.Element; -import org.apache.oodt.cas.filemgr.structs.Product; -import org.apache.oodt.cas.filemgr.structs.ProductPage; -import org.apache.oodt.cas.filemgr.structs.ProductType; +import org.apache.lucene.document.*; +import org.apache.lucene.index.*; +import org.apache.lucene.search.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Version; +import org.apache.oodt.cas.filemgr.structs.*; import org.apache.oodt.cas.filemgr.structs.Query; -import org.apache.oodt.cas.filemgr.structs.QueryCriteria; -import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria; -import org.apache.oodt.cas.filemgr.structs.Reference; -import org.apache.oodt.cas.filemgr.structs.TermQueryCriteria; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.ValidationLayerException; import org.apache.oodt.cas.filemgr.validation.ValidationLayer; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.commons.pagination.PaginationUtils; - +import org.apache.poi.hssf.record.formula.functions.Text; +import org.apache.solr.schema.FieldType; import org.safehaus.uuid.UUID; import org.safehaus.uuid.UUIDGenerator; import java.io.File; import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -63,10 +48,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; -//Lucene imports -//OODT imports -//JUG imports - /** * @author mattmann * @author bfoster @@ -81,7 +62,9 @@ import java.util.logging.Logger; * */ public class LuceneCatalog implements Catalog { + Directory indexDir = null; + private DirectoryReader reader; /* the path to the index directory for this catalog */ private String indexFilePath = null; @@ -92,7 +75,7 @@ public class LuceneCatalog implements Catalog { * temporary Cache of product/metadata/reference information before it is * written to the index */ - private static ConcurrentHashMap<String, CompleteProduct> CATALOG_CACHE = new ConcurrentHashMap<String, CompleteProduct>(); + private static ConcurrentHashMap<String, CompleteProduct> CATALOG_CACHE = new ConcurrentHashMap<>(); /* our product ID generator */ private static UUIDGenerator generator = UUIDGenerator.getInstance(); @@ -112,6 +95,8 @@ public class LuceneCatalog implements Catalog { /* lucene index merge factor */ private int mergeFactor = -1; + + /** * * @param idxFilePath @@ -144,6 +129,17 @@ public class LuceneCatalog implements Catalog { this.writeLockTimeout = writeTimeout; this.commitLockTimeout = commitTimeout; this.mergeFactor = mergeFactor; + + try { + indexDir = FSDirectory.open(new File( indexFilePath ).toPath()); + } catch (IOException e) { + e.printStackTrace(); + } + + + + + } /* @@ -170,7 +166,7 @@ public class LuceneCatalog implements Catalog { else{ // move product from index to cache // it will be moved back after metadata is added - CompleteProduct p = getCompleteProductById(product.getProductId(), true, true); + getCompleteProductById(product.getProductId(), true, true); LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); removeProduct(product); @@ -185,7 +181,7 @@ public class LuceneCatalog implements Catalog { */ public synchronized void removeMetadata(Metadata m, Product product) throws CatalogException { - CompleteProduct p=null; + CompleteProduct p; if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { p = CATALOG_CACHE.get(product.getProductId()); @@ -350,7 +346,7 @@ public class LuceneCatalog implements Catalog { else{ // move product from index to cache // it will be moved back after metadata is added - CompleteProduct p = getCompleteProductById(product.getProductId(), true, true); + getCompleteProductById(product.getProductId(), true, true); LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); removeProduct(product); @@ -387,19 +383,28 @@ public class LuceneCatalog implements Catalog { boolean getRefs, boolean getMet) throws CatalogException { IndexSearcher searcher = null; try { - searcher = new IndexSearcher(indexFilePath); + try { + reader = DirectoryReader.open(indexDir); + } catch (IOException e) { + e.printStackTrace(); + } + + searcher = new IndexSearcher(reader); Term productIdTerm = new Term("product_id", productId); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); - Hits hits = searcher.search(query); + TopDocs topDocs = searcher.search(query,1); + + ScoreDoc[] hits = topDocs.scoreDocs; // should be exactly 1 hit - if (hits.length() == 0) { + if (topDocs.totalHits == 0) { throw new CatalogException("Product: [" + productId + "] NOT found in the catalog!"); - } else if (hits.length() > 1) { + } + if (topDocs.totalHits > 1) { throw new CatalogException("Product: [" + productId+ "] is not unique in the catalog!"); } - Document productDoc = hits.doc(0); + Document productDoc = searcher.doc(hits[0].doc); return toCompleteProduct(productDoc, getRefs, getMet); } catch (IOException e) { @@ -411,7 +416,7 @@ public class LuceneCatalog implements Catalog { } finally { if (searcher != null) { try { - searcher.close(); + //TODO shutdown reader } catch (Exception ignore) { } } @@ -431,17 +436,25 @@ public class LuceneCatalog implements Catalog { throws CatalogException { IndexSearcher searcher = null; try { - searcher = new IndexSearcher(indexFilePath); + try { + reader = DirectoryReader.open(indexDir); + } catch (IOException e) { + e.printStackTrace(); + } + searcher = new IndexSearcher(reader); Term productIdTerm = new Term("product_name", productName); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.STRING, true)); - Hits hits = searcher.search(query, sort); + SortField.Type.STRING, true)); + //TODO FIX NUMBER OF RECORDS + TopDocs topDocs = searcher.search(query, 1, sort); + + ScoreDoc[] hits = topDocs.scoreDocs; // should be > 0 hits - if (hits.length() > 0) { + if (hits.length > 0) { // just get the first hit back - Document productDoc = hits.doc(0); + Document productDoc = searcher.doc(hits[0].doc); CompleteProduct prod = toCompleteProduct(productDoc, getRefs, false); return prod.getProduct(); @@ -460,7 +473,7 @@ public class LuceneCatalog implements Catalog { } finally { if (searcher != null) { try { - searcher.close(); +//TODO CLOSE SEARCHER } catch (Exception ignore) { } } @@ -495,18 +508,26 @@ public class LuceneCatalog implements Catalog { List<Product> products = null; try { - searcher = new IndexSearcher(indexFilePath); + try { + reader = DirectoryReader.open(indexDir); + } catch (IOException e) { + e.printStackTrace(); + } + searcher = new IndexSearcher(reader); Term productIdTerm = new Term("myfield", "myvalue"); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.STRING, true)); - Hits hits = searcher.search(query, sort); + SortField.Type.STRING, true)); + //TODO FIX NUMBER OF RECORDS + TopDocs topDocs = searcher.search(query, 1, sort); + + ScoreDoc[] hits = topDocs.scoreDocs; // should be > 0 hits - if (hits.length() > 0) { - products = new Vector<Product>(hits.length()); - for (int i = 0; i < hits.length(); i++) { - Document productDoc = hits.doc(i); + if (hits.length > 0) { + products = new Vector<Product>(hits.length); + for (ScoreDoc hit : hits) { + Document productDoc = searcher.doc(hit.doc); CompleteProduct prod = toCompleteProduct(productDoc, getRefs, false); products.add(prod.getProduct()); @@ -526,7 +547,7 @@ public class LuceneCatalog implements Catalog { } finally { if (searcher != null) { try { - searcher.close(); + //TODO close searcher } catch (Exception ignore) { } } @@ -551,19 +572,27 @@ public class LuceneCatalog implements Catalog { List<Product> products = null; try { - searcher = new IndexSearcher(indexFilePath); + try { + reader = DirectoryReader.open(indexDir); + } catch (IOException e) { + e.printStackTrace(); + } + searcher = new IndexSearcher(reader); Term productIdTerm = new Term("product_type_id", type .getProductTypeId()); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.STRING, true)); - Hits hits = searcher.search(query, sort); + SortField.Type.STRING, true)); + //TODO FIX NUMBER OF RECORDS + TopDocs topDocs = searcher.search(query, 1, sort); + + ScoreDoc[] hits = topDocs.scoreDocs; // should be > 0 hits - if (hits.length() > 0) { - products = new Vector<Product>(hits.length()); - for (int i = 0; i < hits.length(); i++) { - Document productDoc = hits.doc(i); + if (hits.length > 0) { + products = new Vector<Product>(hits.length); + for (ScoreDoc hit : hits) { + Document productDoc = searcher.doc(hit.doc); CompleteProduct prod = toCompleteProduct(productDoc, getRefs, false); products.add(prod.getProduct()); @@ -583,8 +612,8 @@ public class LuceneCatalog implements Catalog { } finally { if (searcher != null) { try { - searcher.close(); - } catch (Exception ignore) { +//TODO CLOSE + } catch (Exception ignore) { } } } @@ -595,20 +624,31 @@ public class LuceneCatalog implements Catalog { public Metadata getMetadata(Product product) throws CatalogException { IndexSearcher searcher = null; try { - searcher = new IndexSearcher(indexFilePath); + try { + reader = DirectoryReader.open(indexDir); + } catch (IOException e) { + e.printStackTrace(); + } + searcher = new IndexSearcher(reader); + TermQuery qry = new TermQuery(new Term("*", "*")); + TopDocs tdocks = searcher.search(qry, 100); Term productIdTerm = new Term("product_id", product.getProductId()); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); - Hits hits = searcher.search(query); + //TODO FIX NUMBER OF RECORDS + TopDocs topDocs = searcher.search(query, 1); + + ScoreDoc[] hits = topDocs.scoreDocs; // should be exactly 1 hit - if (hits.length() != 1) { + if (topDocs.totalHits != 1) { throw new CatalogException("Product: [" + product.getProductId() + "] is not unique in the catalog! Num Hits: [" - + hits.length() + "]"); + + hits.length + "]"); } - Document productDoc = hits.doc(0); + Document productDoc = searcher.doc(hits[0].doc); + CompleteProduct prod = toCompleteProduct(productDoc, false, true); return prod.getMetadata(); } catch (IOException e) { @@ -620,8 +660,8 @@ public class LuceneCatalog implements Catalog { } finally { if (searcher != null) { try { - searcher.close(); - } catch (Exception ignore) { +//TODO CLOSE + } catch (Exception ignore) { } } } @@ -671,23 +711,32 @@ public class LuceneCatalog implements Catalog { IndexSearcher searcher = null; try { - searcher = new IndexSearcher(indexFilePath); + try { + reader = DirectoryReader.open(indexDir); + } catch (IOException e) { + e.printStackTrace(); + } + searcher = new IndexSearcher(reader); // construct a Boolean query here - BooleanQuery booleanQuery = new BooleanQuery(); + BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); TermQuery tq = new TermQuery(new Term("myfield", "myvalue")); booleanQuery.add(tq, BooleanClause.Occur.MUST); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.STRING, true)); + SortField.Type.STRING, true)); LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery + "]"); - Hits hits = searcher.search(booleanQuery, sort); - if (hits.length() > 0) { + //TODO FIX NUMBER OF RECORDS + TopDocs topDocs = searcher.search(booleanQuery.build(), 1, sort); + + ScoreDoc[] hits = topDocs.scoreDocs; + + if (hits.length > 0) { products = new Vector<Product>(n); int i = 0; - while (products.size() < Math.min(n, hits.length())) { - Document productDoc = hits.doc(i); + while (products.size() < Math.min(n, hits.length)) { + Document productDoc = searcher.doc(hits[i].doc); CompleteProduct prod = toCompleteProduct(productDoc, false, false); products.add(prod.getProduct()); @@ -706,7 +755,7 @@ public class LuceneCatalog implements Catalog { } finally { if (searcher != null) { try { - searcher.close(); + //TODO CLOSE } catch (Exception ignore) { } } @@ -940,15 +989,28 @@ public class LuceneCatalog implements Catalog { private synchronized void removeProductDocument(Product product) throws CatalogException { - IndexReader reader = null; try { - reader = IndexReader.open(indexFilePath); + reader = DirectoryReader.open(indexDir); + } catch (IOException e) { + e.printStackTrace(); + } + try { LOG.log(Level.FINE, "LuceneCatalog: remove document from index for product: [" + product.getProductId() + "]"); - reader.deleteDocuments(new Term("product_id", product + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + LogMergePolicy lmp =new LogDocMergePolicy(); + lmp.setMergeFactor(mergeFactor); + config.setMergePolicy(lmp); + + IndexWriter writer = new IndexWriter(indexDir, config); + writer.deleteDocuments(new Term("product_id", product .getProductId())); + writer.close(); + } catch (IOException e) { LOG.log(Level.WARNING, "Exception removing product: [" + product.getProductName() + "] from index: Message: " @@ -969,24 +1031,24 @@ public class LuceneCatalog implements Catalog { private synchronized void addCompleteProductToIndex(CompleteProduct cp) throws CatalogException { IndexWriter writer = null; + try { + /*writer = new IndexWriter(indexFilePath, new StandardAnalyzer(), + createIndex);*/ + //writer.setCommitLockTimeout(this.commitLockTimeout * 1000); + //writer.setWriteLockTimeout(this.writeLockTimeout * 1000); + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - File indexDir = new File(indexFilePath); - - boolean createIndex; - - createIndex = !(indexDir.exists() && indexDir.isDirectory()); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + LogMergePolicy lmp =new LogDocMergePolicy(); + lmp.setMergeFactor(mergeFactor); + config.setMergePolicy(lmp); - try { - writer = new IndexWriter(indexFilePath, new StandardAnalyzer(), - createIndex); - writer.setCommitLockTimeout(this.commitLockTimeout * 1000); - writer.setWriteLockTimeout(this.writeLockTimeout * 1000); - writer.setMergeFactor(this.mergeFactor); + writer = new IndexWriter(indexDir, config); Document doc = toDoc(cp.getProduct(), cp.getMetadata()); writer.addDocument(doc); // TODO: determine a better way to optimize the index - } catch (IOException e) { + } catch (Exception e) { LOG.log(Level.WARNING, "Unable to index product: [" + cp.getProduct().getProductName() + "]: Message: " + e.getMessage(), e); @@ -998,7 +1060,8 @@ public class LuceneCatalog implements Catalog { if (writer != null) { writer.close(); } - } catch (Exception ignore) { + } catch (Exception e) { + System.out.println("failed"+e.getLocalizedMessage()); } } @@ -1044,13 +1107,12 @@ public class LuceneCatalog implements Catalog { } } else { // add all metadata elements found in document - Enumeration<Field> fields = doc.fields(); - while (fields.hasMoreElements()) { - Field field = fields.nextElement(); - if (!names.contains(field.name())) { - names.add(field.name()); - } - } + List<IndexableField> fields = doc.getFields(); + for(IndexableField field: fields){ + if (!names.contains(field.name())) { + names.add(field.name()); + } + } } @@ -1108,36 +1170,33 @@ public class LuceneCatalog implements Catalog { private Document toDoc(Product product, Metadata metadata) { Document doc = new Document(); - +//TODO CHECK STORED TYPES // add the product information doc.add(new Field("product_id", product.getProductId(), - Field.Store.YES, Field.Index.UN_TOKENIZED)); + StringField.TYPE_STORED)); doc.add(new Field("product_name", product.getProductName(), - Field.Store.YES, Field.Index.UN_TOKENIZED)); + TextField.TYPE_STORED)); doc.add(new Field("product_structure", product.getProductStructure(), - Field.Store.YES, Field.Index.UN_TOKENIZED)); + TextField.TYPE_STORED)); doc .add(new Field("product_transfer_status", product - .getTransferStatus(), Field.Store.YES, - Field.Index.UN_TOKENIZED)); + .getTransferStatus(), TextField.TYPE_STORED)); // product type doc .add(new Field("product_type_id", product.getProductType() - .getProductTypeId(), Field.Store.YES, - Field.Index.UN_TOKENIZED)); + .getProductTypeId(), StringField.TYPE_STORED)); doc.add(new Field("product_type_name", product.getProductType() - .getName(), Field.Store.YES, Field.Index.UN_TOKENIZED)); + .getName(), TextField.TYPE_STORED)); doc.add(new Field("product_type_desc", product.getProductType() .getDescription() != null ? product.getProductType() - .getDescription() : "", Field.Store.YES, Field.Index.NO)); + .getDescription() : "", TextField.TYPE_STORED)); doc.add(new Field("product_type_repoPath", product.getProductType() .getProductRepositoryPath() != null ? product.getProductType() - .getProductRepositoryPath() : "", Field.Store.YES, - Field.Index.NO)); + .getProductRepositoryPath() : "", StringField.TYPE_STORED)); doc.add(new Field("product_type_versioner", product.getProductType() .getVersioner() != null ? product.getProductType() - .getVersioner() : "", Field.Store.YES, Field.Index.NO)); + .getVersioner() : "", StringField.TYPE_STORED)); // write metadata fields to the Lucene document List<String> keys = new ArrayList<String>(); @@ -1175,30 +1234,29 @@ public class LuceneCatalog implements Catalog { } for (String val : values) { - doc.add(new Field(key, val, Field.Store.YES, - Field.Index.UN_TOKENIZED)); + doc.add(new Field(key, val, StringField.TYPE_STORED)); + if(values.size()==1) { + doc.add(new SortedDocValuesField(key, new BytesRef(val))); + } } } // add the product references for (Reference r : product.getProductReferences()) { doc.add(new Field("reference_orig", r.getOrigReference(), - Field.Store.YES, Field.Index.NO)); + StringField.TYPE_STORED)); doc .add(new Field("reference_data_store", r - .getDataStoreReference(), Field.Store.YES, - Field.Index.NO)); + .getDataStoreReference(), StringField.TYPE_STORED)); doc.add(new Field("reference_fileSize", String.valueOf(r - .getFileSize()), Field.Store.YES, Field.Index.NO)); + .getFileSize()), StringField.TYPE_STORED)); doc.add(new Field("reference_mimeType", r.getMimeType() != null ? r - .getMimeType().getName() : "", Field.Store.YES, - Field.Index.UN_TOKENIZED)); + .getMimeType().getName() : "", StringField.TYPE_STORED)); } // add special field for all products // then can use that field to retrieve back all products - doc.add(new Field("myfield", "myvalue", Field.Store.NO, - Field.Index.TOKENIZED)); + doc.add(new Field("myfield", "myvalue", TextField.TYPE_STORED)); return doc; } @@ -1228,15 +1286,19 @@ public class LuceneCatalog implements Catalog { IndexSearcher searcher = null; int numHits = -1; - try { - searcher = new IndexSearcher(indexFilePath); + reader = DirectoryReader.open(indexDir); + } catch (IOException e) { + e.printStackTrace(); + } + try { + searcher = new IndexSearcher(reader); // construct a Boolean query here - BooleanQuery booleanQuery = new BooleanQuery(); + BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); // add the product type as the first clause - TermQuery prodTypeTermQuery = new TermQuery(new Term( + org.apache.lucene.search.Query prodTypeTermQuery = new TermQuery(new Term( "product_type_id", type.getProductTypeId())); booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST); @@ -1247,8 +1309,12 @@ public class LuceneCatalog implements Catalog { LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery + "]"); - Hits hits = searcher.search(booleanQuery); - numHits = hits.length(); + + //TODO FIX returned records + TopDocs hits = searcher.search(booleanQuery.build(), 1); + + + numHits = hits.totalHits; } catch (IOException e) { LOG.log(Level.WARNING, "IOException when opening index directory: [" @@ -1258,7 +1324,7 @@ public class LuceneCatalog implements Catalog { } finally { if (searcher != null) { try { - searcher.close(); + //TODO CLOSE } catch (Exception ignore) { } } @@ -1277,12 +1343,16 @@ public class LuceneCatalog implements Catalog { if (pageNum == -1) { doSkip = false; } - try { - searcher = new IndexSearcher(indexFilePath); + reader = DirectoryReader.open(indexDir); + } catch (IOException e) { + e.printStackTrace(); + } + try { + searcher = new IndexSearcher(reader); // construct a Boolean query here - BooleanQuery booleanQuery = new BooleanQuery(); + BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); // add the product type as the first clause TermQuery prodTypeTermQuery = new TermQuery(new Term( @@ -1295,37 +1365,42 @@ public class LuceneCatalog implements Catalog { } Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.STRING, true)); + SortField.Type.STRING, true)); LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery + "]"); - Hits hits = searcher.search(booleanQuery, sort); + //TODO FIX NUMBER OF RECORDS + TopDocs topDocs = searcher.search(booleanQuery.build(),1, sort); // Calculate page size and set it while we have the results if (page != null) { - page.setTotalPages(PaginationUtils.getTotalPage(hits.length(), pageSize)); + page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize)); } - - if (hits.length() > 0) { + + ScoreDoc[] hits = topDocs.scoreDocs; + + if (hits.length > 0) { int startNum = (pageNum - 1) * pageSize; if (doSkip) { - if (startNum > hits.length()) { + if (startNum > hits.length) { startNum = 0; } products = new Vector<Product>(pageSize); - for (int i = startNum; i < Math.min(hits.length(), + for (int i = startNum; i < Math.min(hits.length, (startNum + pageSize)); i++) { - Document productDoc = hits.doc(i); + Document productDoc = searcher.doc(hits[i].doc); + CompleteProduct prod = toCompleteProduct(productDoc, false, false); products.add(prod.getProduct()); } } else { - products = new Vector<Product>(hits.length()); - for (int i = 0; i < hits.length(); i++) { - Document productDoc = hits.doc(i); + products = new Vector<Product>(hits.length); + for (int i = 0; i < hits.length; i++) { + Document productDoc = searcher.doc(hits[i].doc); + CompleteProduct prod = toCompleteProduct(productDoc, false, false); products.add(prod.getProduct()); @@ -1347,7 +1422,7 @@ public class LuceneCatalog implements Catalog { } finally { if (searcher != null) { try { - searcher.close(); + //TODO CLOSE } catch (Exception ignore) { } } @@ -1358,7 +1433,7 @@ public class LuceneCatalog implements Catalog { private org.apache.lucene.search.Query getQuery(QueryCriteria queryCriteria) throws CatalogException { if (queryCriteria instanceof BooleanQueryCriteria) { - BooleanQuery booleanQuery = new BooleanQuery(); + BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); BooleanClause.Occur occur; switch (((BooleanQueryCriteria) queryCriteria).getOperator()) { case BooleanQueryCriteria.AND: @@ -1380,7 +1455,7 @@ public class LuceneCatalog implements Catalog { booleanQuery.add(this.getQuery(qc), occur); } - return booleanQuery; + return booleanQuery.build(); } else if (queryCriteria instanceof TermQueryCriteria) { String val = ((TermQueryCriteria) queryCriteria).getValue(); return new TermQuery(new Term(queryCriteria.getElementName(), val)); @@ -1397,7 +1472,7 @@ public class LuceneCatalog implements Catalog { endTerm = new Term(queryCriteria.getElementName(), endVal); } - return new RangeQuery(startTerm, endTerm, inclusive); + return TermRangeQuery.newStringRange(startTerm.field(), startVal, endVal, inclusive,inclusive); }else { throw new CatalogException("Invalid QueryCriteria [" + queryCriteria.getClass().getCanonicalName() + "]"); http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java index f70fa71..2512964 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java @@ -19,9 +19,15 @@ package org.apache.oodt.cas.filemgr.catalog; //JDK imports import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; import java.util.logging.Logger; //OODT imports +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory; import org.apache.oodt.cas.metadata.util.PathUtils; import org.apache.oodt.cas.filemgr.validation.ValidationLayer; @@ -47,6 +53,7 @@ public class LuceneCatalogFactory implements CatalogFactory { public static final int VAL3 = 20; /* path to the index directory for lucene catalogs */ private String indexFilePath = null; + private IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); /* our validation layer */ private ValidationLayer validationLayer = null; @@ -119,9 +126,16 @@ public class LuceneCatalogFactory implements CatalogFactory { File indexDir = new File(indexFilePath); // Create the index if it does not already exist IndexWriter writer = null; + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); if (!indexDir.exists()) { - try { - writer = new IndexWriter(indexDir, new StandardAnalyzer(), true); + try { + try { + Directory indexDir2 = FSDirectory.open(new File( indexFilePath ).toPath()); + writer = new IndexWriter(indexDir2, config); + } catch (IOException e) { + e.printStackTrace(); + } + } catch (Exception e) { LOG.severe("Unable to create index: " + e.getMessage()); } finally { http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java index f688478..85e06cd 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java @@ -21,14 +21,10 @@ import com.google.common.collect.Lists; import org.apache.commons.lang.Validate; import org.apache.lucene.index.Term; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.RangeQuery; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.util.BytesRef; import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria; import org.apache.oodt.cas.filemgr.structs.QueryCriteria; import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria; @@ -110,13 +106,14 @@ public class LuceneQueryCliAction extends AbstractQueryCliAction { } return bqc; } - } else if (luceneQuery instanceof RangeQuery) { - Term startT = ((RangeQuery) luceneQuery).getLowerTerm(); - Term endT = ((RangeQuery) luceneQuery).getUpperTerm(); - return new RangeQueryCriteria(startT.field(), startT - .text(), endT.text(), ((RangeQuery) luceneQuery).isInclusive()); + } else if (luceneQuery instanceof TermRangeQuery) { + BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm(); + BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm(); + //TODO CHECK Inclusive + return new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT + .utf8ToString(), endT.utf8ToString()/*, ((TermRangeQuery) luceneQuery).isInclusive()*/); } else if (luceneQuery instanceof BooleanQuery) { - BooleanClause[] clauses = ((BooleanQuery) luceneQuery).getClauses(); + List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses(); BooleanQueryCriteria bqc = new BooleanQueryCriteria(); bqc.setOperator(BooleanQueryCriteria.AND); for (BooleanClause clause : clauses) { http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java index d9bdaaf..9470c62 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java @@ -20,7 +20,12 @@ package org.apache.oodt.cas.filemgr.tools; //Lucene imports import org.apache.lucene.analysis.*; +import org.apache.lucene.analysis.core.StopAnalyzer; +import org.apache.lucene.analysis.core.StopFilter; +import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.apache.lucene.analysis.custom.CustomAnalyzer; import org.apache.lucene.analysis.standard.StandardFilter; +import org.apache.lucene.analysis.util.CharArraySet; //JDK imports import java.io.Reader; @@ -44,26 +49,47 @@ public class CASAnalyzer extends Analyzer { * An array containing some common English words that are usually not useful * for searching. */ - public static final String[] STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS; + public static final CharArraySet STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS_SET; + private Reader reader; /** Builds an analyzer. */ public CASAnalyzer() { this(STOP_WORDS); } + @Override + protected TokenStreamComponents createComponents(String fieldName) { + TokenStream result = new WhitespaceTokenizer(/*reader*/); + result = new StandardFilter(result); + result = new StopFilter(result, STOP_WORDS); + + + //TODO FIX + try { + throw new Exception("needs fixing"); + } catch (Exception e) { + e.printStackTrace(); + } + return null; //new TokenStreamComponents(); + } + + public void tokenStreams(String fname, Reader reader){ + this.reader = reader; + } /** Builds an analyzer with the given stop words. */ - public CASAnalyzer(String[] stopWords) { - stopSet = StopFilter.makeStopSet(stopWords); + public CASAnalyzer(CharArraySet stopWords) { + //stopSet = StopFilter.makeStopSet(stopWords); } /** * Constructs a {@link org.apache.lucene.analysis.standard.StandardTokenizer} filtered by a {@link * StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */ - public TokenStream tokenStream(String fieldName, Reader reader) { + /*public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new WhitespaceTokenizer(reader); result = new StandardFilter(result); result = new StopFilter(result, stopSet); return result; - } + }*/ + } http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java index e086940..5dbb0da 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java @@ -18,14 +18,10 @@ package org.apache.oodt.cas.filemgr.tools; import org.apache.lucene.index.Term; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.RangeQuery; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.util.BytesRef; import org.apache.oodt.cas.filemgr.structs.Element; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductType; @@ -41,6 +37,7 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.URL; +import java.util.List; import java.util.StringTokenizer; import java.util.Vector; import java.util.logging.Level; @@ -255,13 +252,14 @@ public class CatalogSearch { aT.text())); } } - } else if (luceneQuery instanceof RangeQuery) { - Term startT = ((RangeQuery) luceneQuery).getLowerTerm(); - Term endT = ((RangeQuery) luceneQuery).getUpperTerm(); - casQuery.addCriterion(new RangeQueryCriteria(startT.field(), startT - .text(), endT.text())); + } else if (luceneQuery instanceof TermRangeQuery) { + BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm(); + BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm(); + + //TODO CHECK THIS RANGE! + casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString())); } else if (luceneQuery instanceof BooleanQuery) { - BooleanClause[] clauses = ((BooleanQuery) luceneQuery).getClauses(); + List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses(); for (BooleanClause clause : clauses) { GenerateCASQuery(casQuery, (clause).getQuery()); } http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java index dd92f28..16cf966 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java @@ -19,12 +19,14 @@ package org.apache.oodt.cas.filemgr.tools; //JDK imports import java.io.IOException; +import java.nio.file.Paths; import java.util.logging.Level; import java.util.logging.Logger; //Lucene imports import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.*; +import org.apache.lucene.store.FSDirectory; /** * @author mattmann @@ -39,6 +41,9 @@ public class OptimizeLuceneCatalog { public static final double DOUBLE = 1000.0; public static final int INT = 20; + private DirectoryReader reader; + private IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + /* the path to the lucene index directory */ private String catalogPath = null; @@ -55,6 +60,12 @@ public class OptimizeLuceneCatalog { public OptimizeLuceneCatalog(String catPath, int mf) { this.catalogPath = catPath; this.mergeFactor = mf; + try { + reader = DirectoryReader.open(FSDirectory.open(Paths.get(catalogPath))); + } catch (IOException e) { + e.printStackTrace(); + } + } public void doOptimize() { @@ -62,11 +73,14 @@ public class OptimizeLuceneCatalog { boolean createIndex = false; try { - writer = new IndexWriter(catalogPath, new StandardAnalyzer(), - false); - writer.setMergeFactor(this.mergeFactor); + writer = new IndexWriter(reader.directory(), config); + LogMergePolicy lmp =new LogDocMergePolicy(); + lmp.setMergeFactor(this.mergeFactor); + config.setMergePolicy(lmp); + long timeBefore = System.currentTimeMillis(); - writer.optimize(); + //TODO http://blog.trifork.com/2011/11/21/simon-says-optimize-is-bad-for-you/ + //writer.optimize(); long timeAfter = System.currentTimeMillis(); double numSeconds = ((timeAfter - timeBefore) * 1.0) / DOUBLE; LOG.log(Level.INFO, "LuceneCatalog: [" + this.catalogPath http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java index f7ba6e6..4a9641f 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java @@ -18,14 +18,10 @@ package org.apache.oodt.cas.filemgr.tools; import org.apache.lucene.index.Term; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.RangeQuery; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.util.BytesRef; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria; @@ -139,13 +135,12 @@ public final class QueryTool { aT.field(), aT.text())); } } - } else if (luceneQuery instanceof RangeQuery) { - Term startT = ((RangeQuery) luceneQuery).getLowerTerm(); - Term endT = ((RangeQuery) luceneQuery).getUpperTerm(); - casQuery.addCriterion(new RangeQueryCriteria(startT - .field(), startT.text(), endT.text())); + } else if (luceneQuery instanceof TermRangeQuery) { + BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm(); + BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm(); + casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString())); } else if (luceneQuery instanceof BooleanQuery) { - BooleanClause[] clauses = ((BooleanQuery) luceneQuery).getClauses(); + List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses(); for (BooleanClause clause : clauses) { generateCASQuery(casQuery, (clause).getQuery()); } http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java index 3b476e4..a765b44 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java @@ -19,6 +19,7 @@ package org.apache.oodt.cas.filemgr.tools; //JDK imports import java.io.IOException; +import java.nio.file.Paths; import java.util.List; import java.util.Vector; import java.util.logging.Level; @@ -26,15 +27,10 @@ import java.util.logging.Logger; //Lucene imports import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Hits; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.RangeQuery; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.*; +import org.apache.lucene.store.FSDirectory; /** * @author mattmann @@ -66,6 +62,7 @@ public final class RangeQueryTester { private static final Logger LOG = Logger.getLogger(RangeQueryTester.class .getName()); + DirectoryReader reader; /** * */ @@ -75,12 +72,16 @@ public final class RangeQueryTester { public List doRangeQuery(String productTypeId) { List products = null; IndexSearcher searcher = null; - try { - searcher = new IndexSearcher(this.indexPath); + reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.indexPath))); + } catch (IOException e) { + e.printStackTrace(); + } + try { + searcher = new IndexSearcher(reader); // construct a Boolean query here - BooleanQuery booleanQuery = new BooleanQuery(); + BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); // add the product type as the first clause TermQuery prodTypeTermQuery = new TermQuery(new Term( @@ -99,8 +100,8 @@ public final class RangeQueryTester { this.startFieldEndValue); } - RangeQuery query1 = new RangeQuery(startFieldStartTerm, - startFieldEndTerm, true); + TermRangeQuery query1 = new TermRangeQuery(startFieldEndTerm.field(),startFieldStartTerm.bytes(), + startFieldEndTerm.bytes(), true, true); booleanQuery.add(query1, BooleanClause.Occur.MUST); if (this.endFieldName != null @@ -117,18 +118,22 @@ public final class RangeQueryTester { this.endFieldEndValue); } - RangeQuery query2 = new RangeQuery(endFieldStartTerm, - endFieldEndTerm, true); + TermRangeQuery query2 = new TermRangeQuery(endFieldEndTerm.field(),endFieldStartTerm.bytes(), + endFieldEndTerm.bytes(), true, true); booleanQuery.add(query2, BooleanClause.Occur.MUST); } Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.STRING, true)); - Hits hits = searcher.search(booleanQuery, sort); - if (hits.length() > 0) { - products = new Vector(hits.length()); - for (int i = 0; i < hits.length(); i++) { - Document productDoc = hits.doc(i); + SortField.Type.STRING, true)); + //TODO Fix number + TopFieldDocs topDocs = searcher.search(booleanQuery.build(), 1, sort); + ScoreDoc[] hits = topDocs.scoreDocs; + + if (topDocs.totalHits > 0) { + products = new Vector(topDocs.totalHits); + for (int i = 0; i < topDocs.totalHits; i++) { + Document productDoc = searcher.doc(hits[i].doc); + products.add(productDoc.get("reference_data_store")); } } else { @@ -146,7 +151,8 @@ public final class RangeQueryTester { } finally { if (searcher != null) { try { - searcher.close(); + //TODO CLOSE SEARCH + // searcher.close(); } catch (Exception ignore) { } }
