Repository: oodt Updated Branches: refs/heads/curator [created] 9897fcff9
slight tweaks to fix build Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/9897fcff Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/9897fcff Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/9897fcff Branch: refs/heads/curator Commit: 9897fcff9f4e9efb7b25eb17b1ba9093e4f2b8c5 Parents: a47b088 Author: Tom Barber <[email protected]> Authored: Thu Mar 9 12:35:19 2017 +0000 Committer: Tom Barber <[email protected]> Committed: Thu Mar 9 12:35:19 2017 +0000 ---------------------------------------------------------------------- core/pom.xml | 9 +- curator2/pom.xml | 2 +- .../oodt/cas/filemgr/catalog/LuceneCatalog.java | 540 ++++++++++--------- 3 files changed, 281 insertions(+), 270 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/9897fcff/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 35f3a70..514e51c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -70,6 +70,11 @@ the License. <dependencyManagement> <dependencies> <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-solrj</artifactId> + <version>6.4.2</version> + </dependency> + <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk</artifactId> <version>1.7.4</version> @@ -416,6 +421,7 @@ the License. <version>6.2.1</version> </dependency> <dependency> + <groupId>org.apache.tika</groupId> <artifactId>tika-core</artifactId> <version>1.10</version> @@ -619,7 +625,6 @@ mm <artifactId>maven-javadoc-plugin</artifactId> <version>2.9</version> <configuration> - <outputEncoding>UTF-8</outputEncoding> <charset>UTF-8</charset> <docencoding>UTF-8</docencoding> <encoding>UTF-8</encoding> @@ -662,7 +667,6 @@ mm <artifactId>maven-javadoc-plugin</artifactId> <version>2.10.4</version> <configuration> - <outputEncoding>UTF-8</outputEncoding> <charset>UTF-8</charset> <docencoding>UTF-8</docencoding> <encoding>UTF-8</encoding> @@ -972,7 +976,6 @@ mm <artifactId>maven-javadoc-plugin</artifactId> <version>2.10.3</version> <configuration> - <outputEncoding>UTF-8</outputEncoding> <charset>UTF-8</charset> <docencoding>UTF-8</docencoding> <encoding>UTF-8</encoding> http://git-wip-us.apache.org/repos/asf/oodt/blob/9897fcff/curator2/pom.xml ---------------------------------------------------------------------- diff --git a/curator2/pom.xml b/curator2/pom.xml index fa2017e..7cd7edd 100644 --- a/curator2/pom.xml +++ b/curator2/pom.xml @@ -20,7 +20,7 @@ the License. <parent> <groupId>org.apache.oodt</groupId> <artifactId>oodt-core</artifactId> - <version>0.13-SNAPSHOT</version> + <version>1.1-SNAPSHOT</version> <relativePath>../core/pom.xml</relativePath> </parent> <artifactId>cas-curator-engine</artifactId> http://git-wip-us.apache.org/repos/asf/oodt/blob/9897fcff/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java index 6199a33..1372ae2 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java @@ -53,13 +53,13 @@ import java.util.logging.Logger; * @author bfoster * @author luca * @version $Revision$ - * + * * <p> * An implementation of a File {@link Catalog} using Apache's popular <a * href="http://lucene.apache.org";>Lucene</a> text indexing engine as a * backend. * </p> - * + * */ public class LuceneCatalog implements Catalog { Directory indexDir = null; @@ -98,7 +98,7 @@ public class LuceneCatalog implements Catalog { /** - * + * * @param idxFilePath * A file path pointing to the lucene index directory for this * catalog. @@ -107,22 +107,22 @@ public class LuceneCatalog implements Catalog { * @param pgSize * The size of pages to be used when doing pagination of the * catalog. - * + * * @param commitTimeout * The amount of time (in seconds) that should be flowed down to * the Lucene IndexReader and IndexWriters for their commit lock * timeout property. - * + * * @param writeTimeout * The amount of time (in seconds) that should be flowed down to * the Lucene IndexWriters for their commit lock timeout * property. - * + * * @param mergeFactor * The merge factor to use when writing to the index. */ public LuceneCatalog(String idxFilePath, ValidationLayer vLayer, - int pgSize, long commitTimeout, long writeTimeout, int mergeFactor) { + int pgSize, long commitTimeout, long writeTimeout, int mergeFactor) { this.indexFilePath = idxFilePath; this.valLayer = vLayer; this.pageSize = pgSize; @@ -144,47 +144,55 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addMetadata(org.apache.oodt.cas.metadata.Metadata, * org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void addMetadata(Metadata m, Product product) - throws CatalogException { + throws CatalogException { if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { CompleteProduct p = CATALOG_CACHE.get(product.getProductId()); - p.setMetadata(m); - if (hasMetadataAndRefs(p)) { - LOG.log(Level.FINE, - "metadata and references present for product: [" + p.setMetadata(m); + if (hasMetadataAndRefs(p)) { + LOG.log(Level.FINE, + "metadata and references present for product: [" + product.getProductId() + "]"); - addCompleteProductToIndex(p); - // now remove its entry from the cache - CATALOG_CACHE.remove(product.getProductId()); + addCompleteProductToIndex(p); + // now remove its entry from the cache + CATALOG_CACHE.remove(product.getProductId()); } } else{ - // move product from index to cache - // it will be moved back after metadata is added - getCompleteProductById(product.getProductId(), true, true); - LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); - removeProduct(product); + // move product from index to cache + // it will be moved back after metadata is added + CompleteProduct p = getCompleteProductById(product.getProductId(), true, true); + LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); + removeProduct(product); + if (hasMetadataAndRefs(p)) { + LOG.log(Level.FINE, + "metadata and references present for product: [" + + product.getProductId() + "]"); + p.setMetadata(m); + addCompleteProductToIndex(p); + // now remove its entry from the cache + } } } /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#removeMetadata(org.apache.oodt.cas.metadata.Metadata, * org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void removeMetadata(Metadata m, Product product) - throws CatalogException { + throws CatalogException { CompleteProduct p; if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { - p = CATALOG_CACHE.get(product.getProductId()); + p = CATALOG_CACHE.get(product.getProductId()); } else{ String prodId = product.getProductId(); @@ -199,21 +207,21 @@ public class LuceneCatalog implements Catalog { List<String> metadataTypes = new ArrayList<String>(); if (valLayer!=null) { - try { - // remove metadata elements specified by validation layer - for (Element element : valLayer.getElements(product.getProductType())) { - metadataTypes.add(element.getElementName()); - } - } catch (ValidationLayerException e) { - LOG.log(Level.SEVERE, e.getMessage()); - throw new CatalogException( - "ValidationLayerException when trying to obtain element list for product type: " - + product.getProductType().getName() - + ": Message: " + e.getMessage(), e); - } + try { + // remove metadata elements specified by validation layer + for (Element element : valLayer.getElements(product.getProductType())) { + metadataTypes.add(element.getElementName()); + } + } catch (ValidationLayerException e) { + LOG.log(Level.SEVERE, e.getMessage()); + throw new CatalogException( + "ValidationLayerException when trying to obtain element list for product type: " + + product.getProductType().getName() + + ": Message: " + e.getMessage(), e); + } } else { - // remove all metadata - metadataTypes = currMet.getAllKeys(); + // remove all metadata + metadataTypes = currMet.getAllKeys(); } for (String name : metadataTypes) { @@ -224,8 +232,8 @@ public class LuceneCatalog implements Catalog { if (hasMetadataAndRefs(p)) { LOG.log(Level.FINE, - "metadata and references present for product: [" - + product.getProductId() + "]"); + "metadata and references present for product: [" + + product.getProductId() + "]"); addCompleteProductToIndex(p); // now remove its entry from the cache CATALOG_CACHE.remove(product.getProductId()); @@ -234,15 +242,15 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addProduct(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void addProduct(Product product) - throws CatalogException { + throws CatalogException { if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { throw new CatalogException( "Attempt to add a product that already existed: product: [" - + product.getProductName() + "]"); + + product.getProductName() + "]"); @@ -270,16 +278,16 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#modifyProduct(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void modifyProduct(Product product) - throws CatalogException { + throws CatalogException { if (product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { LOG.log(Level.FINE, "Modifying product: [" + product.getProductId() - + "]: found product in cache!"); + + "]: found product in cache!"); CompleteProduct cp = CATALOG_CACHE.get(product - .getProductId()); + .getProductId()); cp.setProduct(product); } else { // need to grab the metadata for the existing product, and make sure @@ -300,62 +308,62 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#removeProduct(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void removeProduct(Product product) - throws CatalogException { + throws CatalogException { removeProductDocument(product); } /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#setProductTransferStatus(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void setProductTransferStatus(Product product) - throws CatalogException { + throws CatalogException { LOG.log(Level.FINE, - "LuceneCatalog: seting product transfer status to: [" - + product.getTransferStatus() + "] for " + "product: [" - + product.getProductId() + "]"); + "LuceneCatalog: seting product transfer status to: [" + + product.getTransferStatus() + "] for " + "product: [" + + product.getProductId() + "]"); modifyProduct(product); } /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addProductReferences(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void addProductReferences(Product product) - throws CatalogException { + throws CatalogException { if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { CompleteProduct p = CATALOG_CACHE.get(product .getProductId()); p.getProduct().setProductReferences(product.getProductReferences()); - if (hasMetadataAndRefs(p)) { - LOG.log(Level.FINE, - "metadata and references present for product: [" + if (hasMetadataAndRefs(p)) { + LOG.log(Level.FINE, + "metadata and references present for product: [" + product.getProductId() + "]"); - addCompleteProductToIndex(p); - // now remove its entry from the cache - CATALOG_CACHE.remove(product.getProductId()); - } + addCompleteProductToIndex(p); + // now remove its entry from the cache + CATALOG_CACHE.remove(product.getProductId()); + } } else{ - // move product from index to cache - // it will be moved back after metadata is added - getCompleteProductById(product.getProductId(), true, true); - LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); - removeProduct(product); + // move product from index to cache + // it will be moved back after metadata is added + getCompleteProductById(product.getProductId(), true, true); + LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); + removeProduct(product); } } /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductById(java.lang.String) */ public Product getProductById(String productId) throws CatalogException { @@ -364,23 +372,23 @@ public class LuceneCatalog implements Catalog { } private Product getProductById(String productId, boolean getRefs) - throws CatalogException { + throws CatalogException { CompleteProduct prod = getCompleteProductById(productId, getRefs); return prod.getProduct(); } private CompleteProduct getCompleteProductById(String productId) - throws CatalogException { + throws CatalogException { return getCompleteProductById(productId, false); } private CompleteProduct getCompleteProductById(String productId, - boolean getRefs) throws CatalogException { + boolean getRefs) throws CatalogException { return getCompleteProductById(productId, getRefs, false); } private CompleteProduct getCompleteProductById(String productId, - boolean getRefs, boolean getMet) throws CatalogException { + boolean getRefs, boolean getMet) throws CatalogException { IndexSearcher searcher = null; try { try { @@ -398,7 +406,7 @@ public class LuceneCatalog implements Catalog { // should be exactly 1 hit if (topDocs.totalHits == 0) { - throw new CatalogException("Product: [" + productId + "] NOT found in the catalog!"); + throw new CatalogException("Product: [" + productId + "] NOT found in the catalog!"); } if (topDocs.totalHits > 1) { throw new CatalogException("Product: [" + productId+ "] is not unique in the catalog!"); @@ -406,12 +414,12 @@ public class LuceneCatalog implements Catalog { Document productDoc = searcher.doc(hits[0].doc); return toCompleteProduct(productDoc, getRefs, - getMet); + getMet); } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { @@ -425,7 +433,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductByName(java.lang.String) */ public Product getProductByName(String productName) throws CatalogException { @@ -433,7 +441,7 @@ public class LuceneCatalog implements Catalog { } private Product getProductByName(String productName, boolean getRefs) - throws CatalogException { + throws CatalogException { IndexSearcher searcher = null; try { try { @@ -445,7 +453,7 @@ public class LuceneCatalog implements Catalog { Term productIdTerm = new Term("product_name", productName); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(query, 1, sort); if(check.totalHits>0) { @@ -458,11 +466,11 @@ public class LuceneCatalog implements Catalog { // just get the first hit back Document productDoc = searcher.doc(hits[0].doc); CompleteProduct prod = toCompleteProduct(productDoc, getRefs, - false); + false); return prod.getProduct(); } else { LOG.log(Level.FINEST, "Request for product by name: [" - + productName + "] returned no results"); + + productName + "] returned no results"); return null; } } @@ -472,9 +480,9 @@ public class LuceneCatalog implements Catalog { } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { @@ -488,7 +496,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductReferences(org.apache.oodt.cas.filemgr.structs.Product) */ public List<Reference> getProductReferences(Product product) throws CatalogException { @@ -502,7 +510,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProducts() */ public List<Product> getProducts() throws CatalogException { @@ -523,7 +531,7 @@ public class LuceneCatalog implements Catalog { Term productIdTerm = new Term("myfield", "myvalue"); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(query, 1, sort); TopDocs topDocs = searcher.search(query, check.totalHits, sort); @@ -536,20 +544,20 @@ public class LuceneCatalog implements Catalog { for (ScoreDoc hit : hits) { Document productDoc = searcher.doc(hit.doc); CompleteProduct prod = toCompleteProduct(productDoc, - getRefs, false); + getRefs, false); products.add(prod.getProduct()); } } else { LOG.log(Level.FINEST, - "Request for products returned no results"); + "Request for products returned no results"); return null; } } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { @@ -565,16 +573,16 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductsByProductType(org.apache.oodt.cas.filemgr.structs.ProductType) */ public List<Product> getProductsByProductType(ProductType type) - throws CatalogException { + throws CatalogException { return getProductsByProductType(type, false); } private List<Product> getProductsByProductType(ProductType type, boolean getRefs) - throws CatalogException { + throws CatalogException { IndexSearcher searcher = null; List<Product> products = null; @@ -586,10 +594,10 @@ public class LuceneCatalog implements Catalog { } searcher = new IndexSearcher(reader); Term productIdTerm = new Term("product_type_id", type - .getProductTypeId()); + .getProductTypeId()); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(query, 1, sort); TopDocs topDocs = searcher.search(query, check.totalHits, sort); @@ -602,26 +610,26 @@ public class LuceneCatalog implements Catalog { for (ScoreDoc hit : hits) { Document productDoc = searcher.doc(hit.doc); CompleteProduct prod = toCompleteProduct(productDoc, - getRefs, false); + getRefs, false); products.add(prod.getProduct()); } } else { LOG.log(Level.FINEST, "Request for products by type: [" - + type.getProductTypeId() + "] returned no results"); + + type.getProductTypeId() + "] returned no results"); return null; } } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { try { //TODO CLOSE - } catch (Exception ignore) { + } catch (Exception ignore) { } } } @@ -650,9 +658,9 @@ public class LuceneCatalog implements Catalog { // should be exactly 1 hit if (topDocs.totalHits != 1) { throw new CatalogException("Product: [" - + product.getProductId() - + "] is not unique in the catalog! Num Hits: [" - + hits.length + "]"); + + product.getProductId() + + "] is not unique in the catalog! Num Hits: [" + + hits.length + "]"); } Document productDoc = searcher.doc(hits[0].doc); @@ -661,20 +669,20 @@ public class LuceneCatalog implements Catalog { return prod.getMetadata(); } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { try { //TODO CLOSE - } catch (Exception ignore) { + } catch (Exception ignore) { } } } } - + public Metadata getReducedMetadata(Product product, List<String> elements) throws CatalogException { Metadata fullMetadata = getMetadata(product); Metadata reducedMetadata = new Metadata(); @@ -688,7 +696,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#query(org.apache.oodt.cas.filemgr.structs.Query, * org.apache.oodt.cas.filemgr.structs.ProductType) */ @@ -711,7 +719,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getTopNProducts(int) */ public List<Product> getTopNProducts(int n) throws CatalogException { @@ -732,9 +740,9 @@ public class LuceneCatalog implements Catalog { booleanQuery.add(tq, BooleanClause.Occur.MUST); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery - + "]"); + + "]"); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(booleanQuery.build(), 1, sort); if(check.totalHits>0) { @@ -748,7 +756,7 @@ public class LuceneCatalog implements Catalog { while (products.size() < Math.min(n, hits.length)) { Document productDoc = searcher.doc(hits[i].doc); CompleteProduct prod = toCompleteProduct(productDoc, false, - false); + false); products.add(prod.getProduct()); i++; } @@ -762,9 +770,9 @@ public class LuceneCatalog implements Catalog { } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { @@ -780,12 +788,12 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getTopNProducts(int, * org.apache.oodt.cas.filemgr.structs.ProductType) */ public List<Product> getTopNProducts(int n, ProductType type) - throws CatalogException { + throws CatalogException { int numPages = 1; if (n > this.pageSize) { numPages = n / this.pageSize + (n % this.pageSize == 0 ? 0 : 1); @@ -796,11 +804,11 @@ public class LuceneCatalog implements Catalog { for (int pageNum = 1; pageNum < numPages + 1; pageNum++) { List<Product> pageProducts = paginateQuery(query, type, pageNum, null); - products.addAll(pageProducts); + products.addAll(pageProducts); } if(n<=products.size()) { - return products.subList(0, n); + return products.subList(0, n); } return products; @@ -808,7 +816,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getValidationLayer() */ public ValidationLayer getValidationLayer() { @@ -817,7 +825,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getNumProducts(org.apache.oodt.cas.filemgr.structs.ProductType) */ public int getNumProducts(ProductType type) throws CatalogException { @@ -827,29 +835,29 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.util.Pagination#getFirstPage(org.apache.oodt.cas.filemgr.structs.ProductType) */ public ProductPage getFirstPage(ProductType type) { ProductPage firstPage = new ProductPage(); List<Product> products; Query query = new Query(); - + // now construct the page firstPage.setPageNum(1); firstPage.setPageSize(pageSize); try { - products = paginateQuery(query, type, 1, firstPage); + products = paginateQuery(query, type, 1, firstPage); } catch (CatalogException e) { LOG.log(Level.WARNING, - "CatalogException getting first page for product type: [" - + type.getProductTypeId() - + "] from catalog: Message: " + e.getMessage()); + "CatalogException getting first page for product type: [" + + type.getProductTypeId() + + "] from catalog: Message: " + e.getMessage()); return null; } // There are no products and thus no first page if (products == null || (products.size() == 0)) { - return null; + return null; } firstPage.setPageProducts(products); @@ -859,7 +867,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.util.Pagination#getLastProductPage(org.apache.oodt.cas.filemgr.structs.ProductType) */ public ProductPage getLastProductPage(ProductType type) { @@ -867,22 +875,22 @@ public class LuceneCatalog implements Catalog { ProductPage firstPage = getFirstPage(type); List<Product> products; Query query = new Query(); - + // now construct the page lastPage.setPageNum(firstPage.getTotalPages()); lastPage.setPageSize(pageSize); try { products = paginateQuery(query, type, firstPage.getTotalPages(), lastPage); } catch (CatalogException e) { - LOG.log(Level.WARNING, - "CatalogException getting last page for product type: [" - + type.getProductTypeId() - + "] from catalog: Message: " + e.getMessage()); - return null; + LOG.log(Level.WARNING, + "CatalogException getting last page for product type: [" + + type.getProductTypeId() + + "] from catalog: Message: " + e.getMessage()); + return null; } // There are no products thus there is no last page if (products == null || (products.size() == 0)) { - return null; + return null; } lastPage.setPageProducts(products); @@ -891,7 +899,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.util.Pagination#getNextPage(org.apache.oodt.cas.filemgr.structs.ProductType, * org.apache.oodt.cas.filemgr.structs.ProductPage) */ @@ -918,14 +926,14 @@ public class LuceneCatalog implements Catalog { products = paginateQuery(query, type, currentPage.getPageNum() + 1, nextPage); } catch (CatalogException e) { LOG.log(Level.WARNING, - "CatalogException getting next page for product type: [" - + type.getProductTypeId() - + "] from catalog: Message: " + e.getMessage()); + "CatalogException getting next page for product type: [" + + type.getProductTypeId() + + "] from catalog: Message: " + e.getMessage()); return null; } // There are no products and thus no next page if (products == null || (products.size() == 0)) { - return null; + return null; } nextPage.setPageProducts(products); @@ -934,7 +942,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.util.Pagination#getPrevPage(org.apache.oodt.cas.filemgr.structs.ProductType, * org.apache.oodt.cas.filemgr.structs.ProductPage) */ @@ -962,15 +970,15 @@ public class LuceneCatalog implements Catalog { products = paginateQuery(query, type, currentPage.getPageNum() - 1, prevPage); } catch (CatalogException e) { LOG.log(Level.WARNING, - "CatalogException getting prev page for product type: [" - + type.getProductTypeId() - + "] from catalog: Message: " + e.getMessage()); + "CatalogException getting prev page for product type: [" + + type.getProductTypeId() + + "] from catalog: Message: " + e.getMessage()); return null; } - + // There are no products and thus no pages if (products == null || (products.size() == 0)) { - return null; + return null; } prevPage.setPageProducts(products); @@ -979,12 +987,12 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#pagedQuery(org.apache.oodt.cas.filemgr.structs.Query, * org.apache.oodt.cas.filemgr.structs.ProductType, int) */ public ProductPage pagedQuery(Query query, ProductType type, int pageNum) - throws CatalogException { + throws CatalogException { try { ProductPage retPage = new ProductPage(); retPage.setPageNum(pageNum); @@ -994,15 +1002,15 @@ public class LuceneCatalog implements Catalog { } catch (Exception e) { LOG.log(Level.SEVERE, e.getMessage()); LOG.log(Level.WARNING, - "CatalogException when doing paged product query: Message: " - + e.getMessage()); + "CatalogException when doing paged product query: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } } private synchronized void removeProductDocument(Product product) - throws CatalogException { + throws CatalogException { try { reader = DirectoryReader.open(indexDir); @@ -1011,8 +1019,8 @@ public class LuceneCatalog implements Catalog { } try { LOG.log(Level.FINE, - "LuceneCatalog: remove document from index for product: [" - + product.getProductId() + "]"); + "LuceneCatalog: remove document from index for product: [" + + product.getProductId() + "]"); IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); @@ -1020,15 +1028,15 @@ public class LuceneCatalog implements Catalog { lmp.setMergeFactor(mergeFactor); config.setMergePolicy(lmp); - IndexWriter writer = new IndexWriter(indexDir, config); + IndexWriter writer = new IndexWriter(indexDir, config); writer.deleteDocuments(new Term("product_id", product - .getProductId())); + .getProductId())); writer.close(); } catch (IOException e) { LOG.log(Level.WARNING, "Exception removing product: [" - + product.getProductName() + "] from index: Message: " - + e.getMessage()); + + product.getProductName() + "] from index: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (reader != null) { @@ -1043,7 +1051,7 @@ public class LuceneCatalog implements Catalog { } private synchronized void addCompleteProductToIndex(CompleteProduct cp) - throws CatalogException { + throws CatalogException { IndexWriter writer = null; try { /*writer = new IndexWriter(indexFilePath, new StandardAnalyzer(), @@ -1064,11 +1072,11 @@ public class LuceneCatalog implements Catalog { // TODO: determine a better way to optimize the index } catch (Exception e) { LOG.log(Level.WARNING, "Unable to index product: [" - + cp.getProduct().getProductName() + "]: Message: " - + e.getMessage(), e); + + cp.getProduct().getProductName() + "]: Message: " + + e.getMessage(), e); throw new CatalogException("Unable to index product: [" - + cp.getProduct().getProductName() + "]: Message: " - + e.getMessage(), e); + + cp.getProduct().getProductName() + "]: Message: " + + e.getMessage(), e); } finally { try { if (writer != null) { @@ -1086,7 +1094,7 @@ public class LuceneCatalog implements Catalog { } private CompleteProduct toCompleteProduct(Document doc, boolean getRefs, - boolean getMetadata) { + boolean getMetadata) { Product product = new Product(); Metadata metadata = new Metadata(); CompleteProduct completeProduct = new CompleteProduct(); @@ -1108,39 +1116,39 @@ public class LuceneCatalog implements Catalog { List<String> names = new ArrayList<String>(); if (valLayer!=null) { - // only add metadata elements specified by validation layer - try { - for (Element element : valLayer.getElements(type)) { - names.add(element.getElementName()); - } - } catch (ValidationLayerException e) { - LOG.log(Level.WARNING, - "Unable to obtain metadata for product: [" - + product.getProductName() + "]: Message: " - + e.getMessage()); - } + // only add metadata elements specified by validation layer + try { + for (Element element : valLayer.getElements(type)) { + names.add(element.getElementName()); + } + } catch (ValidationLayerException e) { + LOG.log(Level.WARNING, + "Unable to obtain metadata for product: [" + + product.getProductName() + "]: Message: " + + e.getMessage()); + } } else { - // add all metadata elements found in document - List<IndexableField> fields = doc.getFields(); + // add all metadata elements found in document + List<IndexableField> fields = doc.getFields(); for(IndexableField field: fields){ if (!names.contains(field.name())) { names.add(field.name()); } } - + } // loop over field names to add to metadata for (String name : names) { - if (metadata.getAllMetadata(name)==null || metadata.getAllMetadata(name).size()==0) { - String[] elemValues = doc.getValues(name); - - if (elemValues != null && elemValues.length > 0) { + if (metadata.getAllMetadata(name)==null || metadata.getAllMetadata(name).size()==0) { + String[] elemValues = doc.getValues(name); + + if (elemValues != null && elemValues.length > 0) { for (String elemValue : elemValues) { metadata.addMetadata(name, elemValue); } - } - } + } + } } completeProduct.setMetadata(metadata); @@ -1154,7 +1162,7 @@ public class LuceneCatalog implements Catalog { String[] refMimeTypes = doc.getValues("reference_mimeType"); if ((origRefs.length == dataStoreRefs.length) - && (origRefs.length == refLengths.length)) { + && (origRefs.length == refLengths.length)) { List<Reference> references = new Vector<Reference>(); for (int i = 0; i < origRefs.length; i++) { Reference r = new Reference(); @@ -1170,11 +1178,11 @@ public class LuceneCatalog implements Catalog { product.setProductReferences(references); } else { LOG.log(Level.WARNING, "Number of original refs: [" - + origRefs.length + "] for product: [" - + product.getProductName() - + "] not equivalent to number of data store refs: [" - + dataStoreRefs.length - + "]: Skipping product references"); + + origRefs.length + "] for product: [" + + product.getProductName() + + "] not equivalent to number of data store refs: [" + + dataStoreRefs.length + + "]: Skipping product references"); } } @@ -1187,63 +1195,63 @@ public class LuceneCatalog implements Catalog { //TODO CHECK STORED TYPES // add the product information doc.add(new Field("product_id", product.getProductId(), - StringField.TYPE_STORED)); + StringField.TYPE_STORED)); doc.add(new Field("product_name", product.getProductName(), - StringField.TYPE_STORED)); + StringField.TYPE_STORED)); doc.add(new Field("product_structure", product.getProductStructure(), - StringField.TYPE_STORED)); + StringField.TYPE_STORED)); doc - .add(new Field("product_transfer_status", product - .getTransferStatus(), StringField.TYPE_STORED)); + .add(new Field("product_transfer_status", product + .getTransferStatus(), StringField.TYPE_STORED)); // product type doc - .add(new Field("product_type_id", product.getProductType() - .getProductTypeId(), StringField.TYPE_STORED)); + .add(new Field("product_type_id", product.getProductType() + .getProductTypeId(), StringField.TYPE_STORED)); doc.add(new Field("product_type_name", product.getProductType() - .getName(), StringField.TYPE_STORED)); + .getName(), StringField.TYPE_STORED)); doc.add(new Field("product_type_desc", product.getProductType() - .getDescription() != null ? product.getProductType() - .getDescription() : "", StringField.TYPE_STORED)); + .getDescription() != null ? product.getProductType() + .getDescription() : "", StringField.TYPE_STORED)); doc.add(new Field("product_type_repoPath", product.getProductType() - .getProductRepositoryPath() != null ? product.getProductType() - .getProductRepositoryPath() : "", StringField.TYPE_STORED)); + .getProductRepositoryPath() != null ? product.getProductType() + .getProductRepositoryPath() : "", StringField.TYPE_STORED)); doc.add(new Field("product_type_versioner", product.getProductType() - .getVersioner() != null ? product.getProductType() - .getVersioner() : "", StringField.TYPE_STORED)); - + .getVersioner() != null ? product.getProductType() + .getVersioner() : "", StringField.TYPE_STORED)); + // write metadata fields to the Lucene document List<String> keys = new ArrayList<String>(); // validation layer: add only specifically configured keys if (valLayer!=null) { - List<Element> elements = quietGetElements(product.getProductType()); + List<Element> elements = quietGetElements(product.getProductType()); for (Element element : elements) { String key = element.getElementName(); keys.add(key); } - // no validation layer: add all keys that are NOT already in doc - // (otherwise some keys such as the product_* keys are duplicated) + // no validation layer: add all keys that are NOT already in doc + // (otherwise some keys such as the product_* keys are duplicated) } else { - for (String key : metadata.getAllKeys()) { - if (doc.getField(key)==null) { - keys.add(key); - } - } + for (String key : metadata.getAllKeys()) { + if (doc.getField(key)==null) { + keys.add(key); + } + } } for (String key : keys) { - List<String> values = metadata.getAllMetadata(key); + List<String> values = metadata.getAllMetadata(key); if (values == null) { LOG - .log( - Level.WARNING, - "No Metadata specified for product [" - + product.getProductName() - + "] for required field [" - + key - + "]: Attempting to continue processing metadata"); + .log( + Level.WARNING, + "No Metadata specified for product [" + + product.getProductName() + + "] for required field [" + + key + + "]: Attempting to continue processing metadata"); continue; } @@ -1258,7 +1266,7 @@ public class LuceneCatalog implements Catalog { // add the product references for (Reference r : product.getProductReferences()) { doc.add(new Field("reference_orig", r.getOrigReference(), - StringField.TYPE_STORED)); + StringField.TYPE_STORED)); doc .add(new Field("reference_data_store", r .getDataStoreReference(), StringField.TYPE_STORED)); @@ -1296,7 +1304,7 @@ public class LuceneCatalog implements Catalog { } private int getNumHits(Query query, ProductType type) - throws CatalogException { + throws CatalogException { IndexSearcher searcher = null; int numHits = -1; @@ -1313,7 +1321,7 @@ public class LuceneCatalog implements Catalog { // add the product type as the first clause org.apache.lucene.search.Query prodTypeTermQuery = new TermQuery(new Term( - "product_type_id", type.getProductTypeId())); + "product_type_id", type.getProductTypeId())); booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST); //convert filemgr query into a lucene query @@ -1322,7 +1330,7 @@ public class LuceneCatalog implements Catalog { } LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery - + "]"); + + "]"); //TODO FIX returned records TopDocs hits = searcher.search(booleanQuery.build(), 1); @@ -1331,9 +1339,9 @@ public class LuceneCatalog implements Catalog { numHits = hits.totalHits; } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage()); } finally { if (searcher != null) { @@ -1348,7 +1356,7 @@ public class LuceneCatalog implements Catalog { } private List<Product> paginateQuery(Query query, ProductType type, int pageNum, ProductPage page) - throws CatalogException { + throws CatalogException { List<Product> products = null; IndexSearcher searcher = null; @@ -1370,25 +1378,25 @@ public class LuceneCatalog implements Catalog { // add the product type as the first clause TermQuery prodTypeTermQuery = new TermQuery(new Term( - "product_type_id", type.getProductTypeId())); + "product_type_id", type.getProductTypeId())); booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST); - + //convert filemgr query into a lucene query for (QueryCriteria queryCriteria : query.getCriteria()) { booleanQuery.add(this.getQuery(queryCriteria), BooleanClause.Occur.MUST); } - + Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery - + "]"); + + "]"); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(booleanQuery.build(),1, sort); TopDocs topDocs = searcher.search(booleanQuery.build(),check.totalHits, sort); // Calculate page size and set it while we have the results if (page != null) { - page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize)); + page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize)); } ScoreDoc[] hits = topDocs.scoreDocs; @@ -1404,11 +1412,11 @@ public class LuceneCatalog implements Catalog { products = new Vector<Product>(pageSize); for (int i = startNum; i < Math.min(hits.length, - (startNum + pageSize)); i++) { + (startNum + pageSize)); i++) { Document productDoc = searcher.doc(hits[i].doc); CompleteProduct prod = toCompleteProduct(productDoc, - false, false); + false, false); products.add(prod.getProduct()); } } else { @@ -1417,22 +1425,22 @@ public class LuceneCatalog implements Catalog { Document productDoc = searcher.doc(hits[i].doc); CompleteProduct prod = toCompleteProduct(productDoc, - false, false); + false, false); products.add(prod.getProduct()); } } } else { LOG.log(Level.WARNING, "Query: [" + query - + "] for Product Type: [" + type.getProductTypeId() - + "] returned no results"); + + "] for Product Type: [" + type.getProductTypeId() + + "] returned no results"); } } catch (Exception e) { LOG.log(Level.SEVERE, e.getMessage()); LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage()); } finally { if (searcher != null) { @@ -1451,19 +1459,19 @@ public class LuceneCatalog implements Catalog { BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); BooleanClause.Occur occur; switch (((BooleanQueryCriteria) queryCriteria).getOperator()) { - case BooleanQueryCriteria.AND: - occur = BooleanClause.Occur.MUST; - break; - case BooleanQueryCriteria.OR: - occur = BooleanClause.Occur.SHOULD; - break; - case BooleanQueryCriteria.NOT: - occur = BooleanClause.Occur.MUST_NOT; - booleanQuery.add(new WildcardQuery(new Term(((BooleanQueryCriteria) queryCriteria) + case BooleanQueryCriteria.AND: + occur = BooleanClause.Occur.MUST; + break; + case BooleanQueryCriteria.OR: + occur = BooleanClause.Occur.SHOULD; + break; + case BooleanQueryCriteria.NOT: + occur = BooleanClause.Occur.MUST_NOT; + booleanQuery.add(new WildcardQuery(new Term(((BooleanQueryCriteria) queryCriteria) .getTerms().get(0).getElementName(), "*")), BooleanClause.Occur.SHOULD); - break; - default: - throw new CatalogException("Invalid BooleanQueryCriteria opertor [" + break; + default: + throw new CatalogException("Invalid BooleanQueryCriteria opertor [" + ((BooleanQueryCriteria) queryCriteria).getOperator() + "]"); } for (QueryCriteria qc : ((BooleanQueryCriteria) queryCriteria).getTerms()) { @@ -1486,10 +1494,10 @@ public class LuceneCatalog implements Catalog { return TermRangeQuery.newStringRange(startTerm.field(), startVal, endVal, inclusive,inclusive); }else { throw new CatalogException("Invalid QueryCriteria [" - + queryCriteria.getClass().getCanonicalName() + "]"); + + queryCriteria.getClass().getCanonicalName() + "]"); } } - + private List<Element> quietGetElements(ProductType type) { List<Element> elementList = new Vector<Element>(); @@ -1497,8 +1505,8 @@ public class LuceneCatalog implements Catalog { elementList = valLayer.getElements(type); } catch (Exception e) { LOG.log(Level.WARNING, - "Exception obtaining elements for product type: [" - + type.getName() + "]: Message: " + e.getMessage()); + "Exception obtaining elements for product type: [" + + type.getName() + "]: Message: " + e.getMessage()); } return elementList; @@ -1565,4 +1573,4 @@ public class LuceneCatalog implements Catalog { } -} \ No newline at end of file +}
