http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java index 4c9ed96..6199a33 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java @@ -53,13 +53,13 @@ import java.util.logging.Logger; * @author bfoster * @author luca * @version $Revision$ - * + * * <p> * An implementation of a File {@link Catalog} using Apache's popular <a * href="http://lucene.apache.org";>Lucene</a> text indexing engine as a * backend. * </p> - * + * */ public class LuceneCatalog implements Catalog { Directory indexDir = null; @@ -98,7 +98,7 @@ public class LuceneCatalog implements Catalog { /** - * + * * @param idxFilePath * A file path pointing to the lucene index directory for this * catalog. @@ -107,22 +107,22 @@ public class LuceneCatalog implements Catalog { * @param pgSize * The size of pages to be used when doing pagination of the * catalog. - * + * * @param commitTimeout * The amount of time (in seconds) that should be flowed down to * the Lucene IndexReader and IndexWriters for their commit lock * timeout property. - * + * * @param writeTimeout * The amount of time (in seconds) that should be flowed down to * the Lucene IndexWriters for their commit lock timeout * property. - * + * * @param mergeFactor * The merge factor to use when writing to the index. */ public LuceneCatalog(String idxFilePath, ValidationLayer vLayer, - int pgSize, long commitTimeout, long writeTimeout, int mergeFactor) { + int pgSize, long commitTimeout, long writeTimeout, int mergeFactor) { this.indexFilePath = idxFilePath; this.valLayer = vLayer; this.pageSize = pgSize; @@ -144,47 +144,47 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addMetadata(org.apache.oodt.cas.metadata.Metadata, * org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void addMetadata(Metadata m, Product product) - throws CatalogException { + throws CatalogException { if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { CompleteProduct p = CATALOG_CACHE.get(product.getProductId()); - p.setMetadata(m); - if (hasMetadataAndRefs(p)) { - LOG.log(Level.FINE, - "metadata and references present for product: [" + p.setMetadata(m); + if (hasMetadataAndRefs(p)) { + LOG.log(Level.FINE, + "metadata and references present for product: [" + product.getProductId() + "]"); - addCompleteProductToIndex(p); - // now remove its entry from the cache - CATALOG_CACHE.remove(product.getProductId()); + addCompleteProductToIndex(p); + // now remove its entry from the cache + CATALOG_CACHE.remove(product.getProductId()); } } else{ - // move product from index to cache - // it will be moved back after metadata is added - getCompleteProductById(product.getProductId(), true, true); - LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); - removeProduct(product); + // move product from index to cache + // it will be moved back after metadata is added + getCompleteProductById(product.getProductId(), true, true); + LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); + removeProduct(product); } } /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#removeMetadata(org.apache.oodt.cas.metadata.Metadata, * org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void removeMetadata(Metadata m, Product product) - throws CatalogException { + throws CatalogException { CompleteProduct p; if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { - p = CATALOG_CACHE.get(product.getProductId()); + p = CATALOG_CACHE.get(product.getProductId()); } else{ String prodId = product.getProductId(); @@ -199,21 +199,21 @@ public class LuceneCatalog implements Catalog { List<String> metadataTypes = new ArrayList<String>(); if (valLayer!=null) { - try { - // remove metadata elements specified by validation layer - for (Element element : valLayer.getElements(product.getProductType())) { - metadataTypes.add(element.getElementName()); - } - } catch (ValidationLayerException e) { - LOG.log(Level.SEVERE, e.getMessage()); - throw new CatalogException( - "ValidationLayerException when trying to obtain element list for product type: " - + product.getProductType().getName() - + ": Message: " + e.getMessage(), e); - } + try { + // remove metadata elements specified by validation layer + for (Element element : valLayer.getElements(product.getProductType())) { + metadataTypes.add(element.getElementName()); + } + } catch (ValidationLayerException e) { + LOG.log(Level.SEVERE, e.getMessage()); + throw new CatalogException( + "ValidationLayerException when trying to obtain element list for product type: " + + product.getProductType().getName() + + ": Message: " + e.getMessage(), e); + } } else { - // remove all metadata - metadataTypes = currMet.getAllKeys(); + // remove all metadata + metadataTypes = currMet.getAllKeys(); } for (String name : metadataTypes) { @@ -224,8 +224,8 @@ public class LuceneCatalog implements Catalog { if (hasMetadataAndRefs(p)) { LOG.log(Level.FINE, - "metadata and references present for product: [" - + product.getProductId() + "]"); + "metadata and references present for product: [" + + product.getProductId() + "]"); addCompleteProductToIndex(p); // now remove its entry from the cache CATALOG_CACHE.remove(product.getProductId()); @@ -234,15 +234,15 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addProduct(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void addProduct(Product product) - throws CatalogException { + throws CatalogException { if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { throw new CatalogException( "Attempt to add a product that already existed: product: [" - + product.getProductName() + "]"); + + product.getProductName() + "]"); @@ -270,16 +270,16 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#modifyProduct(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void modifyProduct(Product product) - throws CatalogException { + throws CatalogException { if (product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { LOG.log(Level.FINE, "Modifying product: [" + product.getProductId() - + "]: found product in cache!"); + + "]: found product in cache!"); CompleteProduct cp = CATALOG_CACHE.get(product - .getProductId()); + .getProductId()); cp.setProduct(product); } else { // need to grab the metadata for the existing product, and make sure @@ -300,62 +300,62 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#removeProduct(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void removeProduct(Product product) - throws CatalogException { + throws CatalogException { removeProductDocument(product); } /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#setProductTransferStatus(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void setProductTransferStatus(Product product) - throws CatalogException { + throws CatalogException { LOG.log(Level.FINE, - "LuceneCatalog: seting product transfer status to: [" - + product.getTransferStatus() + "] for " + "product: [" - + product.getProductId() + "]"); + "LuceneCatalog: seting product transfer status to: [" + + product.getTransferStatus() + "] for " + "product: [" + + product.getProductId() + "]"); modifyProduct(product); } /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addProductReferences(org.apache.oodt.cas.filemgr.structs.Product) */ public synchronized void addProductReferences(Product product) - throws CatalogException { + throws CatalogException { if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) { CompleteProduct p = CATALOG_CACHE.get(product .getProductId()); p.getProduct().setProductReferences(product.getProductReferences()); - if (hasMetadataAndRefs(p)) { - LOG.log(Level.FINE, - "metadata and references present for product: [" + if (hasMetadataAndRefs(p)) { + LOG.log(Level.FINE, + "metadata and references present for product: [" + product.getProductId() + "]"); - addCompleteProductToIndex(p); - // now remove its entry from the cache - CATALOG_CACHE.remove(product.getProductId()); - } + addCompleteProductToIndex(p); + // now remove its entry from the cache + CATALOG_CACHE.remove(product.getProductId()); + } } else{ - // move product from index to cache - // it will be moved back after metadata is added - getCompleteProductById(product.getProductId(), true, true); - LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); - removeProduct(product); + // move product from index to cache + // it will be moved back after metadata is added + getCompleteProductById(product.getProductId(), true, true); + LOG.log(Level.FINE, "Product not found in local cache, retrieved from index"); + removeProduct(product); } } /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductById(java.lang.String) */ public Product getProductById(String productId) throws CatalogException { @@ -364,23 +364,23 @@ public class LuceneCatalog implements Catalog { } private Product getProductById(String productId, boolean getRefs) - throws CatalogException { + throws CatalogException { CompleteProduct prod = getCompleteProductById(productId, getRefs); return prod.getProduct(); } private CompleteProduct getCompleteProductById(String productId) - throws CatalogException { + throws CatalogException { return getCompleteProductById(productId, false); } private CompleteProduct getCompleteProductById(String productId, - boolean getRefs) throws CatalogException { + boolean getRefs) throws CatalogException { return getCompleteProductById(productId, getRefs, false); } private CompleteProduct getCompleteProductById(String productId, - boolean getRefs, boolean getMet) throws CatalogException { + boolean getRefs, boolean getMet) throws CatalogException { IndexSearcher searcher = null; try { try { @@ -398,7 +398,7 @@ public class LuceneCatalog implements Catalog { // should be exactly 1 hit if (topDocs.totalHits == 0) { - throw new CatalogException("Product: [" + productId + "] NOT found in the catalog!"); + throw new CatalogException("Product: [" + productId + "] NOT found in the catalog!"); } if (topDocs.totalHits > 1) { throw new CatalogException("Product: [" + productId+ "] is not unique in the catalog!"); @@ -406,12 +406,12 @@ public class LuceneCatalog implements Catalog { Document productDoc = searcher.doc(hits[0].doc); return toCompleteProduct(productDoc, getRefs, - getMet); + getMet); } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { @@ -425,7 +425,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductByName(java.lang.String) */ public Product getProductByName(String productName) throws CatalogException { @@ -433,7 +433,7 @@ public class LuceneCatalog implements Catalog { } private Product getProductByName(String productName, boolean getRefs) - throws CatalogException { + throws CatalogException { IndexSearcher searcher = null; try { try { @@ -445,7 +445,7 @@ public class LuceneCatalog implements Catalog { Term productIdTerm = new Term("product_name", productName); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(query, 1, sort); if(check.totalHits>0) { @@ -458,11 +458,11 @@ public class LuceneCatalog implements Catalog { // just get the first hit back Document productDoc = searcher.doc(hits[0].doc); CompleteProduct prod = toCompleteProduct(productDoc, getRefs, - false); + false); return prod.getProduct(); } else { LOG.log(Level.FINEST, "Request for product by name: [" - + productName + "] returned no results"); + + productName + "] returned no results"); return null; } } @@ -472,9 +472,9 @@ public class LuceneCatalog implements Catalog { } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { @@ -488,7 +488,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductReferences(org.apache.oodt.cas.filemgr.structs.Product) */ public List<Reference> getProductReferences(Product product) throws CatalogException { @@ -502,7 +502,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProducts() */ public List<Product> getProducts() throws CatalogException { @@ -523,7 +523,7 @@ public class LuceneCatalog implements Catalog { Term productIdTerm = new Term("myfield", "myvalue"); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(query, 1, sort); TopDocs topDocs = searcher.search(query, check.totalHits, sort); @@ -536,20 +536,20 @@ public class LuceneCatalog implements Catalog { for (ScoreDoc hit : hits) { Document productDoc = searcher.doc(hit.doc); CompleteProduct prod = toCompleteProduct(productDoc, - getRefs, false); + getRefs, false); products.add(prod.getProduct()); } } else { LOG.log(Level.FINEST, - "Request for products returned no results"); + "Request for products returned no results"); return null; } } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { @@ -565,16 +565,16 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductsByProductType(org.apache.oodt.cas.filemgr.structs.ProductType) */ public List<Product> getProductsByProductType(ProductType type) - throws CatalogException { + throws CatalogException { return getProductsByProductType(type, false); } private List<Product> getProductsByProductType(ProductType type, boolean getRefs) - throws CatalogException { + throws CatalogException { IndexSearcher searcher = null; List<Product> products = null; @@ -586,10 +586,10 @@ public class LuceneCatalog implements Catalog { } searcher = new IndexSearcher(reader); Term productIdTerm = new Term("product_type_id", type - .getProductTypeId()); + .getProductTypeId()); org.apache.lucene.search.Query query = new TermQuery(productIdTerm); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(query, 1, sort); TopDocs topDocs = searcher.search(query, check.totalHits, sort); @@ -602,26 +602,26 @@ public class LuceneCatalog implements Catalog { for (ScoreDoc hit : hits) { Document productDoc = searcher.doc(hit.doc); CompleteProduct prod = toCompleteProduct(productDoc, - getRefs, false); + getRefs, false); products.add(prod.getProduct()); } } else { LOG.log(Level.FINEST, "Request for products by type: [" - + type.getProductTypeId() + "] returned no results"); + + type.getProductTypeId() + "] returned no results"); return null; } } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { try { //TODO CLOSE - } catch (Exception ignore) { + } catch (Exception ignore) { } } } @@ -650,9 +650,9 @@ public class LuceneCatalog implements Catalog { // should be exactly 1 hit if (topDocs.totalHits != 1) { throw new CatalogException("Product: [" - + product.getProductId() - + "] is not unique in the catalog! Num Hits: [" - + hits.length + "]"); + + product.getProductId() + + "] is not unique in the catalog! Num Hits: [" + + hits.length + "]"); } Document productDoc = searcher.doc(hits[0].doc); @@ -661,20 +661,20 @@ public class LuceneCatalog implements Catalog { return prod.getMetadata(); } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { try { //TODO CLOSE - } catch (Exception ignore) { + } catch (Exception ignore) { } } } } - + public Metadata getReducedMetadata(Product product, List<String> elements) throws CatalogException { Metadata fullMetadata = getMetadata(product); Metadata reducedMetadata = new Metadata(); @@ -688,7 +688,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#query(org.apache.oodt.cas.filemgr.structs.Query, * org.apache.oodt.cas.filemgr.structs.ProductType) */ @@ -711,7 +711,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getTopNProducts(int) */ public List<Product> getTopNProducts(int n) throws CatalogException { @@ -732,9 +732,9 @@ public class LuceneCatalog implements Catalog { booleanQuery.add(tq, BooleanClause.Occur.MUST); Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery - + "]"); + + "]"); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(booleanQuery.build(), 1, sort); if(check.totalHits>0) { @@ -748,7 +748,7 @@ public class LuceneCatalog implements Catalog { while (products.size() < Math.min(n, hits.length)) { Document productDoc = searcher.doc(hits[i].doc); CompleteProduct prod = toCompleteProduct(productDoc, false, - false); + false); products.add(prod.getProduct()); i++; } @@ -762,9 +762,9 @@ public class LuceneCatalog implements Catalog { } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (searcher != null) { @@ -780,12 +780,12 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getTopNProducts(int, * org.apache.oodt.cas.filemgr.structs.ProductType) */ public List<Product> getTopNProducts(int n, ProductType type) - throws CatalogException { + throws CatalogException { int numPages = 1; if (n > this.pageSize) { numPages = n / this.pageSize + (n % this.pageSize == 0 ? 0 : 1); @@ -796,11 +796,11 @@ public class LuceneCatalog implements Catalog { for (int pageNum = 1; pageNum < numPages + 1; pageNum++) { List<Product> pageProducts = paginateQuery(query, type, pageNum, null); - products.addAll(pageProducts); + products.addAll(pageProducts); } if(n<=products.size()) { - return products.subList(0, n); + return products.subList(0, n); } return products; @@ -808,7 +808,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getValidationLayer() */ public ValidationLayer getValidationLayer() { @@ -817,7 +817,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getNumProducts(org.apache.oodt.cas.filemgr.structs.ProductType) */ public int getNumProducts(ProductType type) throws CatalogException { @@ -827,29 +827,29 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.util.Pagination#getFirstPage(org.apache.oodt.cas.filemgr.structs.ProductType) */ public ProductPage getFirstPage(ProductType type) { ProductPage firstPage = new ProductPage(); List<Product> products; Query query = new Query(); - + // now construct the page firstPage.setPageNum(1); firstPage.setPageSize(pageSize); try { - products = paginateQuery(query, type, 1, firstPage); + products = paginateQuery(query, type, 1, firstPage); } catch (CatalogException e) { LOG.log(Level.WARNING, - "CatalogException getting first page for product type: [" - + type.getProductTypeId() - + "] from catalog: Message: " + e.getMessage()); + "CatalogException getting first page for product type: [" + + type.getProductTypeId() + + "] from catalog: Message: " + e.getMessage()); return null; } // There are no products and thus no first page if (products == null || (products.size() == 0)) { - return null; + return null; } firstPage.setPageProducts(products); @@ -859,7 +859,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.util.Pagination#getLastProductPage(org.apache.oodt.cas.filemgr.structs.ProductType) */ public ProductPage getLastProductPage(ProductType type) { @@ -867,22 +867,22 @@ public class LuceneCatalog implements Catalog { ProductPage firstPage = getFirstPage(type); List<Product> products; Query query = new Query(); - + // now construct the page lastPage.setPageNum(firstPage.getTotalPages()); lastPage.setPageSize(pageSize); try { products = paginateQuery(query, type, firstPage.getTotalPages(), lastPage); } catch (CatalogException e) { - LOG.log(Level.WARNING, - "CatalogException getting last page for product type: [" - + type.getProductTypeId() - + "] from catalog: Message: " + e.getMessage()); - return null; + LOG.log(Level.WARNING, + "CatalogException getting last page for product type: [" + + type.getProductTypeId() + + "] from catalog: Message: " + e.getMessage()); + return null; } // There are no products thus there is no last page if (products == null || (products.size() == 0)) { - return null; + return null; } lastPage.setPageProducts(products); @@ -891,7 +891,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.util.Pagination#getNextPage(org.apache.oodt.cas.filemgr.structs.ProductType, * org.apache.oodt.cas.filemgr.structs.ProductPage) */ @@ -918,14 +918,14 @@ public class LuceneCatalog implements Catalog { products = paginateQuery(query, type, currentPage.getPageNum() + 1, nextPage); } catch (CatalogException e) { LOG.log(Level.WARNING, - "CatalogException getting next page for product type: [" - + type.getProductTypeId() - + "] from catalog: Message: " + e.getMessage()); + "CatalogException getting next page for product type: [" + + type.getProductTypeId() + + "] from catalog: Message: " + e.getMessage()); return null; } // There are no products and thus no next page if (products == null || (products.size() == 0)) { - return null; + return null; } nextPage.setPageProducts(products); @@ -934,7 +934,7 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.util.Pagination#getPrevPage(org.apache.oodt.cas.filemgr.structs.ProductType, * org.apache.oodt.cas.filemgr.structs.ProductPage) */ @@ -962,15 +962,15 @@ public class LuceneCatalog implements Catalog { products = paginateQuery(query, type, currentPage.getPageNum() - 1, prevPage); } catch (CatalogException e) { LOG.log(Level.WARNING, - "CatalogException getting prev page for product type: [" - + type.getProductTypeId() - + "] from catalog: Message: " + e.getMessage()); + "CatalogException getting prev page for product type: [" + + type.getProductTypeId() + + "] from catalog: Message: " + e.getMessage()); return null; } - + // There are no products and thus no pages if (products == null || (products.size() == 0)) { - return null; + return null; } prevPage.setPageProducts(products); @@ -979,12 +979,12 @@ public class LuceneCatalog implements Catalog { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.catalog.Catalog#pagedQuery(org.apache.oodt.cas.filemgr.structs.Query, * org.apache.oodt.cas.filemgr.structs.ProductType, int) */ public ProductPage pagedQuery(Query query, ProductType type, int pageNum) - throws CatalogException { + throws CatalogException { try { ProductPage retPage = new ProductPage(); retPage.setPageNum(pageNum); @@ -994,15 +994,15 @@ public class LuceneCatalog implements Catalog { } catch (Exception e) { LOG.log(Level.SEVERE, e.getMessage()); LOG.log(Level.WARNING, - "CatalogException when doing paged product query: Message: " - + e.getMessage()); + "CatalogException when doing paged product query: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } } private synchronized void removeProductDocument(Product product) - throws CatalogException { + throws CatalogException { try { reader = DirectoryReader.open(indexDir); @@ -1011,8 +1011,8 @@ public class LuceneCatalog implements Catalog { } try { LOG.log(Level.FINE, - "LuceneCatalog: remove document from index for product: [" - + product.getProductId() + "]"); + "LuceneCatalog: remove document from index for product: [" + + product.getProductId() + "]"); IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); @@ -1020,15 +1020,15 @@ public class LuceneCatalog implements Catalog { lmp.setMergeFactor(mergeFactor); config.setMergePolicy(lmp); - IndexWriter writer = new IndexWriter(indexDir, config); + IndexWriter writer = new IndexWriter(indexDir, config); writer.deleteDocuments(new Term("product_id", product - .getProductId())); + .getProductId())); writer.close(); } catch (IOException e) { LOG.log(Level.WARNING, "Exception removing product: [" - + product.getProductName() + "] from index: Message: " - + e.getMessage()); + + product.getProductName() + "] from index: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage(), e); } finally { if (reader != null) { @@ -1043,7 +1043,7 @@ public class LuceneCatalog implements Catalog { } private synchronized void addCompleteProductToIndex(CompleteProduct cp) - throws CatalogException { + throws CatalogException { IndexWriter writer = null; try { /*writer = new IndexWriter(indexFilePath, new StandardAnalyzer(), @@ -1057,18 +1057,18 @@ public class LuceneCatalog implements Catalog { lmp.setMergeFactor(mergeFactor); config.setMergePolicy(lmp); - writer = new IndexWriter(indexDir, config); + writer = new IndexWriter(indexDir, config); Document doc = toDoc(cp.getProduct(), cp.getMetadata()); writer.addDocument(doc); // TODO: determine a better way to optimize the index } catch (Exception e) { LOG.log(Level.WARNING, "Unable to index product: [" - + cp.getProduct().getProductName() + "]: Message: " - + e.getMessage(), e); + + cp.getProduct().getProductName() + "]: Message: " + + e.getMessage(), e); throw new CatalogException("Unable to index product: [" - + cp.getProduct().getProductName() + "]: Message: " - + e.getMessage(), e); + + cp.getProduct().getProductName() + "]: Message: " + + e.getMessage(), e); } finally { try { if (writer != null) { @@ -1086,7 +1086,7 @@ public class LuceneCatalog implements Catalog { } private CompleteProduct toCompleteProduct(Document doc, boolean getRefs, - boolean getMetadata) { + boolean getMetadata) { Product product = new Product(); Metadata metadata = new Metadata(); CompleteProduct completeProduct = new CompleteProduct(); @@ -1108,39 +1108,39 @@ public class LuceneCatalog implements Catalog { List<String> names = new ArrayList<String>(); if (valLayer!=null) { - // only add metadata elements specified by validation layer - try { - for (Element element : valLayer.getElements(type)) { - names.add(element.getElementName()); - } - } catch (ValidationLayerException e) { - LOG.log(Level.WARNING, - "Unable to obtain metadata for product: [" - + product.getProductName() + "]: Message: " - + e.getMessage()); - } + // only add metadata elements specified by validation layer + try { + for (Element element : valLayer.getElements(type)) { + names.add(element.getElementName()); + } + } catch (ValidationLayerException e) { + LOG.log(Level.WARNING, + "Unable to obtain metadata for product: [" + + product.getProductName() + "]: Message: " + + e.getMessage()); + } } else { - // add all metadata elements found in document - List<IndexableField> fields = doc.getFields(); + // add all metadata elements found in document + List<IndexableField> fields = doc.getFields(); for(IndexableField field: fields){ if (!names.contains(field.name())) { names.add(field.name()); } } - + } // loop over field names to add to metadata for (String name : names) { - if (metadata.getAllMetadata(name)==null || metadata.getAllMetadata(name).size()==0) { - String[] elemValues = doc.getValues(name); - - if (elemValues != null && elemValues.length > 0) { + if (metadata.getAllMetadata(name)==null || metadata.getAllMetadata(name).size()==0) { + String[] elemValues = doc.getValues(name); + + if (elemValues != null && elemValues.length > 0) { for (String elemValue : elemValues) { metadata.addMetadata(name, elemValue); } - } - } + } + } } completeProduct.setMetadata(metadata); @@ -1154,7 +1154,7 @@ public class LuceneCatalog implements Catalog { String[] refMimeTypes = doc.getValues("reference_mimeType"); if ((origRefs.length == dataStoreRefs.length) - && (origRefs.length == refLengths.length)) { + && (origRefs.length == refLengths.length)) { List<Reference> references = new Vector<Reference>(); for (int i = 0; i < origRefs.length; i++) { Reference r = new Reference(); @@ -1170,11 +1170,11 @@ public class LuceneCatalog implements Catalog { product.setProductReferences(references); } else { LOG.log(Level.WARNING, "Number of original refs: [" - + origRefs.length + "] for product: [" - + product.getProductName() - + "] not equivalent to number of data store refs: [" - + dataStoreRefs.length - + "]: Skipping product references"); + + origRefs.length + "] for product: [" + + product.getProductName() + + "] not equivalent to number of data store refs: [" + + dataStoreRefs.length + + "]: Skipping product references"); } } @@ -1187,63 +1187,63 @@ public class LuceneCatalog implements Catalog { //TODO CHECK STORED TYPES // add the product information doc.add(new Field("product_id", product.getProductId(), - StringField.TYPE_STORED)); + StringField.TYPE_STORED)); doc.add(new Field("product_name", product.getProductName(), - StringField.TYPE_STORED)); + StringField.TYPE_STORED)); doc.add(new Field("product_structure", product.getProductStructure(), - StringField.TYPE_STORED)); + StringField.TYPE_STORED)); doc - .add(new Field("product_transfer_status", product - .getTransferStatus(), StringField.TYPE_STORED)); + .add(new Field("product_transfer_status", product + .getTransferStatus(), StringField.TYPE_STORED)); // product type doc - .add(new Field("product_type_id", product.getProductType() - .getProductTypeId(), StringField.TYPE_STORED)); + .add(new Field("product_type_id", product.getProductType() + .getProductTypeId(), StringField.TYPE_STORED)); doc.add(new Field("product_type_name", product.getProductType() - .getName(), StringField.TYPE_STORED)); + .getName(), StringField.TYPE_STORED)); doc.add(new Field("product_type_desc", product.getProductType() - .getDescription() != null ? product.getProductType() - .getDescription() : "", StringField.TYPE_STORED)); + .getDescription() != null ? product.getProductType() + .getDescription() : "", StringField.TYPE_STORED)); doc.add(new Field("product_type_repoPath", product.getProductType() - .getProductRepositoryPath() != null ? product.getProductType() - .getProductRepositoryPath() : "", StringField.TYPE_STORED)); + .getProductRepositoryPath() != null ? product.getProductType() + .getProductRepositoryPath() : "", StringField.TYPE_STORED)); doc.add(new Field("product_type_versioner", product.getProductType() - .getVersioner() != null ? product.getProductType() - .getVersioner() : "", StringField.TYPE_STORED)); - + .getVersioner() != null ? product.getProductType() + .getVersioner() : "", StringField.TYPE_STORED)); + // write metadata fields to the Lucene document List<String> keys = new ArrayList<String>(); // validation layer: add only specifically configured keys if (valLayer!=null) { - List<Element> elements = quietGetElements(product.getProductType()); + List<Element> elements = quietGetElements(product.getProductType()); for (Element element : elements) { String key = element.getElementName(); keys.add(key); } - // no validation layer: add all keys that are NOT already in doc - // (otherwise some keys such as the product_* keys are duplicated) + // no validation layer: add all keys that are NOT already in doc + // (otherwise some keys such as the product_* keys are duplicated) } else { - for (String key : metadata.getAllKeys()) { - if (doc.getField(key)==null) { - keys.add(key); - } - } + for (String key : metadata.getAllKeys()) { + if (doc.getField(key)==null) { + keys.add(key); + } + } } for (String key : keys) { - List<String> values = metadata.getAllMetadata(key); + List<String> values = metadata.getAllMetadata(key); if (values == null) { LOG - .log( - Level.WARNING, - "No Metadata specified for product [" - + product.getProductName() - + "] for required field [" - + key - + "]: Attempting to continue processing metadata"); + .log( + Level.WARNING, + "No Metadata specified for product [" + + product.getProductName() + + "] for required field [" + + key + + "]: Attempting to continue processing metadata"); continue; } @@ -1258,7 +1258,7 @@ public class LuceneCatalog implements Catalog { // add the product references for (Reference r : product.getProductReferences()) { doc.add(new Field("reference_orig", r.getOrigReference(), - StringField.TYPE_STORED)); + StringField.TYPE_STORED)); doc .add(new Field("reference_data_store", r .getDataStoreReference(), StringField.TYPE_STORED)); @@ -1296,7 +1296,7 @@ public class LuceneCatalog implements Catalog { } private int getNumHits(Query query, ProductType type) - throws CatalogException { + throws CatalogException { IndexSearcher searcher = null; int numHits = -1; @@ -1313,7 +1313,7 @@ public class LuceneCatalog implements Catalog { // add the product type as the first clause org.apache.lucene.search.Query prodTypeTermQuery = new TermQuery(new Term( - "product_type_id", type.getProductTypeId())); + "product_type_id", type.getProductTypeId())); booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST); //convert filemgr query into a lucene query @@ -1322,7 +1322,7 @@ public class LuceneCatalog implements Catalog { } LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery - + "]"); + + "]"); //TODO FIX returned records TopDocs hits = searcher.search(booleanQuery.build(), 1); @@ -1331,9 +1331,9 @@ public class LuceneCatalog implements Catalog { numHits = hits.totalHits; } catch (IOException e) { LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage()); } finally { if (searcher != null) { @@ -1348,7 +1348,7 @@ public class LuceneCatalog implements Catalog { } private List<Product> paginateQuery(Query query, ProductType type, int pageNum, ProductPage page) - throws CatalogException { + throws CatalogException { List<Product> products = null; IndexSearcher searcher = null; @@ -1370,25 +1370,25 @@ public class LuceneCatalog implements Catalog { // add the product type as the first clause TermQuery prodTypeTermQuery = new TermQuery(new Term( - "product_type_id", type.getProductTypeId())); + "product_type_id", type.getProductTypeId())); booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST); - + //convert filemgr query into a lucene query for (QueryCriteria queryCriteria : query.getCriteria()) { booleanQuery.add(this.getQuery(queryCriteria), BooleanClause.Occur.MUST); } - + Sort sort = new Sort(new SortField("CAS.ProductReceivedTime", - SortField.Type.STRING, true)); + SortField.Type.STRING, true)); LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery - + "]"); + + "]"); //TODO FIX NUMBER OF RECORDS TopDocs check = searcher.search(booleanQuery.build(),1, sort); TopDocs topDocs = searcher.search(booleanQuery.build(),check.totalHits, sort); // Calculate page size and set it while we have the results if (page != null) { - page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize)); + page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize)); } ScoreDoc[] hits = topDocs.scoreDocs; @@ -1404,11 +1404,11 @@ public class LuceneCatalog implements Catalog { products = new Vector<Product>(pageSize); for (int i = startNum; i < Math.min(hits.length, - (startNum + pageSize)); i++) { + (startNum + pageSize)); i++) { Document productDoc = searcher.doc(hits[i].doc); CompleteProduct prod = toCompleteProduct(productDoc, - false, false); + false, false); products.add(prod.getProduct()); } } else { @@ -1417,22 +1417,22 @@ public class LuceneCatalog implements Catalog { Document productDoc = searcher.doc(hits[i].doc); CompleteProduct prod = toCompleteProduct(productDoc, - false, false); + false, false); products.add(prod.getProduct()); } } } else { LOG.log(Level.WARNING, "Query: [" + query - + "] for Product Type: [" + type.getProductTypeId() - + "] returned no results"); + + "] for Product Type: [" + type.getProductTypeId() + + "] returned no results"); } } catch (Exception e) { LOG.log(Level.SEVERE, e.getMessage()); LOG.log(Level.WARNING, - "IOException when opening index directory: [" - + indexFilePath + "] for search: Message: " - + e.getMessage()); + "IOException when opening index directory: [" + + indexFilePath + "] for search: Message: " + + e.getMessage()); throw new CatalogException(e.getMessage()); } finally { if (searcher != null) { @@ -1451,19 +1451,19 @@ public class LuceneCatalog implements Catalog { BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); BooleanClause.Occur occur; switch (((BooleanQueryCriteria) queryCriteria).getOperator()) { - case BooleanQueryCriteria.AND: - occur = BooleanClause.Occur.MUST; - break; - case BooleanQueryCriteria.OR: - occur = BooleanClause.Occur.SHOULD; - break; - case BooleanQueryCriteria.NOT: - occur = BooleanClause.Occur.MUST_NOT; - booleanQuery.add(new WildcardQuery(new Term(((BooleanQueryCriteria) queryCriteria) + case BooleanQueryCriteria.AND: + occur = BooleanClause.Occur.MUST; + break; + case BooleanQueryCriteria.OR: + occur = BooleanClause.Occur.SHOULD; + break; + case BooleanQueryCriteria.NOT: + occur = BooleanClause.Occur.MUST_NOT; + booleanQuery.add(new WildcardQuery(new Term(((BooleanQueryCriteria) queryCriteria) .getTerms().get(0).getElementName(), "*")), BooleanClause.Occur.SHOULD); - break; - default: - throw new CatalogException("Invalid BooleanQueryCriteria opertor [" + break; + default: + throw new CatalogException("Invalid BooleanQueryCriteria opertor [" + ((BooleanQueryCriteria) queryCriteria).getOperator() + "]"); } for (QueryCriteria qc : ((BooleanQueryCriteria) queryCriteria).getTerms()) { @@ -1486,10 +1486,10 @@ public class LuceneCatalog implements Catalog { return TermRangeQuery.newStringRange(startTerm.field(), startVal, endVal, inclusive,inclusive); }else { throw new CatalogException("Invalid QueryCriteria [" - + queryCriteria.getClass().getCanonicalName() + "]"); + + queryCriteria.getClass().getCanonicalName() + "]"); } } - + private List<Element> quietGetElements(ProductType type) { List<Element> elementList = new Vector<Element>(); @@ -1497,8 +1497,8 @@ public class LuceneCatalog implements Catalog { elementList = valLayer.getElements(type); } catch (Exception e) { LOG.log(Level.WARNING, - "Exception obtaining elements for product type: [" - + type.getName() + "]: Message: " + e.getMessage()); + "Exception obtaining elements for product type: [" + + type.getName() + "]: Message: " + e.getMessage()); } return elementList; @@ -1565,4 +1565,4 @@ public class LuceneCatalog implements Catalog { } -} +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/SpacerCatalog.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/SpacerCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/SpacerCatalog.java new file mode 100644 index 0000000..8fd898f --- /dev/null +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/SpacerCatalog.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.cas.filemgr.catalog; + +//OODT imports +import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria; +import org.apache.oodt.cas.filemgr.structs.Element; +import org.apache.oodt.cas.filemgr.structs.Product; +import org.apache.oodt.cas.filemgr.structs.ProductPage; +import org.apache.oodt.cas.filemgr.structs.ProductType; +import org.apache.oodt.cas.filemgr.structs.Query; +import org.apache.oodt.cas.filemgr.structs.QueryCriteria; +import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria; +import org.apache.oodt.cas.filemgr.structs.Reference; +import org.apache.oodt.cas.filemgr.structs.TermQueryCriteria; +import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; +import org.apache.oodt.cas.filemgr.structs.exceptions.ValidationLayerException; +import org.apache.oodt.cas.filemgr.util.DbStructFactory; +import org.apache.oodt.cas.filemgr.validation.ValidationLayer; +import org.apache.oodt.cas.metadata.Metadata; +import org.apache.oodt.commons.pagination.PaginationUtils; +import org.apache.oodt.commons.util.DateConvert; + +//SPRING imports +import org.springframework.util.StringUtils; + + + +//JDK imports +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.TreeMap; +import java.util.UUID; +import java.util.Vector; +import java.util.logging.Level; +import java.util.logging.Logger; + + +/** + * @author starchmd + * @version $Revision$ + * + * <p> + * Passes through {@link Catalog} calls to a subordinate catalog. Allows individual method overriding/ method hijacking. + * </p> + * + */ +public abstract class SpacerCatalog implements Catalog { + /* our log stream */ + private static final Logger LOG = Logger.getLogger(DataSourceCatalog.class.getName()); + + protected Catalog catalog; + + /** + * <p> + * Default Constructor + * </p>. + * @throws + */ + public SpacerCatalog(Catalog catalog) { + this.catalog = catalog; + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addMetadata(org.apache.oodt.cas.metadata.Metadata, + * org.apache.oodt.cas.filemgr.structs.Product) + */ + public void addMetadata(Metadata m, Product product) + throws CatalogException { + this.catalog.addMetadata(m,product); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addMetadata(org.apache.oodt.cas.metadata.Metadata, + * org.apache.oodt.cas.filemgr.structs.Product) + */ + public void removeMetadata(Metadata m, Product product) + throws CatalogException { + this.catalog.removeMetadata(m,product); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addProduct(org.apache.oodt.cas.filemgr.structs.Product) + */ + public void addProduct(Product product) + throws CatalogException { + this.catalog.addProduct(product); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#modifyProduct(org.apache.oodt.cas.filemgr.structs.Product) + */ + public void modifyProduct(Product product) + throws CatalogException { + this.catalog.modifyProduct(product); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#removeProduct(org.apache.oodt.cas.filemgr.structs.Product) + */ + public void removeProduct(Product product) + throws CatalogException { + this.catalog.removeProduct(product); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#setProductTransferStatus(org.apache.oodt.cas.filemgr.structs.Product) + */ + public void setProductTransferStatus(Product product) + throws CatalogException { + this.catalog.setProductTransferStatus(product); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#addProductReferences(org.apache.oodt.cas.filemgr.structs.Product) + */ + public void addProductReferences(Product product) + throws CatalogException { + this.catalog.addProductReferences(product); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductById(java.lang.String) + */ + public Product getProductById(String productId) throws CatalogException { + return this.catalog.getProductById(productId); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductByName(java.lang.String) + */ + public Product getProductByName(String productName) throws CatalogException { + return this.catalog.getProductByName(productName); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductReferences(org.apache.oodt.cas.filemgr.structs.Product) + */ + public List<Reference> getProductReferences(Product product) throws CatalogException { + return this.catalog.getProductReferences(product); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProducts() + */ + public List<Product> getProducts() throws CatalogException { + return this.catalog.getProducts(); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductsByProductType(org.apache.oodt.cas.filemgr.structs.ProductType) + */ + public List<Product> getProductsByProductType(ProductType type) + throws CatalogException { + return this.catalog.getProductsByProductType(type); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getMetadata(org.apache.oodt.cas.filemgr.structs.Product) + */ + public Metadata getMetadata(Product product) throws CatalogException { + return this.catalog.getMetadata(product); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getReducedMetadata(org.apache.oodt.cas.filemgr.structs.Product,java.util.List<java.lang.String>) + */ + public Metadata getReducedMetadata(Product product, List<String> elems) throws CatalogException { + return this.catalog.getReducedMetadata(product,elems); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#query(org.apache.oodt.cas.filemgr.structs.Query + * org.apache.oodt.cas.filemgr.structs.ProductType) + */ + public List<String> query(Query query, ProductType type) throws CatalogException { + return this.catalog.query(query,type); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getTopNProducts(int) + */ + public List<Product> getTopNProducts(int n) throws CatalogException { + return this.catalog.getTopNProducts(n); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getTopNProducts(int, + * org.apache.oodt.cas.filemgr.structs.ProductType) + */ + public List<Product> getTopNProducts(int n, ProductType type) + throws CatalogException { + return this.catalog.getTopNProducts(n,type); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getValidationLayer() + */ + public ValidationLayer getValidationLayer() throws CatalogException { + return this.catalog.getValidationLayer(); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#getNumProducts(org.apache.oodt.cas.filemgr.structs.ProductType) + */ + public int getNumProducts(ProductType type) throws CatalogException { + return this.catalog.getNumProducts(type); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.util.Pagination#getFirstPage(org.apache.oodt.cas.filemgr.structs.ProductType) + */ + public ProductPage getFirstPage(ProductType type) { + return this.catalog.getFirstPage(type); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.util.Pagination#getLastProductPage(org.apache.oodt.cas.filemgr.structs.ProductType) + */ + public ProductPage getLastProductPage(ProductType type) { + return this.catalog.getLastProductPage(type); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.util.Pagination#getNextPage(org.apache.oodt.cas.filemgr.structs.ProductType, + * org.apache.oodt.cas.filemgr.structs.ProductPage) + */ + public ProductPage getNextPage(ProductType type, ProductPage currentPage) { + return this.catalog.getNextPage(type,currentPage); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.util.Pagination#getPrevPage(org.apache.oodt.cas.filemgr.structs.ProductType, + * org.apache.oodt.cas.filemgr.structs.ProductPage) + */ + public ProductPage getPrevPage(ProductType type, ProductPage currentPage) { + return this.catalog.getPrevPage(type,currentPage); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.Catalog#pagedQuery(org.apache.oodt.cas.filemgr.structs.Query, + * org.apache.oodt.cas.filemgr.structs.ProductType, int) + */ + public ProductPage pagedQuery(Query query, ProductType type, int pageNum) + throws CatalogException { + return this.catalog.pagedQuery(query,type,pageNum); + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/SpacerCatalogFactory.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/SpacerCatalogFactory.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/SpacerCatalogFactory.java new file mode 100644 index 0000000..32289b6 --- /dev/null +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/SpacerCatalogFactory.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.cas.filemgr.catalog; + +//JDK imports +import java.util.logging.Logger; + +//OODT imports +import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory; + +/** + * @author starchmd + * @version $Revision$ + * + * <p> + * A Factory for creating {@link Catalog} with a spacer (intercept) layer. + * </p> + * + */ +public abstract class SpacerCatalogFactory implements CatalogFactory { + + private String catalogFactory; + /* our log stream */ + private static final Logger LOG = Logger.getLogger(SpacerCatalogFactory.class.getName()); + + /** + * + */ + public SpacerCatalogFactory() throws IllegalArgumentException { + this.catalogFactory = System.getProperty("org.apache.oodt.cas.filemgr.catalog.spacer.subfactory"); + } + + /* + * (non-Javadoc) + * + * @see org.apache.oodt.cas.filemgr.catalog.CatalogFactory#createCatalog() + */ + public Catalog createCatalog() { + Catalog sub = GenericFileManagerObjectFactory.getCatalogServiceFromFactory(this.catalogFactory); + return this.getWrapper(sub); + } + /** + * Forces user of Spacer Factory to provide their own catalog implementation. + */ + protected abstract Catalog getWrapper(Catalog sub); + +} http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java index 72180e3..2faeb26 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java @@ -16,12 +16,14 @@ */ package org.apache.oodt.cas.filemgr.catalog.solr; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.structs.Reference; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.metadata.Metadata; +import org.apache.solr.client.solrj.util.ClientUtils; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -84,7 +86,7 @@ public class DefaultProductSerializer implements ProductSerializer { ProductType productType = product.getProductType(); if (productType!=null) { this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_NAME, productType.getName()); - this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_ID, productType.getProductTypeId()); + this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_ID, productType.getProductTypeId()); } if (create) { // only insert date/time when product is first created @@ -116,10 +118,10 @@ public class DefaultProductSerializer implements ProductSerializer { // product root reference if (rootReference!=null) { - addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_ORIGINAL, rootReference.getOrigReference()); - addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_DATASTORE, rootReference.getDataStoreReference()); + addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_ORIGINAL, StringEscapeUtils.escapeXml(rootReference.getOrigReference())); + addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_DATASTORE, StringEscapeUtils.escapeXml(rootReference.getDataStoreReference())); addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_FILESIZE, ""+rootReference.getFileSize()); - addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_MIMETYPE, rootReference.getMimeType().toString()); + addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_MIMETYPE, StringEscapeUtils.escapeXml(rootReference.getMimeType().toString())); } @@ -127,10 +129,10 @@ public class DefaultProductSerializer implements ProductSerializer { // note that Solr will preserve the indexing order. for (Reference reference : references) { - addKeyValueToMap(fields, Parameters.REFERENCE_ORIGINAL, reference.getOrigReference()); - addKeyValueToMap(fields, Parameters.REFERENCE_DATASTORE, reference.getDataStoreReference()); + addKeyValueToMap(fields, Parameters.REFERENCE_ORIGINAL, StringEscapeUtils.escapeXml(reference.getOrigReference())); + addKeyValueToMap(fields, Parameters.REFERENCE_DATASTORE, StringEscapeUtils.escapeXml(reference.getDataStoreReference())); addKeyValueToMap(fields, Parameters.REFERENCE_FILESIZE, ""+reference.getFileSize()); - addKeyValueToMap(fields, Parameters.REFERENCE_MIMETYPE, reference.getMimeType().toString()); + addKeyValueToMap(fields, Parameters.REFERENCE_MIMETYPE, StringEscapeUtils.escapeXml(reference.getMimeType().toString())); } @@ -182,11 +184,11 @@ public class DefaultProductSerializer implements ProductSerializer { for (String key : metadata.getKeys()) { if (! (key.startsWith(Parameters.NS) // skip metadata keys starting with reserved namespace - || Parameters.PRODUCT_TYPE_NAME.contains(key) + //|| Parameters.PRODUCT_TYPE_NAME.contains(key) // skip 'ProductType' as already stored as 'CAS.ProductTypeName' || Parameters.PRODUCT_STRUCTURE.contains(key))) { // skip 'ProductType' as already stored as 'CAS.ProductStructure' for (String value : metadata.getAllMetadata(key)) { - this.addKeyValueToMap(fields, key, value); + this.addKeyValueToMap(fields, key, StringEscapeUtils.escapeXml(value)); } } } @@ -232,7 +234,7 @@ public class DefaultProductSerializer implements ProductSerializer { // all other fields for (Map.Entry<String, List<String>> key : fields.entrySet()) { for (String value : key.getValue()) { - doc.append( encodeIndexField(key.getKey(), value) ); + doc.append( encodeIndexField(key.getKey(), StringEscapeUtils.escapeXml(value)) ); } } @@ -268,13 +270,13 @@ public class DefaultProductSerializer implements ProductSerializer { } else { for (String value : values) { - setFields.add( this.encodeUpdateField(key.getKey(), value, true) ); + setFields.add( this.encodeUpdateField(key.getKey(), StringEscapeUtils.escapeXml(value), true) ); } } } else { for (String value : values) { - addFields.add( this.encodeUpdateField(key.getKey(), value, false) ); + addFields.add( this.encodeUpdateField(key.getKey(), StringEscapeUtils.escapeXml(value), false) ); } } @@ -407,7 +409,7 @@ public class DefaultProductSerializer implements ProductSerializer { List<String> vals = new ArrayList<String>(); for (int k=0; k<values.getLength(); k++) { String value = ((Element)values.item(k)).getTextContent(); - vals.add(value); + vals.add(StringEscapeUtils.unescapeXml(value)); } // CAS.reference.... fields if (name.startsWith(Parameters.NS)) { @@ -440,22 +442,34 @@ public class DefaultProductSerializer implements ProductSerializer { */ } else { - String value = element.getTextContent(); + String value = StringEscapeUtils.unescapeXml(element.getTextContent()); // core CAS fields if (name.startsWith(Parameters.NS)) { if (name.equals(Parameters.PRODUCT_ID)) { product.setProductId(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_NAME)) { product.setProductName(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_STRUCTURE)) { product.setProductStructure(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_TRANSFER_STATUS)) { product.setTransferStatus(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_TYPE_NAME)) { productType.setName(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_TYPE_ID)) { productType.setProductTypeId(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_RECEIVED_TIME)) { product.setProductRecievedTime(value); metadata.addMetadata(name, value); http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java index af554c5..30ec0b3 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java @@ -65,7 +65,9 @@ public class SolrCatalog implements Catalog { public void addMetadata(Metadata metadata, Product product) throws CatalogException { LOG.info("Adding metadata for product:"+product.getProductName()); - + if(metadata.containsKey("_version_")){ + metadata.removeMetadata("_version_"); + } // serialize metadadta to Solr document(s) // replace=false i.e. add metadata to existing values List<String> docs = productSerializer.serialize(product.getProductId(), metadata, false); @@ -110,7 +112,10 @@ public class SolrCatalog implements Catalog { } } - + + if(updateMetadata.containsKey("_version_")){ + updateMetadata.removeMetadata("_version_"); + } // generate Solr update documents // replace=true to override existing values List<String> docs = productSerializer.serialize(product.getProductId(), updateMetadata, true); @@ -127,21 +132,32 @@ public class SolrCatalog implements Catalog { */ @Override public void addProduct(Product product) throws CatalogException { - - LOG.info("Adding product:"+product.getProductName()); - - // generate product identifier if not existing already - if (!StringUtils.hasText(product.getProductId())) { - String productId = this.productIdGenerator.generateId(product); - product.setProductId(productId); + + if(product.getProductId()!=null && this.getCompleteProductById(product.getProductId()) !=null) { + throw new CatalogException( + "Attempt to add a product that already existed: product: [" + + product.getProductName() + "]"); + + + + + + } else { + LOG.info("Adding product:" + product.getProductName()); + + // generate product identifier if not existing already + if (!StringUtils.hasText(product.getProductId())) { + String productId = this.productIdGenerator.generateId(product); + product.setProductId(productId); + } + + // serialize product for ingestion into Solr + List<String> docs = productSerializer.serialize(product, true); // create=true + + // send records to Solr + solrClient.index(docs, true, productSerializer.getMimeType()); } - // serialize product for ingestion into Solr - List<String> docs = productSerializer.serialize(product, true); // create=true - - // send records to Solr - solrClient.index(docs, true, productSerializer.getMimeType()); - } @Override http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java index 1b17b62..be05a2a 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java @@ -16,20 +16,35 @@ */ package org.apache.oodt.cas.filemgr.catalog.solr; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.httpclient.NameValuePair; -import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.methods.PostMethod; -import org.apache.commons.httpclient.methods.StringRequestEntity; +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.HttpClient; +import org.apache.http.client.ResponseHandler; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicResponseHandler; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.message.BasicNameValuePair; import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; +import org.apache.solr.client.solrj.util.ClientUtils; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.concurrent.ConcurrentHashMap; import java.util.List; import java.util.Map; @@ -154,7 +169,7 @@ public class SolrClient { public String queryProductByName(String name, String mimeType) throws CatalogException { ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>(); - params.put("q", new String[]{Parameters.PRODUCT_NAME+":"+name} ); + params.put("q", new String[]{Parameters.PRODUCT_NAME+":"+ ClientUtils.escapeQueryChars(name)} ); return query(params, mimeType); } @@ -237,19 +252,23 @@ public class SolrClient { throws IOException, CatalogException { // build HTTP/GET request - GetMethod method = new GetMethod(url); - List<NameValuePair> nvps = new ArrayList<NameValuePair>(); + + + List<BasicNameValuePair> nvps = new ArrayList<BasicNameValuePair>(); for (Map.Entry<String, String[]> key : parameters.entrySet()) { for (String value : key.getValue()) { - nvps.add(new NameValuePair(key.getKey(), value)); + nvps.add(new BasicNameValuePair(key.getKey(), value)); } } // request results in JSON format if (mimeType.equals(Parameters.MIME_TYPE_JSON)) { - nvps.add(new NameValuePair("wt", "json")); + nvps.add(new BasicNameValuePair("wt", "json")); } - method.setQueryString( nvps.toArray( new NameValuePair[nvps.size()] ) ); - LOG.info("GET url: "+url+" query string: "+method.getQueryString()); + + String paramString = URLEncodedUtils.format(nvps, "utf-8"); + + HttpRequestBase method = new HttpGet(url+"?"+paramString); + LOG.info("GET url: "+url+" query string: "+method.getURI()); // send HTTP/GET request, return response return doHttp(method); @@ -265,10 +284,15 @@ public class SolrClient { private String doPost(String url, String document, String mimeType) throws IOException, CatalogException { // build HTTP/POST request - PostMethod method = new PostMethod(url); - StringRequestEntity requestEntity = new StringRequestEntity(document, mimeType, "UTF-8"); - method.setRequestEntity(requestEntity); - + HttpPost method = new HttpPost(url); + HttpEntity requestEntity = null; + try { + requestEntity = new StringEntity(document, mimeType, "UTF-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + + method.setEntity(requestEntity); // send HTTP/POST request, return response return doHttp(method); @@ -280,34 +304,34 @@ public class SolrClient { * @return * @throws Exception */ - private String doHttp(HttpMethod method) throws IOException, CatalogException { + private String doHttp(HttpRequestBase method) throws IOException, CatalogException { - StringBuilder response = new StringBuilder(); + String response = null; BufferedReader br = null; try { // send request - HttpClient httpClient = new HttpClient(); + HttpClient httpClient = new DefaultHttpClient(); // OODT-719 Prevent httpclient from spawning closewait tcp connections - method.setRequestHeader("Connection", "close"); + method.setHeader("Connection", "close"); - int statusCode = httpClient.executeMethod(method); + HttpResponse statusCode = httpClient.execute(method); // read response - if (statusCode != HttpStatus.SC_OK) { + if (statusCode.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { // still consume the response - method.getResponseBodyAsString(); - throw new CatalogException("HTTP method failed: " + method.getStatusLine()); + ResponseHandler<String> handler = new BasicResponseHandler(); + + handler.handleResponse(statusCode); + throw new CatalogException("HTTP method failed: " + statusCode.getStatusLine().toString()); } else { - - // read the response body. - br = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream())); - String readLine; - while(((readLine = br.readLine()) != null)) { - response.append(readLine); - } + ResponseHandler<String> handler = new BasicResponseHandler(); + + String resp = handler.handleResponse(statusCode); + + response=resp; } @@ -322,7 +346,7 @@ public class SolrClient { } } - return response.toString(); + return response; }
