Repository: oodt Updated Branches: refs/heads/master ebaae1f33 -> 8a3d51944
OODT-940 update solr assets Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/29fc6162 Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/29fc6162 Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/29fc6162 Branch: refs/heads/master Commit: 29fc61620fb11360e4beae6216238e555f2afc35 Parents: e55f2cd Author: Tom Barber <[email protected]> Authored: Thu Nov 10 15:59:24 2016 +0000 Committer: Tom Barber <[email protected]> Committed: Thu Nov 10 15:59:24 2016 +0000 ---------------------------------------------------------------------- core/pom.xml | 4 +- .../catalog/solr/DefaultProductSerializer.java | 310 +++++++++--------- .../cas/filemgr/catalog/solr/SolrCatalog.java | 324 ++++++++++--------- 3 files changed, 334 insertions(+), 304 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/29fc6162/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 6fafae6..3420597 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -382,12 +382,12 @@ the License. <dependency> <groupId>org.apache.solr</groupId> <artifactId>solr-core</artifactId> - <version>1.3.0</version> + <version>6.2.1</version> </dependency> <dependency> <groupId>org.apache.solr</groupId> <artifactId>solr-solrj</artifactId> - <version>1.3.0</version> + <version>6.2.1</version> </dependency> <dependency> <groupId>org.apache.tika</groupId> http://git-wip-us.apache.org/repos/asf/oodt/blob/29fc6162/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java index 72180e3..5b42a8b 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java @@ -16,12 +16,14 @@ */ package org.apache.oodt.cas.filemgr.catalog.solr; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.structs.Reference; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.metadata.Metadata; +import org.apache.solr.client.solrj.util.ClientUtils; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -44,21 +46,21 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; /** - * Default implementation of {@link ProductSerializer} - * that transforms a CAS product into a single Solr document based on the following rules: + * Default implementation of {@link ProductSerializer} + * that transforms a CAS product into a single Solr document based on the following rules: * o) the core product attributes are used to generate Solr fields starting with "CAS...." * o) the product references are converted to Solr fields starting with "CAS.Reference..." or "CAS.RootReference..." - * o) all other metadata fields are converted into Solr fields with the same name and number of values. + * o) all other metadata fields are converted into Solr fields with the same name and number of values. * Note that the field multiplicity must be consistent with its definition in the Solr schema.xml. - * + * * This class generates all Solr documents in XML format. - * + * * @author Luca Cinquini * */ public class DefaultProductSerializer implements ProductSerializer { - private static Logger LOG = Logger.getLogger(DefaultProductSerializer.class.getName()); + private static Logger LOG = Logger.getLogger(DefaultProductSerializer.class.getName()); /** * {@inheritDoc} */ @@ -72,10 +74,10 @@ public class DefaultProductSerializer implements ProductSerializer { */ @Override public List<String> serialize(Product product, boolean create) { - + Map<String, List<String>> fields = new ConcurrentHashMap<String, List<String>>(); List<String> docs = new ArrayList<String>(); - + // add core product attributes to map this.addKeyValueToMap(fields, Parameters.PRODUCT_ID, product.getProductId()); this.addKeyValueToMap(fields, Parameters.PRODUCT_NAME, product.getProductName()); @@ -84,117 +86,117 @@ public class DefaultProductSerializer implements ProductSerializer { ProductType productType = product.getProductType(); if (productType!=null) { this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_NAME, productType.getName()); - this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_ID, productType.getProductTypeId()); + this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_ID, productType.getProductTypeId()); } if (create) { // only insert date/time when product is first created Date productDateTime = new Date(); // current datetime this.addKeyValueToMap(fields, Parameters.PRODUCT_RECEIVED_TIME, Parameters.SOLR_DATE_TIME_FORMATTER.format(productDateTime)); } - + // create new product: use Solr id == CAS id - if (create) { + if (create) { docs.add( this.generateInsertDocuments(product.getProductId(), fields) ); - - // update existing product - } else { + + // update existing product + } else { docs.addAll( this.generateUpdateDocuments(product.getProductId(), fields, true) ); // replace=true - + } - + return docs; - + } - + /** * {@inheritDoc} */ public List<String> serialize(String productId, Reference rootReference, List<Reference> references, boolean replace) { - + Map<String, List<String>> fields = new ConcurrentHashMap<String, List<String>>(); - + // product root reference if (rootReference!=null) { - - addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_ORIGINAL, rootReference.getOrigReference()); - addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_DATASTORE, rootReference.getDataStoreReference()); + + addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_ORIGINAL, StringEscapeUtils.escapeXml(rootReference.getOrigReference())); + addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_DATASTORE, StringEscapeUtils.escapeXml(rootReference.getDataStoreReference())); addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_FILESIZE, ""+rootReference.getFileSize()); - addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_MIMETYPE, rootReference.getMimeType().toString()); - + addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_MIMETYPE, StringEscapeUtils.escapeXml(rootReference.getMimeType().toString())); + } - + // all other product references - // note that Solr will preserve the indexing order. + // note that Solr will preserve the indexing order. for (Reference reference : references) { - - addKeyValueToMap(fields, Parameters.REFERENCE_ORIGINAL, reference.getOrigReference()); - addKeyValueToMap(fields, Parameters.REFERENCE_DATASTORE, reference.getDataStoreReference()); + + addKeyValueToMap(fields, Parameters.REFERENCE_ORIGINAL, StringEscapeUtils.escapeXml(reference.getOrigReference())); + addKeyValueToMap(fields, Parameters.REFERENCE_DATASTORE, StringEscapeUtils.escapeXml(reference.getDataStoreReference())); addKeyValueToMap(fields, Parameters.REFERENCE_FILESIZE, ""+reference.getFileSize()); - addKeyValueToMap(fields, Parameters.REFERENCE_MIMETYPE, reference.getMimeType().toString()); - + addKeyValueToMap(fields, Parameters.REFERENCE_MIMETYPE, StringEscapeUtils.escapeXml(reference.getMimeType().toString())); + } - + return generateUpdateDocuments(productId, fields, replace); - + } - + /** * {@inheritDoc} */ public QueryResponse deserialize(String xml) throws CatalogException { - + try { - + QueryResponse queryResponse = new QueryResponse(); - + // parse XML into DOM Document document = parseXml(xml); - + // extract information from DOM to Product Element response = document.getDocumentElement(); Node result = response.getElementsByTagName("result").item(0); queryResponse.setNumFound( Integer.parseInt( ((Element)result).getAttribute("numFound") ) ); - queryResponse.setStart( Integer.parseInt( ((Element)result).getAttribute("start") ) ); + queryResponse.setStart( Integer.parseInt( ((Element)result).getAttribute("start") ) ); NodeList docs = result.getChildNodes(); for (int i=0; i< docs.getLength(); i++) { Node node = docs.item(i); if (node.getNodeName().equals("doc")) { - Element doc = (Element)node; + Element doc = (Element)node; CompleteProduct cp = this.deserialize(doc); queryResponse.getCompleteProducts().add(cp); } - } + } return queryResponse; - + } catch(Exception e) { LOG.log(Level.SEVERE, e.getMessage()); throw new CatalogException(e.getMessage(), e); } - + } - + /** * {@inheritDoc} */ public List<String> serialize(String productId, Metadata metadata, boolean replace) { - + Map<String, List<String>> fields = new ConcurrentHashMap<String, List<String>>(); - + for (String key : metadata.getKeys()) { if (! (key.startsWith(Parameters.NS) // skip metadata keys starting with reserved namespace - || Parameters.PRODUCT_TYPE_NAME.contains(key) - // skip 'ProductType' as already stored as 'CAS.ProductTypeName' - || Parameters.PRODUCT_STRUCTURE.contains(key))) { // skip 'ProductType' as already stored as 'CAS.ProductStructure' + //|| Parameters.PRODUCT_TYPE_NAME.contains(key) + // skip 'ProductType' as already stored as 'CAS.ProductTypeName' + || Parameters.PRODUCT_STRUCTURE.contains(key))) { // skip 'ProductType' as already stored as 'CAS.ProductStructure' for (String value : metadata.getAllMetadata(key)) { - this.addKeyValueToMap(fields, key, value); + this.addKeyValueToMap(fields, key, StringEscapeUtils.escapeXml(value)); } } } - + return this.generateUpdateDocuments(productId, fields, replace); - + } - + /** * Method to add a (key, value) to a multi-valued map with appropriate checks. * @param map @@ -202,7 +204,7 @@ public class DefaultProductSerializer implements ProductSerializer { * @param value */ protected void addKeyValueToMap(Map<String, List<String>> map, String key, String value) { - + if (!map.containsKey(key)) { map.put(key, new ArrayList<String>()); } @@ -213,33 +215,33 @@ public class DefaultProductSerializer implements ProductSerializer { map.get(key).add(Parameters.NULL); } } - + /** * Utility method to generate a Solr insert document. - * + * * @param productId * @param fields * @return */ protected String generateInsertDocuments(String productId, Map<String,List<String>> fields) { - + StringBuilder doc = new StringBuilder(); doc.append("<doc>"); - + // product Solr id field doc.append( encodeIndexField(Parameters.ID, productId) ); - + // all other fields for (Map.Entry<String, List<String>> key : fields.entrySet()) { for (String value : key.getValue()) { - doc.append( encodeIndexField(key.getKey(), value) ); + doc.append( encodeIndexField(key.getKey(), StringEscapeUtils.escapeXml(value)) ); } } doc.append("</doc>"); return doc.toString(); } - + /** * Utility method to generate Solr update documents. * Note that the requests for setting/adding/deleting fields must be sent as separate documents to Solr @@ -249,51 +251,51 @@ public class DefaultProductSerializer implements ProductSerializer { * @return */ protected List<String> generateUpdateDocuments(String productId, Map<String,List<String>> fields, boolean replace) { - + // list for different instruction types List<String> setFields = new ArrayList<String>(); List<String> addFields = new ArrayList<String>(); List<String> delFields = new ArrayList<String>(); - + // encode update instructions for (Map.Entry<String, List<String>> key : fields.entrySet()) { - + List<String> values = key.getValue(); - + if (replace) { - + if (values.isEmpty()) { // use special value to flag removal delFields.add( this.encodeUpdateField(key.getKey(), Parameters.NULL, true) ); - + } else { for (String value : values) { - setFields.add( this.encodeUpdateField(key.getKey(), value, true) ); + setFields.add( this.encodeUpdateField(key.getKey(), StringEscapeUtils.escapeXml(value), true) ); } } - + } else { for (String value : values) { - addFields.add( this.encodeUpdateField(key.getKey(), value, false) ); + addFields.add( this.encodeUpdateField(key.getKey(), StringEscapeUtils.escapeXml(value), false) ); } } - + } - + List<String> docs = new ArrayList<String>(); if (!delFields.isEmpty()) { - docs.add(toDoc(productId, delFields)); + docs.add(toDoc(productId, delFields)); } if (!setFields.isEmpty()) { - docs.add(toDoc(productId, setFields)); + docs.add(toDoc(productId, setFields)); } if (!addFields.isEmpty()) { - docs.add(toDoc(productId, addFields)); + docs.add(toDoc(productId, addFields)); } return docs; - + } - + /** * Utility method to merge field update instructions into a single document. * @param productId @@ -301,24 +303,24 @@ public class DefaultProductSerializer implements ProductSerializer { * @return */ private String toDoc(String productId, List<String> updates) { - + StringBuilder doc = new StringBuilder(); doc.append("<doc>"); - + // reference product record id doc.append( encodeIndexField(Parameters.ID, productId) ); - + // loop over field update instructions for (String update : updates) { doc.append(update); } - + doc.append("</doc>"); - + return doc.toString(); - + } - + /** * Method to encode a Solr field indexing instruction. * If the value is null, the empty string is returned. @@ -333,11 +335,11 @@ public class DefaultProductSerializer implements ProductSerializer { return "<field name=\""+key+"\">" + value + "</field>"; } } - + /** * Method to encode a field update instruction for the three possible cases: * add new values to a key (1), replace current values for a key (2), remove all values for a key (3). - * + * * @param key * @param value * @param replace @@ -346,123 +348,135 @@ public class DefaultProductSerializer implements ProductSerializer { protected String encodeUpdateField(String key, String value, boolean replace) { StringBuilder sb = new StringBuilder(); sb.append("<field name=\"").append(key).append("\""); - + if (replace) { - + if (value==null || value.equals(Parameters.NULL)) { - + // (3) remove all values for given key sb.append(" update=\"set\" null=\"true\" />"); - + } else { - + // (2) replace existing values with new values sb.append(" update=\"set\">").append(value).append("</field>"); } - + } else { - + // (1) add new values to existing values sb.append(" update=\"add\">").append(value).append("</field>"); - + } return sb.toString(); } - - + + /** * Method that parses a single Solr document snippet * to extract Product and Metadata attributes. - * + * * @param doc * @return */ private CompleteProduct deserialize(Element doc) { - + CompleteProduct cp = new CompleteProduct(); Product product = cp.getProduct(); ProductType productType = product.getProductType(); Metadata metadata = cp.getMetadata(); List<Reference> references = product.getProductReferences(); Reference rootReference = product.getRootRef(); - + NodeList children = doc.getChildNodes(); for (int j=0; j<children.getLength(); j++) { - + Node child = children.item(j); Element element = (Element)child; String name = element.getAttribute("name"); - + /** *<arr name="ScanPointingSource"> * <str>G073.65+0.19</str> * <str>J2015+3410</str> - * .......... + * .......... */ if (child.getNodeName().equals("arr")) { - + NodeList values = element.getChildNodes(); List<String> vals = new ArrayList<String>(); for (int k=0; k<values.getLength(); k++) { String value = ((Element)values.item(k)).getTextContent(); - vals.add(value); + vals.add(StringEscapeUtils.unescapeXml(value)); } // CAS.reference.... fields - if (name.startsWith(Parameters.NS)) { - for (int k=0; k<values.getLength(); k++) { - // create this reference - if (references.size()<=k) { - references.add(new Reference()); - } - if (name.equals(Parameters.REFERENCE_ORIGINAL)) { - references.get(k).setOrigReference(vals.get(k)); - } else if (name.equals(Parameters.REFERENCE_DATASTORE)) { - references.get(k).setDataStoreReference(vals.get(k)); - } else if (name.equals(Parameters.REFERENCE_FILESIZE)) { - references.get(k).setFileSize(Long.parseLong(vals.get(k))); - } else if (name.equals(Parameters.REFERENCE_MIMETYPE)) { - references.get(k).setMimeType(vals.get(k)); - } + if (name.startsWith(Parameters.NS)) { + for (int k=0; k<values.getLength(); k++) { + // create this reference + if (references.size()<=k) { + references.add(new Reference()); } - // all other multi-valued fields + if (name.equals(Parameters.REFERENCE_ORIGINAL)) { + references.get(k).setOrigReference(vals.get(k)); + } else if (name.equals(Parameters.REFERENCE_DATASTORE)) { + references.get(k).setDataStoreReference(vals.get(k)); + } else if (name.equals(Parameters.REFERENCE_FILESIZE)) { + references.get(k).setFileSize(Long.parseLong(vals.get(k))); + } else if (name.equals(Parameters.REFERENCE_MIMETYPE)) { + references.get(k).setMimeType(vals.get(k)); + } + } + // all other multi-valued fields } else { this.deserializeMultiValueField(name, vals, metadata); } - - /** - * <str name="id">6684d79d-a011-4bc0-b3b3-4f11817091c8</str> - * <str name="CAS.ProductId">6684d79d-a011-4bc0-b3b3-4f11817091c8</str> - * <str name="CAS.ProductName">tns_br145x4_20</str> - * <str name="FileLocation">/usr/local/ska-dc/data/archive</str> - * ........... - */ + + /** + * <str name="id">6684d79d-a011-4bc0-b3b3-4f11817091c8</str> + * <str name="CAS.ProductId">6684d79d-a011-4bc0-b3b3-4f11817091c8</str> + * <str name="CAS.ProductName">tns_br145x4_20</str> + * <str name="FileLocation">/usr/local/ska-dc/data/archive</str> + * ........... + */ } else { - - String value = element.getTextContent(); - + + String value = StringEscapeUtils.unescapeXml(element.getTextContent()); + // core CAS fields if (name.startsWith(Parameters.NS)) { if (name.equals(Parameters.PRODUCT_ID)) { product.setProductId(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_NAME)) { product.setProductName(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_STRUCTURE)) { product.setProductStructure(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_TRANSFER_STATUS)) { product.setTransferStatus(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_TYPE_NAME)) { productType.setName(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_TYPE_ID)) { productType.setProductTypeId(value); + metadata.addMetadata(name, value); + } else if (name.equals(Parameters.PRODUCT_RECEIVED_TIME)) { product.setProductRecievedTime(value); metadata.addMetadata(name, value); // CAS root reference } else if (name.startsWith(Parameters.NS+Parameters.ROOT)) { if (rootReference==null) { - rootReference = new Reference(); + rootReference = new Reference(); } if (name.equals(Parameters.ROOT_REFERENCE_ORIGINAL)) { rootReference.setOrigReference(value); @@ -475,48 +489,48 @@ public class DefaultProductSerializer implements ProductSerializer { } } - - // non core single-valued fields + + // non core single-valued fields } else { this.deserializeSingleValueField(name, value, metadata); } // "CAS".... or not - + } // "arr" or anything else - + } // loop over <doc> children - + return cp; - + } - + private Document parseXml(String xml) throws IOException, SAXException, ParserConfigurationException { - + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder parser = factory.newDocumentBuilder(); - return parser.parse( new InputSource(new StringReader(xml)) ); - + return parser.parse( new InputSource(new StringReader(xml)) ); + } - + /** * Method that deserializes a single-valued Solr field into a Metadata element. * This method can be overridden by sub-classes to provide custom behavior. - * + * * @param name : the Solr field name * @param value : the Solr field single value * @param metadata : the metadata container */ protected void deserializeSingleValueField(String name, String value, Metadata metadata) { - // ignore Solr internal identifier (as it is duplicate information of CAS.ProductId) + // ignore Solr internal identifier (as it is duplicate information of CAS.ProductId) if (!name.equals(Parameters.ID)){ metadata.addMetadata(name, value); } } - + /** * Method that deserializes a multi-valued Solr field into a Metadata element. * This method can be overridden by sub-classes to provide custom behavior. - * + * * @param name : the Solr field name * @param values : the Solr field multiple values * @param metadata : the metadata container http://git-wip-us.apache.org/repos/asf/oodt/blob/29fc6162/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java index af554c5..a734e91 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java @@ -38,23 +38,23 @@ import org.springframework.util.StringUtils; /** * Implementation of the CAS {@link Catalog} interface * that uses a Solr back-end metadata store. - * + * * @author Luca Cinquini * */ public class SolrCatalog implements Catalog { - + // Class responsible for serializing/deserializing CAS products into Solr documents ProductSerializer productSerializer; - + // Class responsible for generating unique identifiers for incoming Products ProductIdGenerator productIdGenerator; - + // Class responsible for interacting with the Solr server SolrClient solrClient; - + private static final Logger LOG = Logger.getLogger(SolrCatalog.class.getName()); - + public SolrCatalog(String solrUrl, ProductIdGenerator productIdGenerator, ProductSerializer productSerializer) { this.productIdGenerator = productIdGenerator; this.productSerializer = productSerializer; @@ -63,13 +63,15 @@ public class SolrCatalog implements Catalog { @Override public void addMetadata(Metadata metadata, Product product) throws CatalogException { - + LOG.info("Adding metadata for product:"+product.getProductName()); - + if(metadata.containsKey("_version_")){ + metadata.removeMetadata("_version_"); + } // serialize metadadta to Solr document(s) - // replace=false i.e. add metadata to existing values - List<String> docs = productSerializer.serialize(product.getProductId(), metadata, false); - + // replace=false i.e. add metadata to existing values + List<String> docs = productSerializer.serialize(product.getProductId(), metadata, false); + // send documents to Solr server solrClient.index(docs, true, productSerializer.getMimeType()); @@ -82,39 +84,42 @@ public class SolrCatalog implements Catalog { */ @Override public void removeMetadata(Metadata metadata, Product product) throws CatalogException { - + // retrieve full existing metadata Metadata currentMetadata = getMetadata(product); - + // metadata to be updated Metadata updateMetadata = new Metadata(); - + // loop over keys of metadata be updated for (String key : metadata.getKeys()) { - + // list of values remaining after removal List<String> values = new ArrayList<String>(); if (currentMetadata.containsKey(key)) { for (String value : currentMetadata.getAllMetadata(key)) { if (!metadata.getAllMetadata(key).contains(value)) { values.add(value); - } + } } // add remaining values to updated metadata if (values.isEmpty()) { // special value because Metadata will NOT store an empty list values.add(Parameters.NULL); - } + } updateMetadata.addMetadata(key, values); - + } } - + + if(updateMetadata.containsKey("_version_")){ + updateMetadata.removeMetadata("_version_"); + } // generate Solr update documents // replace=true to override existing values List<String> docs = productSerializer.serialize(product.getProductId(), updateMetadata, true); - + // send documents to Solr server solrClient.index(docs, true, productSerializer.getMimeType()); @@ -127,40 +132,51 @@ public class SolrCatalog implements Catalog { */ @Override public void addProduct(Product product) throws CatalogException { - - LOG.info("Adding product:"+product.getProductName()); - - // generate product identifier if not existing already - if (!StringUtils.hasText(product.getProductId())) { - String productId = this.productIdGenerator.generateId(product); - product.setProductId(productId); + + if(product.getProductId()!=null && this.getCompleteProductById(product.getProductId()) !=null) { + throw new CatalogException( + "Attempt to add a product that already existed: product: [" + + product.getProductName() + "]"); + + + + + + } else { + LOG.info("Adding product:" + product.getProductName()); + + // generate product identifier if not existing already + if (!StringUtils.hasText(product.getProductId())) { + String productId = this.productIdGenerator.generateId(product); + product.setProductId(productId); + } + + // serialize product for ingestion into Solr + List<String> docs = productSerializer.serialize(product, true); // create=true + + // send records to Solr + solrClient.index(docs, true, productSerializer.getMimeType()); } - - // serialize product for ingestion into Solr - List<String> docs = productSerializer.serialize(product, true); // create=true - - // send records to Solr - solrClient.index(docs, true, productSerializer.getMimeType()); - + } @Override public void modifyProduct(Product product) throws CatalogException { - + LOG.info("Modifying product:"+product.getProductName()); - + // serialize the update product information to Solr document(s) List<String> docs = productSerializer.serialize(product, false); // create=false - + // send records to Solr solrClient.index(docs, true, productSerializer.getMimeType()); - + } @Override public void removeProduct(Product product) throws CatalogException { - + // send message to Solr server solrClient.delete(product.getProductId(), true); @@ -168,17 +184,17 @@ public class SolrCatalog implements Catalog { @Override public void setProductTransferStatus(Product product) throws CatalogException { - + this.modifyProduct(product); } @Override public void addProductReferences(Product product) throws CatalogException { - + // generate update documents (with replace=true) - List<String> docs = productSerializer.serialize(product.getProductId(), product.getRootRef(), product.getProductReferences(), true); - + List<String> docs = productSerializer.serialize(product.getProductId(), product.getRootRef(), product.getProductReferences(), true); + // send documents to Solr server solrClient.index(docs, true, productSerializer.getMimeType()); @@ -186,51 +202,51 @@ public class SolrCatalog implements Catalog { @Override public Product getProductById(String productId) throws CatalogException { - - CompleteProduct cp = getCompleteProductById(productId); - return cp.getProduct(); - + + CompleteProduct cp = getCompleteProductById(productId); + return cp.getProduct(); + } @Override public Product getProductByName(String productName) throws CatalogException { - - CompleteProduct cp = getCompleteProductByName(productName); + + CompleteProduct cp = getCompleteProductByName(productName); if (cp!=null) { - LOG.info("Found product name="+productName+" id="+cp.getProduct().getProductId()); - return cp.getProduct(); + LOG.info("Found product name="+productName+" id="+cp.getProduct().getProductId()); + return cp.getProduct(); } else { - LOG.info("Product with name="+productName+" not found"); - return null; + LOG.info("Product with name="+productName+" not found"); + return null; } - + } @Override public List<Reference> getProductReferences(Product product) throws CatalogException { - + CompleteProduct cp = getCompleteProductById(product.getProductId()); return cp.getProduct().getProductReferences(); - + } @Override public List<Product> getProducts() throws CatalogException { - + // build query parameters Map<String, String[]> params = new ConcurrentHashMap<String, String[]>(); params.put("q", new String[] { "*:*" } ); //params.put("rows", new String[] { "20" } ); // control pagination ? - + // query Solr for all matching products return getProducts(params, 0, -1).getProducts(); // get ALL products - + } - + /** * Common utility to retrieve a range of products matching the specified {@link Query} and {@link ProductType}. * This method transforms the given constraints in a map of HTTP (name, value) pairs and delegates to the following method. - * + * * @param query * @param type * @param offset @@ -239,25 +255,25 @@ public class SolrCatalog implements Catalog { * @throws CatalogException */ private QueryResponse getProducts(Query query, ProductType type, int offset, int limit) throws CatalogException { - + // build HTTP request ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>(); // product type constraint params.put("q", new String[]{Parameters.PRODUCT_TYPE_NAME+":"+type.getName()} ); - // convert filemgr query into a Solr query + // convert filemgr query into a Solr query List<String> qc = new ArrayList<String>(); - for (QueryCriteria queryCriteria : query.getCriteria()) { - LOG.info("Query criteria="+queryCriteria.toString()); - qc.add(queryCriteria.toString()); - } - params.put("fq", qc.toArray( new String[ qc.size() ] )); - // sort - params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} ); - - return this.getProducts(params, offset, limit); - - } - + for (QueryCriteria queryCriteria : query.getCriteria()) { + LOG.info("Query criteria="+queryCriteria.toString()); + qc.add(queryCriteria.toString()); + } + params.put("fq", qc.toArray( new String[ qc.size() ] )); + // sort + params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} ); + + return this.getProducts(params, offset, limit); + + } + /** * Common utility to retrieve a range of products matching the specified query parameters * @param params : HTTP query parameters used for query to Solr @@ -266,38 +282,38 @@ public class SolrCatalog implements Catalog { * @return */ private QueryResponse getProducts(Map<String, String[]> params, int offset, int limit) throws CatalogException { - + // combined results from pagination QueryResponse queryResponse = new QueryResponse(); queryResponse.setStart(offset); int start = offset; while (queryResponse.getCompleteProducts().size()<limit || limit<0) { - + params.put("start", new String[] { ""+start } ); String response = solrClient.query(params, productSerializer.getMimeType()); QueryResponse qr = productSerializer.deserialize(response); - + for (CompleteProduct cp : qr.getCompleteProducts()) { if (queryResponse.getCompleteProducts().size()<limit) { queryResponse.getCompleteProducts().add( cp ); } } - + queryResponse.setNumFound( qr.getNumFound() ); start = offset+queryResponse.getCompleteProducts().size(); if (limit<0) { - limit = queryResponse.getNumFound(); // retrieve ALL results + limit = queryResponse.getNumFound(); // retrieve ALL results } if (start>=queryResponse.getNumFound()) { - break; // don't query any longer + break; // don't query any longer } - + } - + LOG.info("Total number of products found="+queryResponse.getNumFound()); LOG.info("Total number of products returned="+queryResponse.getCompleteProducts().size()); return queryResponse; - + } @Override @@ -308,7 +324,7 @@ public class SolrCatalog implements Catalog { params.put("q", new String[] { "*:*" } ); // use the product type name as query parameter params.put("fq", new String[] { Parameters.PRODUCT_TYPE_NAME+":"+type.getName() } ); - + // query Solr for all matching products return getProducts(params, 0, -1).getProducts(); // get ALL products @@ -316,28 +332,28 @@ public class SolrCatalog implements Catalog { @Override public Metadata getMetadata(Product product) throws CatalogException { - + CompleteProduct cp = getCompleteProductById(product.getProductId()); return cp.getMetadata(); - + } @Override public Metadata getReducedMetadata(Product product, List<String> elements) throws CatalogException { - + // build HTTP request ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>(); params.put("q", new String[]{Parameters.PRODUCT_ID+":"+product.getProductId()} ); // request metadata elements explicitly params.put("fl", elements.toArray(new String[elements.size()]) ); - + // execute request String doc = solrClient.query(params, productSerializer.getMimeType()); - + // parse response CompleteProduct cp = extractCompleteProduct(doc); return cp.getMetadata(); - + } /** @@ -346,50 +362,50 @@ public class SolrCatalog implements Catalog { */ @Override public List<String> query(Query query, ProductType type) throws CatalogException { - - // execute request for ALL results - QueryResponse queryResponse = this.getProducts(query, type, 0, -1); // get ALL products + + // execute request for ALL results + QueryResponse queryResponse = this.getProducts(query, type, 0, -1); // get ALL products // extract ids from products List<String> ids = new ArrayList<String>(); for (CompleteProduct cp : queryResponse.getCompleteProducts()) { ids.add(cp.getProduct().getProductId()); } - + return ids; - + } @Override - public List<Product> getTopNProducts(int n) throws CatalogException { - + public List<Product> getTopNProducts(int n) throws CatalogException { + // retrieve most recent n products from Solr String doc = solrClient.queryProductsByDate(n, productSerializer.getMimeType()); - + // parse Solr response into Product objects return this.getProductsFromDocument(doc); - + } - + @Override public ProductPage pagedQuery(Query query, ProductType type, int pageNum) throws CatalogException { - - - // execute request for one page of results - int offset = (pageNum-1)*Parameters.PAGE_SIZE; - int limit = Parameters.PAGE_SIZE; - QueryResponse queryResponse = this.getProducts(query, type, offset, limit); - - // build product page from query response - return newProductPage(pageNum, queryResponse); - - } - + + + // execute request for one page of results + int offset = (pageNum-1)*Parameters.PAGE_SIZE; + int limit = Parameters.PAGE_SIZE; + QueryResponse queryResponse = this.getProducts(query, type, offset, limit); + + // build product page from query response + return newProductPage(pageNum, queryResponse); + + } + @Override public ProductPage getFirstPage(ProductType type) { try { return this.pagedQuery(new Query(), type, 1); - + } catch(CatalogException e) { throw new RuntimeException(e.getMessage(), e); } @@ -397,82 +413,82 @@ public class SolrCatalog implements Catalog { @Override public ProductPage getLastProductPage(ProductType type) { - + try { - + // query for total number of products of this type int numTotalResults = this.getNumProducts(type); - + // compute last page number int numOfPages = PaginationUtils.getTotalPage(numTotalResults, Parameters.PAGE_SIZE); - + // request last page return pagedQuery(new Query(), type, numOfPages); - + } catch(CatalogException e) { throw new RuntimeException(e.getMessage(), e); } - + } @Override public ProductPage getNextPage(ProductType type, ProductPage currentPage) { - + int nextPageNumber = currentPage.getPageNum()+1; if (nextPageNumber>currentPage.getTotalPages()) { - throw new RuntimeException("Invalid next page number: " + nextPageNumber); + throw new RuntimeException("Invalid next page number: " + nextPageNumber); } try { return this.pagedQuery(new Query(), type, currentPage.getPageNum()+1); - + } catch(CatalogException e) { throw new RuntimeException(e.getMessage(), e); } - + } @Override public ProductPage getPrevPage(ProductType type, ProductPage currentPage) { - + int prevPageNumber = currentPage.getPageNum()-1; if (prevPageNumber<=0) { - throw new RuntimeException("Invalid previous page number: " + prevPageNumber); + throw new RuntimeException("Invalid previous page number: " + prevPageNumber); } - + try { return this.pagedQuery(new Query(), type, prevPageNumber); - + } catch(CatalogException e) { throw new RuntimeException(e.getMessage(), e); } - + } - + /** * Common functionality for extracting products from a Solr response document. * @param doc * @return */ private List<Product> getProductsFromDocument(String doc) throws CatalogException { - + // extract full product information from Solr response QueryResponse queryResponse = productSerializer.deserialize(doc); - + // return products only return queryResponse.getProducts(); - + } @Override public List<Product> getTopNProducts(int n, ProductType type) throws CatalogException { - + // retrieve most recent n products from Solr String doc = solrClient.queryProductsByDateAndType(n, type, productSerializer.getMimeType()); - + // parse Solr response into Product objects return this.getProductsFromDocument(doc); - + } @Override @@ -484,43 +500,43 @@ public class SolrCatalog implements Catalog { @Override public int getNumProducts(ProductType type) throws CatalogException { - + // build query parameters Map<String, String[]> params = new ConcurrentHashMap<String, String[]>(); params.put("q", new String[] { "CAS.ProductTypeName:"+type.getName() } ); params.put("rows", new String[] { "0" } ); // don't return any results - + // execute query String response = solrClient.query(params, productSerializer.getMimeType()); - + // parse response QueryResponse queryResponse = productSerializer.deserialize(response); return queryResponse.getNumFound(); - + } - + private CompleteProduct getCompleteProductById(String productId) throws CatalogException { - + // request document with given id String doc = solrClient.queryProductById(productId, productSerializer.getMimeType()); - + // parse document into complete product return extractCompleteProduct(doc); - + } - - private CompleteProduct getCompleteProductByName(String productName) throws CatalogException { - + + private CompleteProduct getCompleteProductByName(String productName) throws CatalogException { + // request document with given id String doc = solrClient.queryProductByName(productName, productSerializer.getMimeType()); - + // parse document into complete product return extractCompleteProduct(doc); - + } - + private CompleteProduct extractCompleteProduct(String doc) throws CatalogException { - + // deserialize document into Product LOG.info("Parsing Solr document: "+doc); QueryResponse queryResponse = productSerializer.deserialize(doc); @@ -532,7 +548,7 @@ public class SolrCatalog implements Catalog { } else { return queryResponse.getCompleteProducts().get(0); } - + } /** @@ -541,7 +557,7 @@ public class SolrCatalog implements Catalog { * @return */ private ProductPage newProductPage(int pageNum, QueryResponse queryResponse) { - + ProductPage page = new ProductPage(); page.setPageNum(pageNum); page.setPageSize(queryResponse.getProducts().size()); @@ -549,7 +565,7 @@ public class SolrCatalog implements Catalog { page.setPageProducts(queryResponse.getProducts()); page.setTotalPages(PaginationUtils.getTotalPage(queryResponse.getNumFound(), Parameters.PAGE_SIZE)); return page; - + } }
