Completely removed XmlRpcFileManager from all the unnecessary locations. Replaced with RpcCommunicationsFactory.createClient()
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/2d19275c Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/2d19275c Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/2d19275c Branch: refs/heads/development Commit: 2d19275c1f142d4d42f9765aad8904ebbe7e5a12 Parents: dc79494 Author: Imesha Sudasingha <[email protected]> Authored: Thu May 17 18:04:53 2018 +0530 Committer: Imesha Sudasingha <[email protected]> Committed: Thu May 17 18:04:53 2018 +0530 ---------------------------------------------------------------------- .../FilemgrUniquenessCheckComparator.java | 10 +- .../curation/service/CurationServiceConfig.java | 22 +- .../cas/curation/service/MetadataResource.java | 317 +++++++++---------- .../cas/curation/service/PolicyResource.java | 9 +- .../cli/action/DeleteProductByIdCliAction.java | 1 - .../cli/action/FileManagerCliAction.java | 22 +- .../cli/action/IngestProductCliAction.java | 1 - .../datatransfer/LocalDataTransferer.java | 11 +- .../cas/filemgr/metadata/ProductMetKeys.java | 9 +- .../oodt/cas/filemgr/tools/DeleteProduct.java | 37 ++- .../oodt/cas/filemgr/tools/ExpImpCatalog.java | 1 - .../tools/MetadataBasedProductMover.java | 1 - .../oodt/cas/filemgr/tools/QueryTool.java | 29 +- .../oodt/cas/filemgr/tools/SolrIndexer.java | 35 +- .../org/apache/oodt/cas/pge/util/XmlHelper.java | 21 +- .../oodt/cas/product/CASProductHandler.java | 1 - .../cas/product/data/DataDeliveryServlet.java | 33 +- .../product/data/DatasetDeliveryServlet.java | 3 +- .../jaxrs/services/CasProductJaxrsService.java | 1 - .../oodt/cas/product/rdf/RDFDatasetServlet.java | 1 - .../oodt/cas/product/rdf/RDFProductServlet.java | 1 - .../oodt/cas/product/rss/RSSProductServlet.java | 34 +- .../product/rss/RSSProductTransferServlet.java | 1 - 23 files changed, 298 insertions(+), 303 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java ---------------------------------------------------------------------- diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java index 95512c1..f0bd4dc 100644 --- a/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java +++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java @@ -23,7 +23,7 @@ import java.io.File; import java.net.URL; //OODT imports -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import org.apache.oodt.cas.filemgr.system.FileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.exceptions.PreconditionComparatorException; import org.apache.oodt.cas.metadata.preconditions.PreConditionComparator; @@ -41,17 +41,15 @@ import org.springframework.beans.factory.annotation.Required; * product exists before generating metadata for it. * </p>. */ -public class FilemgrUniquenessCheckComparator extends - PreConditionComparator<Boolean> { +public class FilemgrUniquenessCheckComparator extends PreConditionComparator<Boolean> { private String filemgrUrl; @Override protected int performCheck(File product, Boolean compareItem) throws PreconditionComparatorException { - try { - boolean returnVal = new XmlRpcFileManagerClient(new URL( - this.filemgrUrl)).hasProduct(product.getName()); + try (FileManagerClient fmClient= RpcCommunicationFactory.createClient(new URL(filemgrUrl))){ + boolean returnVal = fmClient.hasProduct(product.getName()); return Boolean.valueOf(returnVal).compareTo(compareItem); } catch (Exception e) { throw new PreconditionComparatorException( http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java ---------------------------------------------------------------------- diff --git a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java index 1fbfb9b..70f24d7 100644 --- a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java +++ b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java @@ -20,7 +20,8 @@ package org.apache.oodt.cas.curation.service; //OODT imports import org.apache.oodt.cas.curation.metadata.CuratorConfMetKeys; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import org.apache.oodt.cas.filemgr.system.FileManagerClient; +import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.util.PathUtils; @@ -57,14 +58,13 @@ import javax.servlet.ServletContext; * */ public class CurationServiceConfig implements CuratorConfMetKeys { - private static CurationServiceConfig instance; - private final Map<String, String> parameters = new ConcurrentHashMap<String, String>(); - - private XmlRpcFileManagerClient fmClient = null; private static final Logger LOG = Logger .getLogger(CurationServiceConfig.class.getName()); + private static CurationServiceConfig instance; + private final Map<String, String> parameters = new ConcurrentHashMap<String, String>(); + /** * Gets a singleton static instance of the global * {@link CurationServiceConfig} for the CAS Curator Webapp. @@ -121,12 +121,12 @@ public class CurationServiceConfig implements CuratorConfMetKeys { /** * - * @return The full {@link XmlRpcFileManagerClient} built from the CAS curator + * @return The full {@link FileManagerClient} built from the CAS curator * property <code>filemgr.url</code>. */ - public XmlRpcFileManagerClient getFileManagerClient() { + public FileManagerClient getFileManagerClient() { try { - return new XmlRpcFileManagerClient(new URL(this.getFileMgrURL())); + return RpcCommunicationFactory.createClient(new URL(this.getFileMgrURL())); } catch (Exception e) { LOG.log(Level.SEVERE, e.getMessage()); return null; @@ -189,12 +189,6 @@ public class CurationServiceConfig implements CuratorConfMetKeys { // Note that the constructor is private private CurationServiceConfig(ServletConfig conf) { readContextParams(conf.getServletContext()); - try { - fmClient = new XmlRpcFileManagerClient(new URL(this.getFileMgrURL())); - } catch (Exception e) { - LOG.log(Level.SEVERE, e.getMessage()); - LOG.log(Level.WARNING, "Unable to build CurationServiceConfig: Message: " + e.getMessage()); - } } @SuppressWarnings("unchecked") http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java ---------------------------------------------------------------------- diff --git a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java index e9e5953..92901c2 100644 --- a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java +++ b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java @@ -20,6 +20,9 @@ package org.apache.oodt.cas.curation.service; //JDK imports +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; +import net.sf.json.JSONSerializer; import org.apache.oodt.cas.curation.structs.ExtractorConfig; import org.apache.oodt.cas.curation.util.CurationXmlStructFactory; import org.apache.oodt.cas.curation.util.ExtractorConfigReader; @@ -33,7 +36,7 @@ import org.apache.oodt.cas.filemgr.structs.Reference; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.structs.exceptions.ValidationLayerException; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import org.apache.oodt.cas.filemgr.system.FileManagerClient; import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory; import org.apache.oodt.cas.filemgr.validation.XMLValidationLayer; import org.apache.oodt.cas.metadata.MetExtractor; @@ -41,13 +44,16 @@ import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.SerializableMetadata; import org.apache.oodt.cas.metadata.exceptions.MetExtractionException; import org.apache.oodt.cas.metadata.util.GenericMetadataObjectFactory; - -import net.sf.json.JSONArray; -import net.sf.json.JSONObject; -import net.sf.json.JSONSerializer; - import org.springframework.util.StringUtils; +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -59,47 +65,28 @@ import java.net.MalformedURLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.concurrent.ConcurrentHashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; -import javax.servlet.ServletContext; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.FormParam; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; - @Path("metadata") /** - * + * * A web-service endpoint for dealing with CAS {@link Metadata} object. - * + * * @author pramirez * @version $Id$ */ public class MetadataResource extends CurationService { - + @Context UriInfo uriInfo; - - @Context + + @Context private ServletContext context; private static final long serialVersionUID = 1930946924218765724L; @@ -109,20 +96,20 @@ public class MetadataResource extends CurationService { public static final String CATALOG = "catalog"; public static final String PRODUCT_TYPE = "productType"; - + public static final String UPDATE = "update"; - + public static final String DELETE = "delete"; private static Logger LOG = Logger.getLogger(MetadataResource.class.getName()); // single instance of CAS catalog shared among all requests private Catalog catalog = null; - + public MetadataResource(){ - + } public MetadataResource(@Context ServletContext context) { - + } @GET @@ -135,19 +122,19 @@ public class MetadataResource extends CurationService { @Context HttpServletRequest req, @Context HttpServletResponse res) { Metadata metadata; - + try { metadata = this.getStagingMetadata(id, configId, overwrite); } catch (Exception e) { return "<div class=\"error\">" + e.getMessage() + "</div>"; } - + if (FORMAT_HTML.equals(format)) { return this.getMetadataAsHTML(metadata); } return this.getMetadataAsJSON(metadata).toString(); } - + @GET @Path("extractor/config") @Produces("text/plain") @@ -179,14 +166,14 @@ public class MetadataResource extends CurationService { } return html.toString(); } - + protected String getExtractorConfigIdsAsJSON(String[] configIds) { // TODO: Support JSON return "Not Implemented..."; } /** - * + * * @param id * Relative path from staging root to product. The met extension will * be added to this id to look up and see if a met file exists with @@ -229,7 +216,7 @@ public class MetadataResource extends CurationService { } /** - * + * * @param id * Relative path from staging root to the product * @param config @@ -246,7 +233,7 @@ public class MetadataResource extends CurationService { .getStagingAreaPath() + "/" + id); } - + @GET @Path(CATALOG) @Produces("text/plain") @@ -259,9 +246,8 @@ public class MetadataResource extends CurationService { Metadata metadata; String productId = id.substring(id.lastIndexOf("/") + 1); - try { - prod = CurationService.config.getFileManagerClient().getProductById( - productId); + try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) { + prod = fmClient.getProductById(productId); metadata = this.getCatalogMetadata(prod); } catch (Exception e) { return "<div class=\"error\">" + e.getMessage() + "</div>"; @@ -279,15 +265,14 @@ public class MetadataResource extends CurationService { @Produces("text/plain") public String setCatalogMetadata(MultivaluedMap<String, String> formParams, @FormParam("id") String id) { - + Product prod; Metadata metadata = this.getMetadataFromMap(formParams); - + String productId = id.substring(id.lastIndexOf("/") + 1); - try { - prod = CurationService.config.getFileManagerClient().getProductById( - productId); + try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) { + prod = fmClient.getProductById(productId); this.updateCatalogMetadata(prod, metadata); } catch (Exception e) { LOG.log(Level.SEVERE, e.getMessage()); @@ -296,7 +281,7 @@ public class MetadataResource extends CurationService { return this.getMetadataAsHTML(metadata); } - + @GET @Path(PRODUCT_TYPE) @Produces("text/plain") @@ -321,7 +306,7 @@ public class MetadataResource extends CurationService { } return this.getMetadataAsJSON(metadata).toString(); } - + @POST @Path(PRODUCT_TYPE) @@ -361,7 +346,7 @@ public class MetadataResource extends CurationService { return "Staging met info"; } - + private JSONObject getMetadataAsJSON(Metadata metadata) { return JSONObject.fromObject(metadata.getMap()); @@ -379,10 +364,10 @@ public class MetadataResource extends CurationService { return metadata; } - + private Metadata getMetadataFromMap(MultivaluedMap<String, String> formParams) { Metadata metadata = new Metadata(); - + for (Map.Entry<String, List<String>> entry : formParams.entrySet()) { if (entry.getKey().startsWith("metadata.")) { String newKey = entry.getKey().substring(entry.getKey().indexOf('.') + 1); @@ -391,10 +376,10 @@ public class MetadataResource extends CurationService { } } } - + return metadata; } - + protected String getMetadataAsHTML(Metadata metadata) { if (metadata == null) { return "<table></table>"; @@ -423,12 +408,12 @@ public class MetadataResource extends CurationService { return html.toString(); } - + /** * Reads a {@link Metadata} object from a String representation of a .met * {@link File}. - * + * * @param file * The full path to the .met {@link File}. * @return The read-in CAS {@link Metadata} object. @@ -454,7 +439,7 @@ public class MetadataResource extends CurationService { /** * Retrieves the cataloged {@link Metadata} associated with a {@link Product}. - * + * * @param product * The {@link Product} to obtain cataloged {@link Metadata} for. * @return The cataloged {@link Metadata} for {@link Product}. @@ -463,14 +448,18 @@ public class MetadataResource extends CurationService { * {@link Catalog}. */ public Metadata getCatalogMetadata(Product product) throws CatalogException { - return CurationService.config.getFileManagerClient().getMetadata( - CurationService.config.getFileManagerClient().getProductById( - product.getProductId())); + try( FileManagerClient fmClient = CurationService.config.getFileManagerClient()){ + return fmClient.getMetadata(fmClient.getProductById(product.getProductId())); + } catch (IOException e){ + LOG.severe("Error occurred when fetching catalog metadata for product: " + + product.getProductName() + " :" + e.getMessage()); + return null; + } } /** * Writes a CAS {@link Metadata} {@link File} using the given identifier. - * + * * @param id * The identifier of the .met file to write. * @param metadata @@ -488,12 +477,12 @@ public class MetadataResource extends CurationService { serMet.writeMetadataToXmlStream(new FileOutputStream(new File(config .getMetAreaPath(), id + config.getMetExtension()))); } - + /** - * Method to update the catalog metadata for a given product. - * All current metadata fields will be preserved, + * Method to update the catalog metadata for a given product. + * All current metadata fields will be preserved, * except those specified in the HTTP POST request as 'metadata.<field_name>=<field_value>'. - * + * * @param id * identifier of CAS product - either 'id' or 'name' must be specified * @param name @@ -507,23 +496,17 @@ public class MetadataResource extends CurationService { @Path(UPDATE) @Consumes("application/x-www-form-urlencoded") @Produces("text/plain") - public String updateMetadata(MultivaluedMap<String, String> formParams, - @FormParam("id") String id, - @FormParam("name") String name, + public String updateMetadata(MultivaluedMap<String, String> formParams, + @FormParam("id") String id, + @FormParam("name") String name, @DefaultValue("true") @FormParam("replace") boolean replace, @DefaultValue("false") @FormParam("remove") boolean remove) { - + // new metadata from HTTP POST request Metadata newMetadata = this.getMetadataFromMap(formParams); - - // client for interacting with remote File Manager - XmlRpcFileManagerClient fmClient = CurationService.config.getFileManagerClient(); - // empty metadata Metadata metadata; - - try { - + try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) { // retrieve product from catalog Product product; if (StringUtils.hasText(id)) { @@ -534,7 +517,7 @@ public class MetadataResource extends CurationService { } else { throw new Exception("Either the HTTP parameter 'id' or the HTTP parameter 'name' must be specified"); } - + // retrieve existing metadata metadata = fmClient.getMetadata(product); @@ -543,44 +526,41 @@ public class MetadataResource extends CurationService { metadata.removeMetadata("reference_data_store"); metadata.removeMetadata("reference_fileSize"); metadata.removeMetadata("reference_mimeType"); - + // merge new and existing metadata metadata.addMetadata(newMetadata); - + // replace metadata values for keys specified in HTTP request (not others) if (replace) { for (String key : newMetadata.getAllKeys()) { metadata.replaceMetadata(key, newMetadata.getAllMetadata(key)); } } - + // remove metadata tags if (remove) { for (String key : newMetadata.getAllKeys()) { metadata.removeMetadata(key); } } - + // insert old and new metadata fmClient.updateMetadata(product, metadata); - + // return product id to downstream processors return "id="+product.getProductId(); - + } catch (Exception e) { - LOG.log(Level.SEVERE, e.getMessage()); // return error message throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); - } - } /** * Updates the cataloged {@link Metadata} for a {@link Product} in the CAS * File Manager. - * + * * @param product * The {@link Product} to update {@link Metadata} for. * @throws CatalogException @@ -598,7 +578,7 @@ public class MetadataResource extends CurationService { is.close(); } Catalog catalog = this.getCatalog(); - + Metadata oldMetadata = catalog.getMetadata(product); List<Reference> references = catalog.getProductReferences(product); Product newProduct = new Product(product.getProductName(), product @@ -617,7 +597,7 @@ public class MetadataResource extends CurationService { /** * Method to delete a specific product from the catalog - * + * * @param id * identifier of CAS product - either 'id' or 'name' must be specified * @param name @@ -629,20 +609,20 @@ public class MetadataResource extends CurationService { @Consumes("application/x-www-form-urlencoded") @Produces("text/plain") public String deleteCatalogMetadata( - @FormParam("id") String id, + @FormParam("id") String id, @FormParam("name") String name) { - try { + try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) { // retrieve product from catalog Product product; - if (StringUtils.hasText(id)) { - id = id.substring(id.lastIndexOf("/") + 1); - product = CurationService.config.getFileManagerClient().getProductById(id); - } else if (StringUtils.hasText(name)) { - product = CurationService.config.getFileManagerClient().getProductByName(name); - } else { - throw new Exception("Either the HTTP parameter 'id' or the HTTP parameter 'name' must be specified"); - } + if (StringUtils.hasText(id)) { + id = id.substring(id.lastIndexOf("/") + 1); + product = fmClient.getProductById(id); + } else if (StringUtils.hasText(name)) { + product = fmClient.getProductByName(name); + } else { + throw new Exception("Either the HTTP parameter 'id' or the HTTP parameter 'name' must be specified"); + } // remove product from catalog this.deleteCatalogProduct(product); @@ -651,17 +631,16 @@ public class MetadataResource extends CurationService { return "id="+product.getProductId(); } catch (Exception e) { - LOG.log(Level.SEVERE, e.getMessage()); // return error message throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); } - } + } /** * Deletes a given product from the catalog - * + * * @param product * The {@link Product} to delete * @throws FileNotFoundException @@ -669,10 +648,14 @@ public class MetadataResource extends CurationService { * @throws CatalogException * If any error occurs during this delete operation. */ - public void deleteCatalogProduct(Product product) - throws CatalogException { - CurationService.config.getFileManagerClient().removeProduct(product); - } + public void deleteCatalogProduct(Product product) throws CatalogException { + try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()){ + fmClient.removeProduct(product); + } catch (IOException e) { + LOG.severe(String.format("Couldn't detele product - %s : %s", product.getProductName(), e.getMessage())); + throw new CatalogException("Unable to delete product", e); + } + } private Metadata getProductTypeMetadataForPolicy(String policy, String productTypeName) throws MalformedURLException, @@ -685,34 +668,38 @@ public class MetadataResource extends CurationService { XMLRepositoryManager repMgr = new XMLRepositoryManager(Arrays .asList(policies)); ProductType productType = repMgr.getProductTypeByName(productTypeName); - - return productType.getTypeMetadata(); - } - - private Metadata writeProductTypeMetadata(String policy, - String productTypeName, Metadata metadata) - throws MalformedURLException, InstantiationException, RepositoryManagerException, UnsupportedEncodingException, - CurationException { - String rootPolicyPath = this.cleanse(CurationService.config - .getPolicyUploadPath()); - String policyPath = new File(rootPolicyPath + policy).toURL() - .toExternalForm(); - String[] policies = { policyPath }; - XMLRepositoryManager repMgr = new XMLRepositoryManager(Arrays - .asList(policies)); - ProductType productType = repMgr.getProductTypeByName(productTypeName); - productType.setTypeMetadata(metadata); - - CurationXmlStructFactory.writeProductTypeXmlDocument(repMgr - .getProductTypes(), rootPolicyPath + policy + "/product-types.xml"); - - // refresh the config on the fm end - CurationService.config.getFileManagerClient().refreshConfigAndPolicy(); - return productType.getTypeMetadata(); } - + + private Metadata writeProductTypeMetadata(String policy, + String productTypeName, Metadata metadata) + throws MalformedURLException, InstantiationException, RepositoryManagerException, UnsupportedEncodingException, + CurationException { + String rootPolicyPath = this.cleanse(CurationService.config + .getPolicyUploadPath()); + String policyPath = new File(rootPolicyPath + policy).toURL() + .toExternalForm(); + String[] policies = {policyPath}; + XMLRepositoryManager repMgr = new XMLRepositoryManager(Arrays + .asList(policies)); + + ProductType productType = repMgr.getProductTypeByName(productTypeName); + productType.setTypeMetadata(metadata); + + CurationXmlStructFactory.writeProductTypeXmlDocument(repMgr + .getProductTypes(), rootPolicyPath + policy + "/product-types.xml"); + + // refresh the config on the fm end + try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) { + fmClient.refreshConfigAndPolicy(); + } catch (IOException e) { + LOG.severe(String.format("Unable to refresh config and policy: %s", e.getMessage())); + } + + return productType.getTypeMetadata(); + } + private String cleanse(String origPath) { String retStr = origPath; if (!retStr.endsWith("/")) { @@ -720,10 +707,10 @@ public class MetadataResource extends CurationService { } return retStr; } - + // Method to instantiate the CAS catalog, if not done already. private synchronized Catalog getCatalog() { - + if (catalog==null) { String catalogFactoryClass = this.context.getInitParameter(CATALOG_FACTORY_CLASS); // preserve backward compatibility @@ -732,15 +719,15 @@ public class MetadataResource extends CurationService { } catalog = GenericFileManagerObjectFactory.getCatalogServiceFromFactory(catalogFactoryClass); } - + return catalog; } - + @DELETE @Path(PRODUCT_TYPE+"/remove") @Produces("text/plain") public boolean removeProductType( - @FormParam("policy") String policy, + @FormParam("policy") String policy, @FormParam("id") String id) { XMLRepositoryManager xmlRepo = getRepo(policy); try { @@ -752,7 +739,7 @@ public class MetadataResource extends CurationService { return false; } } - + @GET @Path(PRODUCT_TYPE+"/parentmap") @Produces("text/plain") @@ -761,13 +748,13 @@ public class MetadataResource extends CurationService { XMLValidationLayer vLayer = getValidationLayer(policy); return JSONSerializer.toJSON(vLayer.getSubToSuperMap()).toString(); } - + @POST @Path(PRODUCT_TYPE+"/parent/add") @Produces("text/plain") public boolean addParentForProductType( - @FormParam("policy") String policy, - @FormParam("id") String id, + @FormParam("policy") String policy, + @FormParam("id") String id, @FormParam("parentId") String parentId) { XMLValidationLayer vLayer = getValidationLayer(policy); XMLRepositoryManager xmlRepo = getRepo(policy); @@ -780,13 +767,13 @@ public class MetadataResource extends CurationService { } return false; } - + @DELETE @Path(PRODUCT_TYPE+"/parent/remove") @Produces("text/plain") public boolean removeParentForProductType( - @FormParam("policy") String policy, - @FormParam("id") String id) + @FormParam("policy") String policy, + @FormParam("id") String id) throws ValidationLayerException { XMLValidationLayer vLayer = getValidationLayer(policy); XMLRepositoryManager xmlRepo = getRepo(policy); @@ -799,13 +786,13 @@ public class MetadataResource extends CurationService { } return false; } - + @POST @Path(PRODUCT_TYPE+"/elements/add") @Produces("text/plain") public boolean addElementsForProductType( - @FormParam("policy") String policy, - @FormParam("id") String id, + @FormParam("policy") String policy, + @FormParam("id") String id, @FormParam("elementIds") String elementIds) { XMLValidationLayer vLayer = getValidationLayer(policy); XMLRepositoryManager xmlRepo = getRepo(policy); @@ -825,13 +812,13 @@ public class MetadataResource extends CurationService { } return false; } - + @GET @Path(PRODUCT_TYPE+"/elements") @Produces("text/plain") public String getElementsForProductType( - @FormParam("policy") String policy, - @FormParam("id") String id, + @FormParam("policy") String policy, + @FormParam("id") String id, @FormParam("direct") boolean direct) { XMLValidationLayer vLayer = getValidationLayer(policy); XMLRepositoryManager xmlRepo = getRepo(policy); @@ -847,12 +834,12 @@ public class MetadataResource extends CurationService { } return null; } - + @DELETE @Path(PRODUCT_TYPE+"/elements/remove/all") @Produces("text/plain") public boolean removeAllElementsForProductType( - @FormParam("policy") String policy, + @FormParam("policy") String policy, @FormParam("id") String id) { XMLValidationLayer vLayer = getValidationLayer(policy); XMLRepositoryManager xmlRepo = getRepo(policy); @@ -874,8 +861,8 @@ public class MetadataResource extends CurationService { @Path(PRODUCT_TYPE+"/elements/remove") @Produces("text/plain") public boolean removeElementsForProductType( - @FormParam("policy") String policy, - @FormParam("id") String id, + @FormParam("policy") String policy, + @FormParam("id") String id, @FormParam("elementIds") String elementIds) { XMLValidationLayer vLayer = getValidationLayer(policy); XMLRepositoryManager xmlRepo = getRepo(policy); @@ -896,12 +883,12 @@ public class MetadataResource extends CurationService { } return false; } - + @GET @Path(PRODUCT_TYPE+"/typeswithelement/{elementId}") @Produces("text/plain") public String getProductTypeIdsHavingElement( - @FormParam("policy") String policy, + @FormParam("policy") String policy, @PathParam("elementId") String elementId) { XMLValidationLayer vLayer = getValidationLayer(policy); XMLRepositoryManager xmlRepo = getRepo(policy); @@ -919,19 +906,19 @@ public class MetadataResource extends CurationService { } return JSONSerializer.toJSON(typeids).toString(); } - - + + /* * Private helper functions */ - private void removeUnusedElements(List<Element> elements, - XMLRepositoryManager xmlRepo, XMLValidationLayer vLayer) + private void removeUnusedElements(List<Element> elements, + XMLRepositoryManager xmlRepo, XMLValidationLayer vLayer) throws ValidationLayerException, RepositoryManagerException { // Remove Elements that aren't used in any product type List<ProductType> ptypelist = xmlRepo.getProductTypes(); ConcurrentHashMap<String, Boolean> usedElementIds = new ConcurrentHashMap<String, Boolean>(); for(ProductType ptype: ptypelist) { - List<Element> ptypeElements = + List<Element> ptypeElements = vLayer.getElements(ptype); for(Element el: ptypeElements) { usedElementIds.put(el.getElementId(), true); @@ -942,7 +929,7 @@ public class MetadataResource extends CurationService { vLayer.removeElement(el); } } - } + } private XMLRepositoryManager getRepo(String policy) { XMLRepositoryManager xmlRepo = null; http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java ---------------------------------------------------------------------- diff --git a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java index bde259b..c3bd174 100644 --- a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java +++ b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java @@ -28,6 +28,7 @@ import org.apache.oodt.cas.filemgr.structs.Query; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import net.sf.json.JSONObject; +import org.apache.oodt.cas.filemgr.system.FileManagerClient; import java.io.File; import java.io.FilenameFilter; @@ -116,11 +117,9 @@ public class PolicyResource extends CurationService { ProductType productType; ProductPage page; - try { - productType = config.getFileManagerClient().getProductTypeByName( - productTypeName); - page = config.getFileManagerClient().pagedQuery(new Query(), - productType, pageNum); + try (FileManagerClient fmClient = config.getFileManagerClient()){ + productType = fmClient.getProductTypeByName(productTypeName); + page = fmClient.pagedQuery(new Query(), productType, pageNum); } catch (Exception e) { LOG.log(Level.SEVERE, e.getMessage()); LOG.log(Level.WARNING, "Unable to obtain products for product type: [" http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java index a202a0c..03eb54e 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java @@ -23,7 +23,6 @@ import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import java.io.IOException; import java.net.MalformedURLException; http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java index c3e9032..8630e3d 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java @@ -17,18 +17,20 @@ package org.apache.oodt.cas.filemgr.cli.action; //JDK imports -import java.net.MalformedURLException; -import java.net.URL; -//Apache imports import org.apache.commons.lang.Validate; - -//OODT imports import org.apache.oodt.cas.cli.action.CmdLineAction; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +//Apache imports +//OODT imports + /** * Base {@link CmdLineAction} for File Manager. * @@ -42,8 +44,7 @@ public abstract class FileManagerCliAction extends CmdLineAction { return System.getProperty("org.apache.oodt.cas.filemgr.url"); } - protected FileManagerClient getClient() - throws MalformedURLException, ConnectionException { + protected FileManagerClient getClient() throws MalformedURLException, ConnectionException { Validate.notNull(getUrl(), "Must specify url"); if (client != null) { @@ -56,4 +57,11 @@ public abstract class FileManagerCliAction extends CmdLineAction { public void setClient(FileManagerClient client) { this.client = client; } + + @Override + public void finalize() throws IOException { + if (client != null) { + client.close(); + } + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java index 4bc326b..e0ec33e 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java @@ -23,7 +23,6 @@ import org.apache.oodt.cas.cli.exception.CmdLineActionException; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.metadata.SerializableMetadata; import com.google.common.collect.Lists; http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java index b6f2beb..06eb300 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java @@ -24,7 +24,6 @@ import org.apache.oodt.cas.filemgr.structs.Reference; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.filemgr.versioning.VersioningUtils; import org.apache.tika.mime.MimeTypeException; @@ -84,8 +83,7 @@ public class LocalDataTransferer implements DataTransfer { public void setFileManagerUrl(URL url) { try { client = RpcCommunicationFactory.createClient(url); - LOG.log(Level.INFO, "Local Data Transfer to: [" - + client.getFileManagerUrl().toString() + "] enabled"); + LOG.log(Level.INFO, "Local Data Transfer to: [" + client.getFileManagerUrl().toString() + "] enabled"); } catch (ConnectionException e) { LOG.log(Level.SEVERE, e.getMessage()); } @@ -439,4 +437,11 @@ public class LocalDataTransferer implements DataTransfer { + p.getProductId() + "]: Message: " + e.getMessage()); } } + + @Override + public void finalize() throws IOException { + if (client != null) { + client.close(); + } + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java index 558c6eb..cfd8377 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -20,10 +20,11 @@ package org.apache.oodt.cas.filemgr.metadata; /** * - * Met key field names used to augment {@link Product} {@link Metadata} in - * {@link org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient#getMetadata(org.apache.oodt.cas.filemgr.structs.Product)} + * Met key field names used to augment {@link org.apache.oodt.cas.filemgr.structs.Product} + * {@link org.apache.oodt.cas.metadata.Metadata} in + * {@link org.apache.oodt.cas.filemgr.system.FileManagerClient#getMetadata(org.apache.oodt.cas.filemgr.structs.Product)} * and in - * {@link org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient#getReducedMetadata(org.apache.oodt.cas.filemgr.structs.Product, java.util.List)} + * {@link org.apache.oodt.cas.filemgr.system.FileManagerClient#getReducedMetadata(org.apache.oodt.cas.filemgr.structs.Product, java.util.List)} * . * * @see http://issues.apache.org/jira/browse/OODT-72 http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java index 654c85a..772a5ee 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java @@ -18,13 +18,14 @@ package org.apache.oodt.cas.filemgr.tools; //OODT imports -import org.apache.oodt.cas.filemgr.structs.Reference; + import org.apache.oodt.cas.filemgr.structs.Product; +import org.apache.oodt.cas.filemgr.structs.Reference; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import org.apache.oodt.cas.filemgr.system.FileManagerClient; +import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; -//JDK imports import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -32,10 +33,12 @@ import java.io.InputStreamReader; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.List; import java.util.Vector; +import java.util.logging.Level; +import java.util.logging.Logger; + +//JDK imports /** * @author woollard @@ -53,22 +56,19 @@ public class DeleteProduct { private static final Logger LOG = Logger .getLogger(DeleteProduct.class.getName()); - /* our File Manager client */ - private XmlRpcFileManagerClient client = null; - + private FileManagerClient client; /* whether or not we should commit our deletions */ private boolean commit = true; public DeleteProduct(String fileManagerUrl, boolean commit) { + this.commit = commit; + try { - client = new XmlRpcFileManagerClient(new URL(fileManagerUrl)); + client = RpcCommunicationFactory.createClient(new URL(fileManagerUrl)); } catch (Exception e) { - LOG.log(Level.SEVERE, "Unable to create file manager client: Message: " - + e.getMessage() + ": errors to follow"); + LOG.severe("Unable to create client: " + e.getMessage()); } - this.commit = commit; - if (!this.commit) { LOG.log(Level.INFO, "Commit disabled."); } else { @@ -230,4 +230,15 @@ public class DeleteProduct { return prodIds; } + + /** + * Not the best place to do this. But, no other option at the moment. Mandatory to close the client once done. + * @throws IOException + */ + @Override + public void finalize() throws IOException { + if (client != null) { + client.close(); + } + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java index e64e560..c2db95f 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java @@ -26,7 +26,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.Metadata; http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java index 989dbd7..64e2c67 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java @@ -26,7 +26,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.util.PathUtils; http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java index 27e37d2..cf065e4 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java @@ -32,10 +32,11 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.QueryFormulationException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.structs.query.ComplexQuery; import org.apache.oodt.cas.filemgr.structs.query.QueryResult; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import org.apache.oodt.cas.filemgr.system.FileManagerClient; +import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.filemgr.util.SqlParser; -import java.net.MalformedURLException; +import java.io.IOException; import java.net.URL; import java.util.List; import java.util.Vector; @@ -55,7 +56,7 @@ public final class QueryTool { private static String freeTextBlock = "__FREE__"; - private XmlRpcFileManagerClient client = null; + private FileManagerClient client = null; private enum QueryType { LUCENE, SQL } @@ -64,7 +65,7 @@ public final class QueryTool { public QueryTool(URL fmUrl) throws InstantiationException { try { - client = new XmlRpcFileManagerClient(fmUrl); + client = RpcCommunicationFactory.createClient(fmUrl); } catch (ConnectionException e) { throw new InstantiationException(e.getMessage()); } @@ -151,8 +152,8 @@ public final class QueryTool { } public static void main(String[] args) - throws MalformedURLException, InstantiationException, CatalogException, QueryFormulationException, - ConnectionException { + throws IOException, InstantiationException, CatalogException, QueryFormulationException, + ConnectionException { String usage = "Usage: QueryTool [options] \n" + "options: \n" + "--url <fm url> \n" @@ -241,17 +242,19 @@ public final class QueryTool { return prodTypes; } - private static String performSqlQuery(String query, String sortBy, String outputFormat, String delimiter, String filemgrUrl) - throws MalformedURLException, CatalogException, ConnectionException, QueryFormulationException { + private static String performSqlQuery(String query, String sortBy, String outputFormat, String delimiter, String filemgrUrl) + throws IOException, CatalogException, ConnectionException, QueryFormulationException { ComplexQuery complexQuery = SqlParser.parseSqlQuery(query); complexQuery.setSortByMetKey(sortBy); complexQuery.setToStringResultFormat(outputFormat); - List<QueryResult> results = new XmlRpcFileManagerClient(new URL(filemgrUrl)).complexQuery(complexQuery); - StringBuilder returnString = new StringBuilder(""); - for (QueryResult qr : results) { - returnString.append(qr.toString()).append(delimiter); + try(FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(filemgrUrl))){ + List<QueryResult> results = fmClient.complexQuery(complexQuery); + StringBuilder returnString = new StringBuilder(""); + for (QueryResult qr : results) { + returnString.append(qr.toString()).append(delimiter); + } + return returnString.substring(0, returnString.length() - delimiter.length()); } - return returnString.substring(0, returnString.length() - delimiter.length()); } private static void exit(String msg) { http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java index 16ca7f1..9361ead 100755 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java @@ -34,7 +34,8 @@ import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import org.apache.oodt.cas.filemgr.system.FileManagerClient; +import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.SerializableMetadata; import org.apache.oodt.cas.metadata.util.PathUtils; @@ -67,6 +68,9 @@ import java.util.logging.Logger; * specific documentation. */ public class SolrIndexer { + + private static Logger LOG = Logger.getLogger(SolrIndexer.class.getName()); + private final static String SOLR_INDEXER_CONFIG = "SOLR_INDEXER_CONFIG"; private final static String SOLR_URL = "solr.url"; private final static String FILEMGR_URL = "filemgr.url"; @@ -77,7 +81,6 @@ public class SolrIndexer { private final SolrServer server; private String fmUrl; private String solrUrl; - private static Logger LOG = Logger.getLogger(SolrIndexer.class.getName()); private final static SimpleDateFormat solrFormat = new SimpleDateFormat( "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); @@ -245,9 +248,7 @@ public class SolrIndexer { */ public void indexProductTypes(boolean delete) { LOG.info("Indexing product types..."); - try { - XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL( - this.fmUrl)); + try (FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl))) { LOG.info("Retrieving list of product types."); List<ProductType> types = fmClient.getProductTypes(); for (ProductType type : types) { @@ -290,6 +291,8 @@ public class SolrIndexer { } catch (RepositoryManagerException e) { LOG.severe("Could not retrieve product types from File Manager: " + e.getMessage()); + } catch (IOException e) { + LOG.severe(String.format("Error occurred when indexing product types: %s", e.getMessage())); } LOG.info("Finished indexing product types."); } @@ -297,7 +300,7 @@ public class SolrIndexer { /** * Suppresses exception that occurred with older file managers. */ - private ProductPage safeFirstPage(XmlRpcFileManagerClient fmClient, ProductType type) { + private ProductPage safeFirstPage(FileManagerClient fmClient, ProductType type) { ProductPage page = null; try { page = fmClient.getFirstPage(type); @@ -320,9 +323,7 @@ public class SolrIndexer { */ public void indexAll(boolean delete) { LOG.info("Indexing products..."); - try { - XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL( - this.fmUrl)); + try (FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl))) { LOG.info("Retrieving list of product types."); List<ProductType> types = fmClient.getProductTypes(); for (ProductType type : types) { @@ -358,6 +359,8 @@ public class SolrIndexer { } catch (RepositoryManagerException e) { LOG.severe("Could not retrieve product types from File Manager: " + e.getMessage()); + } catch (IOException e) { + LOG.severe(String.format("Error occurred when indexing: %s", e.getMessage())); } } @@ -374,9 +377,7 @@ public class SolrIndexer { public void indexProduct(String productId) throws SolrServerException { LOG.info("Attempting to index product: " + productId); - try { - XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL( - this.fmUrl)); + try (FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl))) { Product product = fmClient.getProductById(productId); Metadata productMetadata = fmClient.getMetadata(product); indexProduct(product.getProductId(), productMetadata, product @@ -390,6 +391,8 @@ public class SolrIndexer { + e.getMessage()); } catch (java.text.ParseException e) { LOG.severe("Could not format date: " + e.getMessage()); + } catch (IOException e) { + LOG.severe(String.format("Error occurred when indexing product types: %s", e.getMessage())); } } @@ -409,8 +412,7 @@ public class SolrIndexer { public void indexProductByName(String productName, boolean delete) throws SolrServerException { LOG.info("Attempting to index product: " + productName); - try { - + try (FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl))) { // Try to delete product by name // Note: the standard field "CAS.ProductName" must be mapped to some Solr field in file indexer.properties if (delete) { @@ -425,9 +427,6 @@ public class SolrIndexer { LOG.warning("Could not delete product: "+productName+" from Solr index"); } } - - XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL( - this.fmUrl)); Product product = fmClient.getProductByName(productName); Metadata productMetadata = fmClient.getMetadata(product); // NOTE: delete (by id) is now false @@ -442,6 +441,8 @@ public class SolrIndexer { + e.getMessage()); } catch (java.text.ParseException e) { LOG.severe("Could not format date: " + e.getMessage()); + } catch (IOException e) { + LOG.severe(String.format("Error occurred when indexing product types: %s", e.getMessage())); } } http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java ---------------------------------------------------------------------- diff --git a/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java b/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java index 8c53aff..9f01d99 100644 --- a/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java +++ b/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java @@ -16,7 +16,10 @@ */ package org.apache.oodt.cas.pge.util; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.apache.oodt.cas.filemgr.system.FileManagerClient; import org.apache.oodt.cas.filemgr.util.QueryUtils; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.filemgr.util.SqlParser; @@ -28,11 +31,6 @@ import org.apache.oodt.cas.pge.config.OutputDir; import org.apache.oodt.cas.pge.config.RegExprOutputFiles; import org.apache.oodt.cas.pge.exceptions.PGEException; import org.apache.oodt.commons.xml.XMLUtils; - -import com.google.common.base.Splitter; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; - import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -442,20 +440,15 @@ public class XmlHelper { public static String fillIn(String value, Metadata inputMetadata, boolean envReplaceRecur) throws PGEException { - try { + try (FileManagerClient fmClient=RpcCommunicationFactory.createClient( + new URL(inputMetadata.getMetadata(QUERY_FILE_MANAGER_URL.getName())))){ while ((value = PathUtils .doDynamicReplacement(value, inputMetadata)).contains("[") && envReplaceRecur) { } if (value.toUpperCase().matches( "^\\s*SQL\\s*\\(.*\\)\\s*\\{.*\\}\\s*$")) { - value = QueryUtils - .getQueryResultsAsString(new XmlRpcFileManagerClient( - new URL(inputMetadata - .getMetadata(QUERY_FILE_MANAGER_URL - .getName()))) - .complexQuery(SqlParser - .parseSqlQueryMethod(value))); + value = QueryUtils.getQueryResultsAsString(fmClient.complexQuery(SqlParser.parseSqlQueryMethod(value))); } return value; } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java ---------------------------------------------------------------------- diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java index 4e4d19c..00ef50f 100644 --- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java +++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java @@ -25,7 +25,6 @@ import org.apache.oodt.cas.filemgr.structs.Reference; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.product.data.DataUtils; import org.apache.oodt.product.LargeProductQueryHandler; http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java ---------------------------------------------------------------------- diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java index 62a9b25..282053b 100644 --- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java +++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java @@ -24,7 +24,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.Reference; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.util.PathUtils; @@ -59,18 +58,30 @@ public class DataDeliveryServlet extends HttpServlet implements public static final int INT = 512; + /** Client i/f to filemgr server. */ + private FileManagerClient client; + + /** our log stream */ + private static final Logger LOG = Logger.getLogger(DataDeliveryServlet.class + .getName()); + + /** our working dir path. */ + private String workingDirPath; + + /** serial version UID. */ + private static final long serialVersionUID = -955613407495060869L; + /** {@inheritDoc} */ public void init(ServletConfig config) throws ServletException { super.init(config); try { String fileMgrURL; try { - fileMgrURL = PathUtils.replaceEnvVariables(config.getServletContext().getInitParameter( - "filemgr.url") ); + fileMgrURL = PathUtils.replaceEnvVariables(config.getServletContext().getInitParameter("filemgr.url") ); } catch (Exception e) { throw new ServletException("Failed to get filemgr url : " + e.getMessage(), e); } - client = new XmlRpcFileManagerClient(new URL(fileMgrURL)); + client = RpcCommunicationFactory.createClient(new URL(fileMgrURL)); } catch (MalformedURLException ex) { throw new ServletException(ex); } catch (ConnectionException ex) { @@ -265,18 +276,4 @@ public class DataDeliveryServlet extends HttpServlet implements in.close(); out.close(); } - - /** Client i/f to filemgr server. */ - private FileManagerClient client; - - /** our log stream */ - private static final Logger LOG = Logger.getLogger(DataDeliveryServlet.class - .getName()); - - /** our working dir path. */ - private String workingDirPath; - - /** serial version UID. */ - private static final long serialVersionUID = -955613407495060869L; - } http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java ---------------------------------------------------------------------- diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java index 00c89c1..9685e36 100644 --- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java +++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java @@ -24,7 +24,6 @@ import org.apache.oodt.cas.filemgr.structs.ProductPage; import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.util.PathUtils; @@ -247,7 +246,7 @@ public class DatasetDeliveryServlet extends HttpServlet implements } catch (Exception e) { throw new ServletException("Failed to get filemgr url : " + e.getMessage(), e); } - client = new XmlRpcFileManagerClient(new URL(fileMgrURL)); + client = RpcCommunicationFactory.createClient(new URL(fileMgrURL)); } catch (MalformedURLException ex) { throw new ServletException(ex); } catch (ConnectionException ex) { http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java ---------------------------------------------------------------------- diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java index f8d32d3..ddb98de 100755 --- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java +++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java @@ -22,7 +22,6 @@ import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.structs.Reference; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.product.exceptions.CasProductException; import org.apache.oodt.cas.product.jaxrs.exceptions.BadRequestException; http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java ---------------------------------------------------------------------- diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java index 03fb813..e15faf4 100644 --- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java +++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java @@ -26,7 +26,6 @@ import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.util.PathUtils; import org.apache.oodt.commons.xml.XMLUtils; http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java ---------------------------------------------------------------------- diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java index df1eeac..f32211a 100644 --- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java +++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java @@ -29,7 +29,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.util.PathUtils; http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java ---------------------------------------------------------------------- diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java index 94772d3..4a2636c 100644 --- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java +++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java @@ -20,6 +20,7 @@ package org.apache.oodt.cas.product.rss; //OODT imports + import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductType; import org.apache.oodt.cas.filemgr.structs.Reference; @@ -27,7 +28,7 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.util.PathUtils; import org.apache.oodt.commons.util.DateConvert; @@ -35,17 +36,6 @@ import org.apache.oodt.commons.xml.XMLUtils; import org.w3c.dom.Document; import org.w3c.dom.Element; -//JDK imports -import java.io.FileNotFoundException; -import java.net.MalformedURLException; -import java.net.URL; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - import javax.servlet.ServletConfig; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -58,6 +48,18 @@ import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +//JDK imports /** * A Servlet that supports the <a @@ -317,7 +319,7 @@ public class RSSProductServlet extends HttpServlet { private void getFileManager(String fileManagerUrl) { try { - fm = new XmlRpcFileManagerClient(new URL(fileManagerUrl)); + fm = RpcCommunicationFactory.createClient(new URL(fileManagerUrl)); } catch (MalformedURLException e) { LOG.log(Level.SEVERE, "Unable to initialize file manager url in RSS Servlet: [url=" @@ -329,4 +331,10 @@ public class RSSProductServlet extends HttpServlet { } } + @Override + public void finalize() throws IOException { + if (fm != null) { + fm.close(); + } + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java ---------------------------------------------------------------------- diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java index 9aaecaf..44eb712 100644 --- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java +++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java @@ -23,7 +23,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.util.PathUtils;
