Repository: oodt Updated Branches: refs/heads/development 835f16d08 -> 9b0b61ec0
Merge master versions of files. Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/9b0b61ec Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/9b0b61ec Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/9b0b61ec Branch: refs/heads/development Commit: 9b0b61ec0e1fd5c36a1ea9c9902a947ba0d20ba1 Parents: 835f16d Author: Chris Mattmann <mattm...@apache.org> Authored: Sun Sep 3 11:56:59 2017 -0700 Committer: Chris Mattmann <mattm...@apache.org> Committed: Sun Sep 3 11:56:59 2017 -0700 ---------------------------------------------------------------------- .../oodt/cas/filemgr/catalog/LuceneCatalog.java | 10 +- .../oodt/cas/filemgr/tools/DeleteProduct.java | 303 ++++++++++--------- .../oodt/cas/filemgr/tools/QueryTool.java | 127 ++++---- .../oodt/cas/filemgr/tools/SolrIndexer.java | 228 +++++++------- .../LuceneWorkflowInstanceRepository.java | 53 +++- 5 files changed, 384 insertions(+), 337 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java index a8c0253..59ded25 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java @@ -902,7 +902,8 @@ public class LuceneCatalog implements Catalog { */ public ProductPage getNextPage(ProductType type, ProductPage currentPage) { if(type==null){ - return null; + LOG.warning("getNextPage: Provided type was null: Returning blank page."); + return ProductPage.blankPage(); } if (currentPage == null) { return getFirstPage(type); @@ -926,11 +927,11 @@ public class LuceneCatalog implements Catalog { "CatalogException getting next page for product type: [" + type.getProductTypeId() + "] from catalog: Message: " + e.getMessage()); - return null; + return ProductPage.blankPage(); } // There are no products and thus no next page if (products == null || (products.size() == 0)) { - return null; + return ProductPage.blankPage(); } nextPage.setPageProducts(products); @@ -945,7 +946,8 @@ public class LuceneCatalog implements Catalog { */ public ProductPage getPrevPage(ProductType type, ProductPage currentPage) { if(type==null){ - return null; + LOG.warning("getPrevPage: Provided type was null: Returning blank page."); + return ProductPage.blankPage(); } if (currentPage == null) { http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java index 42b07e9..654c85a 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java @@ -22,9 +22,7 @@ import org.apache.oodt.cas.filemgr.structs.Reference; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException; -import org.apache.oodt.cas.filemgr.system.FileManagerClient; import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; -import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; //JDK imports import java.io.BufferedReader; @@ -43,180 +41,193 @@ import java.util.Vector; * @author woollard * @version $Revision$ * - * <p> - * A utility class that deletes products in the File Manager Catalog based on - * productID. - * </p> + * <p> + * A utility class that deletes products in the File Manager Catalog + * based on productID. + * </p> * */ -@Deprecated public class DeleteProduct { - /* our log stream */ - private static final Logger LOG = Logger.getLogger(DeleteProduct.class.getName()); + /* our log stream */ + private static final Logger LOG = Logger + .getLogger(DeleteProduct.class.getName()); - /* our File Manager client */ - private FileManagerClient client = null; + /* our File Manager client */ + private XmlRpcFileManagerClient client = null; - /* whether or not we should commit our deletions */ - private boolean commit = true; + /* whether or not we should commit our deletions */ + private boolean commit = true; - public DeleteProduct(String fileManagerUrl, boolean commit) { - try { - client = RpcCommunicationFactory.createClient(new URL(fileManagerUrl)); - } catch (Exception e) { - LOG.log(Level.SEVERE, - "Unable to create file manager client: Message: " - + e.getMessage() + ": errors to follow"); - } + public DeleteProduct(String fileManagerUrl, boolean commit) { + try { + client = new XmlRpcFileManagerClient(new URL(fileManagerUrl)); + } catch (Exception e) { + LOG.log(Level.SEVERE, "Unable to create file manager client: Message: " + + e.getMessage() + ": errors to follow"); + } - this.commit = commit; + this.commit = commit; - if (!this.commit) { - LOG.log(Level.INFO, "Commit disabled."); - } else { - LOG.log(Level.INFO, "Commit enabled."); - } + if (!this.commit) { + LOG.log(Level.INFO, "Commit disabled."); + } else { + LOG.log(Level.INFO, "Commit enabled."); } + } - /** - * @param args - */ - public static void main(String[] args) { - String productId = null; - String fileManagerUrl = null; - boolean commitChanges = true; - boolean readFromStdIn = false; - - String usage = "DeleteProduct --productID <product id> " - + "--fileManagerUrl <url to file manager> [--read] [--nocommit]\n"; - - for (int i = 0; i < args.length; i++) { - if (args[i].equals("--productID")) { - productId = args[++i]; - } else if (args[i].equals("--fileManagerUrl")) { - fileManagerUrl = args[++i]; - } else if (args[i].equals("--read")) { - readFromStdIn = true; - } else if (args[i].equals("--nocommit")) { - commitChanges = false; - } - } - - if ((productId == null && !readFromStdIn) || fileManagerUrl == null) { - System.err.println(usage); - System.exit(1); - } + public void remove(String productId) { + Product target = null; - DeleteProduct remover = new DeleteProduct(fileManagerUrl, commitChanges); - if (readFromStdIn) { - List prodIds = readProdIdsFromStdin(); - for (Object prodId1 : prodIds) { - String prodId = (String) prodId1; - remover.remove(prodId); - } - } else { - remover.remove(productId); - } + try { + target = client.getProductById(productId); + } catch (CatalogException e) { + LOG.log(Level.WARNING, + "Unable to obtain product : [" + productId + "] from file manager: [" + + client.getFileManagerUrl() + "]: Message: " + e.getMessage()); + } + if (target == null) { + // could not file product + return; } - private static List readProdIdsFromStdin() { - List prodIds = new Vector(); - BufferedReader br; + // delete references first + Vector refs = new Vector(); - br = new BufferedReader(new InputStreamReader(System.in)); + try { + refs = (Vector) client.getProductReferences(target); + } catch (CatalogException e) { + LOG.log(Level.WARNING, + "Unable to obtain references for product : [" + productId + + "] from file manager: [" + client.getFileManagerUrl() + + "]: Message: " + e.getMessage()); + } - String line = null; + for (Object ref1 : refs) { + Reference ref = (Reference) ref1; + if (commit) { try { - while ((line = br.readLine()) != null) { - prodIds.add(line); - } - } catch (IOException e) { - LOG.log(Level.WARNING, "Error reading prod id: line: [" + line - + "]: Message: " + e.getMessage(), e); - } finally { - try { - br.close(); - } catch (Exception ignore) { - } - + client.removeFile( + new File(new URI(ref.getDataStoreReference())).getAbsolutePath()); + } catch (DataTransferException e) { + LOG.log(Level.WARNING, + "Unable to delete reference : [" + ref.getDataStoreReference() + + "] for product : [" + productId + "] from file manager : [" + + client.getFileManagerUrl() + "]: Message: " + + e.getMessage()); + } catch (URISyntaxException e) { + LOG.log(Level.WARNING, + "uri syntax exception getting file absolute path from URI: [" + + ref.getDataStoreReference() + "]: Message: " + + e.getMessage()); } + } else { + LOG.log(Level.INFO, + "Delete file: [" + ref.getDataStoreReference() + "]"); + } + + } - return prodIds; + if (commit) { + + // now delete product + try { + client.removeProduct(target); + } catch (CatalogException e) { + LOG.log(Level.WARNING, + "Unable to remove product : [" + productId + + "] from file manager: [" + client.getFileManagerUrl() + + "]: Message: " + e.getMessage()); + } + } else { + LOG.log(Level.INFO, "Remote catalog entry for product: [" + + target.getProductName() + "]"); } - private void remove(String productId) { - Product target = null; + } + + /** + * @return the commit + */ + public boolean isCommit() { + return commit; + } + + /** + * @param commit + * the commit to set + */ + public void setCommit(boolean commit) { + this.commit = commit; + } + + /** + * @param args + */ + public static void main(String[] args) { + String productId = null; + String fileManagerUrl = null; + boolean commitChanges = true; + boolean readFromStdIn = false; + + String usage = "DeleteProduct --productID <product id> " + + "--fileManagerUrl <url to file manager> [--read] [--nocommit]\n"; + + for (int i = 0; i < args.length; i++) { + if (args[i].equals("--productID")) { + productId = args[++i]; + } else if (args[i].equals("--fileManagerUrl")) { + fileManagerUrl = args[++i]; + } else if (args[i].equals("--read")) { + readFromStdIn = true; + } else if (args[i].equals("--nocommit")) { + commitChanges = false; + } + } - try { - target = client.getProductById(productId); - } catch (CatalogException e) { - LOG.log(Level.WARNING, "Unable to obtain product : [" + productId - + "] from file manager: [" + client.getFileManagerUrl() - + "]: Message: " + e.getMessage()); - } + if ((productId == null && !readFromStdIn) || fileManagerUrl == null) { + System.err.println(usage); + System.exit(1); + } - if (target == null) { - // could not file product - return; - } + DeleteProduct remover = new DeleteProduct(fileManagerUrl, commitChanges); + if (readFromStdIn) { + List prodIds = readProdIdsFromStdin(); + for (Object prodId1 : prodIds) { + String prodId = (String) prodId1; + remover.remove(prodId); + } + } else { + remover.remove(productId); + } - // delete references first - Vector refs = new Vector(); + } - try { - refs = (Vector) client.getProductReferences(target); - } catch (CatalogException e) { - LOG.log(Level.WARNING, - "Unable to obtain references for product : [" + productId - + "] from file manager: [" - + client.getFileManagerUrl() + "]: Message: " - + e.getMessage()); - } + private static List readProdIdsFromStdin() { + List prodIds = new Vector(); + BufferedReader br; - for (Object ref1 : refs) { - Reference ref = (Reference) ref1; - - if (commit) { - try { - client.removeFile(new File(new URI(ref - .getDataStoreReference())).getAbsolutePath()); - } catch (DataTransferException e) { - LOG.log(Level.WARNING, "Unable to delete reference : [" - + ref.getDataStoreReference() + "] for product : [" - + productId + "] from file manager : [" - + client.getFileManagerUrl() + "]: Message: " - + e.getMessage()); - } catch (URISyntaxException e) { - LOG.log(Level.WARNING, - "uri syntax exception getting file absolute path from URI: [" - + ref.getDataStoreReference() - + "]: Message: " + e.getMessage()); - } - } else { - LOG.log(Level.INFO, "Delete file: [" - + ref.getDataStoreReference() + "]"); - } + br = new BufferedReader(new InputStreamReader(System.in)); - } + String line = null; - if (commit) { - - // now delete product - try { - client.removeProduct(target); - } catch (CatalogException e) { - LOG.log(Level.WARNING, "Unable to remove product : [" - + productId + "] from file manager: [" - + client.getFileManagerUrl() + "]: Message: " - + e.getMessage()); - } - } else { - LOG.log(Level.INFO, "Remote catalog entry for product: [" - + target.getProductName() + "]"); - } + try { + while ((line = br.readLine()) != null) { + prodIds.add(line); + } + } catch (IOException e) { + LOG.log(Level.WARNING, "Error reading prod id: line: [" + line + + "]: Message: " + e.getMessage(), e); + } finally { + try { + br.close(); + } catch (Exception ignore) { + } } + + return prodIds; + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java index 7b37f90..27e37d2 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java @@ -32,9 +32,7 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.QueryFormulationException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.structs.query.ComplexQuery; import org.apache.oodt.cas.filemgr.structs.query.QueryResult; -import org.apache.oodt.cas.filemgr.system.FileManagerClient; import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; -import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.filemgr.util.SqlParser; import java.net.MalformedURLException; @@ -53,12 +51,11 @@ import java.util.logging.Logger; * A tool to return product ids given a {@link Query} against the File Manager. * </p> */ -@Deprecated public final class QueryTool { private static String freeTextBlock = "__FREE__"; - private FileManagerClient client = null; + private XmlRpcFileManagerClient client = null; private enum QueryType { LUCENE, SQL } @@ -67,26 +64,12 @@ public final class QueryTool { public QueryTool(URL fmUrl) throws InstantiationException { try { - client = RpcCommunicationFactory.createClient(fmUrl); + client = new XmlRpcFileManagerClient(fmUrl); } catch (ConnectionException e) { throw new InstantiationException(e.getMessage()); } } - public static Query parseQuery(String query) { - QueryParser parser; - // note that "__FREE__" is a control work for free text searching - parser = new QueryParser(freeTextBlock, new CASAnalyzer()); - - Query luceneQ = null; - try { - luceneQ = (Query) parser.parse(query); - } catch (ParseException e) { - System.out.println("Error parsing query text."); - System.exit(-1); - } - return luceneQ; - } public List query(org.apache.oodt.cas.filemgr.structs.Query query) { List prodIds = new Vector(); @@ -118,55 +101,55 @@ public final class QueryTool { return prodIds; } - + + + public static Query parseQuery(String query) { + QueryParser parser; + // note that "__FREE__" is a control work for free text searching + parser = new QueryParser(freeTextBlock, new CASAnalyzer()); + Query luceneQ = null; + try { + luceneQ = (Query) parser.parse(query); + } catch (ParseException e) { + System.out.println("Error parsing query text."); + System.exit(-1); + } + return luceneQ; + } + public void generateCASQuery( - org.apache.oodt.cas.filemgr.structs.Query casQuery, - Query luceneQuery) { - if (luceneQuery instanceof TermQuery) { - Term t = ((TermQuery) luceneQuery).getTerm(); - if (!t.field().equals(freeTextBlock)) { - casQuery.addCriterion(new TermQueryCriteria(t.field(), - t.text())); - } - } else if (luceneQuery instanceof PhraseQuery) { - Term[] t = ((PhraseQuery) luceneQuery).getTerms(); - if (!t[0].field().equals(freeTextBlock)) { - for (Term aT : t) { - casQuery.addCriterion(new TermQueryCriteria( - aT.field(), aT.text())); - } - } - } else if (luceneQuery instanceof TermRangeQuery) { - BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm(); - BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm(); - casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString())); - } else if (luceneQuery instanceof BooleanQuery) { - List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses(); - for (BooleanClause clause : clauses) { - generateCASQuery(casQuery, (clause).getQuery()); + org.apache.oodt.cas.filemgr.structs.Query casQuery, + Query luceneQuery) { + if (luceneQuery instanceof TermQuery) { + Term t = ((TermQuery) luceneQuery).getTerm(); + if (!t.field().equals(freeTextBlock)) { + casQuery.addCriterion(new TermQueryCriteria(t.field(), + t.text())); + } + } else if (luceneQuery instanceof PhraseQuery) { + Term[] t = ((PhraseQuery) luceneQuery).getTerms(); + if (!t[0].field().equals(freeTextBlock)) { + for (Term aT : t) { + casQuery.addCriterion(new TermQueryCriteria( + aT.field(), aT.text())); } - } else { - throw new RuntimeException( - "Error parsing query! Cannot determine clause type: [" - + luceneQuery.getClass().getName() + "] !"); } - } - - private List safeGetProductTypes() { - List prodTypes = null; - - try { - prodTypes = client.getProductTypes(); - } catch (RepositoryManagerException e) { - LOG.log(Level.WARNING, - "Error obtaining product types from file manager: [" - + client.getFileManagerUrl() + "]: Message: " - + e.getMessage()); + } else if (luceneQuery instanceof TermRangeQuery) { + BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm(); + BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm(); + casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString())); + } else if (luceneQuery instanceof BooleanQuery) { + List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses(); + for (BooleanClause clause : clauses) { + generateCASQuery(casQuery, (clause).getQuery()); } - - return prodTypes; + } else { + throw new RuntimeException( + "Error parsing query! Cannot determine clause type: [" + + luceneQuery.getClass().getName() + "] !"); } - +} + public static void main(String[] args) throws MalformedURLException, InstantiationException, CatalogException, QueryFormulationException, ConnectionException { @@ -181,7 +164,7 @@ public final class QueryTool { + " -query <query> \n" + " -sortBy <metadata-key> \n" + " -outputFormat <output-format-string> \n"; - + String fmUrlStr = null, queryStr = null, sortBy = null, outputFormat = null, delimiter = null; QueryType queryType = null; for (int i = 0; i < args.length; i++) { @@ -241,6 +224,22 @@ public final class QueryTool { } } + + + private List safeGetProductTypes() { + List prodTypes = null; + + try { + prodTypes = client.getProductTypes(); + } catch (RepositoryManagerException e) { + LOG.log(Level.WARNING, + "Error obtaining product types from file manager: [" + + client.getFileManagerUrl() + "]: Message: " + + e.getMessage()); + } + + return prodTypes; + } private static String performSqlQuery(String query, String sortBy, String outputFormat, String delimiter, String filemgrUrl) throws MalformedURLException, CatalogException, ConnectionException, QueryFormulationException { http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java index 6d453bd..16ca7f1 100755 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java @@ -27,7 +27,6 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.OptionGroup; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.http.impl.client.SystemDefaultHttpClient; import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductPage; @@ -39,8 +38,9 @@ import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.SerializableMetadata; import org.apache.oodt.cas.metadata.util.PathUtils; +import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; import org.apache.solr.common.SolrInputDocument; import org.springframework.util.StringUtils; @@ -73,31 +73,31 @@ public class SolrIndexer { private final static String ACCESS_KEY = "access.key"; private final static String ACCESS_URL = "access.url"; private final static String PRODUCT_NAME = "CAS.ProductName"; - private final HttpSolrClient server; private IndexerConfig config = null; + private final SolrServer server; private String fmUrl; private String solrUrl; private static Logger LOG = Logger.getLogger(SolrIndexer.class.getName()); private final static SimpleDateFormat solrFormat = new SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); /** * Constructor reads in the configuration and initiates the connection to the * Solr instance. - * + * * @param solrUrl * URL for the Solr instance. * @param fmUrl * URL for the File Manager instance. */ public SolrIndexer(String solrUrl, String fmUrl) - throws InstantiationException { + throws InstantiationException { InputStream input = null; String filename; try { LOG.info("System property " + SOLR_INDEXER_CONFIG + " set to " - + System.getProperty(SOLR_INDEXER_CONFIG)); + + System.getProperty(SOLR_INDEXER_CONFIG)); filename = System.getProperty(SOLR_INDEXER_CONFIG); if (filename != null) { LOG.info("Reading config from " + filename); @@ -109,7 +109,7 @@ public class SolrIndexer { config = new IndexerConfig(input); } catch (IOException e) { LOG - .severe("Could not read in configuration for indexer from classpath or file"); + .severe("Could not read in configuration for indexer from classpath or file"); throw new InstantiationException(e.getMessage()); } finally { if (input != null) { @@ -132,10 +132,14 @@ public class SolrIndexer { } LOG.info("Using Solr: " + this.solrUrl + " FileManager: " + this.fmUrl); - SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient(); - server = new HttpSolrClient.Builder(this.solrUrl).build(); -// server = new SolrClient(this.solrUrl); + + try { + server = new CommonsHttpSolrServer(this.solrUrl); + } catch (MalformedURLException e) { + LOG.severe("Could not connect to Solr server " + this.solrUrl); + throw new InstantiationException(e.getMessage()); + } } @@ -162,7 +166,7 @@ public class SolrIndexer { /** * This method transforms the product metadata into a Solr document. - * + * * @param metadata * The metadata object for the product to index. * @return Returns the SolrInputDocument containing product metadata. @@ -170,8 +174,6 @@ public class SolrIndexer { private SolrInputDocument getSolrDocument(Metadata metadata) { SolrInputDocument doc = new SolrInputDocument(); // Only grab metadata which have a mapping in the indexer.properties - String official = ""; - for (Object objKey : config.getMapProperties().keySet()) { // The key in the metadata object String key = (String) objKey; @@ -182,21 +184,19 @@ public class SolrIndexer { for (String value : values) { // Add each metadata value into the if (value != null && !config.getIgnoreValues().contains(value.trim())) { - official += " Key: " + key + " Value: " + value; + LOG.fine("Adding field: " + fieldName + " value: " + value); doc.addField(fieldName, value); } } } } - // LOG.info("KEYS WERE:" +official); - return doc; } /** * This method adds a single product extracted from a metadata file to the * Solr index. - * + * * @param file * The file containing product metadata. * @param delete @@ -206,24 +206,24 @@ public class SolrIndexer { * When an error occurs communicating with the Solr server instance. */ public void indexMetFile(File file, boolean delete) - throws - SolrServerException { + throws + SolrServerException { LOG.info("Attempting to index product from metadata file."); try { SerializableMetadata metadata = new SerializableMetadata("UTF-8", false); metadata.loadMetadataFromXmlStream(new FileInputStream(file)); metadata.addMetadata("id", metadata.getMetadata("CAS." - + CoreMetKeys.PRODUCT_ID)); + + CoreMetKeys.PRODUCT_ID)); metadata.addMetadata(config.getProperty(ACCESS_KEY), config - .getProperty(ACCESS_URL) - + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID)); + .getProperty(ACCESS_URL) + + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID)); if (delete) { server - .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID)); + .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID)); } server.add(this.getSolrDocument(metadata)); LOG.info("Indexed product: " - + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID)); + + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID)); } catch (InstantiationException e) { LOG.severe("Could not instantiate metadata object: " + e.getMessage()); } catch (FileNotFoundException e) { @@ -236,7 +236,7 @@ public class SolrIndexer { /** * This method indexes all product types retrieved from the File Manager to * the Solr index. - * + * * @param delete * Flag indicating whether each product type retrieved from the File * Manager should be deleted from the index. @@ -247,7 +247,7 @@ public class SolrIndexer { LOG.info("Indexing product types..."); try { XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL( - this.fmUrl)); + this.fmUrl)); LOG.info("Retrieving list of product types."); List<ProductType> types = fmClient.getProductTypes(); for (ProductType type : types) { @@ -256,9 +256,9 @@ public class SolrIndexer { metadata.addMetadata("id", type.getProductTypeId()); metadata.addMetadata("CAS.ProductTypeId", type.getProductTypeId()); metadata.addMetadata("CAS.ProductTypeDescription", type - .getDescription()); + .getDescription()); metadata.addMetadata("CAS.ProductTypeRepositoryPath", type - .getProductRepositoryPath()); + .getProductRepositoryPath()); metadata.addMetadata("CAS.ProductTypeVersioner", type.getVersioner()); metadata.addMetadata("CAS.ProductTypeName", type.getName()); metadata.addMetadata("ProductType", "ProductType"); @@ -268,7 +268,7 @@ public class SolrIndexer { server.deleteById(type.getProductTypeId()); } catch (Exception e) { LOG.severe("Could not delete product type " + type.getName() - + " from index: " + e.getMessage()); + + " from index: " + e.getMessage()); } } try { @@ -277,7 +277,7 @@ public class SolrIndexer { LOG.info("Indexed product type: " + type.getName()); } catch (Exception e) { LOG.severe("Could not index " + type.getName() + ": " - + e.getMessage()); + + e.getMessage()); } } else { LOG.info("Ignoring product type: " + type.getName()); @@ -289,13 +289,13 @@ public class SolrIndexer { LOG.severe("Could not connect to File Manager: " + e.getMessage()); } catch (RepositoryManagerException e) { LOG.severe("Could not retrieve product types from File Manager: " - + e.getMessage()); + + e.getMessage()); } LOG.info("Finished indexing product types."); } - + /** - * Suppresses exception that occurred with older file managers. + * Suppresses exception that occurred with older file managers. */ private ProductPage safeFirstPage(XmlRpcFileManagerClient fmClient, ProductType type) { ProductPage page = null; @@ -311,7 +311,7 @@ public class SolrIndexer { * This method indexes all products retrieved from the File Manager to the * Solr index. Metadata from the product's associated ProductType is also * included. - * + * * @param delete * Flag indicating whether each product retrieved from the File * Manager should be deleted from the index. @@ -322,29 +322,28 @@ public class SolrIndexer { LOG.info("Indexing products..."); try { XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL( - this.fmUrl)); + this.fmUrl)); LOG.info("Retrieving list of product types."); List<ProductType> types = fmClient.getProductTypes(); for (ProductType type : types) { if (!config.getIgnoreTypes().contains(type.getName().trim())) { - ProductPage page = safeFirstPage(fmClient, type); LOG.info("Paging through products for product type: " - + type.getName()+" Total pages are "+page.getTotalPages()); + + type.getName()); + ProductPage page = safeFirstPage(fmClient, type); while (page != null) { - for (Product product : page.getPageProducts()) { + for (Product product : page.getPageProducts()) { try { - String p = product.getProductId(); - Metadata m = fmClient.getMetadata(product); - this.indexProduct(p, m, type.getTypeMetadata()); + this.indexProduct(product.getProductId(), fmClient + .getMetadata(product), type.getTypeMetadata()); } catch (Exception e) { LOG.severe("Could not index " + product.getProductId() + ": " - + e.getLocalizedMessage()); + + e.getMessage()); } } - if (page.getPageNum() >= page.getTotalPages() || page.isLastPage()) { - break; - } - page = fmClient.getNextPage(type, page); + if (page.isLastPage()) { + break; + } + page = fmClient.getNextPage(type, page); } } } @@ -355,10 +354,10 @@ public class SolrIndexer { LOG.severe("Could not connect to File Manager: " + e.getMessage()); } catch (CatalogException e) { LOG.severe("Could not retrieve products from File Manager: " - + e.getMessage()); + + e.getMessage()); } catch (RepositoryManagerException e) { LOG.severe("Could not retrieve product types from File Manager: " - + e.getMessage()); + + e.getMessage()); } } @@ -366,50 +365,39 @@ public class SolrIndexer { * This method adds a single product retrieved from the File Manager by its * product identifier to the Solr index. Metadata from the ProductType is also * included. - * + * * @param productId * The identifier of the product (CAS.ProductId). * @throws SolrServerException * When an error occurs communicating with the Solr server instance. */ public void indexProduct(String productId) - throws SolrServerException { + throws SolrServerException { LOG.info("Attempting to index product: " + productId); try { XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL( - this.fmUrl)); + this.fmUrl)); Product product = fmClient.getProductById(productId); Metadata productMetadata = fmClient.getMetadata(product); - List<String> keys = productMetadata.getAllKeys(); - String logger = ""; - for(String k :keys){ - logger+=" key: "+keys; - List<String> values = productMetadata.getAllValues(k); - - for(String v : values){ - logger+=" value: "+v + ", "; - } - } - LOG.info("Metadata: "+ logger); indexProduct(product.getProductId(), productMetadata, product - .getProductType().getTypeMetadata()); + .getProductType().getTypeMetadata()); } catch (MalformedURLException e) { LOG.severe("File Manager URL is malformed: " + e.getMessage()); } catch (ConnectionException e) { LOG.severe("Could not connect to File Manager: " + e.getMessage()); } catch (CatalogException e) { LOG.severe("Could not retrieve product from File Manager: " - + e.getMessage()); + + e.getMessage()); } catch (java.text.ParseException e) { LOG.severe("Could not format date: " + e.getMessage()); } } - + /** * This method adds a single product retrieved from the File Manager by its * product name to the Solr index. Metadata from the ProductType is also * included. - * + * * @param productName * The identifier of the product (CAS.ProductId). * @param delete @@ -419,7 +407,7 @@ public class SolrIndexer { * When an error occurs communicating with the Solr server instance. */ public void indexProductByName(String productName, boolean delete) throws SolrServerException { - + LOG.info("Attempting to index product: " + productName); try { @@ -439,27 +427,27 @@ public class SolrIndexer { } XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL( - this.fmUrl)); + this.fmUrl)); Product product = fmClient.getProductByName(productName); Metadata productMetadata = fmClient.getMetadata(product); // NOTE: delete (by id) is now false indexProduct(product.getProductId(), productMetadata, product.getProductType().getTypeMetadata()); - + } catch (MalformedURLException e) { LOG.severe("File Manager URL is malformed: " + e.getMessage()); } catch (ConnectionException e) { LOG.severe("Could not connect to File Manager: " + e.getMessage()); } catch (CatalogException e) { LOG.severe("Could not retrieve product from File Manager: " - + e.getMessage()); + + e.getMessage()); } catch (java.text.ParseException e) { LOG.severe("Could not format date: " + e.getMessage()); } } private void indexProduct(String productId, Metadata productMetadata, - Metadata typeMetadata) throws SolrServerException, - java.text.ParseException { + Metadata typeMetadata) throws SolrServerException, + java.text.ParseException { Metadata metadata = new Metadata(); metadata.addMetadata("id", productId); // Add in product type metadata @@ -485,7 +473,7 @@ public class SolrIndexer { server.add(this.getSolrDocument(metadata)); LOG.info("Indexed product: " + productId); } catch (IOException e) { - LOG.severe("Could not index product: " + productId+ "Exception:"+e.getLocalizedMessage()); + LOG.severe("Could not index product: " + productId); } } else { LOG.info("Could not find metadata for product: " + productId); @@ -494,7 +482,7 @@ public class SolrIndexer { /** * This method deletes a single product identified by a productID from the Solr index - * + * * @param productId * @throws IOException * @throws SolrServerException @@ -506,7 +494,7 @@ public class SolrIndexer { /** * This method deletes a product(s) from the Solr index identified by a given name - * + * * @param productName * @throws IOException * @throws SolrServerException @@ -530,16 +518,16 @@ public class SolrIndexer { private void deleteProductFromIndex(String productId) throws IOException, SolrServerException { server.deleteById(productId); } - + /** * Quick helper method to do substitution on the keys specified in the config - * + * * @param metadata * to substitute on * @throws java.text.ParseException */ private void performSubstitution(Metadata metadata) - throws java.text.ParseException { + throws java.text.ParseException { // Do metadata replacement for (String key : config.getReplacementKeys()) { List<String> values = metadata.getAllMetadata(key); @@ -559,12 +547,10 @@ public class SolrIndexer { List<String> values = metadata.getAllMetadata(keyString); if (values != null) { List<String> newValues = new ArrayList<String>(); - - /* SimpleDateFormat format = new SimpleDateFormat(config - .getFormatProperties().getProperty(keyString).trim());*/ + SimpleDateFormat format = new SimpleDateFormat(config + .getFormatProperties().getProperty(keyString).trim()); for (String value : values) { - String d = value.trim(); - newValues.add(d); + newValues.add(formatDate(format, value)); } metadata.removeMetadata(keyString); metadata.addMetadata(keyString, newValues); @@ -574,17 +560,17 @@ public class SolrIndexer { } private String formatDate(SimpleDateFormat format, String value) - throws java.text.ParseException { + throws java.text.ParseException { // Ignore formating if its an ignore value if (config.getIgnoreValues().contains(value.trim())) { - return value; + return value; } return solrFormat.format(format.parse(value)); } /** * This method builds the command-line options. - * + * * @return Returns the supported Options. */ @SuppressWarnings("static-access") @@ -593,31 +579,31 @@ public class SolrIndexer { options.addOption(new Option("h", "help", false, "Print this message")); options.addOption(new Option("o", "optimize", false, - "Optimize the Solr index")); + "Optimize the Solr index")); options.addOption(new Option("d", "delete", false, - "Delete item before indexing")); + "Delete item before indexing")); options.addOption(OptionBuilder.withArgName("Solr URL").hasArg() - .withDescription("URL to the Solr instance").withLongOpt("solrUrl") - .create("su")); + .withDescription("URL to the Solr instance").withLongOpt("solrUrl") + .create("su")); options.addOption(OptionBuilder.withArgName("Filemgr URL").hasArg() - .withDescription("URL to the File Manager").withLongOpt("fmUrl") - .create("fmu")); + .withDescription("URL to the File Manager").withLongOpt("fmUrl") + .create("fmu")); OptionGroup group = new OptionGroup(); Option all = new Option("a", "all", false, - "Index all products from the File Manager"); + "Index all products from the File Manager"); Option product = OptionBuilder.withArgName("productId").hasArg() - .withDescription("Index the product from the File Manager") - .withLongOpt("product").create("p"); + .withDescription("Index the product from the File Manager") + .withLongOpt("product").create("p"); Option met = OptionBuilder.withArgName("file").hasArg().withDescription( - "Index the product from a metadata file").withLongOpt("metFile") - .create("mf"); + "Index the product from a metadata file").withLongOpt("metFile") + .create("mf"); Option read = new Option("r", "read", false, - "Index all products based on a list of product identifiers passed in"); + "Index all products based on a list of product identifiers passed in"); Option types = new Option("t", "types", false, - "Index all product types from the File Manager"); + "Index all product types from the File Manager"); Option deleteAll = new Option("da", "deleteAll", false, - "Delete all products/types from the Solr index"); + "Delete all products/types from the Solr index"); group.addOption(all); group.addOption(product); @@ -632,7 +618,7 @@ public class SolrIndexer { /** * The main method. Execution without argument displays help message. - * + * * @param args * Command-line arguments. */ @@ -651,8 +637,8 @@ public class SolrIndexer { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("java " + SolrIndexer.class.getName(), options); } else if (line.hasOption("all") || line.hasOption("product") - || line.hasOption("metFile") || line.hasOption("read") - || line.hasOption("types") || line.hasOption("deleteAll")) { + || line.hasOption("metFile") || line.hasOption("read") + || line.hasOption("types") || line.hasOption("deleteAll")) { SolrIndexer indexer; String solrUrl = null; String fmUrl = null; @@ -670,7 +656,7 @@ public class SolrIndexer { indexer.indexProduct(line.getOptionValue("product")); } else if (line.hasOption("metFile")) { indexer.indexMetFile(new File(line.getOptionValue("metFile")), line - .hasOption("delete")); + .hasOption("delete")); } else if (line.hasOption("read")) { for (String productId : readProductIdsFromStdin()) { indexer.indexProduct(productId); @@ -689,14 +675,14 @@ public class SolrIndexer { } catch (Exception e) { LOG.severe("An error occurred indexing: " + e.getMessage()); LOG - .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information."); + .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information."); } } } /** * This method reads product identifiers from the standard input. - * + * * @return Returns a List of product identifiers. */ private static List<String> readProductIdsFromStdin() { @@ -712,12 +698,12 @@ public class SolrIndexer { } } catch (IOException e) { LOG.severe("Error reading product id: line: [" + line + "]: Message: " - + e.getMessage()); + + e.getMessage()); } finally { - try { - br.close(); - } catch (Exception ignore) { - } + try { + br.close(); + } catch (Exception ignore) { + } } return productIds; } @@ -749,32 +735,32 @@ public class SolrIndexer { String key = (String) objKey; if (key.startsWith(PREFIX_CONFIG)) { properties.put(key.substring(PREFIX_CONFIG.length()), props - .getProperty(key)); + .getProperty(key)); } else if (key.startsWith(PREFIX_MET)) { mapProperties.put(key.substring(PREFIX_MET.length()), props - .getProperty(key)); + .getProperty(key)); } else if (key.startsWith(PREFIX_FORMAT)) { formatProperties.put(key.substring(PREFIX_FORMAT.length()), props - .getProperty(key)); + .getProperty(key)); } } if (properties.getProperty(IGNORE_TYPES) != null) { String[] values = properties.getProperty(IGNORE_TYPES).trim() - .split(","); - Collections.addAll(ignoreTypes, values); + .split(","); + Collections.addAll(ignoreTypes, values); } if (properties.getProperty(IGNORE_VALUES) != null) { String[] values = properties.getProperty(IGNORE_VALUES).trim().split( - ","); - Collections.addAll(ignoreValues, values); + ","); + Collections.addAll(ignoreValues, values); } if (properties.getProperty(REPLACEMENT_KEYS) != null) { String[] values = properties.getProperty(REPLACEMENT_KEYS).trim() - .split(","); - Collections.addAll(replacementKeys, values); + .split(","); + Collections.addAll(replacementKeys, values); } } http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java ---------------------------------------------------------------------- diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java index c3bf291..93f3afe 100644 --- a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java +++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java @@ -23,8 +23,21 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LogDocMergePolicy; +import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.BytesRef; @@ -308,6 +321,42 @@ public class LuceneWorkflowInstanceRepository extends return wInsts; } + + @Override + public synchronized boolean clearWorkflowInstances() throws InstanceRepositoryException { + IndexWriter writer = null; + try { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + LogMergePolicy lmp =new LogDocMergePolicy(); + lmp.setMergeFactor(mergeFactor); + config.setMergePolicy(lmp); + + writer = new IndexWriter(indexDir, config); + LOG.log(Level.FINE, + "LuceneWorkflowEngine: remove all workflow instances"); + writer.deleteDocuments(new Term("myfield", "myvalue")); + } catch (IOException e) { + LOG.log(Level.SEVERE, e.getMessage()); + LOG + .log(Level.WARNING, + "Exception removing workflow instances from index: Message: " + + e.getMessage()); + throw new InstanceRepositoryException(e.getMessage()); + } finally { + if (writer != null){ + try{ + writer.close(); + } + catch(Exception ignore){} + + writer = null; + } + + } + + return true; + } /* * (non-Javadoc)
