Author: luca
Date: Mon Dec 17 12:47:38 2012
New Revision: 1422909
URL: http://svn.apache.org/viewvc?rev=1422909&view=rev
Log:
Changes to improve DataSourceCatalog back-end for File Manager: support of
lenient validation layer, and of "product_id" column of type string (OODT-541
and OODT-544).
Added:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java
(with props)
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/DbStructFactory.java
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java?rev=1422909&r1=1422908&r2=1422909&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java
Mon Dec 17 12:47:38 2012
@@ -17,7 +17,7 @@
package org.apache.oodt.cas.filemgr.catalog;
-//OODT imports
+// OODT imports
import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria;
import org.apache.oodt.cas.filemgr.structs.Element;
import org.apache.oodt.cas.filemgr.structs.Product;
@@ -31,9 +31,9 @@ import org.apache.oodt.cas.filemgr.struc
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ValidationLayerException;
import org.apache.oodt.cas.filemgr.util.DbStructFactory;
-import org.apache.oodt.commons.pagination.PaginationUtils;
import org.apache.oodt.cas.filemgr.validation.ValidationLayer;
import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.commons.pagination.PaginationUtils;
import org.apache.oodt.commons.util.DateConvert;
//JDK imports
@@ -45,6 +45,7 @@ import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.TreeMap;
+import java.util.UUID;
import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -53,6 +54,7 @@ import javax.sql.DataSource;
/**
* @author mattmann
* @author bfoster
+ * @author luca
* @version $Revision$
*
* <p>
@@ -77,6 +79,9 @@ public class DataSourceCatalog implement
/* size of pages of products within the catalog */
protected int pageSize = -1;
+
+ /* flag to indicate whether the "product_id" key type should be treated as
a string */
+ boolean productIdString = false;
/*
* cache of products per product type: [productTypeId]=>([ISO8601 time of
@@ -97,12 +102,27 @@ public class DataSourceCatalog implement
* @throws
*/
public DataSourceCatalog(DataSource ds, ValidationLayer valLayer,
- boolean fieldId, int pageSize, long cacheUpdateMin) {
+ boolean fieldId, int pageSize, long cacheUpdateMin, boolean
productIdString) {
this.dataSource = ds;
this.validationLayer = valLayer;
fieldIdStringFlag = fieldId;
this.pageSize = pageSize;
cacheUpdateMinutes = cacheUpdateMin;
+ this.productIdString = productIdString;
+ }
+
+ /**
+ * Constructor that assumes productIdString=false
+ * to support current subclasses.
+ * @param ds
+ * @param valLayer
+ * @param fieldId
+ * @param pageSize
+ * @param cacheUpdateMin
+ */
+ public DataSourceCatalog(DataSource ds, ValidationLayer valLayer,
+ boolean fieldId, int pageSize, long cacheUpdateMin) {
+ this(ds, valLayer, fieldId, pageSize, cacheUpdateMin, false);
}
/*
@@ -240,8 +260,44 @@ public class DataSourceCatalog implement
productTypeIdStr = product.getProductType().getProductTypeId();
}
- addProductSql = "INSERT INTO products (product_name,
product_structure, product_transfer_status, product_type_id) "
+ if (productIdString==false) {
+
+ addProductSql = "INSERT INTO products (product_name,
product_structure, product_transfer_status, product_type_id) "
+ + "VALUES ('"
+ + product.getProductName()
+ + "', '"
+ + product.getProductStructure()
+ + "', '"
+ + product.getTransferStatus()
+ + "', "
+ + productTypeIdStr
+ + ")";
+
+ LOG.log(Level.FINE, "addProduct:
Executing: " + addProductSql);
+ statement.execute(addProductSql);
+
+ // read "product_id" value that was
automatically assigned by the database
+ String productId = new String();
+
+ String getProductIdSql = "SELECT
MAX(product_id) AS max_id FROM products";
+
+ rs =
statement.executeQuery(getProductIdSql);
+
+ while (rs.next()) {
+ productId =
String.valueOf(rs.getInt("max_id"));
+ }
+
+ product.setProductId(productId);
+ conn.commit();
+
+ } else {
+
+ // generate a new UUID
string, insert in database
+ String productId = UUID.randomUUID().toString();
+ addProductSql = "INSERT INTO products (product_id,
product_name, product_structure, product_transfer_status, product_type_id) "
+ "VALUES ('"
+ + productId
+ + "', '"
+ product.getProductName()
+ "', '"
+ product.getProductStructure()
@@ -249,23 +305,16 @@ public class DataSourceCatalog implement
+ product.getTransferStatus()
+ "', "
+ productTypeIdStr
- + ")";
+ + ")";
- LOG.log(Level.FINE, "addProduct: Executing: " + addProductSql);
- statement.execute(addProductSql);
+ LOG.log(Level.FINE, "addProduct: Executing: " + addProductSql);
+ statement.execute(addProductSql);
+
+ product.setProductId(productId);
+ conn.commit();
- String productId = new String();
+ }
- String getProductIdSql = "SELECT MAX(product_id) AS max_id FROM
products";
-
- rs = statement.executeQuery(getProductIdSql);
-
- while (rs.next()) {
- productId = String.valueOf(rs.getInt("max_id"));
- }
-
- product.setProductId(productId);
- conn.commit();
} catch (Exception e) {
e.printStackTrace();
@@ -332,7 +381,7 @@ public class DataSourceCatalog implement
+ product.getProductStructure()
+ "', product_transfer_status='"
+ product.getTransferStatus() + "' "
- + "WHERE product_id = " + product.getProductId();
+ + "WHERE product_id = " + quoteIt(product.getProductId());
LOG
.log(Level.FINE, "modifyProduct: Executing: "
@@ -395,7 +444,7 @@ public class DataSourceCatalog implement
statement = conn.createStatement();
String deleteProductSql = "DELETE FROM products WHERE product_id =
"
- + product.getProductId();
+ + quoteIt(product.getProductId());
LOG
.log(Level.FINE, "removeProduct: Executing: "
@@ -403,14 +452,14 @@ public class DataSourceCatalog implement
statement.execute(deleteProductSql);
deleteProductSql = "DELETE FROM "
+ product.getProductType().getName() + "_metadata "
- + " WHERE product_id = " + product.getProductId();
+ + " WHERE product_id = " + quoteIt(product.getProductId());
LOG
.log(Level.FINE, "removeProduct: Executing: "
+ deleteProductSql);
statement.execute(deleteProductSql);
deleteProductSql = "DELETE FROM "
+ product.getProductType().getName() + "_reference "
- + " WHERE product_id = " + product.getProductId();
+ + " WHERE product_id = " + quoteIt(product.getProductId());
LOG
.log(Level.FINE, "removeProduct: Executing: "
+ deleteProductSql);
@@ -470,7 +519,7 @@ public class DataSourceCatalog implement
String modifyProductSql = "UPDATE products SET
product_transfer_status='"
+ product.getTransferStatus()
+ "' "
- + "WHERE product_id = " + product.getProductId();
+ + "WHERE product_id = " + quoteIt(product.getProductId());
LOG.log(Level.FINE, "setProductTransferStatus: Executing: "
+ modifyProductSql);
@@ -541,7 +590,7 @@ public class DataSourceCatalog implement
+ " "
+ "(product_id, product_orig_reference,
product_datastore_reference, product_reference_filesize,
product_reference_mimetype) "
+ "VALUES ("
- + product.getProductId()
+ + quoteIt(product.getProductId())
+ ", '"
+ r.getOrigReference()
+ "', '"
@@ -622,13 +671,13 @@ public class DataSourceCatalog implement
statement = conn.createStatement();
String getProductSql = "SELECT * " + "FROM products "
- + "WHERE product_id = " + productId;
+ + "WHERE product_id = " + quoteIt(productId);
LOG.log(Level.FINE, "getProductById: Executing: " + getProductSql);
rs = statement.executeQuery(getProductSql);
while (rs.next()) {
- product = DbStructFactory.getProduct(rs, false);
+ product = DbStructFactory.getProduct(rs, false,
productIdString);
}
} catch (Exception e) {
@@ -702,7 +751,7 @@ public class DataSourceCatalog implement
rs = statement.executeQuery(getProductSql);
while (rs.next()) {
- product = DbStructFactory.getProduct(rs, false);
+ product = DbStructFactory.getProduct(rs, false,
productIdString);
}
} catch (Exception e) {
@@ -769,7 +818,7 @@ public class DataSourceCatalog implement
String getProductRefSql = "SELECT * FROM "
+ product.getProductType().getName() + "_reference"
- + " WHERE product_id = " + product.getProductId();
+ + " WHERE product_id = " + quoteIt(product.getProductId());
LOG.log(Level.FINE, "getProductReferences: Executing: "
+ getProductRefSql);
@@ -851,7 +900,7 @@ public class DataSourceCatalog implement
products = new Vector<Product>();
while (rs.next()) {
- Product product = DbStructFactory.getProduct(rs, false);
+ Product product = DbStructFactory.getProduct(rs, false,
productIdString);
products.add(product);
}
@@ -940,7 +989,7 @@ public class DataSourceCatalog implement
products = new Vector<Product>();
while (rs.next()) {
- Product product = DbStructFactory.getProduct(rs, false);
+ Product product = DbStructFactory.getProduct(rs, false,
productIdString);
products.add(product);
}
@@ -1006,7 +1055,7 @@ public class DataSourceCatalog implement
String metadataSql = "SELECT * FROM "
+ product.getProductType().getName() + "_metadata "
- + " WHERE product_id = " + product.getProductId();
+ + " WHERE product_id = " + quoteIt(product.getProductId());
LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
rs = statement.executeQuery(metadataSql);
@@ -1097,7 +1146,7 @@ public class DataSourceCatalog implement
}
String metadataSql = "SELECT element_id,metadata_value FROM "
+ product.getProductType().getName() + "_metadata"
- + " WHERE product_id = " + product.getProductId() +
elementIds;
+ + " WHERE product_id = " + quoteIt(product.getProductId())
+ elementIds;
LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
rs = statement.executeQuery(metadataSql);
@@ -1229,7 +1278,7 @@ public class DataSourceCatalog implement
products = new Vector<Product>();
while (rs.next()) {
- Product product = DbStructFactory.getProduct(rs, false);
+ Product product = DbStructFactory.getProduct(rs, false,
productIdString);
products.add(product);
}
@@ -2047,7 +2096,7 @@ public class DataSourceCatalog implement
}
- private String getSqlQuery(QueryCriteria queryCriteria, ProductType type)
throws ValidationLayerException, CatalogException {
+ protected String getSqlQuery(QueryCriteria queryCriteria, ProductType
type) throws ValidationLayerException, CatalogException {
String sqlQuery = null;
if (queryCriteria instanceof BooleanQueryCriteria) {
BooleanQueryCriteria bqc = (BooleanQueryCriteria) queryCriteria;
@@ -2061,7 +2110,7 @@ public class DataSourceCatalog implement
sqlQuery += ")";
}
}else {
- String elementIdStr =
this.validationLayer.getElementByName(queryCriteria.getElementName()).getElementId();
+ String elementIdStr =
this.validationLayer.getElementByName(queryCriteria.getElementName()).getElementId();
if (fieldIdStringFlag)
elementIdStr = "'" + elementIdStr + "'";
sqlQuery = "SELECT DISTINCT product_id FROM " + type.getName() +
"_metadata WHERE element_id = " + elementIdStr + " AND ";
@@ -2104,7 +2153,7 @@ public class DataSourceCatalog implement
// first remove the refs
String deleteProductSql = "DELETE FROM "
+ product.getProductType().getName() + "_reference "
- + " WHERE product_id = " + product.getProductId();
+ + " WHERE product_id = " + quoteIt(product.getProductId());
LOG.log(Level.FINE, "updateProductReferences: Executing: "
+ deleteProductSql);
statement.execute(deleteProductSql);
@@ -2176,5 +2225,19 @@ public class DataSourceCatalog implement
}
}
+
+ /**
+ * Utility method to quote the "productId" value
+ * if the column type is "string".
+ * @param productId
+ * @return
+ */
+ protected String quoteIt(String productId) {
+ if (this.productIdString) {
+ return "'"+productId+"'";
+ } else {
+ return productId;
+ }
+ }
}
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java?rev=1422909&r1=1422908&r2=1422909&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java
Mon Dec 17 12:47:38 2012
@@ -53,6 +53,14 @@ public class DataSourceCatalogFactory im
/* the amount of minutes to allow between updating the product cache */
protected long cacheUpdateMinutes = -1L;
+
+ /* Whether or not to enforce strict definition of metadata fields:
+ * 'lenient=false' means that all metadata fields need to be explicitly
defined in the XML configuration file */
+ protected boolean lenientFields = false;
+
+ /* Flag to indicate whether the "product_id" key type should be treated as
a string */
+ protected boolean productIdString = false;
+
/**
* <p>
@@ -60,6 +68,8 @@ public class DataSourceCatalogFactory im
* </p>.
*/
public DataSourceCatalogFactory() {
+
+ // instantiate data source
String jdbcUrl = null, user = null, pass = null, driver = null;
jdbcUrl = PathUtils
@@ -78,22 +88,8 @@ public class DataSourceCatalogFactory im
dataSource = DatabaseConnectionBuilder.buildDataSource(user, pass,
driver, jdbcUrl);
- String validationLayerFactoryClass = System
- .getProperty("filemgr.validationLayer.factory",
-
"org.apache.oodt.cas.filemgr.validation.DataSourceValidationLayerFactory");
- validationLayer = GenericFileManagerObjectFactory
- .getValidationLayerFromFactory(validationLayerFactoryClass);
- fieldIdStr = Boolean
-
.getBoolean("org.apache.oodt.cas.filemgr.catalog.datasource.quoteFields");
-
- pageSize = Integer
- .getInteger(
-
"org.apache.oodt.cas.filemgr.catalog.datasource.pageSize",
- 20).intValue();
- cacheUpdateMinutes = Long
- .getLong(
-
"org.apache.oodt.cas.filemgr.catalog.datasource.cacheUpdateMinutes",
- 5L).longValue();
+ this.configure();
+
}
/**
@@ -105,12 +101,42 @@ public class DataSourceCatalogFactory im
* The DataSource to construct this factory from.
*/
public DataSourceCatalogFactory(DataSource ds) {
- this.dataSource = ds;
- String validationLayerFactoryClass = System
- .getProperty("filemgr.validationLayer.factory",
-
"org.apache.oodt.cas.filemgr.validation.DataSourceValidationLayerFactory");
- validationLayer = GenericFileManagerObjectFactory
- .getValidationLayerFromFactory(validationLayerFactoryClass);
+
+ this.dataSource = ds;
+
+ this.configure();
+
+ }
+
+ /** Method to configure the factory (and validation layer) from the
environment properties,
+ * before any Catalog instance is created
+ **/
+ private void configure() {
+
+ lenientFields = Boolean.parseBoolean(
System.getProperty("org.apache.oodt.cas.filemgr.catalog.datasource.lenientFields",
"false") );
+ if (!lenientFields) {
+ String validationLayerFactoryClass = System
+ .getProperty("filemgr.validationLayer.factory",
+
"org.apache.oodt.cas.filemgr.validation.DataSourceValidationLayerFactory");
+ validationLayer = GenericFileManagerObjectFactory
+ .getValidationLayerFromFactory(validationLayerFactoryClass);
+ }
+
+ fieldIdStr = Boolean
+
.getBoolean("org.apache.oodt.cas.filemgr.catalog.datasource.quoteFields");
+
+ pageSize = Integer
+ .getInteger(
+
"org.apache.oodt.cas.filemgr.catalog.datasource.pageSize",
+ 20).intValue();
+ cacheUpdateMinutes = Long
+ .getLong(
+
"org.apache.oodt.cas.filemgr.catalog.datasource.cacheUpdateMinutes",
+ 5L).longValue();
+
+ productIdString = Boolean.parseBoolean(
+
System.getProperty("org.apache.oodt.cas.filemgr.catalog.datasource.productId.string",
"false") );
+
}
/*
@@ -119,8 +145,13 @@ public class DataSourceCatalogFactory im
* @see org.apache.oodt.cas.filemgr.catalog.CatalogFactory#createCatalog()
*/
public Catalog createCatalog() {
+ if (validationLayer==null) {
+ return new LenientDataSourceCatalog(dataSource,
validationLayer, fieldIdStr,
+ pageSize, cacheUpdateMinutes, productIdString);
+ } else {
return new DataSourceCatalog(dataSource, validationLayer, fieldIdStr,
- pageSize, cacheUpdateMinutes);
+ pageSize, cacheUpdateMinutes, productIdString);
+ }
}
}
Added:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java?rev=1422909&view=auto
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java
(added)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java
Mon Dec 17 12:47:38 2012
@@ -0,0 +1,795 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.filemgr.catalog;
+
+// OODT imports
+import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.Element;
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.ProductType;
+import org.apache.oodt.cas.filemgr.structs.Query;
+import org.apache.oodt.cas.filemgr.structs.QueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.TermQueryCriteria;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.filemgr.structs.exceptions.ValidationLayerException;
+import org.apache.oodt.cas.filemgr.validation.ValidationLayer;
+import org.apache.oodt.cas.metadata.Metadata;
+
+//JDK imports
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.sql.DataSource;
+
+/**
+ * @author luca
+ * @version $Revision$
+ *
+ * <p>
+ * Extension of {@link DataSourceCatalog} that can accomodate dynamic fields.
+ * </p>
+ *
+ */
+public class LenientDataSourceCatalog extends DataSourceCatalog {
+
+ /* our log stream */
+ private static final Logger LOG =
Logger.getLogger(LenientDataSourceCatalog.class.getName());
+
+ /**
+ * <p>
+ * Default Constructor
+ * </p>.
+ * @throws
+ */
+ public LenientDataSourceCatalog(DataSource ds, ValidationLayer valLayer,
+ boolean fieldId, int pageSize, long cacheUpdateMin, boolean
productIdString) {
+
+ super(ds, valLayer, fieldId, pageSize, cacheUpdateMin,
productIdString);
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
org.apache.oodt.cas.filemgr.catalog.Catalog#addMetadata(org.apache.oodt.cas.metadata.Metadata,
+ * org.apache.oodt.cas.filemgr.structs.Product)
+ */
+ public synchronized void addMetadata(Metadata m, Product product)
+ throws CatalogException {
+
+ // replace "CAS.ProductId"
+ m.removeMetadata("CAS.ProductId");
+ m.addMetadata("CAS.ProductId", product.getProductId());
+
+ // map containing metadata type (id, name) pairs
+ Map<String, String> metadataTypes = getMetadataTypes(m, product);
+
+ // loop over metadata types
+ for (String metadataId : metadataTypes.keySet()) {
+ String metadataName = metadataTypes.get(metadataId);
+
+ List<String> values = m.getAllMetadata(metadataName);
+
+ if (values == null) {
+ LOG.log(Level.WARNING, "No Metadata specified for product ["
+ + product.getProductName() + "] for required field ["
+ + metadataName
+ + "]: Attempting to continue processing metadata");
+ continue;
+ }
+
+ for (Iterator<String> j = values.iterator(); j.hasNext();) {
+ String value = j.next();
+
+ try {
+ addMetadataValue(metadataId, product, value);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG
+ .log(
+ Level.WARNING,
+ "Exception ingesting metadata. Error
inserting field: ["
+ + metadataId
+ + "=>"
+ + value
+ + "]: for product: ["
+ + product.getProductName()
+ + "]: Message: "
+ + e.getMessage()
+ + ": Attempting to continue
processing metadata");
+ }
+ }
+ }
+
+ }
+
+ // Utility method to return a map of metadata (field id, field name)
+ private Map<String, String> getMetadataTypes(Metadata m, Product product)
throws CatalogException {
+
+ // map containing metadata type (id, name) pairs
+ Map<String, String> metadataTypes = new LinkedHashMap<String,String>();
+
+ if (getValidationLayer()!=null) {
+ // validation layer: add valid metadata elements
+ try {
+ for (Element element :
getValidationLayer().getElements(product.getProductType())) {
+ metadataTypes.put(element.getElementId(),
element.getElementName());
+ }
+
+ } catch (ValidationLayerException e) {
+ e.printStackTrace();
+ throw new CatalogException(
+ "ValidationLayerException when trying to obtain element
list for product type: "
+ + product.getProductType().getName()
+ + ": Message: " + e.getMessage());
+ }
+
+ } else {
+
+ // no validation layer: add ALL metadata elements
+ // use (key, key) pairs (i.e. metadata id == metadata name)
+ for (String key : m.getAllKeys()) {
+ metadataTypes.put(key, key);
+ }
+
+ }
+
+ return metadataTypes;
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
org.apache.oodt.cas.filemgr.catalog.Catalog#addMetadata(org.apache.oodt.cas.metadata.Metadata,
+ * org.apache.oodt.cas.filemgr.structs.Product)
+ */
+ public synchronized void removeMetadata(Metadata m, Product product)
+ throws CatalogException {
+
+ // map containing metadata type (id, name) pairs
+ Map<String, String> metadataTypes = getMetadataTypes(m, product);
+
+ // loop over metadata types
+ for (String metadataId : metadataTypes.keySet()) {
+ String metadataName = metadataTypes.get(metadataId);
+
+ List<String> values = m.getAllMetadata(metadataName);
+
+ if (values != null) {
+ for (Iterator<String> j = values.iterator(); j.hasNext();) {
+ String value = j.next();
+
+ try {
+ removeMetadataValue(metadataId, product, value);
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG
+ .log(
+ Level.WARNING,
+ "Exception removing metadata. Error
deleting field: ["
+ + metadataId
+ + "=>"
+ + value
+ + "]: for product: ["
+ + product.getProductName()
+ + "]: Message: "
+ + e.getMessage()
+ + ": Attempting to continue
processing metadata");
+ }
+ }
+ }
+ }
+ }
+
+ public Metadata getMetadata(Product product) throws CatalogException {
+ Connection conn = null;
+ Statement statement = null;
+ ResultSet rs = null;
+ Metadata m = null;
+
+ try {
+ conn = dataSource.getConnection();
+ statement = conn.createStatement();
+
+ String metadataSql = "SELECT * FROM "
+ + product.getProductType().getName() + "_metadata "
+ + " WHERE product_id = '" + product.getProductId()+"'";
+
+ LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
+ rs = statement.executeQuery(metadataSql);
+
+ // parse SQL results
+ m = populateProductMetadata(rs, product);
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING, "Exception getting metadata. Message: "
+ + e.getMessage());
+ throw new CatalogException(e.getMessage());
+ } finally {
+
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException ignore) {
+ }
+
+ rs = null;
+ }
+
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException ignore) {
+ }
+
+ statement = null;
+ }
+
+ if (conn != null) {
+ try {
+ conn.close();
+
+ } catch (SQLException ignore) {
+ }
+
+ conn = null;
+ }
+ }
+
+ return m;
+ }
+
+ /** Method to populate the product metadata from a SQL ResultSet. **/
+ private Metadata populateProductMetadata(ResultSet rs, Product product)
throws CatalogException, SQLException {
+
+ Metadata m = new Metadata();
+
+ if (getValidationLayer()!=null) {
+
+ // validation layer: retrieve valid metadata elements
+ List<Element> elements = null;
+
+ try {
+ elements =
getValidationLayer().getElements(product.getProductType());
+ } catch (ValidationLayerException e) {
+ e.printStackTrace();
+ throw new CatalogException(
+ "ValidationLayerException when trying to obtain element
list for product type: "
+ + product.getProductType().getName()
+ + ": Message: " + e.getMessage());
+ }
+
+ while (rs.next()) {
+ for (Iterator<Element> i = elements.iterator(); i.hasNext();) {
+ Element e = i.next();
+
+ // right now, we just support STRING
+ String elemValue = rs.getString("metadata_value");
+ String elemId = rs.getString("element_id");
+
+ if (elemId.equals(e.getElementId())) {
+ elemValue = (elemValue != null ? elemValue : "");
+ m.addMetadata(e.getElementName(), elemValue);
+ }
+ }
+ }
+
+ } else {
+
+ // no validation layer - add all (name, value) pairs for this
product_id query
+ while (rs.next()) {
+ // right now, we just support STRING
+ String elemValue = rs.getString("metadata_value");
+ String elemId = rs.getString("element_id");
+
+ elemValue = (elemValue != null ? elemValue : "");
+ m.addMetadata(elemId, elemValue);
+ }
+
+ }
+
+ return m;
+
+ }
+
+ public Metadata getReducedMetadata(Product product, List<String> elems)
throws CatalogException {
+ Connection conn = null;
+ Statement statement = null;
+ ResultSet rs = null;
+ Metadata m = null;
+
+ try {
+ conn = dataSource.getConnection();
+ statement = conn.createStatement();
+
+ String elementIds = "";
+ if (elems.size() > 0) {
+
+ if (getValidationLayer()!=null) {
+ // validation layer: column "element_id" contains the
element identifier (e.g. "urn:oodt:ProductReceivedTime")
+ elementIds += " AND (element_id = '" +
this.getValidationLayer().getElementByName(elems.get(0)).getElementId() + "'";
+ for (int i = 1; i < elems.size(); i++)
+ elementIds += " OR element_id = '" +
this.getValidationLayer().getElementByName(elems.get(i)).getElementId() + "'";
+ elementIds += ")";
+
+ } else {
+ // no validation layer: column "element_id" contains
the element name (e.g. "CAS.ProductReceivedTime")
+ elementIds += " AND (element_id = '" + elems.get(0) +
"'";
+ for (int i = 1; i < elems.size(); i++)
+ elementIds += " OR element_id = '" + elems.get(i) +
"'";
+ elementIds += ")";
+
+ }
+
+ }
+ String metadataSql = "SELECT element_id,metadata_value FROM "
+ + product.getProductType().getName() + "_metadata"
+ + " WHERE product_id = " + quoteIt(product.getProductId())
+ elementIds;
+
+ LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
+ rs = statement.executeQuery(metadataSql);
+
+ // parse SQL results
+ m = populateProductMetadata(rs, product);
+
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING, "Exception getting metadata. Message: "
+ + e.getMessage());
+ throw new CatalogException(e.getMessage());
+ } finally {
+
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException ignore) {
+ }
+
+ rs = null;
+ }
+
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException ignore) {
+ }
+
+ statement = null;
+ }
+
+ if (conn != null) {
+ try {
+ conn.close();
+
+ } catch (SQLException ignore) {
+ }
+
+ conn = null;
+ }
+ }
+
+ return m;
+ }
+
+ private synchronized void addMetadataValue(String key,
+ Product product, String value) throws CatalogException {
+
+ Connection conn = null;
+ Statement statement = null;
+
+ String metadataTable = product.getProductType().getName() +
"_metadata";
+
+
+ try {
+ conn = dataSource.getConnection();
+ conn.setAutoCommit(false);
+ statement = conn.createStatement();
+
+ // build up the sql statement
+ StringBuffer insertClauseSql = new StringBuffer();
+ StringBuffer valueClauseSql = new StringBuffer();
+
+ insertClauseSql.append("INSERT INTO " + metadataTable
+ + " (product_id, element_id, metadata_value) ");
+ valueClauseSql.append("VALUES ");
+
+ // now do the value clause
+ if (fieldIdStringFlag) {
+ valueClauseSql.append("(" + quoteIt(product.getProductId()) +
", '"
+ + key + "', '" + value + "')");
+ } else {
+ valueClauseSql.append("(" + product.getProductId() + ", "
+ + key + ", '" + value + "')");
+ }
+
+ String metaIngestSql = insertClauseSql.toString()
+ + valueClauseSql.toString();
+ LOG
+ .log(Level.FINE, "addMetadataValue: Executing: "
+ + metaIngestSql);
+ statement.execute(metaIngestSql);
+ conn.commit();
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING, "Exception adding metadata value. Message: "
+ + e.getMessage());
+ try {
+ conn.rollback();
+ } catch (SQLException e2) {
+ LOG.log(Level.SEVERE,
+ "Unable to rollback add metadata value. Message: "
+ + e2.getMessage());
+ }
+ throw new CatalogException(e.getMessage());
+ } finally {
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException ignore) {
+ }
+
+ statement = null;
+ }
+
+ if (conn != null) {
+ try {
+ conn.close();
+
+ } catch (SQLException ignore) {
+ }
+
+ conn = null;
+ }
+ }
+ }
+
+ private synchronized void removeMetadataValue(String elementId,
+ Product product, String value) throws CatalogException {
+
+ Connection conn = null;
+ Statement statement = null;
+
+ String metadataTable = product.getProductType().getName() +
"_metadata";
+
+ try {
+ conn = dataSource.getConnection();
+ conn.setAutoCommit(false);
+ statement = conn.createStatement();
+
+ // build up the sql statement
+ String metRemoveSql = "DELETE FROM " + metadataTable + " WHERE ";
+ if (fieldIdStringFlag) {
+ metRemoveSql += "PRODUCT_ID = '" + product.getProductId()
+ + "' AND ";
+ metRemoveSql += "ELEMENT_ID = '" + elementId
+ + "' AND ";
+ metRemoveSql += "METADATA_VALUE = '" + value + "'";
+ } else {
+ metRemoveSql += "PRODUCT_ID = " + product.getProductId()
+ + " AND ";
+ metRemoveSql += "ELEMENT_ID = " + elementId
+ + " AND ";
+ metRemoveSql += "METADATA_VALUE = " + value;
+ }
+
+ LOG.log(Level.FINE, "removeMetadataValue: Executing: "
+ + metRemoveSql);
+ statement.execute(metRemoveSql);
+ conn.commit();
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING,
+ "Exception removing metadata value. Message: "
+ + e.getMessage());
+ try {
+ conn.rollback();
+ } catch (SQLException e2) {
+ LOG.log(Level.SEVERE,
+ "Unable to rollback remove metadata value. Message: "
+ + e2.getMessage());
+ }
+ throw new CatalogException(e.getMessage());
+ } finally {
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException ignore) {
+ }
+
+ statement = null;
+ }
+
+ if (conn != null) {
+ try {
+ conn.close();
+
+ } catch (SQLException ignore) {
+ }
+
+ conn = null;
+ }
+ }
+ }
+
+ /**
+ * Overridden method from superclass to allow for null validation layer.
+ */
+ protected int getResultListSize(Query query, ProductType type)
+ throws CatalogException {
+ Connection conn = null;
+ Statement statement = null;
+ ResultSet rs = null;
+
+ int resultCount = 0;
+
+ try {
+ conn = dataSource.getConnection();
+ statement = conn.createStatement();
+
+ String getProductSql = "";
+ String tableName = type.getName() + "_metadata";
+ String subSelectQueryBase = "SELECT product_id FROM " + tableName
+ + " ";
+ StringBuffer selectClause = new StringBuffer(
+ "SELECT COUNT(DISTINCT p.product_id) AS numResults ");
+ StringBuffer fromClause = new StringBuffer("FROM " + tableName
+ + " p ");
+ StringBuffer whereClause = new StringBuffer("WHERE ");
+
+ boolean gotFirstClause = false;
+ int clauseNum = 0;
+
+ if (query.getCriteria() != null && query.getCriteria().size() > 0)
{
+ for (Iterator<QueryCriteria> i =
query.getCriteria().iterator(); i.hasNext();) {
+ QueryCriteria criteria = i.next();
+ clauseNum++;
+
+ String elementIdStr = null;
+
+ if (fieldIdStringFlag) {
+ if (getValidationLayer()!=null) {
+ elementIdStr = "'" +
this.getValidationLayer().getElementByName(criteria.getElementName()).getElementId()
+ "'";
+ } else {
+ elementIdStr = "'" +
criteria.getElementName() + "'";
+ }
+ } else {
+ if (getValidationLayer()!=null) {
+ elementIdStr =
this.getValidationLayer().getElementByName(criteria.getElementName()).getElementId();
+ } else {
+ elementIdStr = criteria.getElementName();
+ }
+ }
+
+ String clause = null;
+
+ if (!gotFirstClause) {
+ clause = "(p.element_id = " + elementIdStr + " AND ";
+ if (criteria instanceof TermQueryCriteria) {
+ clause += " metadata_value LIKE '%"
+ + ((TermQueryCriteria) criteria).getValue()
+ + "%') ";
+ } else if (criteria instanceof RangeQueryCriteria) {
+ String startVal = ((RangeQueryCriteria) criteria)
+ .getStartValue();
+ String endVal = ((RangeQueryCriteria) criteria)
+ .getEndValue();
+ boolean inclusive = ((RangeQueryCriteria) criteria)
+ .getInclusive();
+
+ if ((startVal != null && !startVal.equals(""))
+ || (endVal != null && !endVal.equals("")))
{
+ clause += " metadata_value ";
+
+ boolean gotStart = false;
+
+ if (startVal != null && !startVal.equals("")) {
+ if (inclusive)
+ clause += ">= '" + startVal + "'";
+ else
+ clause += "> '" + startVal + "'";
+ gotStart = true;
+ }
+
+ if (endVal != null && !endVal.equals("")) {
+ if (gotStart) {
+ if (inclusive)
+ clause += " AND metadata_value <=
'"
+ + endVal + "'";
+ else
+ clause += " AND metadata_value < '"
+ + endVal + "'";
+ } else if (inclusive)
+ clause += "<= '" + endVal + "'";
+ else
+ clause += "< '" + endVal + "'";
+ }
+
+ clause += ") ";
+ }
+ }
+ whereClause.append(clause);
+ gotFirstClause = true;
+ } else {
+ String subSelectTblName = "p" + clauseNum;
+ String subSelectQuery = subSelectQueryBase
+ + "WHERE (element_id = " + elementIdStr
+ + " AND ";
+ if (criteria instanceof TermQueryCriteria) {
+ subSelectQuery += " metadata_value LIKE '%"
+ + ((TermQueryCriteria) criteria).getValue()
+ + "%')";
+ } else if (criteria instanceof RangeQueryCriteria) {
+ String startVal = ((RangeQueryCriteria) criteria)
+ .getStartValue();
+ String endVal = ((RangeQueryCriteria) criteria)
+ .getEndValue();
+
+ if (startVal != null || endVal != null) {
+ subSelectQuery += " metadata_value ";
+
+ boolean gotStart = false;
+
+ if (startVal != null && !startVal.equals("")) {
+ subSelectQuery += ">= '" + startVal + "'";
+ gotStart = true;
+ }
+
+ if (endVal != null && !endVal.equals("")) {
+ if (gotStart) {
+ subSelectQuery += " AND metadata_value
<= '"
+ + endVal + "'";
+ } else
+ subSelectQuery += "<= '" + endVal +
"'";
+ }
+
+ subSelectQuery += ") ";
+
+ }
+ }
+
+ fromClause.append("INNER JOIN (" + subSelectQuery
+ + ") " + subSelectTblName + " ON "
+ + subSelectTblName
+ + ".product_id = p.product_id ");
+
+ }
+ }
+ }
+
+ getProductSql = selectClause.toString() + fromClause.toString();
+ if (gotFirstClause) {
+ getProductSql += whereClause.toString();
+ }
+
+
+ LOG.log(Level.FINE, "catalog get num results: executing: "
+ + getProductSql);
+
+ rs = statement.executeQuery(getProductSql);
+
+ while (rs.next()) {
+ resultCount = rs.getInt("numResults");
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING,
+ "Exception performing get num results. Message: "
+ + e.getMessage());
+ try {
+ conn.rollback();
+ } catch (SQLException e2) {
+ LOG.log(Level.SEVERE,
+ "Unable to rollback get num results transaction.
Message: "
+ + e2.getMessage());
+ }
+ throw new CatalogException(e.getMessage());
+ } finally {
+
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException ignore) {
+ }
+
+ rs = null;
+ }
+
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException ignore) {
+ }
+
+ statement = null;
+ }
+
+ if (conn != null) {
+ try {
+ conn.close();
+
+ } catch (SQLException ignore) {
+ }
+
+ conn = null;
+ }
+ }
+
+ return resultCount;
+ }
+
+ /**
+ * Overridden method from superclass to allow for null validation layer.
+ */
+ protected String getSqlQuery(QueryCriteria queryCriteria, ProductType
type) throws ValidationLayerException, CatalogException {
+ String sqlQuery = null;
+ if (queryCriteria instanceof BooleanQueryCriteria) {
+ BooleanQueryCriteria bqc = (BooleanQueryCriteria) queryCriteria;
+ if (bqc.getOperator() == BooleanQueryCriteria.NOT) {
+ sqlQuery = "SELECT DISTINCT product_id FROM " + type.getName() +
"_metadata WHERE product_id NOT IN (" + this.getSqlQuery(bqc.getTerms().get(0),
type) + ")";
+ }else {
+ sqlQuery = "(" + this.getSqlQuery(bqc.getTerms().get(0), type);
+ String op = bqc.getOperator() == BooleanQueryCriteria.AND ?
"INTERSECT" : "UNION";
+ for (int i = 1; i < bqc.getTerms().size(); i++)
+ sqlQuery += ") " + op + " (" +
this.getSqlQuery(bqc.getTerms().get(i), type);
+ sqlQuery += ")";
+ }
+ }else {
+ String elementIdStr = queryCriteria.getElementName();
+ if (this.getValidationLayer()!=null) {
+ elementIdStr =
this.getValidationLayer().getElementByName(queryCriteria.getElementName()).getElementId();
+ }
+
+ if (fieldIdStringFlag)
+ elementIdStr = "'" + elementIdStr + "'";
+ sqlQuery = "SELECT DISTINCT product_id FROM " + type.getName() +
"_metadata WHERE element_id = " + elementIdStr + " AND ";
+ if (queryCriteria instanceof TermQueryCriteria) {
+ sqlQuery += "metadata_value = '" + ((TermQueryCriteria)
queryCriteria).getValue() + "'";
+ } else if (queryCriteria instanceof RangeQueryCriteria) {
+ RangeQueryCriteria rqc = (RangeQueryCriteria) queryCriteria;
+ String rangeSubQuery = null;
+ if (rqc.getStartValue() != null)
+ rangeSubQuery = "metadata_value" + (rqc.getInclusive() ? "
>= " : " > ") + "'" + rqc.getStartValue() + "'";
+ if (rqc.getEndValue() != null) {
+ if (rangeSubQuery == null)
+ rangeSubQuery = "metadata_value" + (rqc.getInclusive() ?
" <= " : " < ") + "'" + rqc.getEndValue() + "'";
+ else
+ rangeSubQuery = "(" + rangeSubQuery + " AND
metadata_value" + (rqc.getInclusive() ? " <= " : " < ") + "'" +
rqc.getEndValue() + "')";
+ }
+ sqlQuery += rangeSubQuery;
+ } else {
+ throw new CatalogException("Invalid QueryCriteria [" +
queryCriteria.getClass().getCanonicalName() + "]");
+ }
+ }
+
+ return sqlQuery;
+ }
+
+}
Propchange:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/DbStructFactory.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/DbStructFactory.java?rev=1422909&r1=1422908&r2=1422909&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/DbStructFactory.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/DbStructFactory.java
Mon Dec 17 12:47:38 2012
@@ -53,13 +53,17 @@ public final class DbStructFactory {
}
public static Product getProduct(ResultSet rs) throws SQLException {
- return getProduct(rs, true);
+ return getProduct(rs, true, false);
}
- public static Product getProduct(ResultSet rs, boolean getType)
+ public static Product getProduct(ResultSet rs, boolean getType, boolean
productIdString)
throws SQLException {
Product product = new Product();
- product.setProductId(String.valueOf(rs.getInt("product_id")));
+ if (productIdString) {
+ product.setProductId(rs.getString("product_id"));
+ } else {
+ product.setProductId(String.valueOf(rs.getInt("product_id")));
+ }
product.setProductName(rs.getString("product_name"));
product.setProductStructure(rs.getString("product_structure"));
product.setTransferStatus(rs.getString("product_transfer_status"));
