Author: mattmann
Date: Sun Feb 3 06:18:20 2013
New Revision: 1441861
URL: http://svn.apache.org/viewvc?rev=1441861&view=rev
Log:
Fix for OODT-551: DataSourceCatalog implementation does not preserve order of
metadata values
Added:
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestOrderedDataSourceCatalog.java
oodt/trunk/filemgr/src/testdata/testcat.ordered.sql
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java
oodt/trunk/filemgr/src/main/resources/cas-filemgr-dyn-catalog-schema.sql
oodt/trunk/filemgr/src/main/resources/filemgr.properties
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestDataSourceCatalog.java
Modified: oodt/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1441861&r1=1441860&r2=1441861&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Sun Feb 3 06:18:20 2013
@@ -7,7 +7,7 @@ Release 0.6 - Current Development
(Michael Starch via mattmann)
* OODT-551 Insert primary key in metadata table for database-based File
Manager, to always return metadata values in proper order
- (luca)
+ (luca, mattmann, bfoster)
* OODT-548 Be more resilient to table definitions matching the master table
in the Mapping config file (mattmann,joyce)
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java?rev=1441861&r1=1441860&r2=1441861&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalog.java
Sun Feb 3 06:18:20 2013
@@ -17,7 +17,7 @@
package org.apache.oodt.cas.filemgr.catalog;
-// OODT imports
+//OODT imports
import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria;
import org.apache.oodt.cas.filemgr.structs.Element;
import org.apache.oodt.cas.filemgr.structs.Product;
@@ -86,6 +86,8 @@ public class DataSourceCatalog implement
/* flag to indicate whether the "product_id" key type should be treated as
a string */
boolean productIdString = false;
+ protected boolean orderedValues = false;
+
/*
* cache of products per product type: [productTypeId]=>([ISO8601 time of
* last update]=>[List of products])
@@ -105,13 +107,14 @@ public class DataSourceCatalog implement
* @throws
*/
public DataSourceCatalog(DataSource ds, ValidationLayer valLayer,
- boolean fieldId, int pageSize, long cacheUpdateMin, boolean
productIdString) {
+ boolean fieldId, int pageSize, long
cacheUpdateMin, boolean productIdString, boolean orderedValues) {
this.dataSource = ds;
this.validationLayer = valLayer;
fieldIdStringFlag = fieldId;
this.pageSize = pageSize;
cacheUpdateMinutes = cacheUpdateMin;
this.productIdString = productIdString;
+ this.orderedValues = orderedValues;
}
/**
@@ -125,7 +128,7 @@ public class DataSourceCatalog implement
*/
public DataSourceCatalog(DataSource ds, ValidationLayer valLayer,
boolean fieldId, int pageSize, long cacheUpdateMin) {
- this(ds, valLayer, fieldId, pageSize, cacheUpdateMin, false);
+ this(ds, valLayer, fieldId, pageSize, cacheUpdateMin, false, false);
}
/*
@@ -824,7 +827,9 @@ public class DataSourceCatalog implement
String getProductRefSql = "SELECT * FROM "
+ product.getProductType().getName() + "_reference"
- + " WHERE product_id = " + quoteIt(product.getProductId());
+ + " WHERE product_id = " + quoteIt(product.getProductId());
+
+ if(this.orderedValues) getProductRefSql += " ORDER BY pkey";
LOG.log(Level.FINE, "getProductReferences: Executing: "
+ getProductRefSql);
@@ -1060,8 +1065,10 @@ public class DataSourceCatalog implement
statement = conn.createStatement();
String metadataSql = "SELECT * FROM "
- + product.getProductType().getName() + "_metadata "
- + " WHERE product_id = " + quoteIt(product.getProductId());
+ + product.getProductType().getName() + "_metadata"
+ + " WHERE product_id = " + quoteIt(product.getProductId());
+
+ if(this.orderedValues) metadataSql += " ORDER BY pkey";
LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
rs = statement.executeQuery(metadataSql);
@@ -1152,7 +1159,8 @@ public class DataSourceCatalog implement
}
String metadataSql = "SELECT element_id,metadata_value FROM "
+ product.getProductType().getName() + "_metadata"
- + " WHERE product_id = " + quoteIt(product.getProductId())
+ elementIds;
+ + " WHERE product_id = " + quoteIt(product.getProductId()) +
elementIds;
+ if(this.orderedValues) metadataSql += " ORDER BY pkey";
LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
rs = statement.executeQuery(metadataSql);
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java?rev=1441861&r1=1441860&r2=1441861&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/DataSourceCatalogFactory.java
Sun Feb 3 06:18:20 2013
@@ -54,13 +54,14 @@ public class DataSourceCatalogFactory im
/* the amount of minutes to allow between updating the product cache */
protected long cacheUpdateMinutes = -1L;
- /* Whether or not to enforce strict definition of metadata fields:
+ /* Whether or not to enforce strict definition of metadata fields:
* 'lenient=false' means that all metadata fields need to be explicitly
defined in the XML configuration file */
- protected boolean lenientFields = false;
+ protected boolean lenientFields = false;
/* Flag to indicate whether the "product_id" key type should be treated as
a string */
- protected boolean productIdString = false;
+ protected boolean productIdString = false;
+ protected boolean orderedValues = false;
/**
* <p>
@@ -136,7 +137,8 @@ public class DataSourceCatalogFactory im
productIdString = Boolean.parseBoolean(
System.getProperty("org.apache.oodt.cas.filemgr.catalog.datasource.productId.string",
"false") );
-
+
+ orderedValues =
Boolean.parseBoolean(System.getProperty("org.apache.oodt.cas.filemgr.catalog.datasource.orderedValues"));
}
/*
@@ -147,10 +149,10 @@ public class DataSourceCatalogFactory im
public Catalog createCatalog() {
if (validationLayer==null) {
return new LenientDataSourceCatalog(dataSource,
validationLayer, fieldIdStr,
- pageSize, cacheUpdateMinutes, productIdString);
+ pageSize,
cacheUpdateMinutes, productIdString, orderedValues);
} else {
return new DataSourceCatalog(dataSource, validationLayer, fieldIdStr,
- pageSize, cacheUpdateMinutes, productIdString);
+ pageSize, cacheUpdateMinutes,
productIdString, orderedValues);
}
}
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java?rev=1441861&r1=1441860&r2=1441861&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LenientDataSourceCatalog.java
Sun Feb 3 06:18:20 2013
@@ -74,9 +74,9 @@ public class LenientDataSourceCatalog ex
* @throws
*/
public LenientDataSourceCatalog(DataSource ds, ValidationLayer valLayer,
- boolean fieldId, int pageSize, long cacheUpdateMin, boolean
productIdString) {
+ boolean fieldId, int pageSize, long
cacheUpdateMin, boolean productIdString, boolean orderedValues) {
- super(ds, valLayer, fieldId, pageSize, cacheUpdateMin,
productIdString);
+ super(ds, valLayer, fieldId, pageSize, cacheUpdateMin, productIdString,
orderedValues);
}
@@ -226,7 +226,8 @@ public class LenientDataSourceCatalog ex
String metadataSql = "SELECT * FROM "
+ product.getProductType().getName() + "_metadata "
- + " WHERE product_id = '" + product.getProductId()+"'";
+ + "WHERE product_id = '" + product.getProductId()+"'";
+ if(this.orderedValues) metadataSql += " ORDER BY pkey" ;
LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
rs = statement.executeQuery(metadataSql);
@@ -358,7 +359,8 @@ public class LenientDataSourceCatalog ex
}
String metadataSql = "SELECT element_id,metadata_value FROM "
+ product.getProductType().getName() + "_metadata"
- + " WHERE product_id = " + quoteIt(product.getProductId())
+ elementIds;
+ + " WHERE product_id = " + quoteIt(product.getProductId()) +
elementIds;
+ if(this.orderedValues) metadataSql += " ORDER BY pkey";
LOG.log(Level.FINE, "getMetadata: Executing: " + metadataSql);
rs = statement.executeQuery(metadataSql);
Modified:
oodt/trunk/filemgr/src/main/resources/cas-filemgr-dyn-catalog-schema.sql
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/resources/cas-filemgr-dyn-catalog-schema.sql?rev=1441861&r1=1441860&r2=1441861&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/resources/cas-filemgr-dyn-catalog-schema.sql
(original)
+++ oodt/trunk/filemgr/src/main/resources/cas-filemgr-dyn-catalog-schema.sql
Sun Feb 3 06:18:20 2013
@@ -16,6 +16,7 @@
CREATE TABLE YourProductTypeName_metadata
(
+ pkey int(10) unsigned primary KEY AUTO_INCREMENT,
product_id int NOT NULL,
element_id varchar(1000) NOT NULL,
metadata_value varchar(2500) NOT NULL
@@ -23,6 +24,7 @@ CREATE TABLE YourProductTypeName_metadat
CREATE TABLE YourProductTypeName_reference
(
+ pkey int(10) unsigned primary KEY AUTO_INCREMENT,
product_id int NOT NULL,
product_orig_reference varchar(2000) NOT NULL,
product_datastore_reference varchar(2000),
Modified: oodt/trunk/filemgr/src/main/resources/filemgr.properties
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/resources/filemgr.properties?rev=1441861&r1=1441860&r2=1441861&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/resources/filemgr.properties (original)
+++ oodt/trunk/filemgr/src/main/resources/filemgr.properties Sun Feb 3
06:18:20 2013
@@ -41,6 +41,7 @@ org.apache.oodt.cas.filemgr.catalog.data
org.apache.oodt.cas.filemgr.catalog.datasource.quoteFields=false
org.apache.oodt.cas.filemgr.catalog.datasource.pageSize=20
org.apache.oodt.cas.filemgr.catalog.datasource.cacheUpdateMinutes=5
+org.apache.oodt.cas.filemgr.catalog.datasource.orderedValues=false
# set the following property to 'true' to allow dynamic metadata fields,
# effectively bypassing the validation layer.
# by default the property is false
Modified:
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestDataSourceCatalog.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestDataSourceCatalog.java?rev=1441861&r1=1441860&r2=1441861&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestDataSourceCatalog.java
(original)
+++
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestDataSourceCatalog.java
Sun Feb 3 06:18:20 2013
@@ -50,7 +50,7 @@ import junit.framework.TestCase;
*/
public class TestDataSourceCatalog extends TestCase {
- private Catalog myCat;
+ protected Catalog myCat;
private String tmpDirPath = null;
Added:
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestOrderedDataSourceCatalog.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestOrderedDataSourceCatalog.java?rev=1441861&view=auto
==============================================================================
---
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestOrderedDataSourceCatalog.java
(added)
+++
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/catalog/TestOrderedDataSourceCatalog.java
Sun Feb 3 06:18:20 2013
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.oodt.cas.filemgr.catalog;
+
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.metadata.Metadata;
+
+/**
+ * @author mattmann
+ * @version $Revision$
+ *
+ * <p>
+ * Describe your class here
+ * </p>.
+ */
+public class TestOrderedDataSourceCatalog extends TestDataSourceCatalog {
+
+ /**
+ *
+ */
+ public TestOrderedDataSourceCatalog() {
+ super();
+
System.setProperty("org.apache.oodt.cas.filemgr.catalog.datasource.orderedValues",
"true");
+ setCatalog(getCatalog());
+
+ }
+
+ protected Catalog getCatalog() {
+ try {
+ return new DataSourceCatalogFactory().createCatalog();
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
org.apache.oodt.cas.filemgr.catalog.TestDataSourceCatalog#getSchemaPath()
+ */
+ @Override
+ protected String getSchemaPath() {
+ return "./src/testdata/testcat.ordered.sql";
+ }
+
+ public void testOrdering(){
+ Product testProduct = getTestProduct();
+ Metadata testMet = getTestMetadata("test");
+
+ try {
+ myCat.addProduct(testProduct);
+ myCat.addMetadata(testMet, testProduct);
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+
+
+
+ Product retProduct;
+ try {
+ Metadata retMet = myCat.getMetadata(testProduct);
+ assertNotNull(retMet);
+ assertNotNull(retMet.getAllMetadata("CAS.ProductName"));
+ assertEquals(3, retMet.getAllMetadata("CAS.ProductName").size());
+ assertEquals("test",
retMet.getAllMetadata("CAS.ProductName").get(0));
+ assertEquals("test2",
retMet.getAllMetadata("CAS.ProductName").get(1));
+ assertEquals("test3",
retMet.getAllMetadata("CAS.ProductName").get(2));
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+
+
+ }
+
+
+ private static Product getTestProduct() {
+ Product testProduct = Product.getDefaultFlatProduct("test",
+
"urn:oodt:GenericFile");
+ testProduct.getProductType().setName("GenericFile");
+ return testProduct;
+ }
+
+ private static Metadata getTestMetadata(String prodName) {
+ Metadata met = new Metadata();
+ met.addMetadata("CAS.ProductName", prodName);
+ met.addMetadata("CAS.ProductName", prodName+"2");
+ met.addMetadata("CAS.ProductName", prodName+"3");
+ return met;
+ }
+
+}
+
+
Added: oodt/trunk/filemgr/src/testdata/testcat.ordered.sql
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/testdata/testcat.ordered.sql?rev=1441861&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/testdata/testcat.ordered.sql (added)
+++ oodt/trunk/filemgr/src/testdata/testcat.ordered.sql Sun Feb 3 06:18:20 2013
@@ -0,0 +1,29 @@
+DROP TABLE GenericFile_metadata IF EXISTS;
+DROP TABLE GenericFile_reference IF EXISTS;
+DROP TABLE products IF EXISTS;
+
+CREATE TABLE GenericFile_metadata
+(
+ pkey int GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+ product_id int NOT NULL,
+ element_id varchar(1000) NOT NULL,
+ metadata_value varchar(2500) NOT NULL
+);
+
+CREATE TABLE GenericFile_reference
+(
+ pkey int GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+ product_id int NOT NULL,
+ product_orig_reference varchar(2000) NOT NULL,
+ product_datastore_reference varchar(2000),
+ product_reference_filesize int NOT NULL,
+ product_reference_mimetype varchar(50)
+);
+
+CREATE TABLE products (
+ product_id INTEGER GENERATED BY DEFAULT AS IDENTITY (START WITH 1, INCREMENT
BY 1) PRIMARY KEY,
+ product_structure varchar(20) NOT NULL,
+ product_type_id varchar(255) NOT NULL,
+ product_name varchar(255) NOT NULL,
+ product_transfer_status varchar(255) NOT NULL
+);
