Author: starchmd
Date: Tue Oct 28 23:55:27 2014
New Revision: 1635009
URL: http://svn.apache.org/r1635009
Log:
OODT-701, strem structure in fm. Added structure check, and fixed tests.
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/CoreMetExtractor.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Product.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Reference.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/BasicVersioner.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/DateTimeVersioner.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/SingleFileBasicVersioner.java
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/VersioningUtils.java
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/ProductResourceTest.java
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransferResourceTest.java
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransfersResourceTest.java
Modified: oodt/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Tue Oct 28 23:55:27 2014
@@ -3,6 +3,8 @@ Apache OODT Change Log
Release 0.8 - Current Development
+* OODT-701 Adding in stream product structure for filemanager. Add in check
for valid product structures, and fixed tests breaking from that check.
+
* OODT-699 Adding in cluster managment scripts for mesos.
* OODT-764 Adding in multiplexing resource manager backend.
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java
Tue Oct 28 23:55:27 2014
@@ -18,6 +18,7 @@ package org.apache.oodt.cas.filemgr.cli.
//OODT static imports
import static
org.apache.oodt.cas.filemgr.structs.Product.STRUCTURE_HIERARCHICAL;
+import static org.apache.oodt.cas.filemgr.structs.Product.STRUCTURE_STREAM;
import static
org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory.getDataTransferServiceFromFactory;
import static
org.apache.oodt.cas.filemgr.versioning.VersioningUtils.addRefsFromUris;
import static
org.apache.oodt.cas.filemgr.versioning.VersioningUtils.getURIsFromDir;
@@ -88,6 +89,15 @@ public class IngestProductCliAction exte
uriRefs.add(hierRefUri.toString());
uriRefs.addAll(getURIsFromDir(new File(hierRefUri.getPath())));
references = uriRefs;
+ } else if (product.getProductStructure().equals(STRUCTURE_STREAM)) {
+ List<String> uriRefs = Lists.newArrayList();
+ for (String ref : references) {
+ URI uri = URI.create(ref);
+ if (!uri.getScheme().equals("stream"))
+ throw new IllegalArgumentException("Streaming data must use
'stream' scheme not "+uri.getScheme());
+ uriRefs.add(uri.toString());
+ }
+ references = uriRefs;
} else {
List<String> uriRefs = Lists.newArrayList();
for (String reference : references) {
@@ -104,6 +114,7 @@ public class IngestProductCliAction exte
new SerializableMetadata(getUri(metadataFile).toURL()
.openStream()), dataTransferer != null));
} catch (Exception e) {
+ e.printStackTrace();
throw new CmdLineActionException("Failed to ingest product '"
+ productName + "' : " + e.getMessage(), e);
}
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
Tue Oct 28 23:55:27 2014
@@ -123,6 +123,8 @@ public class LocalDataTransferer impleme
+ e.getMessage());
throw new DataTransferException(e);
}
+ } else if
(product.getProductStructure().equals(Product.STRUCTURE_STREAM)) {
+ LOG.log(Level.INFO,"Streaming products are not moved.");
} else {
throw new DataTransferException(
"Cannot transfer product on unknown ProductStructure: "
@@ -163,6 +165,8 @@ public class LocalDataTransferer impleme
+ e.getMessage());
throw new DataTransferException(e);
}
+ } else if
(product.getProductStructure().equals(Product.STRUCTURE_STREAM)) {
+ LOG.log(Level.INFO,"Streaming products are not transfered.");
} else {
throw new DataTransferException(
"Cannot transfer product on unknown ProductStructure: "
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/CoreMetExtractor.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/CoreMetExtractor.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/CoreMetExtractor.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/CoreMetExtractor.java
Tue Oct 28 23:55:27 2014
@@ -29,7 +29,7 @@ import java.io.File;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
-
+import java.util.UUID;
/**
* @author mattmann
* @author bfoster
@@ -66,18 +66,22 @@ public class CoreMetExtractor extends Ab
Metadata extractMet = new Metadata();
/* copy through original metadata */
merge(met, extractMet);
-
- File prodFile = getProductFile(product);
-
+ File prodFile = null;
+ if (!product.getProductStructure().equals(Product.STRUCTURE_STREAM))
+ {
+ prodFile = getProductFile(product);
+ }
+ //GUID used when no file is associated with product i.e. stream
+ String guid=UUID.randomUUID().toString();
extractMet
.addMetadata(isNsReplace(PRODUCT_ID) ? elementNs + nsSeparator
+ PRODUCT_ID : PRODUCT_ID, product.getProductId());
addMetadataIfUndefined(met, extractMet,
isNsReplace(FILENAME) ? elementNs + nsSeparator + FILENAME
- : FILENAME, prodFile.getName());
+ : FILENAME, (prodFile ==
null)?guid:prodFile.getName());
addMetadataIfUndefined(met, extractMet,
isNsReplace(FILE_LOCATION) ? elementNs + nsSeparator
- + FILE_LOCATION : FILE_LOCATION, prodFile
+ + FILE_LOCATION : FILE_LOCATION,
(prodFile==null)?guid:prodFile
.getParentFile().getAbsolutePath());
addMetadataIfUndefined(met, extractMet,
isNsReplace(PRODUCT_NAME) ? elementNs + nsSeparator
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Product.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Product.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Product.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Product.java
Tue Oct 28 23:55:27 2014
@@ -27,9 +27,11 @@ import java.util.List;
import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;
+
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@@ -89,6 +91,10 @@ public class Product {
public static final String STRUCTURE_HIERARCHICAL = "Hierarchical";
+ public static final String STRUCTURE_STREAM = "Stream";
+
+ private static String[] VALID_STRUCTURES = new
String[]{STRUCTURE_FLAT,STRUCTURE_HIERARCHICAL,STRUCTURE_STREAM};
+
/* our log stream */
private static final Logger LOG =
Logger.getLogger(Product.class.getName());
@@ -176,6 +182,9 @@ public class Product {
* The productStructure to set.
*/
public void setProductStructure(String productStructure) {
+ //Guard clause, according to a unit test null is a valid value
+ if
(!java.util.Arrays.asList(VALID_STRUCTURES).contains(productStructure) &&
productStructure != null)
+ throw new IllegalArgumentException("Undefined product structure:
"+productStructure);
this.productStructure = productStructure;
}
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Reference.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Reference.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Reference.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/Reference.java
Tue Oct 28 23:55:27 2014
@@ -60,6 +60,8 @@ public class Reference {
private static MimeTypes mimeTypeRepository;
+ public static String STREAM_REFERENCE_DELIMITER = "-";
+
/* the static reference to the Mime-Type repository */
static {
try {
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
Tue Oct 28 23:55:27 2014
@@ -931,7 +931,7 @@ public class XmlRpcFileManager {
}
} else
throw new UnsupportedOperationException(
- "Moving of heirarhical products not supported yet");
+ "Moving of heirarhical and stream products not supported
yet");
}
public boolean removeFile(String filePath) throws DataTransferException,
IOException {
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/BasicVersioner.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/BasicVersioner.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/BasicVersioner.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/BasicVersioner.java
Tue Oct 28 23:55:27 2014
@@ -1,132 +1,135 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oodt.cas.filemgr.versioning;
-
-//JDK imports
-import java.io.File;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.net.MalformedURLException;
-import java.util.logging.Logger;
-import java.util.logging.Level;
-
-//OODT imports
-import org.apache.oodt.cas.filemgr.structs.Product;
-import org.apache.oodt.cas.filemgr.structs.Reference;
-import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.oodt.cas.filemgr.structs.exceptions.VersioningException;
-
-/**
- * @author mattmann
- * @version $Revision$
- *
- * <p>
- * A basic versioner that doesn't do anything special: it just creates data
- * store refs from product refs using the product name and product repo path.
- * </p>
- *
- */
-public class BasicVersioner implements Versioner {
-
- /* our log stream */
- private static final Logger LOG =
Logger.getLogger(BasicVersioner.class.getName());
-
- /**
- *
- */
- public BasicVersioner() {
- super();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
org.apache.oodt.cas.versioning.Versioner#createDataStoreReferences(org.apache.oodt.cas.data.structs.Product)
- */
- public void createDataStoreReferences(Product product, Metadata metadata)
- throws VersioningException {
-
- String productName = product.getProductName();
- String productRepoPath = product.getProductType()
- .getProductRepositoryPath();
-
- if (product.getProductStructure()
- .equals(Product.STRUCTURE_HIERARCHICAL)) {
-
- if (product.getProductReferences() == null
- || (product.getProductReferences() != null && product
- .getProductReferences().size() == 0)) {
- throw new VersioningException(
- "Hierarchical product and references not set!");
- }
-
- // get the first reference, it tells us what directory to move it
- // to
- // TODO: fix that hack :-)
- Reference r = (Reference) product.getProductReferences().get(0);
-
- String dataStoreRef = null;
-
- try {
- dataStoreRef = new File(new URI(productRepoPath)).toURL()
- .toExternalForm();
- if(!dataStoreRef.endsWith("/")){
- dataStoreRef+="/";
- }
-
- dataStoreRef+= URLEncoder.encode(productName, "UTF-8") + "/";
- LOG.log(Level.INFO, "BasicVersioner: generated DataStore ref: "
- + dataStoreRef + " from origRef: "
- + r.getOrigReference());
- r.setDataStoreReference(dataStoreRef);
- VersioningUtils.createBasicDataStoreRefsHierarchical(product
- .getProductReferences());
- } catch (URISyntaxException e) {
- LOG.log(Level.WARNING,
- "BasicVersioner: URISyntaxException while generating
initial "
- + "data store ref for origRef: "
- + r.getOrigReference());
- throw new VersioningException(e);
- } catch (MalformedURLException e) {
- LOG.log(Level.WARNING,
- "BasicVersioner: MalformedURLException while
generating initial "
- + "data store ref for origRef: "
- + r.getOrigReference());
- throw new VersioningException(e);
- } catch (UnsupportedEncodingException e) {
- LOG.log(Level.WARNING,
- "BasicVersioner: UnsupportedEncodingException while
generating "
- + "initial data store ref for origRef: "
- + r.getOrigReference());
- throw new VersioningException(e);
- }
-
- } else if
(product.getProductStructure().equals(Product.STRUCTURE_FLAT)) {
- // just use the VersioningUtils
- VersioningUtils.createBasicDataStoreRefsFlat(productName,
- productRepoPath, product.getProductReferences());
- } else {
- throw new VersioningException("Unsupported product structure: "
- + product.getProductStructure());
- }
-
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.filemgr.versioning;
+
+//JDK imports
+import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.net.MalformedURLException;
+import java.util.logging.Logger;
+import java.util.logging.Level;
+
+//OODT imports
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.filemgr.structs.exceptions.VersioningException;
+
+/**
+ * @author mattmann
+ * @version $Revision$
+ *
+ * <p>
+ * A basic versioner that doesn't do anything special: it just creates data
+ * store refs from product refs using the product name and product repo path.
+ * </p>
+ *
+ */
+public class BasicVersioner implements Versioner {
+
+ /* our log stream */
+ private static final Logger LOG =
Logger.getLogger(BasicVersioner.class.getName());
+
+ /**
+ *
+ */
+ public BasicVersioner() {
+ super();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
org.apache.oodt.cas.versioning.Versioner#createDataStoreReferences(org.apache.oodt.cas.data.structs.Product)
+ */
+ public void createDataStoreReferences(Product product, Metadata metadata)
+ throws VersioningException {
+
+ String productName = product.getProductName();
+ String productRepoPath = product.getProductType()
+ .getProductRepositoryPath();
+
+ if (product.getProductStructure()
+ .equals(Product.STRUCTURE_HIERARCHICAL)) {
+
+ if (product.getProductReferences() == null
+ || (product.getProductReferences() != null && product
+ .getProductReferences().size() == 0)) {
+ throw new VersioningException(
+ "Hierarchical product and references not set!");
+ }
+
+ // get the first reference, it tells us what directory to move it
+ // to
+ // TODO: fix that hack :-)
+ Reference r = (Reference) product.getProductReferences().get(0);
+
+ String dataStoreRef = null;
+
+ try {
+ dataStoreRef = new File(new URI(productRepoPath)).toURL()
+ .toExternalForm();
+ if(!dataStoreRef.endsWith("/")){
+ dataStoreRef+="/";
+ }
+
+ dataStoreRef+= URLEncoder.encode(productName, "UTF-8") + "/";
+ LOG.log(Level.INFO, "BasicVersioner: generated DataStore ref: "
+ + dataStoreRef + " from origRef: "
+ + r.getOrigReference());
+ r.setDataStoreReference(dataStoreRef);
+ VersioningUtils.createBasicDataStoreRefsHierarchical(product
+ .getProductReferences());
+ } catch (URISyntaxException e) {
+ LOG.log(Level.WARNING,
+ "BasicVersioner: URISyntaxException while generating
initial "
+ + "data store ref for origRef: "
+ + r.getOrigReference());
+ throw new VersioningException(e);
+ } catch (MalformedURLException e) {
+ LOG.log(Level.WARNING,
+ "BasicVersioner: MalformedURLException while
generating initial "
+ + "data store ref for origRef: "
+ + r.getOrigReference());
+ throw new VersioningException(e);
+ } catch (UnsupportedEncodingException e) {
+ LOG.log(Level.WARNING,
+ "BasicVersioner: UnsupportedEncodingException while
generating "
+ + "initial data store ref for origRef: "
+ + r.getOrigReference());
+ throw new VersioningException(e);
+ }
+
+ } else if
(product.getProductStructure().equals(Product.STRUCTURE_FLAT)) {
+ // just use the VersioningUtils
+ VersioningUtils.createBasicDataStoreRefsFlat(productName,
+ productRepoPath, product.getProductReferences());
+ } else if
(product.getProductStructure().equals(Product.STRUCTURE_STREAM)) {
+ VersioningUtils.createBasicDataStoreRefsStream(productName,
+ productRepoPath, product.getProductReferences(),"");
+ } else {
+ throw new VersioningException("Unsupported product structure: "
+ + product.getProductStructure());
+ }
+
+ }
+
+}
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/DateTimeVersioner.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/DateTimeVersioner.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/DateTimeVersioner.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/DateTimeVersioner.java
Tue Oct 28 23:55:27 2014
@@ -143,6 +143,9 @@ public class DateTimeVersioner implement
}
}
+ } else if
(product.getProductStructure().equals(Product.STRUCTURE_STREAM)) {
+
VersioningUtils.createBasicDataStoreRefsStream(product.getProductName(),
product.getProductType().getProductRepositoryPath(),
+ product.getProductReferences(), productionDateTime);
} else if (product.getProductStructure().equals(
Product.STRUCTURE_HIERARCHICAL)) {
// if its heirarchical, then we'll version the files within the
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/SingleFileBasicVersioner.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/SingleFileBasicVersioner.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/SingleFileBasicVersioner.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/SingleFileBasicVersioner.java
Tue Oct 28 23:55:27 2014
@@ -71,7 +71,7 @@ public class SingleFileBasicVersioner im
throw new VersioningException(
"SingleFileVersioner: unable to version" + " Product: ["
+ product.getProductName()
- + "] with heirarchical structure");
+ + "] with heirarchical/stream structure");
}
// we need the Filename Metadata parameter for this to work
Modified:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/VersioningUtils.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/VersioningUtils.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/VersioningUtils.java
(original)
+++
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/versioning/VersioningUtils.java
Tue Oct 28 23:55:27 2014
@@ -1,287 +1,300 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oodt.cas.filemgr.versioning;
-
-//OODT imports
-import org.apache.oodt.cas.filemgr.structs.Reference;
-import org.apache.oodt.cas.filemgr.structs.Product;
-
-//JDK imports
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Stack;
-import java.util.Vector;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * @author mattmann
- * @author bfoster
- * @version $Revision$
- *
- * <p>
- * A Utility component to help out with versioning.
- * </p>
- *
- *
- */
-public final class VersioningUtils {
-
- /* our log stream */
- private static final Logger LOG = Logger.getLogger(VersioningUtils.class
- .getName());
-
- // filter to only find directories when doing a listFiles
- private static FileFilter DIR_FILTER = new FileFilter() {
- public boolean accept(File file) {
- return file.isDirectory();
- }
- };
-
- // filter to only find files when doing a listFiles
- private static FileFilter FILE_FILTER = new FileFilter() {
- public boolean accept(File file) {
- return file.isFile();
- }
- };
-
- public static List<Reference> getReferencesFromDir(File dirRoot) {
- List<Reference> references = null;
-
- if (dirRoot == null)
- throw new IllegalArgumentException("null");
- if (!dirRoot.isDirectory())
- dirRoot = dirRoot.getParentFile();
-
- references = new Vector<Reference>();
-
- Stack<File> stack = new Stack<File>();
- stack.push(dirRoot);
- while (!stack.isEmpty()) {
- File dir = stack.pop();
- // add the reference for the dir
- // except if it's the rootDir, then, skip it
- if (!dir.equals(dirRoot)) {
- try {
- Reference r = new Reference();
- r.setOrigReference(dir.toURL().toExternalForm());
- r.setFileSize(dir.length());
- references.add(r);
- } catch (MalformedURLException e) {
- e.printStackTrace();
- LOG.log(Level.WARNING,
- "MalformedURLException when generating reference
for dir: "
- + dir);
- }
- }
-
- File[] files = dir.listFiles(FILE_FILTER);
-
- for (int i = 0; i < files.length; i++) {
- // add the file references
- try {
- Reference r = new Reference();
- r.setOrigReference(files[i].toURL().toExternalForm());
- r.setFileSize(files[i].length());
- references.add(r);
- } catch (MalformedURLException e) {
- e.printStackTrace();
- LOG.log(Level.WARNING,
- "MalformedURLException when generating reference
for file: "
- + files[i]);
- }
-
- }
- File[] subdirs = dir.listFiles(DIR_FILTER);
- if (subdirs != null)
- for (int i = 0; i < subdirs.length; i++) {
- stack.push(subdirs[i]);
- }
- }
-
- return references;
- }
-
- public static List<String> getURIsFromDir(File dirRoot) {
- List<String> uris = null;
-
- if (dirRoot == null)
- throw new IllegalArgumentException("null");
- if (!dirRoot.isDirectory())
- dirRoot = dirRoot.getParentFile();
-
- uris = new Vector<String>();
-
- Stack<File> stack = new Stack<File>();
- stack.push(dirRoot);
- while (!stack.isEmpty()) {
- File dir = stack.pop();
- // add the reference for the dir
- // except if it's the rootDir, then, skip it
- if (!dir.equals(dirRoot)) {
- uris.add(dir.toURI().toString());
- }
-
- File[] files = dir.listFiles(FILE_FILTER);
-
- for (int i = 0; i < files.length; i++) {
- // add the file references
- uris.add(files[i].toURI().toString());
- }
-
- File[] subdirs = dir.listFiles(DIR_FILTER);
- if (subdirs != null)
- for (int i = 0; i < subdirs.length; i++) {
- stack.push(subdirs[i]);
- }
- }
-
- return uris;
- }
-
- public static void createBasicDataStoreRefsHierarchical(List<Reference>
references) {
- // file:///www/folder1
- // file:///www/folder1/file1
- // file:///www/folder1/file2
- // file:///www/folder1/folder2/
- // file:///www/folder1/folder2/file3
-
- // toDir: file:///www/myfolder/product1
- // origDir: file:///www/folder1
-
- String toDirRef = references.get(0)
- .getDataStoreReference();
- String origDirRef = references.get(0).getOrigReference();
- String origDirRefName = new File(origDirRef).getName();
-
- for (Iterator<Reference> i = references.iterator(); i.hasNext();) {
- Reference r = i.next();
-
- // don't bother with the first one, because it's already set
- // correctly
- if (r.getOrigReference().equals(origDirRef)) {
- continue;
- }
-
- // get the first occurence of the origDir name in the string
- // then, the ref becomes:
- // toDir+r.getOrigRef.substring(first occurence of
- // origDir).substring(first occurence of '/'+1)
-
- String dataStoreRef = toDirRef;
- int firstOccurenceOfOrigDir = r.getOrigReference().indexOf(
- origDirRefName);
- String tmpRef = r.getOrigReference().substring(
- firstOccurenceOfOrigDir);
- LOG.log(Level.FINER, "tmpRef: " + tmpRef);
- int firstOccurenceSlash = tmpRef.indexOf("/");
- dataStoreRef += tmpRef.substring(firstOccurenceSlash + 1);
-
- LOG.log(Level.FINE, "VersioningUtils: Generated data store ref: "
- + dataStoreRef + " from origRef: " + r.getOrigReference());
- r.setDataStoreReference(dataStoreRef);
- }
-
- }
-
- public static void createBasicDataStoreRefsFlat(String productName,
- String productRepoPath, List<Reference> references) {
- for (Iterator<Reference> i = references.iterator(); i.hasNext();) {
- Reference r = i.next();
-
- String dataStoreRef = null;
- String productRepoPathRef = null;
-
- try {
- productRepoPathRef = new File(new URI(productRepoPath)).toURL()
- .toExternalForm();
-
- if (!productRepoPathRef.endsWith("/")) {
- productRepoPathRef += "/";
- }
-
- dataStoreRef = productRepoPathRef
- + URLEncoder.encode(productName, "UTF-8") + "/"
- + new File(new URI(r.getOrigReference())).getName();
- } catch (IOException e) {
- LOG.log(Level.WARNING,
- "VersioningUtils: Error generating dataStoreRef for "
- + r.getOrigReference() + ": Message: "
- + e.getMessage());
- } catch (URISyntaxException e) {
- LOG.log(Level.WARNING,
- "VersioningUtils: Error generating dataStoreRef for "
- + r.getOrigReference() + ": Message: "
- + e.getMessage());
- }
-
- LOG.log(Level.FINE, "VersioningUtils: Generated data store ref: "
- + dataStoreRef + " from origRef: " + r.getOrigReference());
- r.setDataStoreReference(dataStoreRef);
- }
-
- }
-
- public static void addRefsFromUris(Product p, List<String> uris) {
- // add the refs to the Product
- for (Iterator<String> i = uris.iterator(); i.hasNext();) {
- String ref = i.next();
- Reference r = new Reference(ref, null,
quietGetFileSizeFromUri(ref));
- p.getProductReferences().add(r);
- }
- }
-
- public static String getAbsolutePathFromUri(String uriStr) {
- URI uri = null;
- String absPath = null;
-
- try {
- uri = new URI(uriStr);
- absPath = new File(uri).getAbsolutePath();
- } catch (URISyntaxException e) {
- LOG.log(Level.WARNING,
- "URISyntaxException getting URI from URI str: [" + uriStr
- + "]");
- }
-
- return absPath;
- }
-
- private static long quietGetFileSizeFromUri(String uri) {
- File fileRef = null;
-
- try {
- fileRef = new File(new URI(uri));
- } catch (URISyntaxException e) {
- LOG.log(Level.WARNING,
- "URISyntaxException when getting file size from uri: ["
- + uri + "]: Message: " + e.getMessage());
- return -1L;
- }
-
- return fileRef.length();
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.filemgr.versioning;
+
+//OODT imports
+import org.apache.commons.lang.StringUtils;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.Product;
+
+//JDK imports
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Stack;
+import java.util.Vector;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author mattmann
+ * @author bfoster
+ * @version $Revision$
+ *
+ * <p>
+ * A Utility component to help out with versioning.
+ * </p>
+ *
+ *
+ */
+public final class VersioningUtils {
+
+ /* our log stream */
+ private static final Logger LOG = Logger.getLogger(VersioningUtils.class
+ .getName());
+
+ // filter to only find directories when doing a listFiles
+ private static FileFilter DIR_FILTER = new FileFilter() {
+ public boolean accept(File file) {
+ return file.isDirectory();
+ }
+ };
+
+ // filter to only find files when doing a listFiles
+ private static FileFilter FILE_FILTER = new FileFilter() {
+ public boolean accept(File file) {
+ return file.isFile();
+ }
+ };
+
+ public static List<Reference> getReferencesFromDir(File dirRoot) {
+ List<Reference> references = null;
+
+ if (dirRoot == null)
+ throw new IllegalArgumentException("null");
+ if (!dirRoot.isDirectory())
+ dirRoot = dirRoot.getParentFile();
+
+ references = new Vector<Reference>();
+
+ Stack<File> stack = new Stack<File>();
+ stack.push(dirRoot);
+ while (!stack.isEmpty()) {
+ File dir = stack.pop();
+ // add the reference for the dir
+ // except if it's the rootDir, then, skip it
+ if (!dir.equals(dirRoot)) {
+ try {
+ Reference r = new Reference();
+ r.setOrigReference(dir.toURL().toExternalForm());
+ r.setFileSize(dir.length());
+ references.add(r);
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING,
+ "MalformedURLException when generating reference
for dir: "
+ + dir);
+ }
+ }
+
+ File[] files = dir.listFiles(FILE_FILTER);
+
+ for (int i = 0; i < files.length; i++) {
+ // add the file references
+ try {
+ Reference r = new Reference();
+ r.setOrigReference(files[i].toURL().toExternalForm());
+ r.setFileSize(files[i].length());
+ references.add(r);
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ LOG.log(Level.WARNING,
+ "MalformedURLException when generating reference
for file: "
+ + files[i]);
+ }
+
+ }
+ File[] subdirs = dir.listFiles(DIR_FILTER);
+ if (subdirs != null)
+ for (int i = 0; i < subdirs.length; i++) {
+ stack.push(subdirs[i]);
+ }
+ }
+
+ return references;
+ }
+
+ public static List<String> getURIsFromDir(File dirRoot) {
+ List<String> uris = null;
+
+ if (dirRoot == null)
+ throw new IllegalArgumentException("null");
+ if (!dirRoot.isDirectory())
+ dirRoot = dirRoot.getParentFile();
+
+ uris = new Vector<String>();
+
+ Stack<File> stack = new Stack<File>();
+ stack.push(dirRoot);
+ while (!stack.isEmpty()) {
+ File dir = stack.pop();
+ // add the reference for the dir
+ // except if it's the rootDir, then, skip it
+ if (!dir.equals(dirRoot)) {
+ uris.add(dir.toURI().toString());
+ }
+
+ File[] files = dir.listFiles(FILE_FILTER);
+
+ for (int i = 0; i < files.length; i++) {
+ // add the file references
+ uris.add(files[i].toURI().toString());
+ }
+
+ File[] subdirs = dir.listFiles(DIR_FILTER);
+ if (subdirs != null)
+ for (int i = 0; i < subdirs.length; i++) {
+ stack.push(subdirs[i]);
+ }
+ }
+
+ return uris;
+ }
+
+ public static void createBasicDataStoreRefsHierarchical(List<Reference>
references) {
+ // file:///www/folder1
+ // file:///www/folder1/file1
+ // file:///www/folder1/file2
+ // file:///www/folder1/folder2/
+ // file:///www/folder1/folder2/file3
+
+ // toDir: file:///www/myfolder/product1
+ // origDir: file:///www/folder1
+
+ String toDirRef = references.get(0)
+ .getDataStoreReference();
+ String origDirRef = references.get(0).getOrigReference();
+ String origDirRefName = new File(origDirRef).getName();
+
+ for (Iterator<Reference> i = references.iterator(); i.hasNext();) {
+ Reference r = i.next();
+
+ // don't bother with the first one, because it's already set
+ // correctly
+ if (r.getOrigReference().equals(origDirRef)) {
+ continue;
+ }
+
+ // get the first occurence of the origDir name in the string
+ // then, the ref becomes:
+ // toDir+r.getOrigRef.substring(first occurence of
+ // origDir).substring(first occurence of '/'+1)
+
+ String dataStoreRef = toDirRef;
+ int firstOccurenceOfOrigDir = r.getOrigReference().indexOf(
+ origDirRefName);
+ String tmpRef = r.getOrigReference().substring(
+ firstOccurenceOfOrigDir);
+ LOG.log(Level.FINER, "tmpRef: " + tmpRef);
+ int firstOccurenceSlash = tmpRef.indexOf("/");
+ dataStoreRef += tmpRef.substring(firstOccurenceSlash + 1);
+
+ LOG.log(Level.FINE, "VersioningUtils: Generated data store ref: "
+ + dataStoreRef + " from origRef: " + r.getOrigReference());
+ r.setDataStoreReference(dataStoreRef);
+ }
+
+ }
+
+ public static void createBasicDataStoreRefsFlat(String productName,
+ String productRepoPath, List<Reference> references) {
+ for (Iterator<Reference> i = references.iterator(); i.hasNext();) {
+ Reference r = i.next();
+
+ String dataStoreRef = null;
+ String productRepoPathRef = null;
+
+ try {
+ productRepoPathRef = new File(new URI(productRepoPath)).toURL()
+ .toExternalForm();
+
+ if (!productRepoPathRef.endsWith("/")) {
+ productRepoPathRef += "/";
+ }
+
+ dataStoreRef = productRepoPathRef
+ + URLEncoder.encode(productName, "UTF-8") + "/"
+ + new File(new URI(r.getOrigReference())).getName();
+ } catch (IOException e) {
+ LOG.log(Level.WARNING,
+ "VersioningUtils: Error generating dataStoreRef for "
+ + r.getOrigReference() + ": Message: "
+ + e.getMessage());
+ } catch (URISyntaxException e) {
+ LOG.log(Level.WARNING,
+ "VersioningUtils: Error generating dataStoreRef for "
+ + r.getOrigReference() + ": Message: "
+ + e.getMessage());
+ }
+
+ LOG.log(Level.FINE, "VersioningUtils: Generated data store ref: "
+ + dataStoreRef + " from origRef: " + r.getOrigReference());
+ r.setDataStoreReference(dataStoreRef);
+ }
+
+ }
+ public static void createBasicDataStoreRefsStream(String productName,
+ String productRepoPath, List<Reference> references,String postfix) {
+ for (Iterator<Reference> i = references.iterator(); i.hasNext();) {
+ Reference r = i.next();
+ createDataStoreRefStream(productName,productRepoPath,r,postfix);
+ }
+
+ }
+ public static void createDataStoreRefStream(String pn, String
productRepoPath, Reference ref, String postfix) {
+ URI uri = URI.create(ref.getOrigReference());
+ String[] parts = (postfix.equals(""))?new String[]
{uri.toString()}:new String[] {uri.toString(),postfix};
+
ref.setDataStoreReference(StringUtils.join(parts,Reference.STREAM_REFERENCE_DELIMITER));
+ }
+ public static void addRefsFromUris(Product p, List<String> uris) {
+ // add the refs to the Product
+ for (Iterator<String> i = uris.iterator(); i.hasNext();) {
+ String ref = i.next();
+ Reference r = new Reference(ref, null,
(p.getProductStructure().equals(Product.STRUCTURE_STREAM)?-1:quietGetFileSizeFromUri(ref)));
+ p.getProductReferences().add(r);
+ }
+ }
+
+ public static String getAbsolutePathFromUri(String uriStr) {
+ URI uri = null;
+ String absPath = null;
+
+ try {
+ uri = new URI(uriStr);
+ absPath = new File(uri).getAbsolutePath();
+ } catch (URISyntaxException e) {
+ LOG.log(Level.WARNING,
+ "URISyntaxException getting URI from URI str: [" + uriStr
+ + "]");
+ }
+
+ return absPath;
+ }
+
+ private static long quietGetFileSizeFromUri(String uri) {
+ File fileRef = null;
+
+ try {
+ fileRef = new File(new URI(uri));
+ } catch (URISyntaxException e) {
+ LOG.log(Level.WARNING,
+ "URISyntaxException when getting file size from uri: ["
+ + uri + "]: Message: " + e.getMessage());
+ return -1L;
+ }
+
+ return fileRef.length();
+ }
+
+}
Modified:
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/ProductResourceTest.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/ProductResourceTest.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/ProductResourceTest.java
(original)
+++
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/ProductResourceTest.java
Tue Oct 28 23:55:27 2014
@@ -78,7 +78,7 @@ public class ProductResourceTest
Product product = new Product();
product.setProductId("123");
product.setProductName("test.txt");
- product.setProductStructure("flat");
+ product.setProductStructure(Product.STRUCTURE_FLAT);
product.setProductType(productType);
ProductResource resource = new ProductResource(product, metadata,
Modified:
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransferResourceTest.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransferResourceTest.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransferResourceTest.java
(original)
+++
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransferResourceTest.java
Tue Oct 28 23:55:27 2014
@@ -74,7 +74,7 @@ public class TransferResourceTest
Product product = new Product();
product.setProductId("123");
product.setProductName("test product");
- product.setProductStructure("flat");
+ product.setProductStructure(Product.STRUCTURE_FLAT);
product.setProductType(productType);
FileTransferStatus status = new FileTransferStatus(reference, 1000, 100,
Modified:
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransfersResourceTest.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransfersResourceTest.java?rev=1635009&r1=1635008&r2=1635009&view=diff
==============================================================================
---
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransfersResourceTest.java
(original)
+++
oodt/trunk/webapp/fmprod/src/test/java/org/apache/oodt/cas/product/jaxrs/resources/TransfersResourceTest.java
Tue Oct 28 23:55:27 2014
@@ -76,7 +76,7 @@ public class TransfersResourceTest
Product product1 = new Product();
product1.setProductId("123");
product1.setProductName("test product");
- product1.setProductStructure("flat");
+ product1.setProductStructure(Product.STRUCTURE_FLAT);
product1.setProductType(productType1);
FileTransferStatus status1 = new FileTransferStatus(reference1, 1000, 100,
@@ -99,7 +99,7 @@ public class TransfersResourceTest
Product product2 = new Product();
product2.setProductId("456");
product2.setProductName("test product 2");
- product2.setProductStructure("hierarchical");
+ product2.setProductStructure(Product.STRUCTURE_HIERARCHICAL);
product2.setProductType(productType2);
FileTransferStatus status2 = new FileTransferStatus(reference2, 500, 200,
