Author: mattmann
Date: Sun Jun 19 18:07:11 2011
New Revision: 1137395
URL: http://svn.apache.org/viewvc?rev=1137395&view=rev
Log:
- fix for OODT-278
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetadataKeys.java
Modified: oodt/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1137395&r1=1137394&r2=1137395&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Sun Jun 19 18:07:11 2011
@@ -4,6 +4,8 @@ Apache OODT Change Log
Release 0.3-SNAPSHOT (in progress)
--------------------------------------------
+* OODT-278 CAS-PGE returns success even if product file(s) fail to ingest
(mattmann, bfoster)
+
* OODT-243 Add method is called on already existing jobspec (mattmann, bfoster)
* OODT-199 CoreMetExtractor chokes on Hierarchical Products (mattmann)
Modified:
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java?rev=1137395&r1=1137394&r2=1137395&view=diff
==============================================================================
--- oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
(original)
+++ oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
Sun Jun 19 18:07:11 2011
@@ -40,7 +40,9 @@ import org.apache.oodt.cas.workflow.stru
import
org.apache.oodt.cas.workflow.structs.exceptions.WorkflowTaskInstanceException;
import org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManagerClient;
import org.apache.oodt.cas.workflow.util.ScriptFile;
+import org.apache.oodt.cas.crawl.ProductCrawler;
import org.apache.oodt.cas.crawl.StdProductCrawler;
+import org.apache.oodt.cas.crawl.status.IngestStatus;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
@@ -390,19 +392,51 @@ public class PGETaskInstance implements
protected void runIngestCrawler(StdProductCrawler crawler,
List<File> crawlDirs) {
- try {
- this.updateStatus(PgeTaskMetadataKeys.CRAWLING);
- for (File crawlDir : crawlDirs) {
- LOG.log(Level.INFO,
- "Executing StdProductCrawler in productPath: ["
- + crawlDir + "]");
- crawler.crawl(crawlDir);
- }
- } catch (Exception e) {
- LOG.log(Level.WARNING,
- "Failed to create StdProductCrawler - Products won't be
crawled : "
- + e.getMessage());
- }
+ File currentDir = null;
+ try {
+ this.updateStatus(PgeTaskMetadataKeys.CRAWLING);
+ boolean attemptIngestAll =
Boolean.parseBoolean(this.pgeMetadata
+
.getMetadataValue(PgeTaskMetadataKeys.ATTEMPT_INGEST_ALL));
+ for (File crawlDir : crawlDirs) {
+ currentDir = crawlDir;
+ LOG.log(Level.INFO,
+ "Executing StdProductCrawler in
productPath: ["
+ + crawlDir +
"]");
+ crawler.crawl(crawlDir);
+ if (!attemptIngestAll)
+ this.verifyIngests(crawler);
+ }
+ if (attemptIngestAll)
+ this.verifyIngests(crawler);
+ } catch (Exception e) {
+ LOG.log(Level.WARNING,
+ "Failed while attempting to ingest
products while crawling directory '"
+ + currentDir
+ + "' (all products may
not have been ingested) : "
+ + e.getMessage(), e);
+ }
+ }
+
+ protected void verifyIngests(ProductCrawler crawler) throws Exception {
+ boolean ingestsSuccess = true;
+ String exceptionMsg = "";
+ for (IngestStatus status : crawler.getIngestStatus()) {
+ if (status.getResult().equals(IngestStatus.Result.FAILURE)) {
+ exceptionMsg += (exceptionMsg.equals("") ? "" : " : ")
+ + "Failed to ingest product
[file='"
+ +
status.getProduct().getAbsolutePath() + "',result='"
+ + status.getResult() +
"',msg='" + status.getMessage()
+ + "']";
+ ingestsSuccess = false;
+ }else if
(!status.getResult().equals(IngestStatus.Result.SUCCESS)) {
+ LOG.log(Level.WARNING, "Product was not ingested [file='"
+ +
status.getProduct().getAbsolutePath() + "',result='"
+ + status.getResult() +
"',msg='" + status.getMessage()
+ + "']");
+ }
+ }
+ if (!ingestsSuccess)
+ throw new Exception(exceptionMsg);
}
protected void updateDynamicMetadata() {
Modified:
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetadataKeys.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetadataKeys.java?rev=1137395&r1=1137394&r2=1137395&view=diff
==============================================================================
---
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetadataKeys.java
(original)
+++
oodt/trunk/pge/src/main/java/org/apache/oodt/cas/pge/metadata/PgeTaskMetadataKeys.java
Sun Jun 19 18:07:11 2011
@@ -42,6 +42,9 @@ public interface PgeTaskMetadataKeys {
public static final String PROPERTY_ADDER_CLASSPATH =
"PGETask_PropertyAdderClasspath";
public static final String PGE_RUNTIME = "PGETask_Runtime";
+
+ public static final String ATTEMPT_INGEST_ALL = "PGETask_AttemptIngestAll";
+
/* PGE task statuses */
public static final String STAGING_INPUT = "PGETask_Staging_Input";
