Author: bfoster
Date: Wed Dec 22 18:30:29 2010
New Revision: 1052019
URL: http://svn.apache.org/viewvc?rev=1052019&view=rev
Log:
- added product crawl status feature to cas-crawler
---------------------
OODT-90
Added:
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java
(with props)
Modified:
oodt/trunk/CHANGES.txt
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
Modified: oodt/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/oodt/trunk/CHANGES.txt?rev=1052019&r1=1052018&r2=1052019&view=diff
==============================================================================
--- oodt/trunk/CHANGES.txt (original)
+++ oodt/trunk/CHANGES.txt Wed Dec 22 18:30:29 2010
@@ -4,6 +4,8 @@ Apache OODT Change Log
Release 0.2 (Current Development)
--------------------------------------------
+* OODT-90 (cas-crawler patch) CAS-PGE returns success even if product file(s)
failed to ingest (bfoster)
+
* OODT-103 modify cas-common's cmd-line parser to throw a special
OptionHelpException if no args
are specified so it can be specifically trapped (bfoster)
Modified:
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java?rev=1052019&r1=1052018&r2=1052019&view=diff
==============================================================================
---
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
(original)
+++
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
Wed Dec 22 18:30:29 2010
@@ -22,6 +22,7 @@ package org.apache.oodt.cas.crawl;
import org.apache.oodt.cas.crawl.action.CrawlerAction;
import org.apache.oodt.cas.crawl.action.CrawlerActionRepo;
import org.apache.oodt.cas.crawl.config.ProductCrawlerBean;
+import org.apache.oodt.cas.crawl.status.IngestStatus;
import org.apache.oodt.cas.filemgr.ingest.Ingester;
import org.apache.oodt.cas.filemgr.ingest.StdIngester;
import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
@@ -31,8 +32,10 @@ import org.apache.oodt.cas.metadata.Meta
import java.io.File;
import java.io.FileFilter;
import java.net.URL;
+import java.util.Collections;
import java.util.List;
import java.util.Stack;
+import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -69,7 +72,7 @@ public abstract class ProductCrawler ext
};
private CrawlerActionRepo actionRepo;
-
+ private List<IngestStatus> ingestStatus;
private Ingester ingester;
public void crawl() {
@@ -77,6 +80,8 @@ public abstract class ProductCrawler ext
}
public void crawl(File dirRoot) {
+ this.ingestStatus = new Vector<IngestStatus>();
+
// Load actions
if (this.getApplicationContext() != null)
(this.actionRepo = new CrawlerActionRepo())
@@ -121,6 +126,10 @@ public abstract class ProductCrawler ext
}
}
+ public List<IngestStatus> getIngestStatus() {
+ return Collections.unmodifiableList(this.ingestStatus);
+ }
+
private synchronized boolean containsRequiredMetadata(
Metadata productMetadata) {
for (int i = 0; i < this.getRequiredMetadata().size(); i++) {
@@ -146,8 +155,10 @@ public abstract class ProductCrawler ext
.getAbsoluteFile().getParentFile().getAbsolutePath());
}
- private void handleFile(File product) {
+ private void handleFile(final File product) {
LOG.log(Level.INFO, "Handling file " + product);
+ final IngestStatus.Result ingestResult;
+ final String message;
if (this.passesPreconditions(product)) {
Metadata productMetadata = new Metadata();
productMetadata.addMetadata(this.getGlobalMetadata().getHashtable());
@@ -158,18 +169,29 @@ public abstract class ProductCrawler ext
boolean isPreIngestActionsComplete =
this.performPreIngestActions(product, productMetadata);
if (this.isSkipIngest()) {
+ ingestResult = IngestStatus.Result.SKIPPED;
+ message = "Crawler ingest turned OFF";
LOG.log(Level.INFO, "Skipping ingest of product: ["
+ product.getAbsolutePath() + "]");
} else {
if (isRequiredMetadataPresent
&& isPreIngestActionsComplete
&& this.ingest(product, productMetadata)) {
+ ingestResult = IngestStatus.Result.SUCCESS;
+ message = "Ingest was successful";
LOG.log(Level.INFO, "Successful ingest of product: ["
+ product.getAbsolutePath() + "]");
this
.performPostIngestOnSuccessActions(product,
productMetadata);
} else {
+ ingestResult = IngestStatus.Result.FAILURE;
+ if (!isRequiredMetadataPresent)
+ message = "Missing required metadata";
+ else if (!isPreIngestActionsComplete)
+ message = "PreIngest actions failed to
complete";
+ else
+ message = "Failed to ingest product";
LOG.log(Level.WARNING, "Failed to ingest product: ["
+ product.getAbsolutePath()
+ "]: performing postIngestFail actions");
@@ -177,10 +199,23 @@ public abstract class ProductCrawler ext
}
}
} else {
+ ingestResult = IngestStatus.Result.PRECONDS_FAILED;
+ message = "Failed to pass preconditions";
LOG.log(Level.WARNING,
"Failed to pass preconditions for ingest of product: ["
+ product.getAbsolutePath() + "]");
}
+ this.ingestStatus.add(new IngestStatus() {
+ public File getProduct() {
+ return product;
+ }
+ public Result getResult() {
+ return ingestResult;
+ }
+ public String getMessage() {
+ return message;
+ }
+ });
}
private boolean ingest(File product, Metadata productMetdata) {
Added:
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java?rev=1052019&view=auto
==============================================================================
---
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java
(added)
+++
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java
Wed Dec 22 18:30:29 2010
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.crawl.status;
+
+//JDK imports
+import java.io.File;
+
+/**
+ * @author bfoster
+ * @version $Revision$
+ *
+ * <p>
+ * A Product's Ingest Status
+ * </p>.
+ */
+public interface IngestStatus {
+
+ public static enum Result {
+ SUCCESS, FAILURE, SKIPPED, PRECONDS_FAILED;
+ }
+
+ public File getProduct();
+
+ public Result getResult();
+
+ public String getMessage();
+
+}
Propchange:
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/status/IngestStatus.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
