Author: bfoster
Date: Mon Apr 9 22:05:34 2012
New Revision: 1311490
URL: http://svn.apache.org/viewvc?rev=1311490&view=rev
Log:
- Make CAS-Crawlers MimeExtractorConfigReader relative file paths be relative
to its XML fileMake CAS-Crawlers MimeExtractorConfigReader relative file paths
be relative to its XML file
--------------
OODT-438
Modified:
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/typedetection/MimeExtractorConfigReader.java
oodt/trunk/crawler/src/main/resources/action-beans.xml
oodt/trunk/crawler/src/main/resources/precondition-beans.xml
Modified:
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java?rev=1311490&r1=1311489&r2=1311490&view=diff
==============================================================================
---
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java
(original)
+++
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/MoveFile.java
Mon Apr 9 22:05:34 2012
@@ -92,6 +92,10 @@ public class MoveFile extends CrawlerAct
this.toDir = toDir;
}
+ public String getToDir() {
+ return toDir;
+ }
+
public void setFileExtension(String fileExtension) {
this.fileExtension = fileExtension;
}
Modified:
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/typedetection/MimeExtractorConfigReader.java
URL:
http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/typedetection/MimeExtractorConfigReader.java?rev=1311490&r1=1311489&r2=1311490&view=diff
==============================================================================
---
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/typedetection/MimeExtractorConfigReader.java
(original)
+++
oodt/trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/typedetection/MimeExtractorConfigReader.java
Mon Apr 9 22:05:34 2012
@@ -22,6 +22,7 @@ import org.apache.oodt.cas.metadata.util
import org.apache.oodt.commons.xml.XMLUtils;
//JDK imports
+import java.io.File;
import java.io.FileInputStream;
import java.util.LinkedList;
@@ -55,8 +56,13 @@ public final class MimeExtractorConfigRe
MimeExtractorRepo extractorRepo = new MimeExtractorRepo();
extractorRepo.setMagic(Boolean.valueOf(
root.getAttribute(MAGIC_ATTR)).booleanValue());
- extractorRepo.setMimeRepoFile(PathUtils.replaceEnvVariables(root
- .getAttribute(MIME_REPO_ATTR)));
+ String mimeTypeFile = PathUtils.replaceEnvVariables(root
+ .getAttribute(MIME_REPO_ATTR));
+ if (!mimeTypeFile.startsWith("/")) {
+ mimeTypeFile = new File(new File(mapFilePath).getParentFile(),
+ mimeTypeFile).getAbsolutePath();
+ }
+ extractorRepo.setMimeRepoFile(mimeTypeFile);
Element defaultExtractorElem = XMLUtils.getFirstElement(
DEFAULT_EXTRACTOR_TAG, root);
@@ -67,13 +73,16 @@ public final class MimeExtractorConfigRe
for (int i = 0; i < defaultExtractorElems.getLength(); i++) {
Element extractorElem = (Element) defaultExtractorElems
.item(i);
- NodeList preCondComparators = XMLUtils.getFirstElement(
- PRECONDITION_COMPARATORS_TAG, extractorElem)
- .getElementsByTagName(PRECONDITION_COMPARATOR_TAG);
+ Element preCondsElem = XMLUtils.getFirstElement(
+ PRECONDITION_COMPARATORS_TAG, extractorElem);
LinkedList<String> preCondComparatorIds = new
LinkedList<String>();
- for (int k = 0; k < preCondComparators.getLength(); k++)
- preCondComparatorIds.add(((Element) preCondComparators
- .item(k)).getAttribute(ID_ATTR));
+ if (preCondsElem != null) {
+ NodeList preCondComparators =
+
preCondsElem.getElementsByTagName(PRECONDITION_COMPARATOR_TAG);
+ for (int k = 0; k < preCondComparators.getLength(); k++)
+ preCondComparatorIds.add(((Element)
preCondComparators
+ .item(k)).getAttribute(ID_ATTR));
+ }
// This seems wrong, so added support for CLASS_ATTR while
still
// supporting EXTRACTOR_CLASS_TAG as an attribute for
specifying
// extractor class.
@@ -81,10 +90,15 @@ public final class MimeExtractorConfigRe
if (Strings.isNullOrEmpty(extractorClass)) {
extractorClass =
extractorElem.getAttribute(EXTRACTOR_CLASS_TAG);
}
+ String extractorConfigFile = getFilePathFromElement(
+ extractorElem, EXTRACTOR_CONFIG_TAG);
+ if (extractorConfigFile != null &&
!extractorConfigFile.startsWith("/")) {
+ extractorConfigFile = new File(new
File(mapFilePath).getParentFile(),
+ extractorConfigFile).getAbsolutePath();
+ }
defaultExtractorSpecs
.add(new MetExtractorSpec(extractorClass,
- getFilePathFromElement(extractorElem,
- EXTRACTOR_CONFIG_TAG),
+ extractorConfigFile,
preCondComparatorIds));
}
extractorRepo
@@ -117,21 +131,27 @@ public final class MimeExtractorConfigRe
// get config file if specified
String configFilePath = getFilePathFromElement(
extractorSpecElem, EXTRACTOR_CONFIG_TAG);
- if (configFilePath != null)
- spec.setExtractorConfigFile(configFilePath);
+ if (configFilePath != null) {
+ if (!configFilePath.startsWith("/")) {
+ configFilePath = new File(new
File(mapFilePath).getParentFile(),
+ configFilePath).getAbsolutePath();
+ }
+ spec.setExtractorConfigFile(configFilePath);
+ }
// get preconditions file if specified
- NodeList preCondComparators = XMLUtils
- .getFirstElement(PRECONDITION_COMPARATORS_TAG,
- extractorSpecElem)
- .getElementsByTagName(
- PRECONDITION_COMPARATOR_TAG);
- LinkedList<String> preCondComparatorIds = new
LinkedList<String>();
- for (int k = 0; k < preCondComparators.getLength();
k++)
- preCondComparatorIds
- .add(((Element) preCondComparators.item(k))
- .getAttribute(ID_ATTR));
-
spec.setPreConditionComparatorIds(preCondComparatorIds);
+ Element preCondsElem = XMLUtils.getFirstElement(
+ PRECONDITION_COMPARATORS_TAG, extractorSpecElem);
+ if (preCondsElem != null) {
+ NodeList preCondComparators = preCondsElem
+
.getElementsByTagName(PRECONDITION_COMPARATOR_TAG);
+ LinkedList<String> preCondComparatorIds = new
LinkedList<String>();
+ for (int k = 0; k < preCondComparators.getLength();
k++)
+ preCondComparatorIds
+ .add(((Element)
preCondComparators.item(k))
+ .getAttribute(ID_ATTR));
+
spec.setPreConditionComparatorIds(preCondComparatorIds);
+ }
specs.add(spec);
}
Modified: oodt/trunk/crawler/src/main/resources/action-beans.xml
URL:
http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/resources/action-beans.xml?rev=1311490&r1=1311489&r2=1311490&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/resources/action-beans.xml (original)
+++ oodt/trunk/crawler/src/main/resources/action-beans.xml Mon Apr 9 22:05:34
2012
@@ -20,7 +20,6 @@ the License.
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-2.5.xsd";>
<bean
class="org.apache.oodt.commons.spring.postprocessor.SetIdBeanPostProcessor" />
-
<bean
class="org.apache.oodt.cas.crawl.util.CasPropertyPlaceholderConfigurer">
<!-- Allow for system-level properties to override all
properties below -->
@@ -33,10 +32,9 @@ the License.
<prop
key="crawler.backup.dir">[BACKUP_DIR]</prop>
<prop
key="crawler.workflowmgr.url">[WORKFLOWMGR_URL]</prop>
<prop
key="crawler.filemgr.url">[FILEMGR_URL]</prop>
- <prop
key="crawler.client.transferer">org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory
- </prop>
+ <prop
key="crawler.client.transferer">org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory</prop>
<prop key="crawler.met.file.ext">met</prop>
- <prop
key="crawler.pushpull.met.file.ext">info.tmp</prop>
+ <prop
key="crawler.pushpull.met.file.ext">[PUSHPULL_MET_FILE_EXT]</prop>
<prop key="crawler.anc.file.ext">anc</prop>
<prop key="crawler.anc.file.suffix"></prop>
<prop key="notification.mail.host"></prop>
Modified: oodt/trunk/crawler/src/main/resources/precondition-beans.xml
URL:
http://svn.apache.org/viewvc/oodt/trunk/crawler/src/main/resources/precondition-beans.xml?rev=1311490&r1=1311489&r2=1311490&view=diff
==============================================================================
--- oodt/trunk/crawler/src/main/resources/precondition-beans.xml (original)
+++ oodt/trunk/crawler/src/main/resources/precondition-beans.xml Mon Apr 9
22:05:34 2012
@@ -20,9 +20,22 @@ the License.
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-2.5.xsd";>
<bean
class="org.apache.oodt.commons.spring.postprocessor.SetIdBeanPostProcessor" />
+ <bean
class="org.apache.oodt.cas.crawl.util.CasPropertyPlaceholderConfigurer">
+
+ <!-- Allow for system-level properties to override all properties below -->
+ <property name="systemPropertiesMode" value="2" />
+
+ <!-- Default Properties -->
+ <property name="properties">
+ <props>
+ <prop
key="crawler.pushpull.met.file.ext">[PUSHPULL_MET_FILE_EXT]</prop>
+ </props>
+ </property>
+ </bean>
<!-- Precondition Comparators -->
<bean id="CheckThatPushPullMetFileExists" lazy-init="true"
class="org.apache.oodt.cas.metadata.preconditions.ExistanceCheckComparator">
+ <property name="fileExtension" value="${crawler.pushpull.met.file.ext}" />
<property name="description" value="Checks if the push-pull
metadata file exists for the current data file" />
<property name="compareItem">
<value type="java.lang.Boolean">true</value>
