Revision: 16610
http://sourceforge.net/p/gate/code/16610
Author: valyt
Date: 2013-03-21 16:05:54 +0000 (Thu, 21 Mar 2013)
Log Message:
-----------
- upgraded to use heritrix 3.1.1
- .. which we now get from the archive.org Maven repository.
- added a class for DocumentID which is used instead of plain String values.
This allows more complex data to be associated with a documentID (such as the
archive record offset and length).
Modified Paths:
--------------
gcp/trunk/.classpath
gcp/trunk/build/ivy.xml
gcp/trunk/build/ivysettings.xml
gcp/trunk/src/gate/cloud/batch/Batch.java
gcp/trunk/src/gate/cloud/batch/BatchRunner.java
gcp/trunk/src/gate/cloud/batch/DocumentProcessor.java
gcp/trunk/src/gate/cloud/batch/FailureResult.java
gcp/trunk/src/gate/cloud/batch/PooledDocumentProcessor.java
gcp/trunk/src/gate/cloud/batch/ProcessResult.java
gcp/trunk/src/gate/cloud/batch/SuccessResult.java
gcp/trunk/src/gate/cloud/io/AbstractOutputHandler.java
gcp/trunk/src/gate/cloud/io/AntBasedDocumentEnumerator.java
gcp/trunk/src/gate/cloud/io/DocumentData.java
gcp/trunk/src/gate/cloud/io/DocumentEnumerator.java
gcp/trunk/src/gate/cloud/io/InputHandler.java
gcp/trunk/src/gate/cloud/io/ListDocumentEnumerator.java
gcp/trunk/src/gate/cloud/io/OutputHandler.java
gcp/trunk/src/gate/cloud/io/arc/ARCDocumentEnumerator.java
gcp/trunk/src/gate/cloud/io/arc/ARCInputHandler.java
gcp/trunk/src/gate/cloud/io/file/AbstractFileOutputHandler.java
gcp/trunk/src/gate/cloud/io/file/FileInputHandler.java
gcp/trunk/src/gate/cloud/io/file/GATEInlineOutputHandler.java
gcp/trunk/src/gate/cloud/io/file/GATEStandOffFileOutputHandler.java
gcp/trunk/src/gate/cloud/io/file/NamingStrategy.java
gcp/trunk/src/gate/cloud/io/file/PlainTextOutputHandler.java
gcp/trunk/src/gate/cloud/io/file/SerializedObjectOutputHandler.java
gcp/trunk/src/gate/cloud/io/file/SimpleNamingStrategy.java
gcp/trunk/src/gate/cloud/io/mimir/MimirOutputHandler.java
gcp/trunk/src/gate/cloud/io/xces/XCESOutputHandler.java
gcp/trunk/src/gate/cloud/io/zip/ZipInputHandler.java
gcp/trunk/src/gate/cloud/util/Tools.java
gcp/trunk/src/gate/cloud/util/XMLBatchParser.java
Added Paths:
-----------
gcp/trunk/src/gate/cloud/batch/DocumentID.java
Removed Paths:
-------------
gcp/trunk/lib/fastutil-5.0.3-heritrix-subset-1.0.jar
gcp/trunk/lib/heritrix-1.14.4.jar
Modified: gcp/trunk/.classpath
===================================================================
--- gcp/trunk/.classpath 2013-03-21 02:21:19 UTC (rev 16609)
+++ gcp/trunk/.classpath 2013-03-21 16:05:54 UTC (rev 16610)
@@ -4,8 +4,6 @@
<classpathentry kind="src" output="cli/classes" path="cli/src"/>
<classpathentry kind="con"
path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
<classpathentry kind="lib" path="conf"/>
- <classpathentry kind="lib"
path="lib/fastutil-5.0.3-heritrix-subset-1.0.jar"/>
- <classpathentry kind="lib" path="lib/heritrix-1.14.4.jar"/>
<classpathentry kind="lib" path="lib/mimir-client-4.0.jar"/>
<classpathentry kind="con"
path="org.apache.ivyde.eclipse.cpcontainer.IVYDE_CONTAINER/?project=gcp&ivyXmlPath=build%2Fivy.xml&confs=*&ivySettingsPath=%24%7Bworkspace_loc%3Agcp%2Fbuild%2Fivysettings.xml%7D&loadSettingsOnDemand=false&propertyFiles="/>
<classpathentry kind="output" path="classes"/>
Modified: gcp/trunk/build/ivy.xml
===================================================================
--- gcp/trunk/build/ivy.xml 2013-03-21 02:21:19 UTC (rev 16609)
+++ gcp/trunk/build/ivy.xml 2013-03-21 16:05:54 UTC (rev 16610)
@@ -21,5 +21,8 @@
<artifact name="commons-httpclient" type="jar" />
</dependency>
+ <dependency org="org.archive.heritrix" name="heritrix-commons" rev="3.1.1"
>
+ <artifact name="heritrix-commons" type="jar" />
+ </dependency>
</dependencies>
</ivy-module>
Modified: gcp/trunk/build/ivysettings.xml
===================================================================
--- gcp/trunk/build/ivysettings.xml 2013-03-21 02:21:19 UTC (rev 16609)
+++ gcp/trunk/build/ivysettings.xml 2013-03-21 16:05:54 UTC (rev 16610)
@@ -5,6 +5,8 @@
<ibiblio name="maven-central" m2compatible="true" />
<ibiblio name="gate.ac.uk" m2compatible="true"
root="http://repo.gate.ac.uk/content/groups/public/"; />
+ <ibiblio name="org.archive.heritrix" m2compatible="true"
+ root="http://builds.archive.org:8080/maven2/"; />
</chain>
</resolvers>
</ivysettings>
\ No newline at end of file
Deleted: gcp/trunk/lib/fastutil-5.0.3-heritrix-subset-1.0.jar
===================================================================
(Binary files differ)
Deleted: gcp/trunk/lib/heritrix-1.14.4.jar
===================================================================
(Binary files differ)
Modified: gcp/trunk/src/gate/cloud/batch/Batch.java
===================================================================
--- gcp/trunk/src/gate/cloud/batch/Batch.java 2013-03-21 02:21:19 UTC (rev
16609)
+++ gcp/trunk/src/gate/cloud/batch/Batch.java 2013-03-21 16:05:54 UTC (rev
16610)
@@ -212,11 +212,11 @@
backupFile.delete();
}
// filter the documents already processed
- List<String> unprocessedDocs = new ArrayList<String>();
+ List<DocumentID> unprocessedDocs = new ArrayList<DocumentID>();
unprocessedDocs.addAll(Arrays.asList(documentIDs));
unprocessedDocs.removeAll(completedDocuments);
unprocessedDocumentIDs = unprocessedDocs.toArray(
- new String[unprocessedDocs.size()]);
+ new DocumentID[unprocessedDocs.size()]);
} catch(XMLStreamException e) {
throw new GateException("Cannot write to the report file!", e);
} catch(IOException e) {
@@ -230,9 +230,9 @@
private String batchId;
- private String[] documentIDs;
+ private DocumentID[] documentIDs;
- private String[] unprocessedDocumentIDs;
+ private DocumentID[] unprocessedDocumentIDs;
private CorpusController gateApplication;
@@ -295,14 +295,14 @@
* Gets the list of input document IDs in this batch.
* @return an array of {@link String}s.
*/
- public String[] getDocumentIDs() {
+ public DocumentID[] getDocumentIDs() {
return documentIDs;
}
/**
* Sets the list of input document IDs in this batch.
*/
- public void setDocumentIDs(String[] documentIDs) {
+ public void setDocumentIDs(DocumentID[] documentIDs) {
this.documentIDs = documentIDs;
}
@@ -371,7 +371,7 @@
* been interrupted and restarted the values may be different.
* @return an array of {@link String}s.
*/
- public String[] getUnprocessedDocumentIDs() {
+ public DocumentID[] getUnprocessedDocumentIDs() {
return unprocessedDocumentIDs;
}
Modified: gcp/trunk/src/gate/cloud/batch/BatchRunner.java
===================================================================
--- gcp/trunk/src/gate/cloud/batch/BatchRunner.java 2013-03-21 02:21:19 UTC
(rev 16609)
+++ gcp/trunk/src/gate/cloud/batch/BatchRunner.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -139,7 +139,7 @@
processor.init();
jobPusher = new Thread(new Runnable() {
public void run() {
- for(String id : batch.getUnprocessedDocumentIDs()) {
+ for(DocumentID id : batch.getUnprocessedDocumentIDs()) {
processor.processDocument(id);
if(Thread.interrupted()) { return; }
}
Added: gcp/trunk/src/gate/cloud/batch/DocumentID.java
===================================================================
--- gcp/trunk/src/gate/cloud/batch/DocumentID.java
(rev 0)
+++ gcp/trunk/src/gate/cloud/batch/DocumentID.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -0,0 +1,104 @@
+/*
+ * DocumentID.java
+ * Copyright (c) 2007-2013, The University of Sheffield.
+ *
+ * This file is part of GCP (see http://gate.ac.uk/), and is free
+ * software, licenced under the GNU Affero General Public License,
+ * Version 3, November 2007.
+ *
+ *
+ * $Id$
+ */
+package gate.cloud.batch;
+
+import gate.cloud.io.InputHandler;
+import gate.cloud.io.file.FileInputHandler;
+
+import java.util.Map;
+
+/**
+ * Class representing a document ID. When listed inside a batch XML
+ * representation, document IDs are expressed like:
+ * <pre>
+ * <id attr1=val1, attr2=val2, ...>idText</id>
+ * </pre>
+ * All of the components (the id text and the element attributes) are
+ * optional. At limit, <tt><id/></tt> is a valid document ID; the only
+ * constraint being that whatever {@link InputHandler} implementation is used
+ * can deal with it.
+ *
+ * In practice, most document IDs consist of only the ID text. For example,
+ * when a {@link FileInputHandler} is used, the id text contains the path of
+ * the file used to generate the document, relative to the document root
+ * directory.
+ *
+ * Values of this class are immutable.
+ */
+public class DocumentID {
+
+ /**
+ * The text of this ID
+ */
+ private String idText;
+
+ /**
+ * The array of attributes, if any were provided. Each element in this
+ * array has size 2, and comprises an attribute name and an attribute value.
+ */
+ private Map<String, String> attributes;
+
+ /**
+ * Cached copy of the toString representation.
+ */
+ private String toString;
+
+ public DocumentID(String idText) {
+ this(idText, null);
+ }
+
+ public DocumentID(String idText, Map<String, String> attributes) {
+ this.idText = idText;
+ this.attributes = attributes;
+ }
+
+ /**
+ * @return the idText
+ */
+ public String getIdText() {
+ return idText;
+ }
+
+ /**
+ * @return the attributes
+ */
+ public Map<String, String> getAttributes() {
+ return attributes;
+ }
+
+ /* (non-Javadoc)
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ if(toString == null) {
+ if(attributes != null && attributes.size() > 0) {
+ // xml-like representation
+ StringBuilder str = new StringBuilder("<id");
+ for(Map.Entry<String, String> attr : attributes.entrySet()) {
+ str.append(' ').append(attr.getKey()).append("=\"")
+ .append(attr.getValue()).append("\"");
+ }
+ if(idText != null){
+ str.append(">").append(idText).append("</id>");
+ } else {
+ str.append("/>");
+ }
+ toString = str.toString();
+ } else {
+ // plain text ID
+ toString = idText != null ? idText : "";
+ }
+ }
+ return toString;
+ }
+}
\ No newline at end of file
Property changes on: gcp/trunk/src/gate/cloud/batch/DocumentID.java
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+Id
\ No newline at end of property
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Modified: gcp/trunk/src/gate/cloud/batch/DocumentProcessor.java
===================================================================
--- gcp/trunk/src/gate/cloud/batch/DocumentProcessor.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/batch/DocumentProcessor.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -75,5 +75,5 @@
*
* @param docId the ID of the document to process
*/
- public void processDocument(String docId);
+ public void processDocument(DocumentID docId);
}
Modified: gcp/trunk/src/gate/cloud/batch/FailureResult.java
===================================================================
--- gcp/trunk/src/gate/cloud/batch/FailureResult.java 2013-03-21 02:21:19 UTC
(rev 16609)
+++ gcp/trunk/src/gate/cloud/batch/FailureResult.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -18,7 +18,7 @@
class FailureResult implements ProcessResult {
- private String documentId;
+ private DocumentID documentId;
private String errorDescription;
@@ -26,7 +26,7 @@
private long docLength = -1;
- FailureResult(DocumentData docData, String docId, Throwable e) {
+ FailureResult(DocumentData docData, DocumentID docId, Throwable e) {
documentId = docId;
if(docData != null) {
fileSize = docData.fileSize;
@@ -39,7 +39,7 @@
return null;
}
- public String getDocumentId() {
+ public DocumentID getDocumentId() {
return documentId;
}
Modified: gcp/trunk/src/gate/cloud/batch/PooledDocumentProcessor.java
===================================================================
--- gcp/trunk/src/gate/cloud/batch/PooledDocumentProcessor.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/batch/PooledDocumentProcessor.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -147,7 +147,7 @@
* Process a single document (specified by ID), reporting
* success or failure to the output queue.
*/
- public void processDocument(final String documentId) {
+ public void processDocument(final DocumentID documentId) {
log.debug("processDocument called for ID " + documentId);
try {
final CorpusController controller = appPool.take();
@@ -216,7 +216,7 @@
myCorpus.add(docData.document);
controller.setCorpus(myCorpus);
// do benchmark logging if it has been enabled externally
- String bid = Benchmark.createBenchmarkId(docData.document.getName(),
id);
+ String bid = Benchmark.createBenchmarkId(id,
docData.document.getName());
bid = Benchmark.createBenchmarkId("runApplication", bid);
// store the running time in a document feature
@@ -272,7 +272,7 @@
* @param docId the document ID
* @param e the exception (if any) that caused processing to fail.
*/
- private void reportFailure(String docId, DocumentData docData, Throwable e) {
+ private void reportFailure(DocumentID docId, DocumentData docData, Throwable
e) {
resultQueue.offer(new FailureResult(docData, docId, e));
}
Modified: gcp/trunk/src/gate/cloud/batch/ProcessResult.java
===================================================================
--- gcp/trunk/src/gate/cloud/batch/ProcessResult.java 2013-03-21 02:21:19 UTC
(rev 16609)
+++ gcp/trunk/src/gate/cloud/batch/ProcessResult.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -11,6 +11,7 @@
*/
package gate.cloud.batch;
+
import java.util.Map;
/**
@@ -59,7 +60,7 @@
* Gets the ID for the document that was processed.
* @return a {@link String} value.
*/
- public String getDocumentId();
+ public DocumentID getDocumentId();
/**
* Gets a textual representation of the error that caused a failure. This
Modified: gcp/trunk/src/gate/cloud/batch/SuccessResult.java
===================================================================
--- gcp/trunk/src/gate/cloud/batch/SuccessResult.java 2013-03-21 02:21:19 UTC
(rev 16609)
+++ gcp/trunk/src/gate/cloud/batch/SuccessResult.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -22,7 +22,7 @@
class SuccessResult implements ProcessResult {
- private String documentId;
+ private DocumentID documentId;
private long executionTime;
@@ -94,7 +94,7 @@
return annotationCounts;
}
- public String getDocumentId() {
+ public DocumentID getDocumentId() {
return documentId;
}
Modified: gcp/trunk/src/gate/cloud/io/AbstractOutputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/AbstractOutputHandler.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/AbstractOutputHandler.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -16,6 +16,7 @@
import gate.AnnotationSet;
import gate.Document;
import gate.cloud.batch.AnnotationSetDefinition;
+import gate.cloud.batch.DocumentID;
import gate.util.GateException;
import java.io.IOException;
@@ -142,7 +143,7 @@
* @param document the document being sent for output.
* @param documentId the ID of the document being output.
*/
- public final void outputDocument(Document document, String documentId)
+ public final void outputDocument(Document document, DocumentID documentId)
throws IOException, GateException {
if(conditionalSaveFeatureName != null) {
Object featureValue =
document.getFeatures().get(conditionalSaveFeatureName);
@@ -161,6 +162,6 @@
* @throws GateException
*/
protected abstract void outputDocumentImpl(Document document,
- String documentId) throws IOException, GateException;
+ DocumentID documentId) throws IOException, GateException;
}
Modified: gcp/trunk/src/gate/cloud/io/AntBasedDocumentEnumerator.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/AntBasedDocumentEnumerator.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/AntBasedDocumentEnumerator.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -11,6 +11,7 @@
*/
package gate.cloud.io;
+import gate.cloud.batch.DocumentID;
import gate.util.GateException;
import static gate.cloud.io.IOConstants.*;
@@ -99,8 +100,8 @@
return antIterator.hasNext();
}
- public String next() {
- return prefix + antIterator.next();
+ public DocumentID next() {
+ return new DocumentID(prefix + antIterator.next());
}
public void remove() {
Modified: gcp/trunk/src/gate/cloud/io/DocumentData.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/DocumentData.java 2013-03-21 02:21:19 UTC
(rev 16609)
+++ gcp/trunk/src/gate/cloud/io/DocumentData.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -12,13 +12,14 @@
package gate.cloud.io;
import gate.Document;
+import gate.cloud.batch.DocumentID;
/**
* Holder for a GATE document plus some metadata extracted at loading
* time.
*/
public class DocumentData {
- public DocumentData(Document document, String id) {
+ public DocumentData(Document document, DocumentID id) {
this.document = document;
if(document != null) {
this.documentLength = document.getContent().size();
@@ -26,7 +27,7 @@
this.id = id;
}
- public String id;
+ public DocumentID id;
public Document document;
public long fileSize = -1;
public long documentLength = -1;
Modified: gcp/trunk/src/gate/cloud/io/DocumentEnumerator.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/DocumentEnumerator.java 2013-03-21 02:21:19 UTC
(rev 16609)
+++ gcp/trunk/src/gate/cloud/io/DocumentEnumerator.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -11,6 +11,7 @@
*/
package gate.cloud.io;
+import gate.cloud.batch.DocumentID;
import gate.util.GateException;
import java.io.IOException;
@@ -27,7 +28,7 @@
* iteration {@link String} values that represent document IDs. The remove()
* method does not apply in this context and is not implemented.
*/
-public interface DocumentEnumerator extends Iterator<String> {
+public interface DocumentEnumerator extends Iterator<DocumentID> {
/**
* Configures this document enumerator by providing a {@link Map} containing
Modified: gcp/trunk/src/gate/cloud/io/InputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/InputHandler.java 2013-03-21 02:21:19 UTC
(rev 16609)
+++ gcp/trunk/src/gate/cloud/io/InputHandler.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -14,6 +14,7 @@
import java.io.IOException;
import java.util.Map;
+import gate.cloud.batch.DocumentID;
import gate.util.GateException;
/**
@@ -30,7 +31,7 @@
* @throws IOException if the GATE document data cannot be accessed.
* @throws GateException if the GATE document cannot be created.
*/
- public DocumentData getInputDocument(String id) throws IOException,
GateException;
+ public DocumentData getInputDocument(DocumentID id) throws IOException,
GateException;
/**
* Configures this input handler by providing a {@link Map} containing
Modified: gcp/trunk/src/gate/cloud/io/ListDocumentEnumerator.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/ListDocumentEnumerator.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/ListDocumentEnumerator.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -14,6 +14,7 @@
import static gate.cloud.io.IOConstants.PARAM_BATCH_FILE_LOCATION;
import static gate.cloud.io.IOConstants.PARAM_ENCODING;
import static gate.cloud.io.IOConstants.PARAM_FILE_NAME_PREFIX;
+import gate.cloud.batch.DocumentID;
import gate.util.GateException;
import gate.util.GateRuntimeException;
@@ -110,7 +111,7 @@
return nextId != null;
}
- public String next() {
+ public DocumentID next() {
if(nextId == null) {
throw new NoSuchElementException("No more entries in list file "
+ file.getAbsolutePath());
@@ -123,7 +124,7 @@
throw new GateRuntimeException("Exception reading list file "
+ file.getAbsolutePath(), e);
}
- return prefix + toReturn;
+ return new DocumentID(prefix + toReturn);
}
/**
Modified: gcp/trunk/src/gate/cloud/io/OutputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/OutputHandler.java 2013-03-21 02:21:19 UTC
(rev 16609)
+++ gcp/trunk/src/gate/cloud/io/OutputHandler.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -13,6 +13,7 @@
import gate.Document;
import gate.cloud.batch.AnnotationSetDefinition;
+import gate.cloud.batch.DocumentID;
import gate.util.GateException;
import java.io.IOException;
@@ -80,7 +81,7 @@
* @throws IOException
* @throws GateException
*/
- public void outputDocument(Document document, String documentId) throws
IOException, GateException;
+ public void outputDocument(Document document, DocumentID documentId) throws
IOException, GateException;
/**
* Notifies this handler that all required documents have now been output,
and
Modified: gcp/trunk/src/gate/cloud/io/arc/ARCDocumentEnumerator.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/arc/ARCDocumentEnumerator.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/arc/ARCDocumentEnumerator.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -11,15 +11,19 @@
*/
package gate.cloud.io.arc;
-import static gate.cloud.io.IOConstants.*;
+import static gate.cloud.io.IOConstants.PARAM_ARC_FILE_LOCATION;
+import static gate.cloud.io.IOConstants.PARAM_BATCH_FILE_LOCATION;
+import static gate.cloud.io.IOConstants.PARAM_EXCLUDE_STATUS_CODES;
+import static gate.cloud.io.IOConstants.PARAM_INCLUDE_STATUS_CODES;
+import static gate.cloud.io.IOConstants.PARAM_MIME_TYPES;
+import gate.cloud.batch.DocumentID;
+import gate.cloud.io.DocumentEnumerator;
+import it.unimi.dsi.fastutil.objects.Object2ObjectArrayMap;
+
import java.io.File;
import java.io.IOException;
-import java.io.FilenameFilter;
import java.text.DecimalFormat;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
@@ -33,8 +37,6 @@
import org.archive.io.arc.ARCRecord;
import org.archive.io.arc.ARCRecordMetaData;
-import gate.cloud.io.DocumentEnumerator;
-
/**
* Enumerator to build a list of document IDs from an ARC file.
* The ARC entries to be considered may be constrained by MIME type
@@ -84,7 +86,7 @@
/**
* The next ID (if any) to be returned from the iterator.
*/
- protected String next;
+ protected DocumentID next;
/**
* ARC file reader used to scan the archive.
@@ -182,8 +184,8 @@
return (next != null);
}
- public String next() {
- String lastNext = next;
+ public DocumentID next() {
+ DocumentID lastNext = next;
moveToNext();
return lastNext;
}
@@ -215,7 +217,14 @@
// check the mime type, if required
if(mimeTypes == null || mimeTypes.contains(header.getMimetype())) {
// found a good document
- next = numberPaddingFormat.format(inputSequence++) + "_" +
header.getUrl();
+ Object2ObjectArrayMap<String, String> attrs =
+ new Object2ObjectArrayMap<String, String>(new String[2],
+ new String[2]);
+ attrs.put(ARCInputHandler.RECORD_OFFSET_ATTR,
+ Long.toString(header.getOffset()));
+ attrs.put(ARCInputHandler.RECORD_LENGTH_ATTR,
+ Long.toString(recordLength));
+ next = new DocumentID(header.getUrl(), attrs);
return;
}
}
Modified: gcp/trunk/src/gate/cloud/io/arc/ARCInputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/arc/ARCInputHandler.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/arc/ARCInputHandler.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -42,6 +42,7 @@
import gate.Document;
import gate.Factory;
import gate.FeatureMap;
+import gate.cloud.batch.DocumentID;
import gate.cloud.batch.PooledDocumentProcessor;
import gate.cloud.io.DocumentData;
import gate.cloud.io.InputHandler;
@@ -60,8 +61,33 @@
private static final String HTTP_HEADER_PREFIX = "http_header_";
private static final String HTTP_CONTENT_TYPE_HEADER_NAME = "Content-Type";
+
+ /**
+ * Name for an attribute used when generating {@link DocumentID} values for
+ * the {@link ARCInputHandler}. This attribute stores (a String
+ * representation) of the Long byte offset for the input .record
+ */
+ static final String RECORD_OFFSET_ATTR = "recordOffset";
/**
+ * Name for an attribute used when generating {@link DocumentID} values for
+ * the {@link ARCInputHandler}. This attribute stores (a String
+ * representation) of the Long byte length for the input .record
+ */
+ static final String RECORD_LENGTH_ATTR = "recordLength";
+
+
+ /**
+ * Name for an attribute used when generating {@link DocumentID} values for
+ * the {@link ARCInputHandler}. This attribute stores the URL where the input
+ * .arc file can be located. This can be used to override the arc file
+ * configured on the actual input handler. When a batch uses records from
+ * multiple .arc file as input, the arc file cannot be configured on the
input
+ * handler, so each document ID must include one.
+ */
+ static final String ARC_URL_ATTR = "arcUrl";
+
+ /**
* The ARC file we are loading documents from.
*/
private File arcFile;
@@ -91,7 +117,7 @@
/**
* Pool of archive readers.
*/
- protected BlockingQueue<ARCReader> readers;
+// protected BlockingQueue<ARCReader> readers;
/**
* The index from archive record number to offset.
@@ -173,25 +199,60 @@
logger.info("Finished indexing ARC file " + arcFile.getAbsolutePath()
+ ", " + index.length + " records total.");
- readers = new LinkedBlockingQueue<ARCReader>();
+// readers = new LinkedBlockingQueue<ARCReader>();
}
- public DocumentData getInputDocument(String id) throws IOException,
GateException {
- ARCReader reader = borrowReader();
+ public DocumentData getInputDocument(DocumentID id) throws IOException,
GateException {
+ ARCReader reader = ARCReaderFactory.get(arcFile);
+ reader.setParseHttpHeaders(true);
+
try {
- Matcher idMatcher = STARTING_DIGITS_PATTERN.matcher(id);
- if(!idMatcher.find()) {
- throw new GateException("Document IDs within an ARC must start " +
- "with one or more digits, \"" + id + "\" does not.");
+// Matcher idMatcher = STARTING_DIGITS_PATTERN.matcher(id.getIdText());
+// if(!idMatcher.find()) {
+// throw new GateException("Document IDs within an ARC must start " +
+// "with one or more digits, \"" + id + "\" does not.");
+// }
+// int docIndex = Integer.parseInt(idMatcher.group(1));
+// if(docIndex >= index.length) {
+// throw new GateException("Input document at index " + docIndex + "
requested, " +
+// "but this ARC file only contains " + index.length + "
entries.");
+// }
+//
+//
+// // get the right ARC record
+// ARCRecord record = (ARCRecord)reader.get(index[docIndex]);
+
+ if(id.getAttributes() == null) {
+ throw new IllegalArgumentException(
+ "Document IDs within an ARC must include \"" + RECORD_OFFSET_ATTR +
+ "\" and \"" + RECORD_LENGTH_ATTR + "\" attributes; \"" +
+ id.toString() + "\" does not.");
}
- int docIndex = Integer.parseInt(idMatcher.group(1));
- if(docIndex >= index.length) {
- throw new GateException("Input document at index " + docIndex + "
requested, " +
- "but this ARC file only contains " + index.length + "
entries.");
+ String offsetStr = id.getAttributes().get(RECORD_OFFSET_ATTR);
+ String lengthStr = id.getAttributes().get(RECORD_LENGTH_ATTR);
+ if(offsetStr == null || offsetStr.length() == 0 ||
+ lengthStr == null || lengthStr.length() == 0) {
+ throw new IllegalArgumentException(
+ "Document IDs within an ARC must include \"" + RECORD_OFFSET_ATTR +
+ "\" and \"" + RECORD_LENGTH_ATTR + "\" attributes; \"" +
+ id.toString() + "\" does not.");
}
- // get the right ARC record
- ARCRecord record = (ARCRecord)reader.get(index[docIndex]);
+ long offset = -1;
+ try{
+ offset = Long.parseLong(offsetStr);
+ }catch (NumberFormatException nfe) {
+ throw new IllegalArgumentException(
+ "Could not parse offset \"" + offsetStr + "\" as a long value.");
+ }
+ long length = -1;
+ try{
+ length = Long.parseLong(lengthStr);
+ }catch (NumberFormatException nfe) {
+ throw new IllegalArgumentException(
+ "Could not parse length \"" + lengthStr + "\" as a long value.");
+ }
+ ARCRecord record = (ARCRecord)reader.get(offset);
ARCRecordMetaData header = record.getMetaData();
// extract the content
@@ -237,7 +298,7 @@
docFeatures.put("redirect_to", redirect.toString());
}
docFeatures.put("archive_name", arcFile.getName());
- docFeatures.put("archive_position", idMatcher.group(1));
+// docFeatures.put("archive_position", idMatcher.group(1));
Date documentDate = getDate(header, record);
if(documentDate != null) {
docFeatures.put("retrievedAt", documentDate);
@@ -278,11 +339,16 @@
DocumentData docData = new DocumentData(
(Document)Factory.createResource("gate.corpora.DocumentImpl",
- docParams, docFeatures, id), id);
+ docParams, docFeatures, id.toString()), id);
docData.fileSize = fileSize;
return docData;
} finally {
- releaseReader(reader);
+ try {
+ reader.close();
+ } catch(IOException e) {
+ // ignore
+ logger.error("Error while closing ARC reader.", e);
+ }
}
}
@@ -324,39 +390,8 @@
}
/**
- * Take a spare ARCReader from the pool, or create a new one if there are
- * no spares available.
+ * No operation.
*/
- protected ARCReader borrowReader() throws IOException {
- ARCReader reader = readers.poll();
- if(reader == null) {
- synchronized(this) {
- reader = ARCReaderFactory.get(arcFile);
- reader.setParseHttpHeaders(true);
- }
- }
- return reader;
- }
-
- /**
- * Return an ARCReader to the pool.
- */
- protected void releaseReader(ARCReader reader) {
- readers.add(reader);
- }
-
- /**
- * Close all the ARCReaders in the pool.
- */
- public void close() throws IOException, GateException {
- for(ARCReader r : readers) {
- try {
- r.close();
- }
- catch(Exception e) {
- logger.warn("Exception while closing ARC reader", e);
- }
- }
- }
+ public void close() throws IOException, GateException { }
}
Modified: gcp/trunk/src/gate/cloud/io/file/AbstractFileOutputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/file/AbstractFileOutputHandler.java
2013-03-21 02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/file/AbstractFileOutputHandler.java
2013-03-21 16:05:54 UTC (rev 16610)
@@ -12,6 +12,7 @@
package gate.cloud.io.file;
import gate.Gate;
+import gate.cloud.batch.DocumentID;
import gate.cloud.io.AbstractOutputHandler;
import gate.cloud.io.IOConstants;
import gate.util.GateException;
@@ -107,7 +108,7 @@
* (obtained from the configuration data of this handler).
* @return an output stream to which data can be written.
*/
- protected OutputStream getFileOutputStream(String docId)
+ protected OutputStream getFileOutputStream(DocumentID docId)
throws IOException {
File docFile = namingStrategy.toFile(docId);
File parent = docFile.getParentFile();
Modified: gcp/trunk/src/gate/cloud/io/file/FileInputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/file/FileInputHandler.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/file/FileInputHandler.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -16,6 +16,7 @@
import gate.Factory;
import gate.FeatureMap;
import gate.Gate;
+import gate.cloud.batch.DocumentID;
import gate.cloud.batch.PooledDocumentProcessor;
import gate.cloud.io.DocumentData;
import gate.cloud.io.IOConstants;
@@ -94,7 +95,7 @@
/* (non-Javadoc)
* @see gate.cloud.InputHandler#getInputDocument(java.lang.String)
*/
- public DocumentData getInputDocument(String id) throws IOException,
GateException {
+ public DocumentData getInputDocument(DocumentID id) throws IOException,
GateException {
File docFile = namingStrategy.toFile(id);
FeatureMap params = Factory.newFeatureMap();
URL docUrl = docFile.toURI().toURL();
@@ -116,7 +117,7 @@
DocumentData docData = new DocumentData(
(Document)Factory.createResource("gate.corpora.DocumentImpl",
- params, Factory.newFeatureMap(), id), id);
+ params, Factory.newFeatureMap(), id.toString()), id);
docData.fileSize = docFile.length();
return docData;
}
Modified: gcp/trunk/src/gate/cloud/io/file/GATEInlineOutputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/file/GATEInlineOutputHandler.java
2013-03-21 02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/file/GATEInlineOutputHandler.java
2013-03-21 16:05:54 UTC (rev 16610)
@@ -13,6 +13,7 @@
import gate.Annotation;
import gate.Document;
+import gate.cloud.batch.DocumentID;
import gate.util.Benchmark;
import gate.util.GateException;
@@ -30,11 +31,11 @@
protected boolean includeFeatures;
- protected void outputDocumentImpl(Document document, String documentId)
throws IOException,
+ protected void outputDocumentImpl(Document document, DocumentID documentId)
throws IOException,
GateException {
//get a Benchmark ID
- String baseBenchmarkID = Benchmark.createBenchmarkId("saveStandoff",
- documentId);
+ String baseBenchmarkID = Benchmark.createBenchmarkId("saveInline",
+ document.getName());
Map<String, Collection<Annotation>> anns = collectAnnotations(document);
long startTime = Benchmark.startPoint();
Modified: gcp/trunk/src/gate/cloud/io/file/GATEStandOffFileOutputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/file/GATEStandOffFileOutputHandler.java
2013-03-21 02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/file/GATEStandOffFileOutputHandler.java
2013-03-21 16:05:54 UTC (rev 16610)
@@ -14,6 +14,7 @@
import static gate.cloud.io.IOConstants.PARAM_FILE_EXTENSION;
import gate.Annotation;
import gate.Document;
+import gate.cloud.batch.DocumentID;
import gate.cloud.io.OutputHandler;
import gate.corpora.DocumentStaxUtils;
import gate.util.Benchmark;
@@ -46,11 +47,8 @@
super.configImpl(configData);
}
- protected void outputDocumentImpl(Document document, String documentId)
+ protected void outputDocumentImpl(Document document, DocumentID documentId)
throws IOException, GateException {
- String baseBenchmarkID = Benchmark.createBenchmarkId(document.getName(),
- documentId);
-
Map<String, Collection<Annotation>> annotationSetsMap =
collectAnnotations(document);
// use the default extension
@@ -68,7 +66,7 @@
xsw.writeCharacters("\n");
String saveStandoffBID =
- Benchmark.createBenchmarkId("saveGATEStandoff", baseBenchmarkID);
+ Benchmark.createBenchmarkId("saveGATEStandoff",
document.getName());
long startTime = Benchmark.startPoint();
DocumentStaxUtils.writeDocument(document, annotationSetsMap, xsw, "");
Benchmark.checkPoint(startTime, saveStandoffBID, this, null);
Modified: gcp/trunk/src/gate/cloud/io/file/NamingStrategy.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/file/NamingStrategy.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/file/NamingStrategy.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -11,6 +11,7 @@
*/
package gate.cloud.io.file;
+import gate.cloud.batch.DocumentID;
import gate.util.GateException;
import java.io.File;
@@ -35,5 +36,5 @@
* @return
* @throws IOException
*/
- public File toFile(String id) throws IOException;
+ public File toFile(DocumentID id) throws IOException;
}
Modified: gcp/trunk/src/gate/cloud/io/file/PlainTextOutputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/file/PlainTextOutputHandler.java
2013-03-21 02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/file/PlainTextOutputHandler.java
2013-03-21 16:05:54 UTC (rev 16610)
@@ -13,6 +13,7 @@
import static gate.cloud.io.IOConstants.PARAM_FILE_EXTENSION;
import gate.Document;
+import gate.cloud.batch.DocumentID;
import gate.corpora.DocumentStaxUtils;
import gate.util.Benchmark;
import gate.util.GateException;
@@ -33,11 +34,11 @@
super.configImpl(configData);
}
- protected void outputDocumentImpl(Document document, String documentId)
+ protected void outputDocumentImpl(Document document, DocumentID documentId)
throws IOException, GateException {
//get a Benchmark ID
String saveContentBID = Benchmark.createBenchmarkId("saveText",
- documentId);
+ document.getName());
long startTime = Benchmark.startPoint();
OutputStream outputStream = getFileOutputStream(documentId);
try {
Modified: gcp/trunk/src/gate/cloud/io/file/SerializedObjectOutputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/file/SerializedObjectOutputHandler.java
2013-03-21 02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/file/SerializedObjectOutputHandler.java
2013-03-21 16:05:54 UTC (rev 16610)
@@ -14,6 +14,7 @@
import static gate.cloud.io.IOConstants.PARAM_ENCODING;
import static gate.cloud.io.IOConstants.PARAM_FILE_EXTENSION;
import gate.Document;
+import gate.cloud.batch.DocumentID;
import gate.cloud.io.OutputHandler;
import gate.util.Benchmark;
import gate.util.GateException;
@@ -49,16 +50,13 @@
}
@Override
- protected void outputDocumentImpl(Document document, String documentId)
+ protected void outputDocumentImpl(Document document, DocumentID documentId)
throws IOException, GateException {
- String baseBenchmarkID =
- Benchmark.createBenchmarkId(document.getName(), documentId);
-
ObjectOutputStream outputStream =
new ObjectOutputStream(getFileOutputStream(documentId));
try {
String saveBID =
- Benchmark.createBenchmarkId("saveSerialized", baseBenchmarkID);
+ Benchmark.createBenchmarkId("saveSerialized",
document.getName());
long startTime = Benchmark.startPoint();
outputStream.writeObject(document);
Benchmark.checkPoint(startTime, saveBID, this, null);
Modified: gcp/trunk/src/gate/cloud/io/file/SimpleNamingStrategy.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/file/SimpleNamingStrategy.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/file/SimpleNamingStrategy.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -14,6 +14,7 @@
import static gate.cloud.io.IOConstants.PARAM_BATCH_FILE_LOCATION;
import static gate.cloud.io.IOConstants.PARAM_DOCUMENT_ROOT;
import static gate.cloud.io.IOConstants.PARAM_FILE_EXTENSION;
+import gate.cloud.batch.DocumentID;
import gate.util.GateException;
import java.io.File;
@@ -97,9 +98,9 @@
fileExtension = configData.get(PARAM_FILE_EXTENSION);
}
- public File toFile(String id) throws IOException {
+ public File toFile(DocumentID id) throws IOException {
try {
- String path = relativePathFor(id);
+ String path = relativePathFor(id.getIdText());
if(fileExtension != null) {
path += fileExtension;
}
Modified: gcp/trunk/src/gate/cloud/io/mimir/MimirOutputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/mimir/MimirOutputHandler.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/mimir/MimirOutputHandler.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -12,6 +12,7 @@
package gate.cloud.io.mimir;
import gate.Document;
+import gate.cloud.batch.DocumentID;
import gate.cloud.io.AbstractOutputHandler;
import gate.cloud.io.OutputHandler;
import gate.mimir.index.MimirConnector;
@@ -122,7 +123,7 @@
mimirConnector = new MimirConnector(webUtils);
}
- protected void outputDocumentImpl(Document document, String documentId)
+ protected void outputDocumentImpl(Document document, DocumentID documentId)
throws IOException, GateException {
// send to mimir, using the ID as the document URI.
String uri = null;
Modified: gcp/trunk/src/gate/cloud/io/xces/XCESOutputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/xces/XCESOutputHandler.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/xces/XCESOutputHandler.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -13,6 +13,7 @@
import gate.Annotation;
import gate.Document;
+import gate.cloud.batch.DocumentID;
import gate.cloud.io.file.AbstractFileOutputHandler;
import gate.corpora.DocumentStaxUtils;
import gate.util.Benchmark;
@@ -28,11 +29,11 @@
public class XCESOutputHandler extends AbstractFileOutputHandler {
- protected void outputDocumentImpl(Document document, String documentId)
+ protected void outputDocumentImpl(Document document, DocumentID documentId)
throws IOException, GateException {
//get a Benchmark ID
String baseBenchmarkID = Benchmark.createBenchmarkId("saveStandoff",
- documentId);
+ document.getName());
Map<String, Collection<Annotation>> anns = collectAnnotations(document);
long startTime = Benchmark.startPoint();
Modified: gcp/trunk/src/gate/cloud/io/zip/ZipInputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/zip/ZipInputHandler.java 2013-03-21
02:21:19 UTC (rev 16609)
+++ gcp/trunk/src/gate/cloud/io/zip/ZipInputHandler.java 2013-03-21
16:05:54 UTC (rev 16610)
@@ -32,6 +32,7 @@
import gate.Document;
import gate.Factory;
import gate.FeatureMap;
+import gate.cloud.batch.DocumentID;
import gate.cloud.batch.PooledDocumentProcessor;
import gate.cloud.io.DocumentData;
import gate.cloud.io.InputHandler;
@@ -125,12 +126,12 @@
fileNameEncoding = configData.get(PARAM_FILE_NAME_ENCODING);
}
- public DocumentData getInputDocument(String id) throws IOException,
GateException {
+ public DocumentData getInputDocument(DocumentID id) throws IOException,
GateException {
ZipFile zipFile = borrowZip();
InputStream is = null;
ZipEntryStreamHandler handler = null;
try {
- ZipEntry ze = zipFile.getEntry(id);
+ ZipEntry ze = zipFile.getEntry(id.getIdText());
if(ze == null) {
throw new GateException("Unknown entry " + id
+ " requested from zip file "
@@ -156,7 +157,7 @@
DocumentData docData = new DocumentData(
(Document)Factory.createResource("gate.corpora.DocumentImpl",
- params, Factory.newFeatureMap(), id), id);
+ params, Factory.newFeatureMap(), id.getIdText()), id);
docData.fileSize = ze.getSize();
return docData;
Modified: gcp/trunk/src/gate/cloud/util/Tools.java
===================================================================
--- gcp/trunk/src/gate/cloud/util/Tools.java 2013-03-21 02:21:19 UTC (rev
16609)
+++ gcp/trunk/src/gate/cloud/util/Tools.java 2013-03-21 16:05:54 UTC (rev
16610)
@@ -45,7 +45,7 @@
public static void writeResultToXml(ProcessResult result,
XMLStreamWriter writer) throws XMLStreamException {
writer.writeStartElement(Tools.REPORT_NAMESPACE, "processResult");
- writer.writeAttribute("id", result.getDocumentId());
+ writer.writeAttribute("id", result.getDocumentId().toString());
writer.writeAttribute("returnCode",
String.valueOf(result.getReturnCode()));
//write the size of the file
if(result.getOriginalFileSize() >= 0) {
Modified: gcp/trunk/src/gate/cloud/util/XMLBatchParser.java
===================================================================
--- gcp/trunk/src/gate/cloud/util/XMLBatchParser.java 2013-03-21 02:21:19 UTC
(rev 16609)
+++ gcp/trunk/src/gate/cloud/util/XMLBatchParser.java 2013-03-21 16:05:54 UTC
(rev 16610)
@@ -16,6 +16,7 @@
import gate.Gate;
import gate.cloud.batch.AnnotationSetDefinition;
import gate.cloud.batch.Batch;
+import gate.cloud.batch.DocumentID;
import gate.cloud.io.DocumentEnumerator;
import gate.cloud.io.InputHandler;
import gate.cloud.io.OutputHandler;
@@ -23,6 +24,8 @@
import gate.util.GateException;
import gate.util.persistence.PersistenceManager;
+import it.unimi.dsi.fastutil.objects.Object2ObjectArrayMap;
+
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -175,17 +178,17 @@
}
batch.setOutputHandlers(outputHandlers);
- List<String> docIds = new LinkedList<String>();
+ List<DocumentID> docIds = new LinkedList<DocumentID>();
for(Object item : docIDsOrSpecs) {
- if(item instanceof String) {
- docIds.add((String)item);
+ if(item instanceof DocumentID) {
+ docIds.add((DocumentID)item);
} else if(item instanceof HandlerSpec) {
DocumentEnumerator enumerator =
((HandlerSpec)item).toDocumentEnumerator();
while(enumerator.hasNext()) docIds.add(enumerator.next());
}
}
- batch.setDocumentIDs(docIds.toArray(new String[docIds.size()]));
+ batch.setDocumentIDs(docIds.toArray(new DocumentID[docIds.size()]));
// check the batch got all the data it needed
batch.init();
@@ -338,8 +341,8 @@
* Parses the documents element in the input XML file and converts it into a
* list of values. Values are either:
* <ul>
- * <li>String, representing a document ID</li>
- * <li>a HandlerSpec (that will be converted to a {@link
DocumentEnumerator}
+ * <li>a {@link DocumentID}, representing a document ID</li>
+ * <li>a {@link HandlerSpec} (that will be converted to a {@link
DocumentEnumerator}
* after the GATE application was loaded, and all its libraries were added
* to the GATE classpath</li>
* </ul>
@@ -359,17 +362,18 @@
if(elemName.equals("id")) {
xsr.require(XMLStreamConstants.START_ELEMENT,
IOConstants.BATCH_NAMESPACE, "id");
- String documentID = xsr.getElementText();
- if(documentID != null) {
- docIDsOrSpecs.add(documentID);
- } else {
- Location place = xsr.getLocation();
- throw new GateException(
- "Element id, within the documents list, has no text content
at "
- + "character: " + place.getCharacterOffset()
- + ", line: " + place.getLineNumber() + ", column: "
- + place.getColumnNumber());
+ Map<String, String> docIdAttrs = null;
+ int attrCount = xsr.getAttributeCount();
+ if(attrCount > 0) {
+ docIdAttrs = new Object2ObjectArrayMap<String, String>(
+ new String[attrCount], new String[attrCount]);
+ for(int i = 0; i< attrCount; i++) {
+ docIdAttrs.put(xsr.getAttributeName(i).toString(),
+ xsr.getAttributeValue(i));
+ }
}
+ String documentIdText = xsr.getElementText();
+ docIDsOrSpecs.add(new DocumentID(documentIdText, docIdAttrs));
xsr.require(XMLStreamConstants.END_ELEMENT,
IOConstants.BATCH_NAMESPACE,
"id");
} else if(elemName.equals("documentEnumerator")) {
@@ -457,8 +461,8 @@
System.out.println(aBatch);
if(Boolean.getBoolean("gate.cloud.util.XMLBatchParser.showDocIds")) {
System.out.println("\nDocument IDs:");
- for(String aDocId : aBatch.getDocumentIDs()){
- System.out.println(aDocId);
+ for(DocumentID aDocId : aBatch.getDocumentIDs()){
+ System.out.println(aDocId.toString());
}
}
}
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Everyone hates slow websites. So do we.
Make your web apps faster with AppDynamics
Download AppDynamics Lite for free today:
http://p.sf.net/sfu/appdyn_d2d_mar
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
