Revision: 16613
http://sourceforge.net/p/gate/code/16613
Author: valyt
Date: 2013-03-22 11:57:54 +0000 (Fri, 22 Mar 2013)
Log Message:
-----------
Support for reading byte-ranges from remote .arc files into GATE documents.
Modified Paths:
--------------
gcp/trunk/src/gate/cloud/io/arc/ARCInputHandler.java
Modified: gcp/trunk/src/gate/cloud/io/arc/ARCInputHandler.java
===================================================================
--- gcp/trunk/src/gate/cloud/io/arc/ARCInputHandler.java 2013-03-22
10:27:20 UTC (rev 16612)
+++ gcp/trunk/src/gate/cloud/io/arc/ARCInputHandler.java 2013-03-22
11:57:54 UTC (rev 16613)
@@ -24,7 +24,6 @@
import gate.cloud.io.InputHandler;
import gate.cloud.util.ByteArrayURLStreamHandler;
import gate.util.GateException;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -38,6 +37,7 @@
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.util.DateParseException;
@@ -45,7 +45,6 @@
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.log4j.Logger;
-import org.archive.io.ArchiveRecord;
import org.archive.io.ArchiveRecordHeader;
import org.archive.io.arc.ARCReader;
import org.archive.io.arc.ARCReaderFactory;
@@ -122,6 +121,7 @@
* The ARC file we are loading documents from.
*/
private File arcFile;
+
/**
* The directory containing the batch specification file, or
@@ -230,14 +230,19 @@
// read the arc record
ARCRecord record = null;
ARCReader reader = null;
+ URL arcUrl = null;
try{
if(arcUrlStr != null) {
- URL arcUrl = new URL(arcUrlStr);
+ arcUrl = new URL(arcUrlStr);
URLConnection conn = arcUrl.openConnection();
conn.setRequestProperty("Range", "bytes=" + Long.toString(offset) +
"-" + Long.toString(offset + length - 1));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
InputStream is = conn.getInputStream();
+ if(arcUrlStr.endsWith(".arc.gz")) {
+ // compressed input
+ is = new GZIPInputStream(is);
+ }
IOUtils.copy(is, baos);
is.close();
record = new ARCRecord(new ByteArrayInputStream(baos.toByteArray()),
@@ -292,12 +297,17 @@
if(redirect != null) {
docFeatures.put("redirect_to", redirect.toString());
}
- docFeatures.put("archive_name", arcFile.getName());
+ if(arcUrl != null) {
+ docFeatures.put("archive_name", arcUrl.getFile());
+ } else if(arcFile != null){
+ docFeatures.put("archive_name", arcFile.getName());
+ }
+
// docFeatures.put("archive_position", idMatcher.group(1));
String posStr = id.getAttributes().get(RECORD_POSITION_ATTR);
if(posStr != null) {
try {
- long pos = Long.parseLong(posStr);
+ Long.parseLong(posStr);
docFeatures.put("archive_position", posStr);
} catch(NumberFormatException e) {
// log and ignore
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Everyone hates slow websites. So do we.
Make your web apps faster with AppDynamics
Download AppDynamics Lite for free today:
http://p.sf.net/sfu/appdyn_d2d_mar
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
