This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 4625f19 TIKA-3542 -- add optional range parameters to FetchKey
4625f19 is described below
commit 4625f1994da31c0db37105e41a5eca83f73c5a91
Author: tallison <[email protected]>
AuthorDate: Tue Aug 31 11:33:17 2021 -0400
TIKA-3542 -- add optional range parameters to FetchKey
---
CHANGES.txt | 3 +-
.../java/org/apache/tika/pipes/PipesServer.java | 91 ++++++++++++++++------
.../org/apache/tika/pipes/fetcher/FetchKey.java | 43 ++++++----
.../pipesiterator/jdbc/JDBCPipesIterator.java | 67 +++++++++++++---
.../metadata/serialization/JsonFetchEmitTuple.java | 23 +++++-
.../serialization/JsonFetchEmitTupleTest.java | 23 ++++++
.../tika/server/core/FetcherStreamFactory.java | 40 +++++++++-
7 files changed, 237 insertions(+), 53 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 746e3f7..9adb889 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,7 @@
Release 2.1.1 - ???
- * Exclude dependency on jdom2 in rome-tools in FeedParser (TIKA-3488).
+ * Add optional fetch ranges to FetchEmitTuple to allow range fetching from,
+ e.g. http or s3 (TIKA-3542).
* Exclude dependencies on jsoup and ehcache in ucar grib/cdm (TIKA-3003).
diff --git a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
index daf1a4c..6e03430 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
@@ -51,8 +51,10 @@ import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.emitter.Emitter;
import org.apache.tika.pipes.emitter.EmitterManager;
import org.apache.tika.pipes.emitter.TikaEmitterException;
+import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.fetcher.Fetcher;
import org.apache.tika.pipes.fetcher.FetcherManager;
+import org.apache.tika.pipes.fetcher.RangeFetcher;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.ContentHandlerFactory;
import org.apache.tika.sax.RecursiveParserWrapperHandler;
@@ -318,46 +320,36 @@ public class PipesServer implements Runnable {
}
private void actuallyParse(FetchEmitTuple t) {
- List<Metadata> metadataList = null;
+
long start = System.currentTimeMillis();
- Fetcher fetcher = null;
- try {
- fetcher =
fetcherManager.getFetcher(t.getFetchKey().getFetcherName());
- } catch (IllegalArgumentException e) {
- String noFetcherMsg =
getNoFetcherMsg(t.getFetchKey().getFetcherName());
- LOG.warn(noFetcherMsg);
- write(STATUS.FETCHER_NOT_FOUND, noFetcherMsg);
- return;
- } catch (IOException | TikaException e) {
- LOG.warn("Couldn't initialize fetcher for fetch id '" +
- t.getId() + "'", e);
- write(STATUS.FETCHER_INITIALIZATION_EXCEPTION,
- ExceptionUtils.getStackTrace(e));
+ Fetcher fetcher = getFetcher(t);
+ if (fetcher == null) {
+ //rely on proper logging/exception handling in getFetcher
return;
}
+
if (LOG.isTraceEnabled()) {
long elapsed = System.currentTimeMillis() - start;
LOG.trace("timer -- got fetcher: {}ms", elapsed);
}
+
start = System.currentTimeMillis();
- Metadata metadata = new Metadata();
- try (InputStream stream = fetcher.fetch(t.getFetchKey().getFetchKey(),
metadata)) {
- metadataList = parse(t, stream, metadata);
- } catch (SecurityException e) {
- LOG.error("security exception " + t.getId(), e);
- throw e;
- } catch (TikaException | IOException e) {
- LOG.warn("fetch exception " + t.getId(), e);
- write(STATUS.FETCH_EXCEPTION, ExceptionUtils.getStackTrace(e));
- }
+ List<Metadata> metadataList = parseIt(t, fetcher);
+
if (LOG.isTraceEnabled()) {
LOG.trace("timer -- to parse: {} ms", System.currentTimeMillis() -
start);
}
+
if (metadataIsEmpty(metadataList)) {
write(STATUS.EMPTY_OUTPUT);
return;
}
- start = System.currentTimeMillis();
+
+ emitIt(t, metadataList);
+ }
+
+ private void emitIt(FetchEmitTuple t, List<Metadata> metadataList) {
+ long start = System.currentTimeMillis();
String stack = getContainerStacktrace(t, metadataList);
if (StringUtils.isBlank(stack) || t.getOnParseException() ==
FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT) {
injectUserMetadata(t.getMetadata(), metadataList);
@@ -382,7 +374,56 @@ public class PipesServer implements Runnable {
} else {
write(STATUS.PARSE_EXCEPTION_NO_EMIT, stack);
}
+ }
+
+ private Fetcher getFetcher(FetchEmitTuple t) {
+ try {
+ return fetcherManager.getFetcher(t.getFetchKey().getFetcherName());
+ } catch (IllegalArgumentException e) {
+ String noFetcherMsg =
getNoFetcherMsg(t.getFetchKey().getFetcherName());
+ LOG.warn(noFetcherMsg);
+ write(STATUS.FETCHER_NOT_FOUND, noFetcherMsg);
+ return null;
+ } catch (IOException | TikaException e) {
+ LOG.warn("Couldn't initialize fetcher for fetch id '" +
+ t.getId() + "'", e);
+ write(STATUS.FETCHER_INITIALIZATION_EXCEPTION,
+ ExceptionUtils.getStackTrace(e));
+ return null;
+ }
+ }
+ private List<Metadata> parseIt(FetchEmitTuple t, Fetcher fetcher) {
+ FetchKey fetchKey = t.getFetchKey();
+ if (fetchKey.hasRange()) {
+ if (! (fetcher instanceof RangeFetcher)) {
+ throw new IllegalArgumentException(
+ "fetch key has a range, but the fetcher is not a range
fetcher");
+ }
+ Metadata metadata = new Metadata();
+ try (InputStream stream =
((RangeFetcher)fetcher).fetch(fetchKey.getFetchKey(),
+ fetchKey.getRangeStart(), fetchKey.getRangeEnd(),
metadata)) {
+ return parse(t, stream, metadata);
+ } catch (SecurityException e) {
+ LOG.error("security exception " + t.getId(), e);
+ throw e;
+ } catch (TikaException | IOException e) {
+ LOG.warn("fetch exception " + t.getId(), e);
+ write(STATUS.FETCH_EXCEPTION, ExceptionUtils.getStackTrace(e));
+ }
+ } else {
+ Metadata metadata = new Metadata();
+ try (InputStream stream =
fetcher.fetch(t.getFetchKey().getFetchKey(), metadata)) {
+ return parse(t, stream, metadata);
+ } catch (SecurityException e) {
+ LOG.error("security exception " + t.getId(), e);
+ throw e;
+ } catch (TikaException | IOException e) {
+ LOG.warn("fetch exception " + t.getId(), e);
+ write(STATUS.FETCH_EXCEPTION, ExceptionUtils.getStackTrace(e));
+ }
+ }
+ return null;
}
private String getNoFetcherMsg(String fetcherName) {
diff --git
a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
index 065b11a..148e353 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/fetcher/FetchKey.java
@@ -31,6 +31,8 @@ public class FetchKey implements Serializable {
private String fetcherName;
private String fetchKey;
+ private long rangeStart = -1;
+ private long rangeEnd = -1;
//this is for serialization...yuck
public FetchKey() {
@@ -38,8 +40,14 @@ public class FetchKey implements Serializable {
}
public FetchKey(String fetcherName, String fetchKey) {
+ this(fetcherName, fetchKey, -1, -1);
+ }
+
+ public FetchKey(String fetcherName, String fetchKey, long rangeStart, long
rangeEnd) {
this.fetcherName = fetcherName;
this.fetchKey = fetchKey;
+ this.rangeStart = rangeStart;
+ this.rangeEnd = rangeEnd;
}
public String getFetcherName() {
@@ -50,10 +58,16 @@ public class FetchKey implements Serializable {
return fetchKey;
}
- @Override
- public String toString() {
- return "FetcherKeyPair{" + "fetcherName='" + fetcherName + '\'' + ",
fetchKey='" +
- fetchKey + '\'' + '}';
+ public boolean hasRange() {
+ return rangeStart > -1 && rangeEnd > -1;
+ }
+
+ public long getRangeStart() {
+ return rangeStart;
+ }
+
+ public long getRangeEnd() {
+ return rangeEnd;
}
@Override
@@ -64,19 +78,20 @@ public class FetchKey implements Serializable {
if (o == null || getClass() != o.getClass()) {
return false;
}
-
- FetchKey fetchKey = (FetchKey) o;
-
- if (!Objects.equals(fetcherName, fetchKey.fetcherName)) {
- return false;
- }
- return Objects.equals(this.fetchKey, fetchKey.fetchKey);
+ FetchKey fetchKey1 = (FetchKey) o;
+ return rangeStart == fetchKey1.rangeStart && rangeEnd ==
fetchKey1.rangeEnd &&
+ Objects.equals(fetcherName, fetchKey1.fetcherName) &&
+ Objects.equals(fetchKey, fetchKey1.fetchKey);
}
@Override
public int hashCode() {
- int result = fetcherName != null ? fetcherName.hashCode() : 0;
- result = 31 * result + (fetchKey != null ? fetchKey.hashCode() : 0);
- return result;
+ return Objects.hash(fetcherName, fetchKey, rangeStart, rangeEnd);
+ }
+
+ @Override
+ public String toString() {
+ return "FetchKey{" + "fetcherName='" + fetcherName + '\'' + ",
fetchKey='" + fetchKey +
+ '\'' + ", rangeStart=" + rangeStart + ", rangeEnd=" + rangeEnd
+ '}';
}
}
diff --git
a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/java/org/apache/tika/pipes/pipesiterator/jdbc/JDBCPipesIterator.java
b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/java/org/apache/tika/pipes/pipesiterator/jdbc/JDBCPipesIterator.java
index 3fb45a1..da1821e 100644
---
a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/java/org/apache/tika/pipes/pipesiterator/jdbc/JDBCPipesIterator.java
+++
b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/java/org/apache/tika/pipes/pipesiterator/jdbc/JDBCPipesIterator.java
@@ -71,6 +71,8 @@ public class JDBCPipesIterator extends PipesIterator
implements Initializable {
private String idColumn;
private String fetchKeyColumn;
+ private String fetchKeyRangeStartColumn;
+ private String fetchKeyRangeEndColumn;
private String emitKeyColumn;
private String connection;
private String select;
@@ -168,9 +170,12 @@ public class JDBCPipesIterator extends PipesIterator
implements Initializable {
throws SQLException, TimeoutException, InterruptedException {
Metadata metadata = new Metadata();
String fetchKey = "";
+ long fetchStartRange = -1l;
+ long fetchEndRange = -1l;
String emitKey = "";
String id = "";
for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
+
if (i == fetchEmitKeyIndices.fetchKeyIndex) {
fetchKey = getString(i, rs);
if (fetchKey == null) {
@@ -195,13 +200,21 @@ public class JDBCPipesIterator extends PipesIterator
implements Initializable {
id = (id == null) ? "" : id;
continue;
}
+ if (i == fetchEmitKeyIndices.fetchStartRangeIndex) {
+ fetchStartRange = getLong(i, rs);
+ continue;
+ }
+ if (i == fetchEmitKeyIndices.fetchEndRangeIndex) {
+ fetchEndRange = getLong(i, rs);
+ continue;
+ }
String val = getString(i, rs);
if (val != null) {
metadata.set(headers.get(i - 1), val);
}
}
- tryToAdd(new FetchEmitTuple(id, new FetchKey(fetcherName, fetchKey),
+ tryToAdd(new FetchEmitTuple(id, new FetchKey(fetcherName, fetchKey,
fetchStartRange, fetchEndRange),
new EmitKey(emitterName, emitKey), metadata, handlerConfig,
getOnParseException()));
}
@@ -225,26 +238,43 @@ public class JDBCPipesIterator extends PipesIterator
implements Initializable {
return val;
}
+ private long getLong(int i, ResultSet rs) throws SQLException {
+ long val = rs.getLong(i);
+ if (rs.wasNull()) {
+ return -1l;
+ }
+ return val;
+ }
+
private FetchEmitKeyIndices loadHeaders(ResultSetMetaData metaData,
List<String> headers)
throws SQLException {
int idIndex = -1;
int fetchKeyIndex = -1;
+ int fetchKeyStartRangeIndex = -1;
+ int fetchKeyEndRangeIndex = -1;
int emitKeyIndex = -1;
for (int i = 1; i <= metaData.getColumnCount(); i++) {
- if (metaData.getColumnLabel(i).equalsIgnoreCase(fetchKeyColumn)) {
+ String colLabel = metaData.getColumnLabel(i);
+ if (colLabel.equalsIgnoreCase(fetchKeyColumn)) {
fetchKeyIndex = i;
}
- if (metaData.getColumnLabel(i).equalsIgnoreCase(emitKeyColumn)) {
+ if (colLabel.equalsIgnoreCase(fetchKeyRangeStartColumn)) {
+ fetchKeyStartRangeIndex = i;
+ }
+ if (colLabel.equalsIgnoreCase(fetchKeyRangeEndColumn)) {
+ fetchKeyEndRangeIndex = i;
+ }
+ if (colLabel.equalsIgnoreCase(emitKeyColumn)) {
emitKeyIndex = i;
}
- if (metaData.getColumnLabel(i).equalsIgnoreCase(idColumn)) {
+ if (colLabel.equalsIgnoreCase(idColumn)) {
idIndex = i;
}
-
headers.add(metaData.getColumnLabel(i));
}
- return new FetchEmitKeyIndices(idIndex, fetchKeyIndex, emitKeyIndex);
+ return new FetchEmitKeyIndices(idIndex, fetchKeyIndex,
+ fetchKeyStartRangeIndex, fetchKeyEndRangeIndex, emitKeyIndex);
}
@Override
@@ -262,16 +292,23 @@ public class JDBCPipesIterator extends PipesIterator
implements Initializable {
super.checkInitialization(problemHandler);
mustNotBeEmpty("connection", this.connection);
mustNotBeEmpty("select", this.select);
- mustNotBeEmpty("emitterName", this.getEmitterName());
- mustNotBeEmpty("emitKeyColumn", this.emitKeyColumn);
if (StringUtils.isBlank(getFetcherName()) &&
!StringUtils.isBlank(fetchKeyColumn)) {
throw new TikaConfigException(
"If you specify a 'fetchKeyColumn', you must specify a
'fetcherName'");
}
+ if (StringUtils.isBlank(getEmitterName()) &&
!StringUtils.isBlank(emitKeyColumn)) {
+ throw new TikaConfigException(
+ "If you specify an 'emitKeyColumn', you must specify an
'emitterName'");
+ }
+
+ if (StringUtils.isBlank(getEmitterName()) &&
StringUtils.isBlank(getFetcherName())) {
+ LOGGER.warn("no fetcher or emitter specified?!");
+ }
+
if (StringUtils.isEmpty(fetchKeyColumn)) {
- LOGGER.info("no fetch key column has been specified");
+ LOGGER.warn("no fetch key column has been specified");
}
}
@@ -279,16 +316,24 @@ public class JDBCPipesIterator extends PipesIterator
implements Initializable {
private static class FetchEmitKeyIndices {
private final int idIndex;
private final int fetchKeyIndex;
+ private final int fetchStartRangeIndex;
+ private final int fetchEndRangeIndex;
private final int emitKeyIndex;
- public FetchEmitKeyIndices(int idIndex, int fetchKeyIndex, int
emitKeyIndex) {
+ public FetchEmitKeyIndices(int idIndex, int fetchKeyIndex,
+ int fetchStartRangeIndex, int
fetchEndRangeIndex,
+ int emitKeyIndex) {
this.idIndex = idIndex;
this.fetchKeyIndex = fetchKeyIndex;
+ this.fetchStartRangeIndex = fetchStartRangeIndex;
+ this.fetchEndRangeIndex = fetchEndRangeIndex;
this.emitKeyIndex = emitKeyIndex;
}
public boolean shouldSkip(int index) {
- return idIndex == index || fetchKeyIndex == index || emitKeyIndex
== index;
+ return idIndex == index || fetchKeyIndex == index ||
+ fetchStartRangeIndex == index || fetchEndRangeIndex ==
index ||
+ emitKeyIndex == index;
}
}
}
diff --git
a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
index cb23b34..73c0737 100644
---
a/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
+++
b/tika-serialization/src/main/java/org/apache/tika/metadata/serialization/JsonFetchEmitTuple.java
@@ -40,6 +40,8 @@ public class JsonFetchEmitTuple {
public static final String ID = "id";
public static final String FETCHER = "fetcher";
public static final String FETCHKEY = "fetchKey";
+ public static final String FETCH_RANGE_START = "fetchRangeStart";
+ public static final String FETCH_RANGE_END = "fetchRangeEnd";
public static final String EMITTER = "emitter";
public static final String EMITKEY = "emitKey";
public static final String METADATAKEY = "metadata";
@@ -72,6 +74,9 @@ public class JsonFetchEmitTuple {
String fetchKey = null;
String emitterName = null;
String emitKey = null;
+ long fetchRangeStart = -1l;
+ long fetchRangeEnd = -1l;
+
FetchEmitTuple.ON_PARSE_EXCEPTION onParseException =
FetchEmitTuple.DEFAULT_ON_PARSE_EXCEPTION;
HandlerConfig handlerConfig = HandlerConfig.DEFAULT_HANDLER_CONFIG;
@@ -108,13 +113,17 @@ public class JsonFetchEmitTuple {
}
} else if (HANDLER_CONFIG.equals(name)) {
handlerConfig = getHandlerConfig(jParser);
+ } else if (FETCH_RANGE_START.equals(name)) {
+ fetchRangeStart = getLong(jParser);
+ } else if (FETCH_RANGE_END.equals(name)) {
+ fetchRangeEnd = getLong(jParser);
}
token = jParser.nextToken();
}
if (id == null) {
id = fetchKey;
}
- return new FetchEmitTuple(id, new FetchKey(fetcherName, fetchKey),
+ return new FetchEmitTuple(id, new FetchKey(fetcherName, fetchKey,
fetchRangeStart, fetchRangeEnd),
new EmitKey(emitterName, emitKey), metadata, handlerConfig,
onParseException);
}
@@ -164,6 +173,14 @@ public class JsonFetchEmitTuple {
return jParser.getValueAsString();
}
+ private static long getLong(JsonParser jParser) throws IOException {
+ JsonToken token = jParser.nextToken();
+ if (token != JsonToken.VALUE_NUMBER_INT) {
+ throw new IOException("required value long, but see: " +
token.name());
+ }
+ return jParser.getValueAsLong();
+ }
+
public static String toJson(FetchEmitTuple t) throws IOException {
StringWriter writer = new StringWriter();
toJson(t, writer);
@@ -182,6 +199,10 @@ public class JsonFetchEmitTuple {
jsonGenerator.writeStringField(ID, t.getId());
jsonGenerator.writeStringField(FETCHER,
t.getFetchKey().getFetcherName());
jsonGenerator.writeStringField(FETCHKEY,
t.getFetchKey().getFetchKey());
+ if (t.getFetchKey().hasRange()) {
+ jsonGenerator.writeNumberField(FETCH_RANGE_START,
t.getFetchKey().getRangeStart());
+ jsonGenerator.writeNumberField(FETCH_RANGE_END,
t.getFetchKey().getRangeEnd());
+ }
jsonGenerator.writeStringField(EMITTER,
t.getEmitKey().getEmitterName());
if (!StringUtils.isBlank(t.getEmitKey().getEmitKey())) {
jsonGenerator.writeStringField(EMITKEY,
t.getEmitKey().getEmitKey());
diff --git
a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
index 4e827cc..a95431e 100644
---
a/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
+++
b/tika-serialization/src/test/java/org/apache/tika/metadata/serialization/JsonFetchEmitTupleTest.java
@@ -54,4 +54,27 @@ public class JsonFetchEmitTupleTest {
FetchEmitTuple deserialized = JsonFetchEmitTuple.fromJson(reader);
assertEquals(t, deserialized);
}
+
+ @Test
+ public void testFetchRange() throws Exception {
+ Metadata m = new Metadata();
+ m.add("m1", "v1");
+ m.add("m1", "v1");
+ m.add("m2", "v2");
+ m.add("m2", "v3");
+ m.add("m3", "v4");
+
+ FetchEmitTuple t = new FetchEmitTuple("my_id",
+ new FetchKey("my_fetcher", "fetchKey1", 10, 1000),
+ new EmitKey("my_emitter", "emitKey1"), m,
+ new HandlerConfig(BasicContentHandlerFactory.HANDLER_TYPE.XML,
+ HandlerConfig.PARSE_MODE.CONCATENATE,
+ 10000,10),
+ FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP);
+ StringWriter writer = new StringWriter();
+ JsonFetchEmitTuple.toJson(t, writer);
+ Reader reader = new StringReader(writer.toString());
+ FetchEmitTuple deserialized = JsonFetchEmitTuple.fromJson(reader);
+ assertEquals(t, deserialized);
+ }
}
diff --git
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java
index e145553..2694006 100644
---
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java
+++
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/FetcherStreamFactory.java
@@ -24,7 +24,9 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.pipes.fetcher.Fetcher;
import org.apache.tika.pipes.fetcher.FetcherManager;
+import org.apache.tika.pipes.fetcher.RangeFetcher;
/**
* This class looks for "fileUrl" in the http header. If it is not
null
@@ -54,18 +56,54 @@ public class FetcherStreamFactory implements
InputStreamFactory {
throws IOException {
String fetcherName = httpHeaders.getHeaderString("fetcherName");
String fetchKey = httpHeaders.getHeaderString("fetchKey");
+ long fetchRangeStart =
getLong(httpHeaders.getHeaderString("fetchRangeStart"));
+ long fetchRangeEnd =
getLong(httpHeaders.getHeaderString("fetchRangeEnd"));
if (StringUtils.isBlank(fetcherName) != StringUtils.isBlank(fetchKey))
{
throw new IOException("Must specify both a 'fetcherName' and a
'fetchKey'. I see: " +
" fetcherName:" + fetcherName + " and fetchKey:" +
fetchKey);
}
+ if (fetchRangeStart < 0 && fetchRangeEnd > -1) {
+ throw new IllegalArgumentException("fetchRangeStart must be > -1
if a fetchRangeEnd " +
+ "is specified");
+ }
+
+ if (fetchRangeStart > -1 && fetchRangeEnd < 0) {
+ throw new IllegalArgumentException("fetchRangeEnd must be > -1 if
a fetchRangeStart " +
+ "is specified");
+ }
if (!StringUtils.isBlank(fetcherName)) {
try {
- return fetcherManager.getFetcher(fetcherName).fetch(fetchKey,
metadata);
+ Fetcher fetcher = fetcherManager.getFetcher(fetcherName);
+ if (fetchRangeStart > -1 && fetchRangeEnd > -1) {
+ if (!(fetcher instanceof RangeFetcher)) {
+ throw new IllegalArgumentException(
+ "Requesting a range fetch from a fetcher " +
+ "that doesn't support range
fetching?!");
+ }
+ return ((RangeFetcher) fetcher).fetch(fetchKey,
fetchRangeStart, fetchRangeEnd,
+ metadata);
+ } else {
+ return fetcher.fetch(fetchKey, metadata);
+ }
} catch (TikaException e) {
throw new IOException(e);
}
}
return is;
}
+
+ /**
+ * Tries to parse a long out of the value. If the val is blank, it
returns -1.
+ * Throws {@link NumberFormatException}
+ *
+ * @param val
+ * @return
+ */
+ private static long getLong(String val) {
+ if (StringUtils.isBlank(val)) {
+ return -1;
+ }
+ return Long.parseLong(val);
+ }
}
