This is an automated email from the ASF dual-hosted git repository. tallison pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/tika.git
commit b2e63bf9aaa2396048e30104369343c2cf4a71de Author: tballison <[email protected]> AuthorDate: Wed Sep 7 09:04:54 2022 -0400 add maxLength to s3 fetcher --- .../tika/exception/FileTooLongException.java | 41 ++++++++++++++++++++++ .../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 18 +++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/tika-core/src/main/java/org/apache/tika/exception/FileTooLongException.java b/tika-core/src/main/java/org/apache/tika/exception/FileTooLongException.java new file mode 100644 index 000000000..3ec3294b3 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/exception/FileTooLongException.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tika.exception; + +import java.io.IOException; + +/* + * Thrown when fetcher or similar has a maxLength set, but the + * underlying file is too long. + */ +public class FileTooLongException extends IOException { + + public FileTooLongException(String msg) { + super(msg); + } + + public FileTooLongException(long length, long maxLength) { + super(msg(length, maxLength)); + } + + private static String msg(long length, long maxLength) { + return "File is " + length + " bytes, but " + maxLength + + " is the maximum length allowed. You can modify maxLength via " + + "the setter on the fetcher."; + } +} diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java index e05c067fe..c04a5a68c 100644 --- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java +++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java @@ -42,6 +42,7 @@ import org.apache.tika.config.Field; import org.apache.tika.config.Initializable; import org.apache.tika.config.InitializableProblemHandler; import org.apache.tika.config.Param; +import org.apache.tika.exception.FileTooLongException; import org.apache.tika.exception.TikaConfigException; import org.apache.tika.exception.TikaException; import org.apache.tika.io.TemporaryResources; @@ -71,6 +72,7 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe private AmazonS3 s3Client; private boolean spoolToTemp = true; private int retries = 0; + private long maxLength = -1; @Override public InputStream fetch(String fetchKey, Metadata metadata) throws TikaException, IOException { @@ -137,6 +139,14 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe synchronized (clientLock) { s3Object = s3Client.getObject(objectRequest); } + long length = s3Object.getObjectMetadata().getContentLength(); + metadata.set(Metadata.CONTENT_LENGTH, Long.toString(length)); + if (maxLength > -1) { + if (length > maxLength) { + throw new FileTooLongException(length, maxLength); + } + } + LOGGER.debug("took {} ms to fetch file's metadata", System.currentTimeMillis() - start); if (extractUserMetadata) { for (Map.Entry<String, String> e : s3Object.getObjectMetadata().getUserMetadata() @@ -152,7 +162,8 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe Path tmpPath = tmp.createTempFile(); Files.copy(s3Object.getObjectContent(), tmpPath, StandardCopyOption.REPLACE_EXISTING); TikaInputStream tis = TikaInputStream.get(tmpPath, metadata, tmp); - LOGGER.debug("took {} ms to copy to local tmp file", System.currentTimeMillis() - start); + LOGGER.debug("took {} ms to fetch metadata and copy to local tmp file", + System.currentTimeMillis() - start); return tis; } } catch (Throwable e) { @@ -225,6 +236,11 @@ public class S3Fetcher extends AbstractFetcher implements Initializable, RangeFe this.credentialsProvider = credentialsProvider; } + @Field + public void setMaxLength(long maxLength) { + this.maxLength = maxLength; + } + /** * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions, * e.g. AmazonClientException in a TikaConfigException.
