Author: jukka
Date: Fri May 22 21:11:42 2009
New Revision: 777715
URL: http://svn.apache.org/viewvc?rev=777715&view=rev
Log:
TIKA-204: Use commons-compress for parsing packages
Merged changes from the TIKA-204 development branch
Added:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ArParser.java
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CpioParser.java
Removed:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/bzip2/
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/tar/
Modified:
lucene/tika/trunk/CHANGES.txt
lucene/tika/trunk/tika-parsers/pom.xml
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/Bzip2Parser.java
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/GzipParser.java
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/TarParser.java
Modified: lucene/tika/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/tika/trunk/CHANGES.txt?rev=777715&r1=777714&r2=777715&view=diff
==============================================================================
--- lucene/tika/trunk/CHANGES.txt (original)
+++ lucene/tika/trunk/CHANGES.txt Fri May 22 21:11:42 2009
@@ -14,6 +14,10 @@
* All the three Tika components are packaged as OSGi bundles. (TIKA-228)
+ * Tika now uses the new Commons Compress library for improved support
+ of compression and packaging formats like gzip, bzip2, tar, cpio,
+ ar, zip and jar. (TIKA-204)
+
* The memory use of parsing Excel sheets with lots of numbers
has been considerably reduced. (TIKA-211)
Modified: lucene/tika/trunk/tika-parsers/pom.xml
URL:
http://svn.apache.org/viewvc/lucene/tika/trunk/tika-parsers/pom.xml?rev=777715&r1=777714&r2=777715&view=diff
==============================================================================
--- lucene/tika/trunk/tika-parsers/pom.xml (original)
+++ lucene/tika/trunk/tika-parsers/pom.xml Fri May 22 21:11:42 2009
@@ -48,6 +48,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.0</version>
+ </dependency>
+ <dependency>
<groupId>pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>0.7.3</version>
Added:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ArParser.java
URL:
http://svn.apache.org/viewvc/lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ArParser.java?rev=777715&view=auto
==============================================================================
---
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ArParser.java
(added)
+++
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/ArParser.java
Fri May 22 21:11:42 2009
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pkg;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
+import org.apache.commons.compress.archivers.ar.ArArchiveInputStream;
+import org.apache.commons.io.input.CloseShieldInputStream;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+/**
+ * Ar archive parser.
+ */
+public class ArParser extends PackageParser {
+
+ /**
+ * Parses the given stream as an ar archive.
+ */
+ public void parse(
+ InputStream stream, ContentHandler handler, Metadata metadata)
+ throws IOException, TikaException, SAXException {
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+ xhtml.startDocument();
+
+ // At the end we want to close the ar stream to release any associated
+ // resources, but the underlying document stream should not be closed
+ ArArchiveInputStream ar =
+ new ArArchiveInputStream(new CloseShieldInputStream(stream));
+ try {
+ ArArchiveEntry entry = ar.getNextArEntry();
+ while (entry != null) {
+ if (!entry.isDirectory()) {
+ Metadata entrydata = new Metadata();
+ entrydata.set(Metadata.RESOURCE_NAME_KEY, entry.getName());
+ parseEntry(ar, xhtml, entrydata);
+ }
+ entry = ar.getNextArEntry();
+ }
+ } finally {
+ ar.close();
+ }
+
+ xhtml.endDocument();
+ }
+
+}
Modified:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/Bzip2Parser.java
URL:
http://svn.apache.org/viewvc/lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/Bzip2Parser.java?rev=777715&r1=777714&r2=777715&view=diff
==============================================================================
---
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/Bzip2Parser.java
(original)
+++
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/Bzip2Parser.java
Fri May 22 21:11:42 2009
@@ -19,10 +19,10 @@
import java.io.IOException;
import java.io.InputStream;
+import
org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.pkg.bzip2.CBZip2InputStream;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
@@ -38,11 +38,6 @@
public void parse(
InputStream stream, ContentHandler handler, Metadata metadata)
throws IOException, SAXException, TikaException {
- // The CBZip2InputStream class does not want to see the magic BZ header
- if (stream.read() != 'B' || stream.read() != 'Z') {
- throw new TikaException("Invalid BZip2 magic header");
- }
-
metadata.set(Metadata.CONTENT_TYPE, "application/x-bzip");
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
@@ -51,7 +46,7 @@
// At the end we want to close the bzip2 stream to release any
associated
// resources, but the underlying document stream should not be closed
InputStream gzip =
- new CBZip2InputStream(new CloseShieldInputStream(stream));
+ new BZip2CompressorInputStream(new CloseShieldInputStream(stream));
try {
Metadata entrydata = new Metadata();
String name = metadata.get(Metadata.RESOURCE_NAME_KEY);
Added:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CpioParser.java
URL:
http://svn.apache.org/viewvc/lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CpioParser.java?rev=777715&view=auto
==============================================================================
---
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CpioParser.java
(added)
+++
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/CpioParser.java
Fri May 22 21:11:42 2009
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pkg;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.compress.archivers.cpio.CpioArchiveEntry;
+import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream;
+import org.apache.commons.io.input.CloseShieldInputStream;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+/**
+ * CPIO parser.
+ */
+public class CpioParser extends PackageParser {
+
+ /**
+ * Parses the given stream as a cpio file.
+ */
+ public void parse(
+ InputStream stream, ContentHandler handler, Metadata metadata)
+ throws IOException, TikaException, SAXException {
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
+ xhtml.startDocument();
+
+ // At the end we want to close the cpio stream to release any
associated
+ // resources, but the underlying document stream should not be closed
+ CpioArchiveInputStream cpio =
+ new CpioArchiveInputStream(new CloseShieldInputStream(stream));
+ try {
+ CpioArchiveEntry entry = cpio.getNextCPIOEntry();
+ while (entry != null) {
+ if (!entry.isDirectory()) {
+ Metadata entrydata = new Metadata();
+ entrydata.set(Metadata.RESOURCE_NAME_KEY, entry.getName());
+ parseEntry(cpio, xhtml, entrydata);
+ }
+ entry = cpio.getNextCPIOEntry();
+ }
+ } finally {
+ cpio.close();
+ }
+
+ xhtml.endDocument();
+ }
+
+}
Modified:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/GzipParser.java
URL:
http://svn.apache.org/viewvc/lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/GzipParser.java?rev=777715&r1=777714&r2=777715&view=diff
==============================================================================
---
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/GzipParser.java
(original)
+++
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/GzipParser.java
Fri May 22 21:11:42 2009
@@ -20,6 +20,7 @@
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipUtils;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
@@ -51,18 +52,9 @@
Metadata entrydata = new Metadata();
String name = metadata.get(Metadata.RESOURCE_NAME_KEY);
if (name != null) {
- if (name.endsWith(".tgz")) {
- name = name.substring(0, name.length() - 4) + ".tar";
- } else if (name.endsWith(".gz") || name.endsWith("-gz")) {
- name = name.substring(0, name.length() - 3);
- } else if (name.toLowerCase().endsWith(".svgz")) {
- name = name.substring(0, name.length() - 1);
- } else if (name.toLowerCase().endsWith(".wmz")) {
- name = name.substring(0, name.length() - 1) + "f";
- } else if (name.toLowerCase().endsWith(".emz")) {
- name = name.substring(0, name.length() - 1) + "f";
- }
- entrydata.set(Metadata.RESOURCE_NAME_KEY, name);
+ entrydata.set(
+ Metadata.RESOURCE_NAME_KEY,
+ GzipUtils.getUncompressedFilename(name));
}
parseEntry(gzip, xhtml, entrydata);
} finally {
Modified:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/TarParser.java
URL:
http://svn.apache.org/viewvc/lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/TarParser.java?rev=777715&r1=777714&r2=777715&view=diff
==============================================================================
---
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/TarParser.java
(original)
+++
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/pkg/TarParser.java
Fri May 22 21:11:42 2009
@@ -19,11 +19,11 @@
import java.io.IOException;
import java.io.InputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.pkg.tar.TarEntry;
-import org.apache.tika.parser.pkg.tar.TarInputStream;
import org.apache.tika.sax.XHTMLContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
@@ -46,17 +46,17 @@
// At the end we want to close the tar stream to release any associated
// resources, but the underlying document stream should not be closed
- TarInputStream tar =
- new TarInputStream(new CloseShieldInputStream(stream));
+ TarArchiveInputStream tar =
+ new TarArchiveInputStream(new CloseShieldInputStream(stream));
try {
- TarEntry entry = tar.getNextEntry();
+ TarArchiveEntry entry = tar.getNextTarEntry();
while (entry != null) {
if (!entry.isDirectory()) {
Metadata entrydata = new Metadata();
entrydata.set(Metadata.RESOURCE_NAME_KEY, entry.getName());
parseEntry(tar, xhtml, entrydata);
}
- entry = tar.getNextEntry();
+ entry = tar.getNextTarEntry();
}
} finally {
tar.close();
