Author: ab
Date: Fri Feb 6 13:09:07 2009
New Revision: 741558
URL: http://svn.apache.org/viewvc?rev=741558&view=rev
Log:
NUTCH-643 ClassCastException in PDF parser, upgrade to unofficial PDFBox 0.7.4
Added:
lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.2.0-dev.jar (with
props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-0.2.0.jar (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt (with
props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.4-dev.jar (with
props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt (with
props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-jdk14-132.jar (with
props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_codec.jar (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_core.jar (with props)
lucene/nutch/trunk/src/plugin/parse-pdf/sample/encrypted.pdf (with props)
Removed:
lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.1.0-dev.jar
lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.3.jar
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/src/plugin/parse-pdf/build.xml
lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml
lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt
lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java
lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java
Modified: lucene/nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Fri Feb 6 13:09:07 2009
@@ -333,6 +333,8 @@
124. NUTCH-671 - JSP errors in Nutch searcher webapp (Edwin Chu via ab)
+125. NUTCH-643 - ClassCastException in PDF parser (Guillaume Smet, ab)
+
Release 0.9 - 2007-04-02
Modified: lucene/nutch/trunk/src/plugin/parse-pdf/build.xml
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/build.xml?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/build.xml (original)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/build.xml Fri Feb 6 13:09:07 2009
@@ -28,6 +28,10 @@
<!-- for junit test -->
<mkdir dir="${build.test}/data"/>
- <copy file="sample/pdftest.pdf" todir="${build.test}/data"/>
+ <copy todir="${build.test}/data">
+ <fileset dir="sample">
+ <include name="*.pdf"/>
+ </fileset>
+ </copy>
</project>
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.2.0-dev.jar
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.2.0-dev.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/FontBox-0.2.0-dev.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-0.2.0.jar
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-0.2.0.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-0.2.0.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt?rev=741558&view=auto
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt (added)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt Fri Feb 6
13:09:07 2009
@@ -0,0 +1,25 @@
+Copyright (c) 2006-2007, www.jempbox.org
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+3. Neither the name of pdfbox; nor the names of its
+ contributors may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/JempBox-LICENSE.txt
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt?rev=741558&view=auto
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt (added)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt Fri Feb 6 13:09:07
2009
@@ -0,0 +1,6 @@
+This plugin uses an unreleased version of PDFBox, obtained from SourceForce
+CVS repository (HEAD revision as of Feb 3 2009), because the last released
+version contains a bug related to document decryption (see NUTCH-643).
+
+The libraries should be updated to a regular PDFBox release whenever that
+becomes available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/NOTICE.txt
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.4-dev.jar
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.4-dev.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/PDFBox-0.7.4-dev.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt?rev=741558&view=auto
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt (added)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt Fri Feb 6
13:09:07 2009
@@ -0,0 +1,7 @@
+Copyright (c) 2000 - 2008 The Legion Of The Bouncy Castle
(http://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-LICENSE.txt
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-jdk14-132.jar
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-jdk14-132.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/bcprov-jdk14-132.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_codec.jar
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_codec.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_codec.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_core.jar
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_core.jar?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/lib/jai_core.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml (original)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/plugin.xml Fri Feb 6 13:09:07 2009
@@ -26,8 +26,12 @@
<library name="parse-pdf.jar">
<export name="*"/>
</library>
- <library name="PDFBox-0.7.3.jar"/>
- <library name="FontBox-0.1.0-dev.jar"/>
+ <library name="PDFBox-0.7.4-dev.jar"/>
+ <library name="FontBox-0.2.0-dev.jar"/>
+ <library name="JempBox-0.2.0-dev.jar"/>
+ <library name="bcprov-jdk14-132.jar"/>
+ <library name="jai_codec.jar"/>
+ <library name="jai_core.jar"/>
</runtime>
<requires>
Modified: lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt (original)
+++ lucene/nutch/trunk/src/plugin/parse-pdf/sample/README.txt Fri Feb 6
13:09:07 2009
@@ -1,10 +1,5 @@
-20040710, John Xing
-
-./pdftest.pdf: a small pdf file picked up from internet.
-
-Used for junit test by ./src/test/org/apache/nutch/parse/pdf/TestPdfParser.java
-
-It contains text:
+1. pdftest.pdf:
+A small pdf file picked up from internet. It contains text:
E:\M55\!\1725.fm 2003-01-01 18:15 P Tagg, IPM, University of Liverpool
A VERY SMALL PDF FILE
A VERY SMALL PDF FILE
@@ -12,3 +7,7 @@
A VERY SMALL PDF FILE
A VERY SMALL PDF FILE
A VERY SMALL PDF FILE
+
+2. encrypt.pdf:
+The same file as above, encrypted with owner's password 'owner'
+and empty user's password.
Added: lucene/nutch/trunk/src/plugin/parse-pdf/sample/encrypted.pdf
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/sample/encrypted.pdf?rev=741558&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/parse-pdf/sample/encrypted.pdf
------------------------------------------------------------------------------
svn:mime-type = application/pdf
Modified:
lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
---
lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java
(original)
+++
lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java
Fri Feb 6 13:09:07 2009
@@ -17,14 +17,14 @@
package org.apache.nutch.parse.pdf;
-import org.pdfbox.encryption.DocumentEncryption;
import org.pdfbox.pdfparser.PDFParser;
import org.pdfbox.pdmodel.PDDocument;
import org.pdfbox.pdmodel.PDDocumentInformation;
+import org.pdfbox.pdmodel.encryption.BadSecurityHandlerException;
+import org.pdfbox.pdmodel.encryption.StandardDecryptionMaterial;
import org.pdfbox.util.PDFTextStripper;
import org.pdfbox.exceptions.CryptographyException;
-import org.pdfbox.exceptions.InvalidPasswordException;
// Commons Logging imports
import org.apache.commons.logging.Log;
@@ -93,9 +93,8 @@
pdf = parser.getPDDocument();
if (pdf.isEncrypted()) {
- DocumentEncryption decryptor = new DocumentEncryption(pdf);
//Just try using the default password and move on
- decryptor.decryptDocument("");
+ pdf.openProtection(new StandardDecryptionMaterial(""));
}
// collect text
@@ -106,7 +105,7 @@
PDDocumentInformation info = pdf.getDocumentInformation();
title = info.getTitle();
// more useful info, currently not used. please keep them for future use.
- metadata.add(Metadata.PAGE_COUNT, String.valueOf(pdf.getPageCount()));
+ metadata.add(Metadata.PAGE_COUNT,
String.valueOf(pdf.getNumberOfPages()));
metadata.add(Metadata.AUTHOR, info.getAuthor());
metadata.add(Metadata.SUBJECT, info.getSubject());
metadata.add(Metadata.KEYWORDS, info.getKeywords());
@@ -122,9 +121,9 @@
} catch (CryptographyException e) {
return new ParseStatus(ParseStatus.FAILED,
"Error decrypting document. " +
e).getEmptyParseResult(content.getUrl(), getConf());
- } catch (InvalidPasswordException e) {
+ } catch (BadSecurityHandlerException e) {
return new ParseStatus(ParseStatus.FAILED,
- "Can't decrypt document - invalid password. " +
e).getEmptyParseResult(content.getUrl(), getConf());
+ "Error decrypting document. " +
e).getEmptyParseResult(content.getUrl(), getConf());
} catch (Exception e) { // run time exception
if (LOG.isWarnEnabled()) {
LOG.warn("General exception in PDF parser: "+e.getMessage());
Modified:
lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java
URL:
http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java?rev=741558&r1=741557&r2=741558&view=diff
==============================================================================
---
lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java
(original)
+++
lucene/nutch/trunk/src/plugin/parse-pdf/src/test/org/apache/nutch/parse/pdf/TestPdfParser.java
Fri Feb 6 13:09:07 2009
@@ -46,7 +46,10 @@
// Make sure sample files are copied to "test.data" as specified in
// ./src/plugin/parse-pdf/build.xml during plugin compilation.
// Check ./src/plugin/parse-pdf/sample/README.txt for what they are.
- private String[] sampleFiles = {"pdftest.pdf"};
+ private String[] sampleFiles = {
+ "pdftest.pdf",
+ "encrypted.pdf"
+ };
private String expectedText = "A VERY SMALL PDF FILE";
@@ -64,7 +67,7 @@
Content content;
Parse parse;
- for (int i=0; i<sampleFiles.length; i++) {
+ for (int i = 0; i < sampleFiles.length; i++) {
urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
Configuration conf = NutchConfiguration.create();
