Author: jukka
Date: Fri Oct 16 09:03:43 2009
New Revision: 825818
URL: http://svn.apache.org/viewvc?rev=825818&view=rev
Log:
TIKA-287: HtmlParser should resolve relative paths in <a href="xxx"> elements
Move the HTML body handler to a top level class to avoid too much bloat in an
anonymous class.
Added:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/BodyHandler.java
Modified:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java
Added:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/BodyHandler.java
URL:
http://svn.apache.org/viewvc/lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/BodyHandler.java?rev=825818&view=auto
==============================================================================
---
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/BodyHandler.java
(added)
+++
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/BodyHandler.java
Fri Oct 16 09:03:43 2009
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.html;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.tika.sax.TextContentHandler;
+import org.apache.tika.sax.XHTMLContentHandler;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+
+class BodyHandler extends TextContentHandler {
+
+ /**
+ * Set of safe mappings from incoming HTML elements to outgoing
+ * XHTML elements. Ensures that the output is valid XHTML 1.0 Strict.
+ */
+ private static final Map<String, String> SAFE_ELEMENTS =
+ new HashMap<String, String>();
+
+ /**
+ * Set of HTML elements whose content will be discarded.
+ */
+ private static final Set<String> DISCARD_ELEMENTS = new HashSet<String>();
+
+ static {
+ // Based on http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd
+ SAFE_ELEMENTS.put("P", "p");
+ SAFE_ELEMENTS.put("H1", "h1");
+ SAFE_ELEMENTS.put("H2", "h2");
+ SAFE_ELEMENTS.put("H3", "h3");
+ SAFE_ELEMENTS.put("H4", "h4");
+ SAFE_ELEMENTS.put("H5", "h5");
+ SAFE_ELEMENTS.put("H6", "h6");
+ SAFE_ELEMENTS.put("UL", "ul");
+ SAFE_ELEMENTS.put("OL", "ol");
+ SAFE_ELEMENTS.put("LI", "li");
+ SAFE_ELEMENTS.put("MENU", "ul");
+ SAFE_ELEMENTS.put("DL", "dl");
+ SAFE_ELEMENTS.put("DT", "dt");
+ SAFE_ELEMENTS.put("DD", "dd");
+ SAFE_ELEMENTS.put("PRE", "pre");
+ SAFE_ELEMENTS.put("BLOCKQUOTE", "blockquote");
+ SAFE_ELEMENTS.put("TABLE", "table");
+ SAFE_ELEMENTS.put("THEAD", "thead");
+ SAFE_ELEMENTS.put("TBODY", "tbody");
+ SAFE_ELEMENTS.put("TR", "tr");
+ SAFE_ELEMENTS.put("TH", "th");
+ SAFE_ELEMENTS.put("TD", "td");
+
+ DISCARD_ELEMENTS.add("STYLE");
+ DISCARD_ELEMENTS.add("SCRIPT");
+ }
+
+ private final XHTMLContentHandler xhtml;
+
+ private int discardLevel = 0;
+
+ public BodyHandler(XHTMLContentHandler xhtml) {
+ super(xhtml);
+ this.xhtml = xhtml;
+ }
+
+ @Override
+ public void startElement(
+ String uri, String local, String name, Attributes atts)
+ throws SAXException {
+ if (discardLevel != 0) {
+ discardLevel++;
+ } else if (DISCARD_ELEMENTS.contains(name)) {
+ discardLevel = 1;
+ } else if (SAFE_ELEMENTS.containsKey(name)) {
+ xhtml.startElement(SAFE_ELEMENTS.get(name));
+ } else if ("A".equals(name)) {
+ String href = atts.getValue("href");
+ if (href != null) {
+ xhtml.startElement("a", "href", href);
+ } else {
+ String anchor = atts.getValue("name");
+ if (anchor != null) {
+ xhtml.startElement("a", "name", anchor);
+ } else {
+ xhtml.startElement("a");
+ }
+ }
+ }
+ }
+
+ @Override
+ public void endElement(
+ String uri, String local, String name) throws SAXException {
+ if (discardLevel != 0) {
+ discardLevel--;
+ } else if (SAFE_ELEMENTS.containsKey(name)) {
+ xhtml.endElement(SAFE_ELEMENTS.get(name));
+ } else if ("A".equals(name)) {
+ xhtml.endElement("a");
+ }
+ }
+
+ @Override
+ public void characters(char[] ch, int start, int length)
+ throws SAXException {
+ if (discardLevel == 0) {
+ super.characters(ch, start, length);
+ }
+ }
+
+ @Override
+ public void ignorableWhitespace(char[] ch, int start, int length)
+ throws SAXException {
+ if (discardLevel == 0) {
+ super.ignorableWhitespace(ch, start, length);
+ }
+ }
+
+}
\ No newline at end of file
Modified:
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java
URL:
http://svn.apache.org/viewvc/lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java?rev=825818&r1=825817&r2=825818&view=diff
==============================================================================
---
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java
(original)
+++
lucene/tika/trunk/tika-parsers/src/main/java/org/apache/tika/parser/html/HtmlParser.java
Fri Oct 16 09:03:43 2009
@@ -19,17 +19,13 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
-import java.util.Set;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.CloseShieldInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.TeeContentHandler;
-import org.apache.tika.sax.TextContentHandler;
import org.apache.tika.sax.WriteOutContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.sax.xpath.Matcher;
@@ -47,48 +43,6 @@
*/
public class HtmlParser implements Parser {
- /**
- * Set of safe mappings from incoming HTML elements to outgoing
- * XHTML elements. Ensures that the output is valid XHTML 1.0 Strict.
- */
- private static final Map<String, String> SAFE_ELEMENTS =
- new HashMap<String, String>();
-
- /**
- * Set of HTML elements whose content will be discarded.
- */
- private static final Set<String> DISCARD_ELEMENTS = new HashSet<String>();
-
- static {
- // Based on http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd
- SAFE_ELEMENTS.put("P", "p");
- SAFE_ELEMENTS.put("H1", "h1");
- SAFE_ELEMENTS.put("H2", "h2");
- SAFE_ELEMENTS.put("H3", "h3");
- SAFE_ELEMENTS.put("H4", "h4");
- SAFE_ELEMENTS.put("H5", "h5");
- SAFE_ELEMENTS.put("H6", "h6");
- SAFE_ELEMENTS.put("UL", "ul");
- SAFE_ELEMENTS.put("OL", "ol");
- SAFE_ELEMENTS.put("LI", "li");
- SAFE_ELEMENTS.put("MENU", "ul");
- SAFE_ELEMENTS.put("DL", "dl");
- SAFE_ELEMENTS.put("DT", "dt");
- SAFE_ELEMENTS.put("DD", "dd");
- SAFE_ELEMENTS.put("PRE", "pre");
- SAFE_ELEMENTS.put("BLOCKQUOTE", "blockquote");
- SAFE_ELEMENTS.put("TABLE", "table");
- SAFE_ELEMENTS.put("THEAD", "thead");
- SAFE_ELEMENTS.put("TBODY", "tbody");
- SAFE_ELEMENTS.put("TR", "tr");
- SAFE_ELEMENTS.put("TH", "th");
- SAFE_ELEMENTS.put("TD", "td");
-
- DISCARD_ELEMENTS.add("STYLE");
- DISCARD_ELEMENTS.add("SCRIPT");
-
- }
-
public void parse(
InputStream stream, ContentHandler handler,
Metadata metadata, Map<String, Object> context)
@@ -111,7 +65,7 @@
Matcher title = xpath.parse("/HTML/HEAD/TITLE//node()");
Matcher meta = xpath.parse("/HTML/HEAD/META//node()");
handler = new TeeContentHandler(
- new MatchingContentHandler(getBodyHandler(xhtml), body),
+ new MatchingContentHandler(new BodyHandler(xhtml), body),
new MatchingContentHandler(getTitleHandler(metadata), title),
new MatchingContentHandler(getMetaHandler(metadata), meta));
@@ -157,65 +111,4 @@
};
}
- private ContentHandler getBodyHandler(final XHTMLContentHandler xhtml) {
- return new TextContentHandler(xhtml) {
-
- private int discardLevel = 0;
-
- @Override
- public void startElement(
- String uri, String local, String name, Attributes atts)
- throws SAXException {
- if (discardLevel != 0) {
- discardLevel++;
- } else if (DISCARD_ELEMENTS.contains(name)) {
- discardLevel = 1;
- } else if (SAFE_ELEMENTS.containsKey(name)) {
- xhtml.startElement(SAFE_ELEMENTS.get(name));
- } else if ("A".equals(name)) {
- String href = atts.getValue("href");
- if (href != null) {
- xhtml.startElement("a", "href", href);
- } else {
- String anchor = atts.getValue("name");
- if (anchor != null) {
- xhtml.startElement("a", "name", anchor);
- } else {
- xhtml.startElement("a");
- }
- }
- }
- }
-
- @Override
- public void endElement(
- String uri, String local, String name) throws SAXException
{
- if (discardLevel != 0) {
- discardLevel--;
- } else if (SAFE_ELEMENTS.containsKey(name)) {
- xhtml.endElement(SAFE_ELEMENTS.get(name));
- } else if ("A".equals(name)) {
- xhtml.endElement("a");
- }
- }
-
- @Override
- public void characters(char[] ch, int start, int length)
- throws SAXException {
- if (discardLevel == 0) {
- super.characters(ch, start, length);
- }
- }
-
- @Override
- public void ignorableWhitespace(char[] ch, int start, int length)
- throws SAXException {
- if (discardLevel == 0) {
- super.ignorableWhitespace(ch, start, length);
- }
- }
-
- };
- }
-
}
