Author: siren
Date: Sun Oct 14 04:05:16 2007
New Revision: 584528
URL: http://svn.apache.org/viewvc?rev=584528&view=rev
Log:
TIKA-58 - Replace jtidy html parser with nekohtml based parser
Added:
incubator/tika/trunk/src/main/java/org/apache/tika/parser/html/HtmlParser.java
Modified:
incubator/tika/trunk/CHANGES.txt
incubator/tika/trunk/pom.xml
Modified: incubator/tika/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/tika/trunk/CHANGES.txt?rev=584528&r1=584527&r2=584528&view=diff
==============================================================================
--- incubator/tika/trunk/CHANGES.txt (original)
+++ incubator/tika/trunk/CHANGES.txt Sun Oct 14 04:05:16 2007
@@ -97,3 +97,5 @@
44. TIKA-62 - Use TikaConfig.getDefaultConfig() instead of a hardcoded
config path in TestParsers (jukka)
+
+45. TIKA-58 - Replace jtidy html parser with nekohtml based parser (siren)
Modified: incubator/tika/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/tika/trunk/pom.xml?rev=584528&r1=584527&r2=584528&view=diff
==============================================================================
--- incubator/tika/trunk/pom.xml (original)
+++ incubator/tika/trunk/pom.xml Sun Oct 14 04:05:16 2007
@@ -189,9 +189,9 @@
<version>2.0.8</version>
</dependency>
<dependency>
- <groupId>jtidy</groupId>
- <artifactId>jtidy</artifactId>
- <version>4aug2000r7-dev</version>
+ <groupId>nekohtml</groupId>
+ <artifactId>nekohtml</artifactId>
+ <version>0.9.5</version>
</dependency>
<dependency>
<groupId>com.ibm.icu</groupId>
Added:
incubator/tika/trunk/src/main/java/org/apache/tika/parser/html/HtmlParser.java
URL:
http://svn.apache.org/viewvc/incubator/tika/trunk/src/main/java/org/apache/tika/parser/html/HtmlParser.java?rev=584528&view=auto
==============================================================================
---
incubator/tika/trunk/src/main/java/org/apache/tika/parser/html/HtmlParser.java
(added)
+++
incubator/tika/trunk/src/main/java/org/apache/tika/parser/html/HtmlParser.java
Sun Oct 14 04:05:16 2007
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.html;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ContentHandlerDecorator;
+import org.apache.tika.parser.Parser;
+import org.cyberneko.html.parsers.SAXParser;
+import org.xml.sax.Attributes;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+/**
+ * Simple HTML parser implemented with NekoHTML.
+ */
+public class HtmlParser implements Parser {
+
+ public void parse(InputStream stream, ContentHandler handler,
+ Metadata metadata) throws IOException, SAXException, TikaException
{
+ final SAXParser parser = new SAXParser();
+ final InputSource source = new InputSource(stream);
+ parser.setContentHandler(new TitleExtractingContentHandler(handler,
+ metadata));
+ parser.parse(source);
+ }
+
+ private static class TitleExtractingContentHandler extends
+ ContentHandlerDecorator {
+
+ private static final String TAG_TITLE = "TITLE";
+
+ private static final String TAG_HEAD = "HEAD";
+
+ private static final String TAG_HTML = "HTML";
+
+ private Phase phase = Phase.START;
+
+ private Metadata metadata;
+
+ private StringBuilder title = new StringBuilder();
+
+ private static enum Phase {
+ START, HTML, HEAD, TITLE, IGNORE;
+ }
+
+ public TitleExtractingContentHandler(final ContentHandler handler,
+ final Metadata metadata) {
+ super(handler);
+ this.metadata = metadata;
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name,
+ Attributes atts) throws SAXException {
+
+ switch (phase) {
+ case START:
+ if (TAG_HTML.equals(localName)) {
+ phase = Phase.HTML;
+ }
+ break;
+ case HTML:
+ if (TAG_HEAD.equals(localName)) {
+ phase = Phase.HEAD;
+ }
+ break;
+ case HEAD:
+ if (TAG_TITLE.equals(localName)) {
+ phase = Phase.TITLE;
+ }
+ break;
+ }
+ super.startElement(uri, localName, name, atts);
+ }
+
+ @Override
+ public void characters(char[] ch, int start, int length)
+ throws SAXException {
+ switch (phase) {
+ case TITLE:
+ title.append(ch, start, length);
+ break;
+ }
+ super.characters(ch, start, length);
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name)
+ throws SAXException {
+ switch (phase) {
+ case TITLE:
+ if (TAG_TITLE.equals(localName)) {
+ phase = Phase.IGNORE;
+ }
+ break;
+ }
+ super.endElement(uri, localName, name);
+ }
+
+ @Override
+ public void endDocument() throws SAXException {
+ metadata.set(Metadata.TITLE, title.toString());
+ super.endDocument();
+ }
+ }
+}
