Revision: 16511
http://sourceforge.net/p/gate/code/16511
Author: adamfunk
Date: 2013-02-07 18:14:30 +0000 (Thu, 07 Feb 2013)
Log Message:
-----------
Branch for Format_Twitter plugin...
Modified Paths:
--------------
gate/branches/format-twitter-wip/.classpath
gate/branches/format-twitter-wip/build.xml
gate/branches/format-twitter-wip/plugins/Tagger_Framework/resources/TreeTagger/tree-tagger-german-gate
gate/branches/format-twitter-wip/plugins/Tagger_Framework/resources/TreeTagger/tree-tagger-spanish-gate
Added Paths:
-----------
gate/branches/format-twitter-wip/plugins/Format_Twitter/
gate/branches/format-twitter-wip/plugins/Format_Twitter/build.xml
gate/branches/format-twitter-wip/plugins/Format_Twitter/creole.xml
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-annotations-2.1.1.jar
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-core-2.1.1.jar
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-databind-2.1.1.jar
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/JSONTweetFormat.java
Modified: gate/branches/format-twitter-wip/.classpath
===================================================================
--- gate/branches/format-twitter-wip/.classpath 2013-02-07 18:12:48 UTC (rev
16510)
+++ gate/branches/format-twitter-wip/.classpath 2013-02-07 18:14:30 UTC (rev
16511)
@@ -32,6 +32,7 @@
<classpathentry kind="src" path="plugins/Language_Identification/src"/>
<classpathentry kind="src" output="plugins/Tagger_Numbers/classes"
path="plugins/Tagger_Numbers/src"/>
<classpathentry kind="src" output="plugins/TermRaider/classes"
path="plugins/TermRaider/src"/>
+ <classpathentry kind="src" output="plugins/Format_Twitter/classes"
path="plugins/Format_Twitter/src"/>
<classpathentry kind="lib"
path="plugins/Gazetteer_Ontology_Based/lib/oro-2.0.8.jar"/>
<classpathentry kind="lib" path="plugins/WordNet/lib/jwnl.jar"/>
<classpathentry kind="lib"
path="plugins/LingPipe/lib/lingpipe-4.0.1.jar"/>
@@ -40,6 +41,8 @@
<classpathentry kind="lib"
path="plugins/Ontology/lib/slf4j-api-1.5.6.jar"/>
<classpathentry kind="lib"
path="plugins/Ontology/lib/slf4j-jdk14-1.5.6.jar"/>
<classpathentry kind="lib"
path="plugins/OpenNLP/lib/opennlp-tools-1.5.2-incubating.jar"/>
+ <classpathentry kind="lib"
path="plugins/Format_Twitter/lib/jackson-core-2.1.1.jar"/>
+ <classpathentry kind="lib"
path="plugins/Format_Twitter/lib/jackson-databind-2.1.1.jar"/>
<classpathentry kind="con"
path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="plugins/Learning/lib/weka.jar"/>
<classpathentry kind="lib"
path="plugins/Web_Crawler_Websphinx/lib/websphinx.jar"/>
Modified: gate/branches/format-twitter-wip/build.xml
===================================================================
--- gate/branches/format-twitter-wip/build.xml 2013-02-07 18:12:48 UTC (rev
16510)
+++ gate/branches/format-twitter-wip/build.xml 2013-02-07 18:14:30 UTC (rev
16511)
@@ -364,7 +364,8 @@
<!-- Learning plugin needs to be before Lang_Chinese -->
<filelist id="plugins.to.build" dir="plugins" files="
Alignment Annotation_Merging Copy_Annots_Between_Docs Coref_Tools
Format_MediaWiki
- Format_PubMed Gazetteer_LKB Gazetteer_Ontology_Based GENIA Groovy
+ Format_PubMed Format_Twitter
+ Gazetteer_LKB Gazetteer_Ontology_Based GENIA Groovy
Inter_Annotator_Agreement JAPE_Plus Keyphrase_Extraction_Algorithm
Lang_Arabic Lang_Cebuano Learning Lang_Chinese Lang_French Lang_German
Lang_Hindi Lang_Romanian Language_Identification LingPipe
Index: gate/branches/format-twitter-wip/plugins/Format_Twitter
===================================================================
--- gate/branches/format-twitter-wip/plugins/Format_Twitter 2013-02-07
18:12:48 UTC (rev 16510)
+++ gate/branches/format-twitter-wip/plugins/Format_Twitter 2013-02-07
18:14:30 UTC (rev 16511)
Property changes on: gate/branches/format-twitter-wip/plugins/Format_Twitter
___________________________________________________________________
Added: svn:ignore
## -0,0 +1,3 ##
+classes
+doc
+format-twitter.jar
Copied: gate/branches/format-twitter-wip/plugins/Format_Twitter/build.xml (from
rev 16506, gate/trunk/plugins/Format_MediaWiki/build.xml)
===================================================================
--- gate/branches/format-twitter-wip/plugins/Format_Twitter/build.xml
(rev 0)
+++ gate/branches/format-twitter-wip/plugins/Format_Twitter/build.xml
2013-02-07 18:14:30 UTC (rev 16511)
@@ -0,0 +1,85 @@
+<project name="Format_MediaWiki" basedir="." default="build">
+ <!-- Prevent Ant from warning about includeantruntime not being set -->
+ <property name="build.sysclasspath" value="ignore" />
+
+ <property file="build.properties" />
+
+ <property name="gate.home" location="../.." />
+ <property name="gate.lib" location="${gate.home}/lib" />
+ <property name="gate.jar" location="${gate.home}/bin/gate.jar" />
+ <property name="src.dir" location="src" />
+ <property name="classes.dir" location="classes" />
+ <property name="jar.location" location="format-twitter.jar" />
+ <property name="lib" location="lib" />
+ <property name="doc.dir" location="doc" />
+ <property name="javadoc.dir" location="${doc.dir}/javadoc" />
+
+ <!-- Path to compile - includes gate.jar and GATE/lib/*.jar -->
+ <path id="compile.classpath">
+ <fileset dir="lib">
+ <include name="**/*.jar" />
+ </fileset>
+ <pathelement location="${gate.jar}" />
+ <fileset dir="${gate.lib}">
+ <include name="**/*.jar" />
+ <include name="**/*.zip" />
+ </fileset>
+ </path>
+
+ <!-- create build directory structure -->
+ <target name="prepare">
+ <mkdir dir="${classes.dir}" />
+ </target>
+
+ <target name="resources" depends="prepare">
+ <!-- <copy todir="${classes.dir}/gate/resources" includeEmptyDirs="true">
+ <fileset dir="${src.dir}/gate/resources" />
+ </copy> -->
+ </target>
+
+ <!-- compile the source -->
+ <target name="compile" depends="prepare, resources">
+ <javac classpathref="compile.classpath" srcdir="${src.dir}"
destdir="${classes.dir}" debug="true" debuglevel="lines,source" source="1.5"
target="1.5">
+ </javac>
+ </target>
+
+ <!-- create the JAR file -->
+ <target name="jar" depends="compile">
+ <jar destfile="${jar.location}" update="false" basedir="${classes.dir}" />
+ </target>
+
+ <!-- remove the generated .class files -->
+ <target name="clean.classes">
+ <delete dir="${classes.dir}" />
+ </target>
+
+ <!-- Clean up - remove .class and .jar files -->
+ <target name="clean" depends="clean.classes">
+ <delete file="${jar.location}" />
+ </target>
+
+ <!-- Build JavaDoc documentation -->
+ <target name="doc.prepare">
+ <mkdir dir="${javadoc.dir}" />
+ </target>
+
+ <target name="javadoc" depends="jar, doc.prepare">
+ <javadoc destdir="${javadoc.dir}" packagenames="*"
classpathref="compile.classpath" encoding="UTF-8" windowtitle="MediaWiki
Document Format JavaDoc" source="1.6">
+ <sourcepath>
+ <pathelement location="${src.dir}" />
+ </sourcepath>
+ <link href="http://docs.oracle.com/javase/6/docs/api/"; />
+ <link href="http://gate.ac.uk/gate/doc/javadoc/"; />
+ </javadoc>
+ </target>
+
+ <!-- Targets used by the main GATE build file:
+ build: build the plugin - just calls "jar" target
+ test : run the unit tests - there aren't any
+ distro.prepare: remove intermediate files that shouldn't be in the
+ distribution
+ -->
+ <target name="build" depends="jar" />
+ <target name="test" />
+ <target name="distro.prepare" depends="clean.classes" />
+</project>
Property changes on:
gate/branches/format-twitter-wip/plugins/Format_Twitter/build.xml
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Copied: gate/branches/format-twitter-wip/plugins/Format_Twitter/creole.xml
(from rev 16506, gate/trunk/plugins/Format_MediaWiki/creole.xml)
===================================================================
--- gate/branches/format-twitter-wip/plugins/Format_Twitter/creole.xml
(rev 0)
+++ gate/branches/format-twitter-wip/plugins/Format_Twitter/creole.xml
2013-02-07 18:14:30 UTC (rev 16511)
@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<CREOLE-DIRECTORY>
+ <JAR SCAN="true">format-twitter.jar</JAR>
+ <JAR>lib/jackson-databind-2.1.1.jar</JAR>
+ <JAR>lib/jackson-core-2.1.1.jar</JAR>
+ <JAR>lib/jackson-annotations-2.1.1.jar</JAR>
+</CREOLE-DIRECTORY>
Property changes on:
gate/branches/format-twitter-wip/plugins/Format_Twitter/creole.xml
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added:
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-annotations-2.1.1.jar
===================================================================
(Binary files differ)
Index:
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-annotations-2.1.1.jar
===================================================================
---
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-annotations-2.1.1.jar
2013-02-07 18:12:48 UTC (rev 16510)
+++
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-annotations-2.1.1.jar
2013-02-07 18:14:30 UTC (rev 16511)
Property changes on:
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-annotations-2.1.1.jar
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added:
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-core-2.1.1.jar
===================================================================
(Binary files differ)
Index:
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-core-2.1.1.jar
===================================================================
---
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-core-2.1.1.jar
2013-02-07 18:12:48 UTC (rev 16510)
+++
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-core-2.1.1.jar
2013-02-07 18:14:30 UTC (rev 16511)
Property changes on:
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-core-2.1.1.jar
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added:
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-databind-2.1.1.jar
===================================================================
(Binary files differ)
Index:
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-databind-2.1.1.jar
===================================================================
---
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-databind-2.1.1.jar
2013-02-07 18:12:48 UTC (rev 16510)
+++
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-databind-2.1.1.jar
2013-02-07 18:14:30 UTC (rev 16511)
Property changes on:
gate/branches/format-twitter-wip/plugins/Format_Twitter/lib/jackson-databind-2.1.1.jar
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added:
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/JSONTweetFormat.java
===================================================================
---
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/JSONTweetFormat.java
(rev 0)
+++
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/JSONTweetFormat.java
2013-02-07 18:14:30 UTC (rev 16511)
@@ -0,0 +1,220 @@
+/*
+ * JSONTweetFormat.java
+ *
+ * Copyright (c) 1995-2012, The University of Sheffield. See the file
+ * COPYRIGHT.txt in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt
+ *
+ * This file is part of GATE (see http://gate.ac.uk/), and is free
+ * software, licenced under the GNU Library General Public License,
+ * Version 2, June 1991 (in the distribution as file licence.html,
+ * and also available at http://gate.ac.uk/gate/licence.html).
+ *
+ * $Id$
+ */
+package gate.corpora;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.*;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import gate.*;
+import gate.corpora.DocumentContentImpl;
+import gate.corpora.MimeType;
+import gate.corpora.TextualDocumentFormat;
+import gate.creole.ResourceInstantiationException;
+import gate.creole.metadata.*;
+import gate.util.DocumentFormatException;
+import gate.util.InvalidOffsetException;
+
+
+// JSON API
+// http://json-lib.sourceforge.net/apidocs/jdk15/index.html
+
+// Standard: RFC 4627
+// https://tools.ietf.org/html/rfc4627
+
+/** Document format for handling JSON tweets: either one
+ * object {...} or a list [{tweet...}, {tweet...}, ...].
+ */
+@CreoleResource(name = "GATE JSON Tweet Document Format", isPrivate = true,
+ autoinstances = {@AutoInstance(hidden = true)})
+
+public class JSONTweetFormat extends TextualDocumentFormat {
+ private static final long serialVersionUID = 6878020036304333918L;
+
+ /** Default construction */
+ public JSONTweetFormat() { super();}
+
+ /** Initialise this resource, and return it. */
+ public Resource init() throws ResourceInstantiationException{
+ // Register ad hoc MIME-type
+ // There is an application/json mime type, but I don't think
+ // we want everything to be handled this way?
+ MimeType mime = new MimeType("text","x-json-twitter");
+ // Register the class handler for this MIME-type
+ mimeString2ClassHandlerMap.put(mime.getType()+ "/" + mime.getSubtype(),
this);
+ // Register the mime type with string
+ mimeString2mimeTypeMap.put(mime.getType() + "/" + mime.getSubtype(), mime);
+ // Register file suffixes for this mime type
+ suffixes2mimeTypeMap.put("json", mime);
+ // Register magic numbers for this mime type
+ //magic2mimeTypeMap.put("Subject:",mime);
+ // Set the mimeType for this language resource
+ setMimeType(mime);
+ return this;
+ }
+
+ public void unpackMarkup(gate.Document doc) throws DocumentFormatException{
+ if ( (doc == null) || (doc.getSourceUrl() == null && doc.getContent() ==
null) ) {
+ throw new DocumentFormatException("GATE document is null or no content
found. Nothing to parse!");
+ }
+
+ setNewLineProperty(doc);
+ String jsonString = doc.getContent().toString();
+ StringReader reader = new StringReader(jsonString) {
+ public void close(){}; };
+
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ List<Tweet> tweets = new ArrayList<Tweet>();
+ JsonNode jnode;
+ int i = 0;
+
+ while ((jnode = mapper.readTree(reader)) != null) {
+ System.out.println("Item " + i + "\n" + jnode.toString());
+
+ // This could be a list of tweet objects
+ if (jnode.isArray()) {
+ for (JsonNode item : jnode) {
+ tweets.add(new Tweet(item));
+ }
+ }
+
+ // or it could be a single tweet object
+ else if (jnode.isObject()) {
+ tweets.add(new Tweet(jnode));
+ }
+ }
+
+ // Put them all together to make the unpacked document content
+ StringBuilder concatenation = new StringBuilder();
+ for (Tweet tweet : tweets) {
+ tweet.setStart(concatenation.length());
+ concatenation.append(tweet.getString()).append("\n\n");
+ }
+
+
+ // Set new document content
+ DocumentContent newContent = new
DocumentContentImpl(concatenation.toString());
+ doc.edit(0L, doc.getContent().size(), newContent);
+
+ AnnotationSet originalMarkups =
doc.getAnnotations(GateConstants.ORIGINAL_MARKUPS_ANNOT_SET_NAME);
+ // Create Original markups annotations for each tweet
+ for (Tweet tweet : tweets) {
+ originalMarkups.add(tweet.getStart(), tweet.getEnd(), "Tweet",
tweet.getFeatures());
+ }
+ }
+ catch (InvalidOffsetException e) {
+ throw new DocumentFormatException(e);
+ }
+ catch(IOException e) {
+ throw new DocumentFormatException(e);
+ }
+ }
+
+}
+
+
+class Tweet {
+ private String string;
+ private FeatureMap features;
+ private long start;
+
+ public int getLength() {
+ return this.string.length();
+ }
+
+ public String getString() {
+ return this.string;
+ }
+
+ public FeatureMap getFeatures() {
+ return this.features;
+ }
+
+ public void setStart(long start) {
+ this.start = start;
+ }
+
+ public long getStart() {
+ return this.start;
+ }
+
+ public long getEnd() {
+ return this.start + this.string.length();
+ }
+
+
+ public Tweet(JsonNode json) {
+ string = "";
+ Iterator<String> keys = json.fieldNames();
+ features = Factory.newFeatureMap();
+
+ while (keys.hasNext()) {
+ String key = keys.next();
+ if (key.equals("text")) {
+ string = json.get(key).toString();
+ }
+ else {
+ features.put(key.toString(), process(json.get(key)));
+ }
+ }
+ }
+
+
+ private Object process(JsonNode node) {
+ /* JSON types: number, string, boolean, array, object (dict/map),
+ * null. All map keys are strings.
+ */
+
+ if (node.isBoolean()) {
+ return node.asBoolean();
+ }
+ if (node.isDouble()) {
+ return node.asDouble();
+ }
+ if (node.isInt()) {
+ return node.asInt();
+ }
+ if (node.isTextual()) {
+ return node.asText();
+ }
+
+ if (node.isNull()) {
+ return null;
+ }
+
+ if (node.isArray()) {
+ List<Object> list = new ArrayList<Object>();
+ for (JsonNode item : node) {
+ list.add(process(item));
+ }
+ return list;
+ }
+
+ if (node.isObject()) {
+ Map<String, Object> map = new HashMap<String, Object>();
+ Iterator<String> keys = node.fieldNames();
+ while (keys.hasNext()) {
+ String key = keys.next();
+ map.put(key, process(node.get(key)));
+ }
+ return map;
+ }
+
+ return node.toString();
+ }
+
+}
Property changes on:
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/JSONTweetFormat.java
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+Id
\ No newline at end of property
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Modified:
gate/branches/format-twitter-wip/plugins/Tagger_Framework/resources/TreeTagger/tree-tagger-german-gate
===================================================================
---
gate/branches/format-twitter-wip/plugins/Tagger_Framework/resources/TreeTagger/tree-tagger-german-gate
2013-02-07 18:12:48 UTC (rev 16510)
+++
gate/branches/format-twitter-wip/plugins/Tagger_Framework/resources/TreeTagger/tree-tagger-german-gate
2013-02-07 18:14:30 UTC (rev 16511)
@@ -9,9 +9,9 @@
# spaces. You should install the tree tagger in a directory that does not
# contain spaces in its path (i.e. not under "Program Files").
-BIN=/usr/local/durmtools/TreeTagger/bin
-CMD=/usr/local/durmtools/TreeTagger/cmd
-LIB=/usr/local/durmtools/TreeTagger/lib
+BIN=/opt/adam/treetagger/bin
+CMD=/opt/adam/treetagger/cmd
+LIB=/opt/adam/treetagger/lib
TAGGER=${BIN}/tree-tagger
ABBR_LIST=${LIB}/german-abbreviations
Modified:
gate/branches/format-twitter-wip/plugins/Tagger_Framework/resources/TreeTagger/tree-tagger-spanish-gate
===================================================================
---
gate/branches/format-twitter-wip/plugins/Tagger_Framework/resources/TreeTagger/tree-tagger-spanish-gate
2013-02-07 18:12:48 UTC (rev 16510)
+++
gate/branches/format-twitter-wip/plugins/Tagger_Framework/resources/TreeTagger/tree-tagger-spanish-gate
2013-02-07 18:14:30 UTC (rev 16511)
@@ -9,9 +9,9 @@
# spaces. You should install the tree tagger in a directory that does not
# contain spaces in its path (i.e. not under "Program Files").
-BIN=/usr/local/durmtools/TreeTagger/bin
-CMD=/usr/local/durmtools/TreeTagger/cmd
-LIB=/usr/local/durmtools/TreeTagger/lib
+BIN=/opt/adam/treetagger/bin
+CMD=/opt/adam/treetagger/cmd
+LIB=/opt/adam/treetagger/lib
MWL=${CMD}/mwl-lookup.perl
TAGGER=${BIN}/tree-tagger
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Free Next-Gen Firewall Hardware Offer
Buy your Sophos next-gen firewall before the end March 2013
and get the hardware for free! Learn more.
http://p.sf.net/sfu/sophos-d2d-feb
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
