Revision: 16513
http://sourceforge.net/p/gate/code/16513
Author: adamfunk
Date: 2013-02-08 11:46:56 +0000 (Fri, 08 Feb 2013)
Log Message:
-----------
split on newlines
Modified Paths:
--------------
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/JSONTweetFormat.java
Modified:
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/JSONTweetFormat.java
===================================================================
---
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/JSONTweetFormat.java
2013-02-08 02:19:17 UTC (rev 16512)
+++
gate/branches/format-twitter-wip/plugins/Format_Twitter/src/gate/corpora/JSONTweetFormat.java
2013-02-08 11:46:56 UTC (rev 16513)
@@ -13,12 +13,10 @@
*/
package gate.corpora;
-import java.io.IOException;
-import java.io.StringReader;
+import java.io.*;
import java.util.*;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.databind.*;
import gate.*;
import gate.corpora.DocumentContentImpl;
import gate.corpora.MimeType;
@@ -44,6 +42,8 @@
public class JSONTweetFormat extends TextualDocumentFormat {
private static final long serialVersionUID = 6878020036304333918L;
+ public static final String TEXT_ATTRIBUTE = "text";
+
/** Default construction */
public JSONTweetFormat() { super();}
@@ -73,31 +73,10 @@
setNewLineProperty(doc);
String jsonString = doc.getContent().toString();
- StringReader reader = new StringReader(jsonString) {
- public void close(){}; };
-
try {
- ObjectMapper mapper = new ObjectMapper();
- List<Tweet> tweets = new ArrayList<Tweet>();
- JsonNode jnode;
- int i = 0;
-
- while ((jnode = mapper.readTree(reader)) != null) {
- System.out.println("Item " + i + "\n" + jnode.toString());
-
- // This could be a list of tweet objects
- if (jnode.isArray()) {
- for (JsonNode item : jnode) {
- tweets.add(new Tweet(item));
- }
- }
-
- // or it could be a single tweet object
- else if (jnode.isObject()) {
- tweets.add(new Tweet(jnode));
- }
- }
-
+ // Parse the String
+ List<Tweet> tweets = readTweets(jsonString);
+
// Put them all together to make the unpacked document content
StringBuilder concatenation = new StringBuilder();
for (Tweet tweet : tweets) {
@@ -105,7 +84,6 @@
concatenation.append(tweet.getString()).append("\n\n");
}
-
// Set new document content
DocumentContent newContent = new
DocumentContentImpl(concatenation.toString());
doc.edit(0L, doc.getContent().size(), newContent);
@@ -123,7 +101,53 @@
throw new DocumentFormatException(e);
}
}
+
+
+ private List<Tweet> readTweets(String string) throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ //JsonFactory jfactory = new JsonFactory();
+ //JsonParser parser = jfactory.createJsonParser(reader);
+ List<Tweet> tweets = new ArrayList<Tweet>();
+ int i = 0;
+ // just not null, so we can use it in the loop
+ //Tweet tweet = new Tweet();
+ String[] lines = string.split("[\\n\\r]+");
+ for (String line : lines) {
+ System.out.println("Item " + i + "\n" + line);
+ if (line.length() > 0) {
+ JsonNode jnode = mapper.readTree(line);
+ tweets.add(new Tweet(jnode));
+ }
+ i++;
+ }
+
+
+
+// while (tweet != null) {
+// String currentName = parser.getCurrentName();
+// JsonToken token = parser.nextToken();
+//
+//
+// //! finished && ( (jnode = mapper.readTree(reader)) != null) ) {
+// System.out.println("Item " + i + "\n" + jnode.toString());
+//
+// // This could be a list of tweet objects
+// if (jnode.isArray()) {
+// for (JsonNode item : jnode) {
+// tweets.add(new Tweet(item));
+// }
+// }
+//
+// // or it could be a single tweet object
+// else if (jnode.isObject()) {
+// tweets.add(new Tweet(jnode));
+// }
+// }
+
+ return tweets;
+ }
+
}
@@ -172,6 +196,12 @@
}
}
}
+
+
+ public Tweet() {
+ string = "";
+ features = Factory.newFeatureMap();
+ }
private Object process(JsonNode node) {
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Free Next-Gen Firewall Hardware Offer
Buy your Sophos next-gen firewall before the end March 2013
and get the hardware for free! Learn more.
http://p.sf.net/sfu/sophos-d2d-feb
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
