svn commit: r1589509 - in /jena/trunk/jena-arq: ./ src/main/java/org/apache/jena/riot/ src/main/java/org/openjena/riot/ src/test/java/org/apache/jena/riot/ src/test/java/org/apache/jena/riot/lang/

Wed, 23 Apr 2014 13:21:13 -0700 

Author: sallen
Date: Wed Apr 23 20:19:53 2014
New Revision: 1589509

URL: http://svn.apache.org/r1589509
Log:
JENA-688 Update N-Triples and N-Quad parsers to UTF-8 instead of ASCII to match 
RDF 1.1 W3C Recommendation in all cases

Modified:
    jena/trunk/jena-arq/ReleaseNotes.txt
    jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/RiotReader.java
    jena/trunk/jena-arq/src/main/java/org/openjena/riot/RiotReader.java
    jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/TestRiotReader.java
    
jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java
    
jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java

Modified: jena/trunk/jena-arq/ReleaseNotes.txt
URL: 
http://svn.apache.org/viewvc/jena/trunk/jena-arq/ReleaseNotes.txt?rev=1589509&r1=1589508&r2=1589509&view=diff
==============================================================================
--- jena/trunk/jena-arq/ReleaseNotes.txt (original)
+++ jena/trunk/jena-arq/ReleaseNotes.txt Wed Apr 23 20:19:53 2014
@@ -8,6 +8,7 @@ ChangeLog for ARQ
 + JENA-638 : Improve coverage of TopN optimziation
 + JENA-634 : Add JSON-LD
 + JENA-518 : Add getter/setter for the error handler to ReaderRIOT.
++ JENA-688 : Update N-Triples and N-Quad parsers to UTF-8 instead of ASCII to 
match RDF 1.1 W3C Recommendation in all cases
 
 ==== Jena 2.11.1
 

Modified: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/RiotReader.java
URL: 
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/RiotReader.java?rev=1589509&r1=1589508&r2=1589509&view=diff
==============================================================================
--- jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/RiotReader.java 
(original)
+++ jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/RiotReader.java Wed 
Apr 23 20:19:53 2014
@@ -391,7 +391,7 @@ public class RiotReader
     @Deprecated
     public static LangNTriples createParserNTriples(InputStream input, 
StreamRDF dest)
     {
-        Tokenizer tokenizer = TokenizerFactory.makeTokenizerASCII(input) ;
+        Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input) ;
         return createParserNTriples(tokenizer, dest) ;
     }
     
@@ -408,7 +408,7 @@ public class RiotReader
     @Deprecated
     public static LangNQuads createParserNQuads(InputStream input, StreamRDF 
dest)
     {
-        Tokenizer tokenizer = TokenizerFactory.makeTokenizerASCII(input) ;
+        Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input) ;
         return createParserNQuads(tokenizer, dest) ;
     }
     

Modified: jena/trunk/jena-arq/src/main/java/org/openjena/riot/RiotReader.java
URL: 
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/openjena/riot/RiotReader.java?rev=1589509&r1=1589508&r2=1589509&view=diff
==============================================================================
--- jena/trunk/jena-arq/src/main/java/org/openjena/riot/RiotReader.java 
(original)
+++ jena/trunk/jena-arq/src/main/java/org/openjena/riot/RiotReader.java Wed Apr 
23 20:19:53 2014
@@ -335,7 +335,7 @@ public class RiotReader
     /** Create a parser for N-Triples, with default behaviour */
     public static LangNTriples createParserNTriples(InputStream input, 
StreamRDF dest)
     {
-        Tokenizer tokenizer = TokenizerFactory.makeTokenizerASCII(input) ;
+        Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input) ;
         return createParserNTriples(tokenizer, dest) ;
     }
     
@@ -349,7 +349,7 @@ public class RiotReader
     /** Create a parser for NQuads, with default behaviour */
     public static LangNQuads createParserNQuads(InputStream input, StreamRDF 
dest)
     {
-        Tokenizer tokenizer = TokenizerFactory.makeTokenizerASCII(input) ;
+        Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(input) ;
         return createParserNQuads(tokenizer, dest) ;
     }
     

Modified: 
jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/TestRiotReader.java
URL: 
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/TestRiotReader.java?rev=1589509&r1=1589508&r2=1589509&view=diff
==============================================================================
--- jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/TestRiotReader.java 
(original)
+++ jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/TestRiotReader.java 
Wed Apr 23 20:19:53 2014
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertNot
 import static org.junit.Assert.assertTrue ;
 
 import java.io.ByteArrayInputStream ;
+import java.io.UnsupportedEncodingException ;
 import java.util.Iterator ;
 
 import org.apache.jena.atlas.lib.StrUtils ;
@@ -34,14 +35,32 @@ import com.hp.hpl.jena.graph.Triple ;
 public class TestRiotReader
 {
     @Test
-    public void testCreateIteratorTriples_01()
+    public void testCreateIteratorTriples_01() throws 
UnsupportedEncodingException
     {
-        Iterator<Triple> it = RiotReader.createIteratorTriples(new 
ByteArrayInputStream("".getBytes()), RDFLanguages.NTRIPLES, "http://example/";);
+        Iterator<Triple> it = RiotReader.createIteratorTriples(new 
ByteArrayInputStream("".getBytes("UTF-8")), RDFLanguages.NTRIPLES, 
"http://example/";);
         
         assertFalse(it.hasNext());
     }
     
     @Test
+    public void testEncodedUTF8() throws UnsupportedEncodingException
+    {
+        Iterator<Triple> it = RiotReader.createIteratorTriples(new 
ByteArrayInputStream("<a> <b> \"\\u263A\" .".getBytes("UTF-8")), 
RDFLanguages.NTRIPLES, null);
+        
+        assertTrue(it.hasNext());
+        assertEquals("☺", it.next().getObject().getLiteralLexicalForm());
+    }
+    
+    @Test
+    public void testRawUTF8() throws UnsupportedEncodingException
+    {
+        Iterator<Triple> it = RiotReader.createIteratorTriples(new 
ByteArrayInputStream("<a> <b> \"☺\" .".getBytes("UTF-8")), 
RDFLanguages.NTRIPLES, null);
+        
+        assertTrue(it.hasNext());
+        assertEquals("☺", it.next().getObject().getLiteralLexicalForm());
+    }
+    
+    @Test
     public void testCreateIteratorTriples_02()
     {
         String x = StrUtils.strjoinNL(

Modified: 
jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java
URL: 
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java?rev=1589509&r1=1589508&r2=1589509&view=diff
==============================================================================
--- 
jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java
 (original)
+++ 
jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTriples.java
 Wed Apr 23 20:19:53 2014
@@ -23,7 +23,6 @@ import java.io.StringReader ;
 import org.apache.jena.atlas.lib.StrUtils ;
 import org.apache.jena.riot.RDFLanguages ;
 import org.apache.jena.riot.RDFDataMgr ;
-import org.apache.jena.riot.RiotException ;
 import org.apache.jena.riot.RiotReader ;
 import org.apache.jena.riot.ErrorHandlerTestLib.ErrorHandlerEx ;
 import org.apache.jena.riot.ErrorHandlerTestLib.ExFatal ;
@@ -84,7 +83,7 @@ public class TestLangNTriples extends Te
         parseCount("@base <http://example/> . <x> <p> <s> .") ; 
     }
 
-    @Test(expected=RiotException.class) 
+    @Test
     public void nt_only_5()
     {
         parseCount("<x> <p> \"é\" .") ; 

Modified: 
jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java
URL: 
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java?rev=1589509&r1=1589508&r2=1589509&view=diff
==============================================================================
--- 
jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java
 (original)
+++ 
jena/trunk/jena-arq/src/test/java/org/apache/jena/riot/lang/TestLangNTuples.java
 Wed Apr 23 20:19:53 2014
@@ -151,7 +151,7 @@ abstract public class TestLangNTuples ex
         parseCheck("<http://example/x\\u00E9> <http://example/p> 
<http://example/s> .") ; 
     }
     
-    @Test(expected=RiotException.class) 
+    @Test
     public void tuple_charset_2()
     {
         parseCheck("<http://example/é> <http://example/p> \"é\" .") ; 
@@ -159,11 +159,10 @@ abstract public class TestLangNTuples ex
     
     static protected Tokenizer tokenizer(String string)
     {
-        //Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(string) ;
-        // ASCII
+        // UTF-8
         byte b[] = StrUtils.asUTF8bytes(string) ;
         ByteArrayInputStream in = new ByteArrayInputStream(b) ;
-        Tokenizer tokenizer = TokenizerFactory.makeTokenizerASCII(in) ;
+        Tokenizer tokenizer = TokenizerFactory.makeTokenizerUTF8(in) ;
         return tokenizer ;
     }

Reply via email to