Author: andy
Date: Mon May 19 13:30:24 2014
New Revision: 1595895
URL: http://svn.apache.org/r1595895
Log:
JENA-696 -- a basic CSV parser in support for SPARQL CSV Results.
Added:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParseException.java
(with props)
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParser.java
(with props)
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVToken.java
(with props)
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenIterator.java
(with props)
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenType.java
(with props)
jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/csv/
jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/csv/TestCSVParser.java
(with props)
Modified:
jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/engine/binding/BindingBase.java
jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/resultset/CSVInput.java
jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/TC_Atlas.java
Modified:
jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/engine/binding/BindingBase.java
URL:
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/engine/binding/BindingBase.java?rev=1595895&r1=1595894&r2=1595895&view=diff
==============================================================================
---
jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/engine/binding/BindingBase.java
(original)
+++
jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/engine/binding/BindingBase.java
Mon May 19 13:30:24 2014
@@ -72,6 +72,7 @@ abstract public class BindingBase implem
@Override
final public Iterator<Var> vars()
{
+ // Hidesight - replace with accumulator style vars1(accumulator)
Iterator<Var> iter = vars1() ;
if ( parent != null )
iter = IteratorConcat.concat(parent.vars(), iter ) ;
Modified:
jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/resultset/CSVInput.java
URL:
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/resultset/CSVInput.java?rev=1595895&r1=1595894&r2=1595895&view=diff
==============================================================================
---
jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/resultset/CSVInput.java
(original)
+++
jena/trunk/jena-arq/src/main/java/com/hp/hpl/jena/sparql/resultset/CSVInput.java
Mon May 19 13:30:24 2014
@@ -18,18 +18,27 @@
package com.hp.hpl.jena.sparql.resultset;
-import java.io.BufferedReader ;
-import java.io.IOException ;
import java.io.InputStream ;
import java.util.ArrayList ;
+import java.util.Iterator ;
import java.util.List ;
-import org.apache.jena.atlas.io.IO ;
+import org.apache.jena.atlas.csv.CSVParser ;
+import org.apache.jena.atlas.iterator.Iter ;
+import org.apache.jena.atlas.iterator.Transform ;
+import org.apache.jena.atlas.logging.FmtLog ;
+import org.slf4j.Logger ;
+import org.slf4j.LoggerFactory ;
+import com.hp.hpl.jena.graph.Node ;
+import com.hp.hpl.jena.graph.NodeFactory ;
import com.hp.hpl.jena.query.ResultSet ;
import com.hp.hpl.jena.sparql.ARQException ;
import com.hp.hpl.jena.sparql.core.Var ;
import com.hp.hpl.jena.sparql.engine.ResultSetStream ;
+import com.hp.hpl.jena.sparql.engine.binding.Binding ;
+import com.hp.hpl.jena.sparql.engine.binding.BindingFactory ;
+import com.hp.hpl.jena.sparql.engine.binding.BindingMap ;
/** Convenient comma separated values - see also TSV (tab separated values)
* which outputs full RDF terms (in Turtle-style).
@@ -55,80 +64,78 @@ import com.hp.hpl.jena.sparql.engine.Res
public class CSVInput
{
// This code exists to support the SPARQL WG tests.
-
+ private static Logger log = LoggerFactory.getLogger(CSVInput.class) ;
public static ResultSet fromCSV(InputStream in)
{
- BufferedReader reader = IO.asBufferedUTF8(in);
- List<Var> vars = new ArrayList<Var>();
- List<String> varNames = new ArrayList<String>();
-
- String str = null;
- try
- {
- //Here we try to parse only the Header Row
- str = reader.readLine();
- if (str == null )
- throw new ARQException("CSV Results malformed, input is empty
(no header row)") ;
-
- if ( ! str.isEmpty() )
- {
- String[] tokens = str.split(",") ;
- for ( String token : tokens )
- {
- // It is perfectly valid for a variable name to be
enclosed in quotes as part of CSV encoding even
- // though legal variable names should not contain a
character which would require this
- if (token.startsWith("\"") && token.endsWith("\""))
token = token.substring(1, token.length() - 1);
-
- Var var = Var.alloc(token);
- vars.add(var);
- varNames.add(var.getName());
+ CSVParser parser = CSVParser.create(in) ;
+ final List<Var> vars = vars(parser) ;
+ List<String> varNames = Var.varNames(vars) ;
+ Transform<List<String>, Binding> transform = new
Transform<List<String>, Binding>(){
+ private int count = 1 ;
+ @Override
+ public Binding convert(List<String> row) {
+ if ( row.size() != vars.size() )
+ FmtLog.warn(log, "Row %d: Length=%d: expected=%d", count,
row.size(), vars.size()) ;
+
+ BindingMap binding = BindingFactory.create() ;
+ // Check.
+ for (int i = 0 ; i < vars.size() ; i++ ) {
+ Var v = vars.get(i) ;
+ String field = (i<row.size()) ? row.get(i) : "" ;
+ Node n = NodeFactory.createLiteral(field) ;
+ binding.add(v, n);
}
- }
- }
- catch ( IOException ex )
- {
- throw new ARQException(ex) ;
- }
-
+ count++ ;
+ return binding ;
+ }} ;
+ Iterator<Binding> bindings = Iter.map(parser, transform) ;
+
//Generate an instance of ResultSetStream using TSVInputIterator
//This will parse actual result rows as needed thus minimising memory
usage
- return new ResultSetStream(varNames, null, new
CSVInputIterator(reader, vars));
+ return new ResultSetStream(varNames, null, bindings);
+ }
+
+ private static List<Var> vars(CSVParser parser) {
+ final List<Var> vars = new ArrayList<Var>();
+ List<String> varNames = parser.parse1() ;
+ if ( varNames == null )
+ throw new ARQException("SPARQL CSV Results malformed, input is
empty");
+ for ( String vn : varNames ) {
+ vars.add(Var.alloc(vn)) ;
+ }
+ return vars ;
}
public static boolean booleanFromCSV(InputStream in)
{
- BufferedReader reader = IO.asBufferedUTF8(in);
- String str = null;
- try
- {
- //First try to parse the header
- str = reader.readLine();
- if (str == null) throw new ARQException("CSV Boolean Results
malformed, input is empty");
- str = str.trim(); //Remove extraneous white space
-
- //Expect a header row with single _askResult variable
- if (!str.equals("_askResult")) throw new ARQException("CSV
Boolean Results malformed, did not get expected ?_askResult header row");
-
- //Then try to parse the boolean result
- str = reader.readLine();
- if (str == null) throw new ARQException("CSV Boolean Results
malformed, unexpected end of input after header row");
- str = str.trim();
-
- // It is perfectly valid for a variable name to be enclosed in
quotes as part of CSV encoding even
- // though legal variable names should not contain a character
which would require this
- if (str.startsWith("\"") && str.endsWith("\"")) str =
str.substring(1, str.length() - 1);
-
- if (str.equalsIgnoreCase("true") ||
str.equalsIgnoreCase("yes")) {
- return true;
- } else if (str.equalsIgnoreCase("false") ||
str.equalsIgnoreCase("no")) {
- return false;
- } else {
- throw new ARQException("CSV Boolean Results malformed,
expected one of - true yes false no - but got " + str);
- }
- }
- catch (IOException ex)
- {
- throw new ARQException(ex);
- }
+ CSVParser parser = CSVParser.create(in) ;
+ final List<Var> vars = vars(parser) ;
+ if ( vars.size() != 1 ) {
+ throw new ARQException("CSV Boolean Results malformed: variables
line='"+vars+"'") ;
+ }
+ if ( ! vars.get(0).getName().equals("_askResult")) {
+ FmtLog.warn(log, "Boolean result variable is '%s', not
'_askResult'", vars.get(0).getName()) ;
+ }
+
+
+ List<String> line = parser.parse1() ;
+ if ( line.size() != 1 ) {
+ throw new ARQException("CSV Boolean Results malformed: data
line='"+line+"'") ;
+ }
+ String str = line.get(0) ;
+ boolean b ;
+ if ( str.equalsIgnoreCase("true") || str.equalsIgnoreCase("yes") )
+ b = true ;
+ else if (str.equalsIgnoreCase("false") || str.equalsIgnoreCase("no"))
+ b = false;
+ else {
+ throw new ARQException("CSV Boolean Results malformed, expected
one of - true yes false no - but got " + str);
+ }
+
+ List<String> line2 = parser.parse1() ;
+ if ( line2 != null ) {
+ FmtLog.warn(log, "Extra rows: first is "+line2) ;
+ }
+ return b ;
}
}
Added:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParseException.java
URL:
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParseException.java?rev=1595895&view=auto
==============================================================================
---
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParseException.java
(added)
+++
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParseException.java
Mon May 19 13:30:24 2014
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.atlas.csv;
+
+class CSVParseException extends RuntimeException
+{
+ public CSVParseException(String msg, Throwable cause) { super(msg,
cause) ; }
+ public CSVParseException(String msg) { super(msg) ; }
+ public CSVParseException(Throwable cause) { super(cause) ; }
+ public CSVParseException() { super() ; }
+}
Propchange:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParseException.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParser.java
URL:
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParser.java?rev=1595895&view=auto
==============================================================================
--- jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParser.java
(added)
+++ jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParser.java
Mon May 19 13:30:24 2014
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.atlas.csv ;
+
+import java.io.InputStream ;
+import java.util.ArrayList ;
+import java.util.Iterator ;
+import java.util.List ;
+
+import org.apache.jena.atlas.io.IO ;
+import org.apache.jena.atlas.iterator.IteratorSlotted ;
+import org.apache.jena.atlas.iterator.PeekIterator ;
+
+/** Written specifically to handle SPARQL results CSv files.
+ * Replace with a real parser (e.g. Apache Commons CSV when released)
+ */
+public class CSVParser implements Iterable<List<String>>
+{
+ public static CSVParser create(String filename) {
+ InputStream input = IO.openFile(filename) ;
+ return create(input) ;
+ }
+
+ public static CSVParser create(InputStream input) {
+ CSVTokenIterator iter = new CSVTokenIterator(input) ;
+ CSVParser parser = new CSVParser(iter) ;
+ return parser ;
+ }
+ // ----
+
+ private final CSVTokenIterator iter ;
+ private final PeekIterator<CSVToken> pIter ;
+
+ public CSVParser(CSVTokenIterator iter) {
+ this.iter = iter ;
+ this.pIter = new PeekIterator<CSVToken>(iter) ;
+ }
+
+ @Override
+ public Iterator<List<String>> iterator() {
+ return new IteratorSlotted<List<String>>() {
+ @Override
+ protected List<String> moveToNext() {
+ return CSVParser.this.parse1() ;
+ }
+
+ @Override
+ protected boolean hasMore() {
+ return true ;
+ }};
+ }
+
+ public List<String> parse1() {
+ // Get rid of switches. break problems.
+ List<String> line = new ArrayList<String>(100) ;
+
+ loop: while (pIter.hasNext()) {
+ CSVToken t = pIter.next() ;
+ switch (t.type) {
+ case EOF :
+ return null ;
+ case NL :
+ // Blank line = one or none?
+ line.add("") ;
+ return line ;
+ case STRING :
+ case QSTRING :
+ line.add(t.image) ;
+ break ;
+ case COMMA :
+ // Immediate COMMA is an empty term.
+ line.add("") ;
+ continue loop ;
+ default :
+ exception("Syntax error: expected a string or comma.", t) ;
+ }
+ // Expect COMMA or NL
+ if ( !pIter.hasNext() ) {
+ // File ends, no NL.
+ return line ;
+ }
+ // Look at separateor or end
+ CSVToken t2 = pIter.peek() ;
+ switch (t2.type) {
+ case COMMA :
+ pIter.next() ;
+ continue loop ;
+ case NL :
+ case EOF :
+ pIter.next() ;
+ return line ;
+ default :
+ exception("Syntax error: expect comma or end of line.", t)
;
+ }
+ }
+ return null ;
+ }
+ static void exception(String msg, CSVToken t) {
+ if ( t != null && t.line >= 0 && t.col > 0 )
+ msg = String.format("[%s, %s] %s", t.line, t.col, msg) ;
+ throw new CSVParseException(msg) ;
+ }
+
+ static void exception(String msg, long line, long col) {
+ if ( line >= 0 && col > 0 )
+ msg = String.format("[%s, %s] %s", line, col, msg) ;
+ throw new CSVParseException(msg) ;
+ }
+}
Propchange:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVParser.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVToken.java
URL:
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVToken.java?rev=1595895&view=auto
==============================================================================
--- jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVToken.java
(added)
+++ jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVToken.java
Mon May 19 13:30:24 2014
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.atlas.csv;
+
+import static org.apache.jena.atlas.csv.CSVTokenType.COMMA ;
+import static org.apache.jena.atlas.csv.CSVTokenType.EOF ;
+import static org.apache.jena.atlas.csv.CSVTokenType.NL ;
+import org.apache.jena.atlas.lib.Lib ;
+
+class CSVToken
+{
+ final CSVTokenType type ;
+ final String image ;
+ final long line ;
+ final long col ;
+
+ public boolean same(CSVToken obj) {
+ if ( this == obj )
+ return true ;
+ if ( obj == null )
+ return false ;
+ if ( type != obj.type )
+ return false ;
+ if ( type == COMMA || type == NL || type == EOF )
+ return true ;
+
+ if ( image == null && obj.image != null )
+ return false ;
+ return Lib.equal(this.image, obj.image) ;
+ }
+
+ public CSVToken(long line, long col, CSVTokenType type, String image) {
+ super() ;
+ this.type = type ;
+ this.image = image ;
+ this.line = line ;
+ this.col = col ;
+ }
+
+ @Override
+ public String toString() {
+ switch (type) {
+ case STRING :
+ case QSTRING :
+ return "Token [" + line + ", " + col + "] " + type + " |" +
image + "|" ;
+ default :
+ return "Token [" + line + ", " + col + "] " + type ;
+ }
+ }
+}
Propchange:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVToken.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenIterator.java
URL:
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenIterator.java?rev=1595895&view=auto
==============================================================================
---
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenIterator.java
(added)
+++
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenIterator.java
Mon May 19 13:30:24 2014
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.atlas.csv ;
+
+import static org.apache.jena.atlas.csv.CSVTokenType.* ;
+
+import java.io.InputStream ;
+
+import org.apache.jena.atlas.io.PeekReader ;
+import org.apache.jena.atlas.iterator.IteratorSlotted ;
+
+class CSVTokenIterator extends IteratorSlotted<CSVToken>
+{
+ private PeekReader in ;
+
+ // One EOF?
+
+ CSVTokenIterator(InputStream input) {
+ this.in = PeekReader.makeUTF8(input) ;
+ }
+
+ @Override
+ protected CSVToken moveToNext() {
+ int ch = in.peekChar() ;
+ if ( ch == '\r' ) {
+ in.readChar() ;
+ ch = in.peekChar() ;
+ if ( ch != '\n' )
+ return new CSVToken(in.getLineNum(), in.getColNum(), NL, "\r")
;
+ // '\n' = drop through.
+ }
+
+ if ( ch == '\n' ) {
+ in.readChar() ;
+ return new CSVToken(in.getLineNum(), in.getColNum(), NL, "\n") ;
+ }
+
+ if ( ch == ',' ) {
+ in.readChar() ;
+ return new CSVToken(in.getLineNum(), in.getColNum(), COMMA, ",") ;
+ }
+
+ long line = in.getLineNum() ;
+ long col = in.getColNum() ;
+
+ // Not -1
+ if ( ch == '"' || ch == '\'' )
+ return new CSVToken(line, col, QSTRING, readQuotedString()) ;
+ else
+ return new CSVToken(line, col, STRING, readUnquotedString()) ;
+ }
+
+ StringBuilder builder = new StringBuilder() ;
+
+ private String readQuotedString() {
+ builder.setLength(0) ;
+ int qCh = in.readChar() ;
+ int ch = qCh ;
+ while (true) {
+ ch = in.readChar() ;
+ if ( ch == -1 )
+ CSVParser.exception("Unterminated quoted string at
end-of-file", in.getLineNum(), in.getColNum()) ;
+ // Newlines are allowed in quoted strings.
+ // if ( ch == '\r' || ch == '\n' )
+ // exception("Unterminated quoted string", in.getLineNum(),
+ // in.getColNum()) ;
+ if ( ch == qCh ) {
+ int ch2 = in.peekChar() ;
+ if ( ch2 != qCh )
+ break ;
+ // Escaped quote
+ in.readChar() ;
+ // Fall through.
+ }
+ builder.append((char)ch) ;
+ }
+ return builder.toString() ;
+ }
+
+ private String readUnquotedString() {
+ builder.setLength(0) ;
+ while (true) {
+ int ch = in.peekChar() ;
+ if ( ch == -1 || ch == '\r' || ch == '\n' )
+ break ;
+ if ( ch == ',' )
+ break ;
+ in.readChar() ;
+ builder.append((char)ch) ;
+ }
+ return builder.toString() ;
+ }
+
+ @Override
+ protected boolean hasMore() {
+ return !in.eof() && in.peekChar() != -1 ;
+ }
+}
Propchange:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenIterator.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenType.java
URL:
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenType.java?rev=1595895&view=auto
==============================================================================
---
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenType.java
(added)
+++
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenType.java
Mon May 19 13:30:24 2014
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.atlas.csv;
+
+public enum CSVTokenType { STRING, QSTRING, COMMA, NL, EOF }
Propchange:
jena/trunk/jena-arq/src/main/java/org/apache/jena/atlas/csv/CSVTokenType.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/TC_Atlas.java
URL:
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/TC_Atlas.java?rev=1595895&r1=1595894&r2=1595895&view=diff
==============================================================================
--- jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/TC_Atlas.java
(original)
+++ jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/TC_Atlas.java Mon
May 19 13:30:24 2014
@@ -18,6 +18,7 @@
package org.apache.jena.atlas;
+import org.apache.jena.atlas.csv.TestCSVParser ;
import org.apache.jena.atlas.data.TS_Data ;
import org.apache.jena.atlas.event.TS_Event ;
import org.apache.jena.atlas.io.TS_IO ;
@@ -38,6 +39,7 @@ import org.junit.runners.Suite ;
, TS_JSON.class
, TS_Data.class
, TS_Web.class
+ , TestCSVParser.class
})
public class TC_Atlas
Added:
jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/csv/TestCSVParser.java
URL:
http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/csv/TestCSVParser.java?rev=1595895&view=auto
==============================================================================
---
jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/csv/TestCSVParser.java
(added)
+++
jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/csv/TestCSVParser.java
Mon May 19 13:30:24 2014
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.atlas.csv;
+
+import static org.apache.jena.atlas.csv.CSVTokenType.NL ;
+import static org.apache.jena.atlas.csv.CSVTokenType.QSTRING ;
+import static org.apache.jena.atlas.csv.CSVTokenType.STRING ;
+
+import java.io.ByteArrayInputStream ;
+import java.io.InputStream ;
+import java.io.UnsupportedEncodingException ;
+import java.util.ArrayList ;
+import java.util.List ;
+
+import org.apache.jena.atlas.junit.BaseTest ;
+import org.junit.Test ;
+
+public class TestCSVParser extends BaseTest
+{
+ String[] row1 = {} ;
+ String[] row2 = { "" } ;
+ String[] row3 = { "a", "b" } ;
+ String[] row4 = { "123", "\"aa\"", "'bb'", "\"''\"Z", "A'\"\"'" } ;
+
+ CSVToken t1 = new CSVToken(-1, -1, CSVTokenType.COMMA, ",") ;
+
+
+ @Test public void csv_parse_term_01() { csvTerm("123", STRING, "123") ; }
+ @Test public void csv_parse_term_02() { csvTerm("aa", STRING, "aa") ; }
+ @Test public void csv_parse_term_03() { csvTerm("\" \"", QSTRING, " ") ; }
+ @Test public void csv_parse_term_04() { csvTerm("' '", QSTRING, " ") ; }
+
+ @Test public void csv_parse_term_05() { csvTerm("\"a\"\"b\"", QSTRING,
"a\"b") ; }
+ @Test public void csv_parse_term_06() { csvTerm("'a\"b'", QSTRING,
"a\"b") ; }
+
+ @Test public void csv_parse_term_07() { csvTerm("\n", NL, "\n") ; }
+ @Test public void csv_parse_term_08() { csvTerm("\r", NL, "\n") ; }
+ @Test public void csv_parse_term_09() { csvTerm("\r\n", NL, "\n") ; }
+
+ private static void csvTerm(String input, CSVTokenType type, String output)
+ {
+ try
+ {
+ CSVToken expected = new CSVToken(-1, -1, type, output) ;
+
+ InputStream in = new ByteArrayInputStream(input.getBytes("UTF-8"))
;
+ CSVTokenIterator iter = new CSVTokenIterator(in) ;
+ assertTrue(iter.hasNext()) ;
+ CSVToken t = iter.next() ;
+ assertTrue(expected.same(t)) ;
+ } catch (UnsupportedEncodingException e)
+ {
+ throw new RuntimeException(e) ;
+ }
+ }
+
+ @Test public void csv_parse_01() { csv("\n", new String[][] {{""}}) ; }
+ @Test public void csv_parse_02() { csv("a\n", new String[][] {{"a"}}) ; }
+ @Test public void csv_parse_03() { csv("a,b\n", new String[][] {{"a",
"b"}}) ; }
+ @Test public void csv_parse_04() { csv(",b\n", new String[][] {{"", "b"}})
; }
+ @Test public void csv_parse_05() { csv("a,\n", new String[][] {{"a", ""}})
; }
+ @Test public void csv_parse_06() { csv(",\n", new String[][] {{"", ""}}) ;
}
+ @Test public void csv_parse_07() { csv(",,\n", new String[][] {{"", "",
""}}) ; }
+
+ @Test public void csv_parse_10() { csv("\n\n", new String[][] { {""}, {""}
}) ; }
+ @Test public void csv_parse_11() { csv("'aa'\naa\n", new String[][] {
{"aa"}, {"aa"} }) ; }
+ @Test public void csv_parse_12() { csv("\naa", new String[][] { {""},
{"aa"} }) ; }
+ @Test public void csv_parse_13() { csv("a,b\nc,d", new String[][] { {"a",
"b"}, {"c", "d"} }) ; }
+ @Test public void csv_parse_14() { csv("a,b\rc,d", new String[][] { {"a",
"b"}, {"c", "d"} }) ; }
+
+
+ private void csv(String input, String[][] strings)
+ {
+ List<List<String>> x = new ArrayList<List<String>>() ;
+ for ( String[] a : strings )
+ {
+ List<String> y = new ArrayList<String>() ;
+ for ( String b : a )
+ y.add(b) ;
+ x.add(y) ;
+ }
+ csv(input, x) ;
+ }
+
+ private static void csv(String input, List<List<String>> answers)
+ {
+ List<List<String>> x = new ArrayList<List<String>>() ;
+ try {
+ InputStream in = new ByteArrayInputStream(input.getBytes("UTF-8"))
;
+ CSVTokenIterator iter = new CSVTokenIterator(in) ;
+ CSVParser parser = new CSVParser(iter) ;
+ List<String> row = null ;
+ while ( (row=parser.parse1())!=null) {
+ x.add(row) ;
+ }
+ assertEquals(answers, x) ;
+ } catch (UnsupportedEncodingException e)
+ {
+ throw new RuntimeException(e) ;
+ }
+
+
+ }
+}
+
Propchange:
jena/trunk/jena-arq/src/test/java/org/apache/jena/atlas/csv/TestCSVParser.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
