changed dependency for commons to 2.4, changed parseElements of XMLParser to accept URI for hdfs
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/a04de233 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/a04de233 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/a04de233 Branch: refs/heads/steven/hdfs Commit: a04de233d7fae047432e69c05025ce49571e5c7c Parents: 319b79c Author: efikalti <[email protected]> Authored: Sun Jun 7 11:00:08 2015 +0300 Committer: efikalti <[email protected]> Committed: Sun Jun 7 11:00:08 2015 +0300 ---------------------------------------------------------------------- pom.xml | 2 +- vxquery-core/pom.xml | 2 +- .../VXQueryCollectionOperatorDescriptor.java | 9 ++++-- .../org/apache/vxquery/xmlparser/XMLParser.java | 30 +++++++++----------- 4 files changed, 22 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/a04de233/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 1862890..f223129 100644 --- a/pom.xml +++ b/pom.xml @@ -322,7 +322,7 @@ <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> - <version>1.3.2</version> + <version>2.4</version> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/vxquery/blob/a04de233/vxquery-core/pom.xml ---------------------------------------------------------------------- diff --git a/vxquery-core/pom.xml b/vxquery-core/pom.xml index 1bed170..ea67380 100644 --- a/vxquery-core/pom.xml +++ b/vxquery-core/pom.xml @@ -247,7 +247,7 @@ <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> - </dependency> + </dependency> <dependency> <groupId>commons-codec</groupId> http://git-wip-us.apache.org/repos/asf/vxquery/blob/a04de233/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index 69bca70..9dcc2be 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -21,6 +21,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.Iterator; import java.util.List; @@ -126,6 +128,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO //else check in HDFS file system else { + System.out.println("search in hdfs"); HDFSFileFunctions hdfs = new HDFSFileFunctions(hdfs_conf_dir); FileSystem fs = hdfs.getFileSystem(); Path directory = new Path(collectionModifiedName); @@ -144,8 +147,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO if (LOGGER.isLoggable(Level.FINE)) { LOGGER.fine("Starting to read XML document: " + xmlDocument.getName()); } - BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(xmlDocument))); - parser.parseElements(br, writer, fta, tupleIndex); + parser.parseHDFSElements(new URI(xmlDocument.getName()), writer, fta, tupleIndex); } } } @@ -161,6 +163,9 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO } catch (IOException e) { // TODO Auto-generated catch block System.err.println(e); + } catch (URISyntaxException e) { + // TODO Auto-generated catch block + e.printStackTrace(); } } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/a04de233/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java index e3075d8..ad7463e 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java @@ -20,11 +20,15 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; +import java.io.ObjectInputStream; import java.io.Reader; +import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSInputStream; import org.apache.vxquery.context.StaticContext; import org.apache.vxquery.exceptions.VXQueryFileNotFoundException; import org.apache.vxquery.exceptions.VXQueryParseException; @@ -130,26 +134,18 @@ public class XMLParser { } } - public void parseElements(BufferedReader br, IFrameWriter writer, FrameTupleAccessor fta, int tupleIndex) - throws HyracksDataException { - try { - Reader input; - if (bufferSize > 0) { - input = br; - in.setCharacterStream(input); + public void parseHDFSElements(URI uri, IFrameWriter writer, FrameTupleAccessor fta, int tupleIndex) throws IOException + { handler.setupElementWriter(writer, fta, tupleIndex); - try { - parser.parse(in); + try { + parser.parse(uri.toString()); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); } catch (SAXException e) { - System.err.println(e); + // TODO Auto-generated catch block + e.printStackTrace(); } - input.close(); - } - } catch (IOException e) { - HyracksDataException hde = new HyracksDataException(e); - hde.setNodeId(nodeId); - throw hde; - } } } \ No newline at end of file
