added apache licence to hdfsfunctions class
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/35b29a4e Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/35b29a4e Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/35b29a4e Branch: refs/heads/steven/hdfs Commit: 35b29a4e87eb6562e9e8c4ba3d9ff5bbde9c3016 Parents: a04de23 Author: efikalti <[email protected]> Authored: Mon Jun 8 16:25:07 2015 +0300 Committer: efikalti <[email protected]> Committed: Mon Jun 8 16:25:07 2015 +0300 ---------------------------------------------------------------------- .../apache/vxquery/hdfs2/HDFSFileFunctions.java | 16 +++++++++ .../VXQueryCollectionOperatorDescriptor.java | 1 - .../runtime/functions/util/FunctionHelper.java | 37 +++++++++++++++++++- .../org/apache/vxquery/xmlparser/XMLParser.java | 15 ++++++++ vxquery-xtest/pom.xml | 5 +++ 5 files changed, 72 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/35b29a4e/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFileFunctions.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFileFunctions.java b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFileFunctions.java index 2f1d289..787a70d 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFileFunctions.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFileFunctions.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.vxquery.hdfs2; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/vxquery/blob/35b29a4e/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index 9dcc2be..3caafc7 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -128,7 +128,6 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO //else check in HDFS file system else { - System.out.println("search in hdfs"); HDFSFileFunctions hdfs = new HDFSFileFunctions(hdfs_conf_dir); FileSystem fs = hdfs.getFileSystem(); Path directory = new Path(collectionModifiedName); http://git-wip-us.apache.org/repos/asf/vxquery/blob/35b29a4e/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java index 3a20a86..b7ab9a3 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java @@ -19,10 +19,18 @@ package org.apache.vxquery.runtime.functions.util; import java.io.DataInputStream; import java.io.DataOutput; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.logging.Level; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.vxquery.context.DynamicContext; import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; import org.apache.vxquery.datamodel.accessors.TypedPointables; @@ -33,6 +41,7 @@ import org.apache.vxquery.datamodel.util.DateTime; import org.apache.vxquery.datamodel.values.ValueTag; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.hdfs2.HDFSFileFunctions; import org.apache.vxquery.runtime.functions.arithmetic.AbstractArithmeticOperation; import org.apache.vxquery.runtime.functions.cast.CastToDoubleOperation; import org.apache.vxquery.runtime.functions.comparison.AbstractValueComparisonOperation; @@ -1186,7 +1195,33 @@ public class FunctionHelper { throw new HyracksDataException(e); } File file = new File(fName); - parser.parseDocument(file, abvs); + if(file.exists()) + { + parser.parseDocument(file, abvs); + } + //else check in HDFS file system + else + { + String hdfs_conf_dir = "/home/efi/Utilities/hadoop/etc/hadoop/"; + HDFSFileFunctions hdfs = new HDFSFileFunctions(hdfs_conf_dir); + FileSystem fs = hdfs.getFileSystem(); + Path xmlDocument = new Path(fName); + try { + if (fs.exists(xmlDocument)) + { + parser.parseHDFSDocument(new URI(xmlDocument.getName()), abvs); + } + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + System.err.println(e); + } catch (IOException e) { + // TODO Auto-generated catch block + System.err.println(e); + } catch (URISyntaxException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } } public static boolean transformThenCompareMinMaxTaggedValues(AbstractValueComparisonOperation aOp, http://git-wip-us.apache.org/repos/asf/vxquery/blob/35b29a4e/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java index ad7463e..66d2cf7 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java @@ -147,5 +147,20 @@ public class XMLParser { e.printStackTrace(); } } + + public void parseHDFSDocument(URI uri, ArrayBackedValueStorage abvs) throws HyracksDataException { + try { + System.out.println("read hdfs document"); + parser.parse(uri.toString()); + handler.writeDocument(abvs); + } catch (IOException e) { + HyracksDataException hde = new HyracksDataException(e); + hde.setNodeId(nodeId); + throw hde; + } catch (SAXException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/35b29a4e/vxquery-xtest/pom.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/pom.xml b/vxquery-xtest/pom.xml index 3b6474b..ed53552 100644 --- a/vxquery-xtest/pom.xml +++ b/vxquery-xtest/pom.xml @@ -150,6 +150,11 @@ <groupId>edu.uci.ics.hyracks</groupId> <artifactId>hyracks-dataflow-std</artifactId> </dependency> + + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> <dependency> <groupId>org.mortbay.jetty</groupId>
