http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java index d8c4e68..d394bbc 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java @@ -19,10 +19,21 @@ package org.apache.vxquery.runtime.functions.util; import java.io.DataInputStream; import java.io.DataOutput; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.primitive.DoublePointable; +import org.apache.hyracks.data.std.primitive.LongPointable; +import org.apache.hyracks.data.std.primitive.UTF8StringPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; import org.apache.vxquery.context.DynamicContext; import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; import org.apache.vxquery.datamodel.accessors.TypedPointables; @@ -33,6 +44,7 @@ import org.apache.vxquery.datamodel.util.DateTime; import org.apache.vxquery.datamodel.values.ValueTag; import org.apache.vxquery.exceptions.ErrorCode; import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.hdfs2.HDFSFunctions; import org.apache.vxquery.runtime.functions.arithmetic.AbstractArithmeticOperation; import org.apache.vxquery.runtime.functions.cast.CastToDoubleOperation; import org.apache.vxquery.runtime.functions.comparison.AbstractValueComparisonOperation; @@ -42,14 +54,6 @@ import org.apache.vxquery.types.BuiltinTypeConstants; import org.apache.vxquery.types.BuiltinTypeRegistry; import org.apache.vxquery.xmlparser.XMLParser; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.data.std.api.IPointable; -import org.apache.hyracks.data.std.primitive.DoublePointable; -import org.apache.hyracks.data.std.primitive.LongPointable; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; -import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; -import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; - public class FunctionHelper { public static void arithmeticOperation(AbstractArithmeticOperation aOp, DynamicContext dCtx, @@ -1213,8 +1217,33 @@ public class FunctionHelper { } catch (SystemException e) { throw new HyracksDataException(e); } - File file = new File(fName); - parser.parseDocument(file, abvs); + if (!fName.contains("hdfs:/")) { + File file = new File(fName); + if (file.exists()) { + parser.parseDocument(file, abvs); + } + } + //else check in HDFS file system + else { + fName = fName.replaceAll("hdfs:/", ""); + HDFSFunctions hdfs = new HDFSFunctions(null, null); + FileSystem fs = hdfs.getFileSystem(); + if (fs != null) { + Path xmlDocument = new Path(fName); + try { + if (fs.exists(xmlDocument)) { + InputStream in = fs.open(xmlDocument).getWrappedStream(); + parser.parseHDFSDocument(in, abvs); + } + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + System.err.println(e); + } catch (IOException e) { + // TODO Auto-generated catch block + System.err.println(e); + } + } + } } public static boolean transformThenCompareMinMaxTaggedValues(AbstractValueComparisonOperation aOp,
http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java index 77d500c..a62a26c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.util.ArrayList; @@ -28,6 +29,7 @@ import org.apache.hyracks.api.comm.IFrameFieldAppender; import org.apache.hyracks.api.comm.IFrameWriter; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; import org.apache.vxquery.context.StaticContext; import org.apache.vxquery.exceptions.VXQueryFileNotFoundException; import org.apache.vxquery.exceptions.VXQueryParseException; @@ -98,8 +100,7 @@ public class XMLParser { } } - public void parseElements(File file, IFrameWriter writer, int tupleIndex) - throws HyracksDataException { + public void parseElements(File file, IFrameWriter writer, int tupleIndex) throws HyracksDataException { try { Reader input; if (bufferSize > 0) { @@ -126,4 +127,48 @@ public class XMLParser { } } + public void parseHDFSElements(InputStream inputStream, IFrameWriter writer, FrameTupleAccessor fta, int tupleIndex) + throws IOException { + try { + Reader input; + if (bufferSize > 0) { + input = new BufferedReader(new InputStreamReader(inputStream), bufferSize); + } else { + input = new InputStreamReader(inputStream); + } + in.setCharacterStream(input); + handler.setupElementWriter(writer, tupleIndex); + parser.parse(in); + input.close(); + } catch (IOException e) { + HyracksDataException hde = new HyracksDataException(e); + hde.setNodeId(nodeId); + throw hde; + } catch (SAXException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + public void parseHDFSDocument(InputStream inputStream, ArrayBackedValueStorage abvs) throws HyracksDataException { + try { + Reader input; + if (bufferSize > 0) { + input = new BufferedReader(new InputStreamReader(inputStream), bufferSize); + } else { + input = new InputStreamReader(inputStream); + } + in.setCharacterStream(input); + parser.parse(in); + handler.writeDocument(abvs); + input.close(); + } catch (IOException e) { + HyracksDataException hde = new HyracksDataException(e); + hde.setNodeId(nodeId); + throw hde; + } catch (SAXException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java index e42caba..8a044ea 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java @@ -17,28 +17,7 @@ package org.apache.vxquery.xmlquery.query; import java.io.Reader; import java.util.ArrayList; import java.util.List; - -import org.apache.vxquery.compiler.CompilerControlBlock; -import org.apache.vxquery.compiler.algebricks.VXQueryBinaryBooleanInspectorFactory; -import org.apache.vxquery.compiler.algebricks.VXQueryBinaryIntegerInspectorFactory; -import org.apache.vxquery.compiler.algebricks.VXQueryComparatorFactoryProvider; -import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue; -import org.apache.vxquery.compiler.algebricks.VXQueryExpressionRuntimeProvider; -import org.apache.vxquery.compiler.algebricks.VXQueryNullWriterFactory; -import org.apache.vxquery.compiler.algebricks.VXQueryPrinterFactoryProvider; -import org.apache.vxquery.compiler.algebricks.prettyprint.VXQueryLogicalExpressionPrettyPrintVisitor; -import org.apache.vxquery.compiler.rewriter.RewriteRuleset; -import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext; -import org.apache.vxquery.exceptions.ErrorCode; -import org.apache.vxquery.exceptions.SystemException; -import org.apache.vxquery.metadata.VXQueryMetadataProvider; -import org.apache.vxquery.runtime.provider.VXQueryBinaryHashFunctionFactoryProvider; -import org.apache.vxquery.runtime.provider.VXQueryBinaryHashFunctionFamilyProvider; -import org.apache.vxquery.types.BuiltinTypeRegistry; -import org.apache.vxquery.types.Quantifier; -import org.apache.vxquery.types.SequenceType; -import org.apache.vxquery.xmlquery.ast.ModuleNode; -import org.apache.vxquery.xmlquery.translator.XMLQueryTranslator; +import java.util.Map; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException; @@ -67,16 +46,40 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IOptimizationContextFact import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig; import org.apache.hyracks.algebricks.data.ISerializerDeserializerProvider; import org.apache.hyracks.algebricks.data.ITypeTraitProvider; +import org.apache.hyracks.api.client.NodeControllerInfo; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.vxquery.compiler.CompilerControlBlock; +import org.apache.vxquery.compiler.algebricks.VXQueryBinaryBooleanInspectorFactory; +import org.apache.vxquery.compiler.algebricks.VXQueryBinaryIntegerInspectorFactory; +import org.apache.vxquery.compiler.algebricks.VXQueryComparatorFactoryProvider; +import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue; +import org.apache.vxquery.compiler.algebricks.VXQueryExpressionRuntimeProvider; +import org.apache.vxquery.compiler.algebricks.VXQueryNullWriterFactory; +import org.apache.vxquery.compiler.algebricks.VXQueryPrinterFactoryProvider; +import org.apache.vxquery.compiler.algebricks.prettyprint.VXQueryLogicalExpressionPrettyPrintVisitor; +import org.apache.vxquery.compiler.rewriter.RewriteRuleset; +import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext; +import org.apache.vxquery.exceptions.ErrorCode; +import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.metadata.VXQueryMetadataProvider; +import org.apache.vxquery.runtime.provider.VXQueryBinaryHashFunctionFactoryProvider; +import org.apache.vxquery.runtime.provider.VXQueryBinaryHashFunctionFamilyProvider; +import org.apache.vxquery.types.BuiltinTypeRegistry; +import org.apache.vxquery.types.Quantifier; +import org.apache.vxquery.types.SequenceType; +import org.apache.vxquery.xmlquery.ast.ModuleNode; +import org.apache.vxquery.xmlquery.translator.XMLQueryTranslator; public class XMLQueryCompiler { private final XQueryCompilationListener listener; private final ICompilerFactory cFactory; + private final String hdfsConf; + private LogicalOperatorPrettyPrintVisitor pprinter; private ModuleNode moduleNode; @@ -87,17 +90,27 @@ public class XMLQueryCompiler { private int frameSize; + private Map<String, NodeControllerInfo> nodeControllerInfos; + private String[] nodeList; - public XMLQueryCompiler(XQueryCompilationListener listener, String[] nodeList, int frameSize) { - this(listener, nodeList, frameSize, -1, -1, -1); + public XMLQueryCompiler(XQueryCompilationListener listener, Map<String, NodeControllerInfo> nodeControllerInfos, + int frameSize, String hdfsConf) { + this(listener, nodeControllerInfos, frameSize, -1, -1, -1, hdfsConf); + } + + public XMLQueryCompiler(XQueryCompilationListener listener, Map<String, NodeControllerInfo> nodeControllerInfos, + int frameSize) { + this(listener, nodeControllerInfos, frameSize, -1, -1, -1, ""); } - public XMLQueryCompiler(XQueryCompilationListener listener, String[] nodeList, int frameSize, - int availableProcessors, long joinHashSize, long maximumDataSize) { + public XMLQueryCompiler(XQueryCompilationListener listener, Map<String, NodeControllerInfo> nodeControllerInfos, + int frameSize, int availableProcessors, long joinHashSize, long maximumDataSize, String hdfsConf) { this.listener = listener == null ? NoopXQueryCompilationListener.INSTANCE : listener; this.frameSize = frameSize; - this.nodeList = nodeList; + this.nodeControllerInfos = nodeControllerInfos; + setNodeList(); + this.hdfsConf = hdfsConf; HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder( new IOptimizationContextFactory() { @Override @@ -116,12 +129,12 @@ public class XMLQueryCompiler { builder.getPhysicalOptimizationConfig().setMaxFramesHybridHash((int) (joinHashSize / this.frameSize)); } if (maximumDataSize > 0) { - builder.getPhysicalOptimizationConfig().setMaxFramesLeftInputHybridHash( - (int) (maximumDataSize / this.frameSize)); + builder.getPhysicalOptimizationConfig() + .setMaxFramesLeftInputHybridHash((int) (maximumDataSize / this.frameSize)); } - builder.getPhysicalOptimizationConfig().setMaxFramesLeftInputHybridHash( - (int) (60L * 1024 * 1048576 / this.frameSize)); + builder.getPhysicalOptimizationConfig() + .setMaxFramesLeftInputHybridHash((int) (60L * 1024 * 1048576 / this.frameSize)); builder.setLogicalRewrites(buildDefaultLogicalRewrites()); builder.setPhysicalRewrites(buildDefaultPhysicalRewrites()); @@ -192,15 +205,26 @@ public class XMLQueryCompiler { cFactory = builder.create(); } + /** + * Set Configuration of node controllers as array of Strings. + */ + private void setNodeList() { + nodeList = new String[nodeControllerInfos.size()]; + int index = 0; + for (String node : nodeControllerInfos.keySet()) { + nodeList[index++] = node; + } + } + public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel) throws SystemException { moduleNode = XMLQueryParser.parse(name, query); listener.notifyParseResult(moduleNode); module = new XMLQueryTranslator(ccb).translateModule(moduleNode); - pprinter = new LogicalOperatorPrettyPrintVisitor(new VXQueryLogicalExpressionPrettyPrintVisitor( - module.getModuleContext())); + pprinter = new LogicalOperatorPrettyPrintVisitor( + new VXQueryLogicalExpressionPrettyPrintVisitor(module.getModuleContext())); VXQueryMetadataProvider mdProvider = new VXQueryMetadataProvider(nodeList, ccb.getSourceFileMap(), - module.getModuleContext()); + module.getModuleContext(), this.hdfsConf, nodeControllerInfos); compiler = cFactory.createCompiler(module.getBody(), mdProvider, 0); listener.notifyTranslationResult(module); XMLQueryTypeChecker.typeCheckModule(module); http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java index 165b330..13ca921 100644 --- a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java +++ b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java @@ -20,16 +20,19 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.StringReader; +import java.util.HashMap; +import java.util.Map; import java.util.zip.GZIPInputStream; -import junit.framework.Assert; - +import org.apache.hyracks.api.client.NodeControllerInfo; +import org.apache.hyracks.api.comm.NetworkAddress; +import org.apache.hyracks.api.dataset.ResultSetId; import org.apache.vxquery.compiler.CompilerControlBlock; import org.apache.vxquery.context.RootStaticContextImpl; import org.apache.vxquery.context.StaticContextImpl; import org.junit.Test; -import org.apache.hyracks.api.dataset.ResultSetId; +import junit.framework.Assert; public class SimpleXQueryTest { @Test @@ -96,8 +99,8 @@ public class SimpleXQueryTest { private static String gunzip(String dir, String filename) { try { - GZIPInputStream in = new GZIPInputStream(new BufferedInputStream(new FileInputStream(new File(dir - + filename + ".gz")))); + GZIPInputStream in = new GZIPInputStream( + new BufferedInputStream(new FileInputStream(new File(dir + filename + ".gz")))); File temp = File.createTempFile("vxquery", filename); temp.deleteOnExit(); FileOutputStream out = new FileOutputStream(temp); @@ -133,7 +136,11 @@ public class SimpleXQueryTest { } private static void runTestInternal(String testName, String query) throws Exception { - XMLQueryCompiler compiler = new XMLQueryCompiler(null, new String[] { "nc1" }, 65536); + + Map<String, NodeControllerInfo> nodeControllerInfos = new HashMap<String, NodeControllerInfo>(); + nodeControllerInfos.put("nc1", new NodeControllerInfo("nc1", null, new NetworkAddress("127.0.0.1", 0), null)); + + XMLQueryCompiler compiler = new XMLQueryCompiler(null, nodeControllerInfos, 65536); CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE), new ResultSetId(System.nanoTime()), null); compiler.compile(testName, new StringReader(query), ccb, Integer.MAX_VALUE); http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-server/src/main/resources/conf/cluster.properties ---------------------------------------------------------------------- diff --git a/vxquery-server/src/main/resources/conf/cluster.properties b/vxquery-server/src/main/resources/conf/cluster.properties index fd015d4..6339fd9 100644 --- a/vxquery-server/src/main/resources/conf/cluster.properties +++ b/vxquery-server/src/main/resources/conf/cluster.properties @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -51,4 +51,4 @@ CCJAVA_OPTS="-server -Xmx500M -Djava.util.logging.config.file=./vxquery-benchmar NCJAVA_OPTS="-server -Xmx7G -Djava.util.logging.config.file=./vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/testing_logging.properties" # debug option: NCJAVA_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,address=7002,server=y,suspend=n -Xmx1g -Djava.util.logging.config.file=logging.properties" # Yourkit option: -agentpath:/tools/yjp-2014-build-14114/bin/linux-x86-64/libyjpagent.so=port=20001" -# Yourkit mac option: -agentpath:/Applications/YourKit_Java_Profiler.app/bin/mac/libyjpagent.jnilib=sampling +# Yourkit mac option: -agentpath:/Applications/YourKit_Java_Profiler.app/bin/mac/libyjpagent.jnilib=sampling \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/pom.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/pom.xml b/vxquery-xtest/pom.xml index 0c563e7..4cb7c60 100644 --- a/vxquery-xtest/pom.xml +++ b/vxquery-xtest/pom.xml @@ -161,6 +161,21 @@ </dependency> <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-hdfs-2.x</artifactId> + </dependency> + + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + </dependency> + + <dependency> <groupId>org.mortbay.jetty</groupId> <artifactId>jetty</artifactId> <scope>compile</scope> http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/MiniDFS.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/MiniDFS.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/MiniDFS.java new file mode 100644 index 0000000..0720baa --- /dev/null +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/MiniDFS.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.xtest; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.mapred.JobConf; + +public class MiniDFS { + + private MiniDFSCluster dfsCluster; + + public void miniDFS() { + + } + + public void startHDFS() throws IOException { + + FileSystem lfs = FileSystem.getLocal(new Configuration()); + JobConf conf = new JobConf(); + String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf"; + conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml")); + conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml")); + conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); + int numDataNodes = 1; + int nameNodePort = 40000; + + // cleanup artifacts created on the local file system + lfs.delete(new Path("build"), true); + System.setProperty("hadoop.log.dir", "logs"); + MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf); + build.nameNodePort(nameNodePort); + build.nameNodeHttpPort(nameNodePort + 34); + build.numDataNodes(numDataNodes); + build.checkExitOnShutdown(true); + build.startupOption(StartupOption.REGULAR); + build.format(true); + build.waitSafeMode(true); + dfsCluster = build.build(); + + FileSystem dfs = FileSystem.get(conf); + String DATA_PATH = "src/test/resources/TestSources/ghcnd"; + Path src = new Path(DATA_PATH); + dfs.mkdirs(new Path("/tmp")); + Path dest = new Path("/tmp/vxquery-hdfs-test"); + dfs.copyFromLocalFile(src, dest); + if (dfs.exists(dest)) { + System.err.println("Test files copied to HDFS successfully"); + } + } + + public void shutdownHDFS() { + System.err.println("Tests completed.Shutting down HDFS"); + dfsCluster.shutdown(); + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java index 18db5c1..5b6ddff 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java @@ -18,13 +18,16 @@ import java.io.File; import java.io.FileInputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.net.InetAddress; import java.util.EnumSet; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.apache.hyracks.api.client.HyracksConnection; import org.apache.hyracks.api.client.IHyracksClientConnection; +import org.apache.hyracks.api.client.NodeControllerInfo; import org.apache.hyracks.api.comm.IFrame; import org.apache.hyracks.api.comm.IFrameTupleAccessor; import org.apache.hyracks.api.comm.VSizeFrame; @@ -63,6 +66,7 @@ public class TestRunner { private NodeControllerService nc1; private IHyracksClientConnection hcc; private IHyracksDataset hds; + private final String publicAddress = InetAddress.getLocalHost().getHostAddress(); public TestRunner(XTestOptions opts) throws Exception { this.opts = opts; @@ -70,9 +74,9 @@ public class TestRunner { public void open() throws Exception { CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = "127.0.0.1"; + ccConfig.clientNetIpAddress = publicAddress; ccConfig.clientNetPort = 39000; - ccConfig.clusterNetIpAddress = "127.0.0.1"; + ccConfig.clusterNetIpAddress = publicAddress; ccConfig.clusterNetPort = 39001; ccConfig.profileDumpPeriod = 10000; File outDir = new File("target/ClusterController"); @@ -87,9 +91,9 @@ public class TestRunner { NCConfig ncConfig1 = new NCConfig(); ncConfig1.ccHost = "localhost"; ncConfig1.ccPort = 39001; - ncConfig1.clusterNetIPAddress = "127.0.0.1"; - ncConfig1.dataIPAddress = "127.0.0.1"; - ncConfig1.resultIPAddress = "127.0.0.1"; + ncConfig1.clusterNetIPAddress = publicAddress; + ncConfig1.dataIPAddress = publicAddress; + ncConfig1.resultIPAddress = publicAddress; ncConfig1.nodeId = "nc1"; nc1 = new NodeControllerService(ncConfig1); nc1.start(); @@ -115,7 +119,14 @@ public class TestRunner { VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, opts.showOET, opts.showRP); - XMLQueryCompiler compiler = new XMLQueryCompiler(listener, new String[] { "nc1" }, opts.frameSize); + + Map<String, NodeControllerInfo> nodeControllerInfos = null; + if (hcc != null) { + nodeControllerInfos = hcc.getNodeControllerInfos(); + } + + XMLQueryCompiler compiler = new XMLQueryCompiler(listener, nodeControllerInfos, opts.frameSize, + opts.hdfsConf); Reader in = new InputStreamReader(new FileInputStream(testCase.getXQueryFile()), "UTF-8"); CompilerControlBlock ccb = new CompilerControlBlock( new StaticContextImpl(RootStaticContextImpl.INSTANCE), http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java index 854cbf8..496b74a 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java @@ -76,4 +76,7 @@ public class XTestOptions { @Option(name = "-showresult", usage = "Show query result.") boolean showResult; + + @Option(name = "-hdfs-conf", usage = "Directory path to Hadoop configuration files") + String hdfsConf; } http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java index 12a91a9..5411215 100644 --- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java +++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java @@ -37,6 +37,7 @@ public abstract class AbstractXQueryTest { opts.threads = 1; opts.showQuery = true; opts.showResult = true; + opts.hdfsConf = "src/test/resources/hadoop/conf"; return opts; } http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryTest.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryTest.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryTest.java index 3704d07..4d0ddc0 100644 --- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryTest.java +++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/VXQueryTest.java @@ -17,15 +17,19 @@ package org.apache.vxquery.xtest; import java.io.File; +import java.io.IOException; import java.util.Collection; import org.apache.commons.lang3.StringUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) public class VXQueryTest extends AbstractXQueryTest { + private static MiniDFS dfs; private static String VXQUERY_CATALOG = StringUtils.join(new String[] { "src", "test", "resources", "VXQueryCatalog.xml" }, File.separator); @@ -52,4 +56,19 @@ public class VXQueryTest extends AbstractXQueryTest { return getOptions(); } + @BeforeClass + public static void setupHDFS() { + dfs = new MiniDFS(); + try { + dfs.startHDFS(); + } catch (IOException e) { + System.err.println(e); + } + } + + @AfterClass + public static void shutdownHDFS() { + dfs.shutdownHDFS(); + } + } http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/avgHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/avgHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/avgHDFS.txt new file mode 100644 index 0000000..7ef6ffe --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/avgHDFS.txt @@ -0,0 +1 @@ +12.5 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/countHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/countHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/countHDFS.txt new file mode 100644 index 0000000..d8263ee --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/countHDFS.txt @@ -0,0 +1 @@ +2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/maxHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/maxHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/maxHDFS.txt new file mode 100644 index 0000000..dc7b54a --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/maxHDFS.txt @@ -0,0 +1 @@ +33 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/maxvalueHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/maxvalueHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/maxvalueHDFS.txt new file mode 100644 index 0000000..e37d32a --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/maxvalueHDFS.txt @@ -0,0 +1 @@ +1000 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/minHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/minHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/minHDFS.txt new file mode 100644 index 0000000..ea1acb6 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/minHDFS.txt @@ -0,0 +1 @@ +11.25 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/sumHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/sumHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/sumHDFS.txt new file mode 100644 index 0000000..2b82dfe --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Aggregate/sumHDFS.txt @@ -0,0 +1 @@ +60 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/avgHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/avgHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/avgHDFS.xq new file mode 100644 index 0000000..3214b97 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/avgHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the average minimum temperature. :) +fn:avg( + let $collection := "hdfs://tmp/vxquery-hdfs-test" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "TMIN" + return $r/value +) http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/countHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/countHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/countHDFS.xq new file mode 100644 index 0000000..7940b03 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/countHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the number of wind sensor readings. :) +fn:count( + let $collection := "hdfs://tmp/vxquery-hdfs-test" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "AWND" + return $r/value +) http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/maxHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/maxHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/maxHDFS.xq new file mode 100644 index 0000000..0db1980 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/maxHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the highest max temperature. :) +fn:max( + let $collection := "hdfs://tmp/vxquery-hdfs-test" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "TMAX" + return $r/value +) http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/maxvalueHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/maxvalueHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/maxvalueHDFS.xq new file mode 100644 index 0000000..9d04916 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/maxvalueHDFS.xq @@ -0,0 +1,23 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the max value. :) +fn:max( + for $r in collection-with-tag("hdfs://tmp/vxquery-hdfs-test/half_1/quarter_1/sensors", "data")/data + return $r/value +) http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/minHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/minHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/minHDFS.xq new file mode 100644 index 0000000..869e222 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/minHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the lowest min temperature. :) +fn:min( + let $collection := "hdfs://tmp/vxquery-hdfs-test" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "TMIN" + return $r/value +) http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/sumHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/sumHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/sumHDFS.xq new file mode 100644 index 0000000..1fdf743 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Aggregate/sumHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the total precipitation. :) +fn:sum( + let $collection := "hdfs://tmp/vxquery-hdfs-test" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "PRCP" + return $r/value +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/VXQueryCatalog.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml index 6f7acee..f75ce49 100644 --- a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml +++ b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml @@ -40,6 +40,8 @@ <!ENTITY SingleQuery SYSTEM "cat/SingleQuery.xml"> <!ENTITY SingleAlternateQuery SYSTEM "cat/SingleAlternateQuery.xml"> +<!ENTITY HDFSAggregateQueries SYSTEM "cat/HDFSAggregateQueries.xml"> + ]> <test-suite xmlns="http://www.w3.org/2005/02/query-test-XQTSCatalog"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; @@ -193,4 +195,17 @@ &GhcndRecordsPartition4Queries; </test-group> </test-group> + <test-group name="HDFSAggregateQueries" featureOwner="Efi Kaltirimidou"> + <GroupInfo> + <title>Aggregate Partition Queries in HDFS</title> + <description/> + </GroupInfo> + <test-group name="CollectionReadFromHDFSAggregateTests" featureOwner="Efi Kaltirimidou"> + <GroupInfo> + <title>Aggregate HDFS Execution Tests</title> + <description/> + </GroupInfo> + &HDFSAggregateQueries; + </test-group> + </test-group> </test-suite> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/cat/HDFSAggregateQueries.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/cat/HDFSAggregateQueries.xml b/vxquery-xtest/src/test/resources/cat/HDFSAggregateQueries.xml new file mode 100644 index 0000000..4f925a0 --- /dev/null +++ b/vxquery-xtest/src/test/resources/cat/HDFSAggregateQueries.xml @@ -0,0 +1,53 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<test-group xmlns="http://www.w3.org/2005/02/query-test-XQTSCatalog"; name="AggregateQueries" featureOwner="VXQuery"> + <GroupInfo> + <title>HDFS Aggregate</title> + <description/> + </GroupInfo> + <test-case name="hdfs-aggregate-avg" FilePath="HDFS/Aggregate/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q02 from the weather benchmark.</description> + <query name="avgHDFS" date="2015-06-11"/> + <output-file compare="Text">avgHDFS.txt</output-file> + </test-case> + <test-case name="hdfs-aggregate-count" FilePath="HDFS/Aggregate/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q02 from the weather benchmark.</description> + <query name="countHDFS" date="2015-06-11"/> + <output-file compare="Text">countHDFS.txt</output-file> + </test-case> + <test-case name="hdfs-aggregate-min" FilePath="HDFS/Aggregate/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q05 from the weather benchmark.</description> + <query name="minHDFS" date="2015-06-11"/> + <output-file compare="Text">minHDFS.txt</output-file> + </test-case> + <test-case name="hdfs-aggregate-max" FilePath="HDFS/Aggregate/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q07 from the weather benchmark.</description> + <query name="maxHDFS" date="2015-06-11"/> + <output-file compare="Text">maxHDFS.txt</output-file> + </test-case> + <test-case name="hdfs-aggregate-sum" FilePath="HDFS/Aggregate/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q03 from the weather benchmark.</description> + <query name="sumHDFS" date="2015-06-11"/> + <output-file compare="Text">sumHDFS.txt</output-file> + </test-case> + <test-case name="hdfs-aggregate-max-value" FilePath="HDFS/Aggregate/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q03 from the weather benchmark.</description> + <query name="maxvalueHDFS" date="2015-10-17"/> + <output-file compare="Text">maxvalueHDFS.txt</output-file> + </test-case> +</test-group> http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/hadoop/conf/core-site.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/hadoop/conf/core-site.xml b/vxquery-xtest/src/test/resources/hadoop/conf/core-site.xml new file mode 100644 index 0000000..b1f576c --- /dev/null +++ b/vxquery-xtest/src/test/resources/hadoop/conf/core-site.xml @@ -0,0 +1,34 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + +<property> + <name>fs.default.name</name> + <value>hdfs://localhost:40000</value> +</property> +<property> + <name>hadoop.tmp.dir</name> + <value>/tmp/hadoop</value> +</property> + + +</configuration> http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/hadoop/conf/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/hadoop/conf/hdfs-site.xml b/vxquery-xtest/src/test/resources/hadoop/conf/hdfs-site.xml new file mode 100644 index 0000000..6b6604d --- /dev/null +++ b/vxquery-xtest/src/test/resources/hadoop/conf/hdfs-site.xml @@ -0,0 +1,34 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + +<property> + <name>dfs.replication</name> + <value>1</value> +</property> + +<property> + <name>dfs.block.size</name> + <value>1048576</value> +</property> + +</configuration> http://git-wip-us.apache.org/repos/asf/vxquery/blob/3fc2d6c2/vxquery-xtest/src/test/resources/hadoop/conf/mapred-site.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/hadoop/conf/mapred-site.xml b/vxquery-xtest/src/test/resources/hadoop/conf/mapred-site.xml new file mode 100644 index 0000000..e15ec74 --- /dev/null +++ b/vxquery-xtest/src/test/resources/hadoop/conf/mapred-site.xml @@ -0,0 +1,41 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + + <property> + <name>mapred.job.tracker</name> + <value>localhost:29007</value> + </property> + <property> + <name>mapred.tasktracker.map.tasks.maximum</name> + <value>20</value> + </property> + <property> + <name>mapred.tasktracker.reduce.tasks.maximum</name> + <value>20</value> + </property> + <property> + <name>mapred.max.split.size</name> + <value>128</value> + </property> + +</configuration>
