Repository: vxquery Updated Branches: refs/heads/master 9d26b31af -> 2c881020c
http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java index fcc35ef..b7839d0 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException; import org.apache.hyracks.algebricks.common.utils.Pair; @@ -33,12 +34,14 @@ import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue; +import org.apache.hyracks.algebricks.core.algebra.expressions.IConflictingTypeResolver; import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer; import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer; import org.apache.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory; -import org.apache.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer; +import org.apache.hyracks.algebricks.core.algebra.expressions.IMissableTypeComputer; import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider; +import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable; import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor; import org.apache.hyracks.algebricks.core.rewriter.base.AbstractRuleController; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; @@ -57,11 +60,12 @@ import org.apache.vxquery.compiler.algebricks.VXQueryBinaryIntegerInspectorFacto import org.apache.vxquery.compiler.algebricks.VXQueryComparatorFactoryProvider; import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue; import org.apache.vxquery.compiler.algebricks.VXQueryExpressionRuntimeProvider; -import org.apache.vxquery.compiler.algebricks.VXQueryNullWriterFactory; +import org.apache.vxquery.compiler.algebricks.VXQueryMissingWriterFactory; import org.apache.vxquery.compiler.algebricks.VXQueryPrinterFactoryProvider; import org.apache.vxquery.compiler.algebricks.prettyprint.VXQueryLogicalExpressionPrettyPrintVisitor; import org.apache.vxquery.compiler.rewriter.RewriteRuleset; import org.apache.vxquery.compiler.rewriter.VXQueryOptimizationContext; +import org.apache.vxquery.exceptions.SystemException; import org.apache.vxquery.metadata.VXQueryMetadataProvider; import org.apache.vxquery.runtime.provider.VXQueryBinaryHashFunctionFactoryProvider; import org.apache.vxquery.runtime.provider.VXQueryBinaryHashFunctionFamilyProvider; @@ -115,25 +119,27 @@ public class XMLQueryCompiler { public IOptimizationContext createOptimizationContext(int varCounter, IExpressionEvalSizeComputer expressionEvalSizeComputer, IMergeAggregationExpressionFactory mergeAggregationExpressionFactory, - IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer, - PhysicalOptimizationConfig physicalOptimizationConfig) { + IExpressionTypeComputer expressionTypeComputer, IMissableTypeComputer missableTypeComputer, + IConflictingTypeResolver conflictintTypeResolver, + PhysicalOptimizationConfig physicalOptimizationConfig, + AlgebricksPartitionConstraint clusterLocations) { return new VXQueryOptimizationContext(varCounter, expressionEvalSizeComputer, - mergeAggregationExpressionFactory, expressionTypeComputer, nullableTypeComputer, - physicalOptimizationConfig, pprinter); + mergeAggregationExpressionFactory, expressionTypeComputer, missableTypeComputer, + conflictintTypeResolver, physicalOptimizationConfig, clusterLocations, pprinter); } }); builder.getPhysicalOptimizationConfig().setFrameSize(this.frameSize); if (joinHashSize > 0) { - builder.getPhysicalOptimizationConfig().setMaxFramesHybridHash((int) (joinHashSize / this.frameSize)); + builder.getPhysicalOptimizationConfig().setMaxFramesForJoin((int) (joinHashSize / this.frameSize)); } if (maximumDataSize > 0) { builder.getPhysicalOptimizationConfig() - .setMaxFramesLeftInputHybridHash((int) (maximumDataSize / this.frameSize)); + .setMaxFramesForJoinLeftInput((int) (maximumDataSize / this.frameSize)); + } else { + builder.getPhysicalOptimizationConfig() + .setMaxFramesForJoinLeftInput((int) (60L * 1024 * 1048576 / this.frameSize)); } - builder.getPhysicalOptimizationConfig() - .setMaxFramesLeftInputHybridHash((int) (60L * 1024 * 1048576 / this.frameSize)); - builder.setLogicalRewrites(buildDefaultLogicalRewrites()); builder.setPhysicalRewrites(buildDefaultPhysicalRewrites()); builder.setSerializerDeserializerProvider(new ISerializerDeserializerProvider() { @@ -172,9 +178,9 @@ public class XMLQueryCompiler { return null; } }); - builder.setNullableTypeComputer(new INullableTypeComputer() { + builder.setMissableTypeComputer(new IMissableTypeComputer() { @Override - public Object makeNullableType(Object type) throws AlgebricksException { + public Object makeMissableType(Object type) throws AlgebricksException { SequenceType st = (SequenceType) type; if (st.getQuantifier().allowsEmptySequence()) { return type; @@ -185,7 +191,7 @@ public class XMLQueryCompiler { } @Override - public boolean canBeNull(Object type) { + public boolean canBeMissing(Object type) { return false; } @@ -194,7 +200,7 @@ public class XMLQueryCompiler { throw new NotImplementedException("NullableTypeComputer is not implented (getNonOptionalType)"); } }); - builder.setNullWriterFactory(new VXQueryNullWriterFactory()); + builder.setMissingWriterFactory(new VXQueryMissingWriterFactory()); if (availableProcessors < 1) { builder.setClusterLocations(VXQueryMetadataProvider.getClusterLocations(nodeList)); } else { @@ -215,11 +221,11 @@ public class XMLQueryCompiler { } public void compile(String name, Reader query, CompilerControlBlock ccb, int optimizationLevel) - throws AlgebricksException { + throws AlgebricksException, SystemException { moduleNode = XMLQueryParser.parse(name, query); listener.notifyParseResult(moduleNode); module = new XMLQueryTranslator(ccb).translateModule(moduleNode); - pprinter = new LogicalOperatorPrettyPrintVisitor( + pprinter = new LogicalOperatorPrettyPrintVisitor(new AlgebricksAppendable(), new VXQueryLogicalExpressionPrettyPrintVisitor(module.getModuleContext())); VXQueryMetadataProvider mdProvider = new VXQueryMetadataProvider(nodeList, ccb.getSourceFileMap(), module.getModuleContext(), this.hdfsConf, nodeControllerInfos); @@ -278,7 +284,7 @@ public class XMLQueryCompiler { } private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultPhysicalRewrites() { - List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultPhysicalRewrites = new ArrayList<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>(); + List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultPhysicalRewrites = new ArrayList<>(); SequentialOnceRuleController seqOnceCtrlAllLevels = new SequentialOnceRuleController(true); SequentialOnceRuleController seqOnceCtrlTopLevel = new SequentialOnceRuleController(false); defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlAllLevels, http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java index ff230d3..3d95cb1 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/translator/XMLQueryTranslator.java @@ -2143,7 +2143,7 @@ public class XMLQueryTranslator { private List<LogicalVariable> translateExpressionList(List<ASTNode> expressions, TranslationContext tCtx) throws SystemException { - List<LogicalVariable> result = new ArrayList<LogicalVariable>(); + List<LogicalVariable> result = new ArrayList<>(); for (ASTNode e : expressions) { result.add(translateExpression(e, tCtx)); } @@ -2151,11 +2151,11 @@ public class XMLQueryTranslator { } private static Mutable<ILogicalExpression> mutable(ILogicalExpression expr) { - return new MutableObject<ILogicalExpression>(expr); + return new MutableObject<>(expr); } private static Mutable<ILogicalOperator> mutable(ILogicalOperator op) { - return new MutableObject<ILogicalOperator>(op); + return new MutableObject<>(op); } private LogicalVariable newLogicalVariable() { http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java index 51e6091..7646f97 100644 --- a/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java +++ b/vxquery-core/src/test/java/org/apache/vxquery/xmlquery/query/SimpleXQueryTest.java @@ -137,7 +137,7 @@ public class SimpleXQueryTest { private static void runTestInternal(String testName, String query) throws Exception { Map<String, NodeControllerInfo> nodeControllerInfos = new HashMap<String, NodeControllerInfo>(); - nodeControllerInfos.put("nc1", new NodeControllerInfo("nc1", null, new NetworkAddress("127.0.0.1", 0), null)); + nodeControllerInfos.put("nc1", new NodeControllerInfo("nc1", null, new NetworkAddress("127.0.0.1", 0), null, null, 1)); XMLQueryCompiler compiler = new XMLQueryCompiler(null, nodeControllerInfos, 65536); CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE), http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-server/src/main/java/org/apache/vxquery/cli/VXQueryClusterShutdown.java ---------------------------------------------------------------------- diff --git a/vxquery-server/src/main/java/org/apache/vxquery/cli/VXQueryClusterShutdown.java b/vxquery-server/src/main/java/org/apache/vxquery/cli/VXQueryClusterShutdown.java index ced608a..c93996d 100644 --- a/vxquery-server/src/main/java/org/apache/vxquery/cli/VXQueryClusterShutdown.java +++ b/vxquery-server/src/main/java/org/apache/vxquery/cli/VXQueryClusterShutdown.java @@ -17,17 +17,16 @@ package org.apache.vxquery.cli; import java.util.ArrayList; import java.util.List; +import org.apache.hyracks.api.client.HyracksConnection; +import org.apache.hyracks.api.client.IHyracksClientConnection; import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.CmdLineParser; import org.kohsuke.args4j.Option; -import org.apache.hyracks.api.client.HyracksConnection; -import org.apache.hyracks.api.client.IHyracksClientConnection; - public class VXQueryClusterShutdown { /** * Main method to get command line options and execute query process. - * + * * @param args * @throws Exception */ @@ -42,16 +41,16 @@ public class VXQueryClusterShutdown { parser.printUsage(System.err); return; } - + // give error message if missing arguments if (opts.clientNetIpAddress == null) { parser.printUsage(System.err); return; } - + try { IHyracksClientConnection hcc = new HyracksConnection(opts.clientNetIpAddress, opts.clientNetPort); - hcc.stopCluster(); + hcc.stopCluster(true); } catch (Exception e) { System.err.println("Unable to connect and shutdown the Hyracks cluster."); System.err.println(e); http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-xtest/pom.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/pom.xml b/vxquery-xtest/pom.xml index 27c2bf8..a00bec2 100644 --- a/vxquery-xtest/pom.xml +++ b/vxquery-xtest/pom.xml @@ -188,6 +188,27 @@ <artifactId>jetty</artifactId> <scope>compile</scope> </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>2.7.0</version> + <classifier>tests</classifier> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>2.7.0</version> + <classifier>tests</classifier> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> <reporting> http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java index 1cd2696..e4ba6eb 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunner.java @@ -29,6 +29,7 @@ import org.apache.hyracks.api.client.NodeControllerInfo; import org.apache.hyracks.api.comm.IFrame; import org.apache.hyracks.api.comm.IFrameTupleAccessor; import org.apache.hyracks.api.comm.VSizeFrame; +import org.apache.hyracks.api.dataset.DatasetJobRecord; import org.apache.hyracks.api.dataset.IHyracksDataset; import org.apache.hyracks.api.dataset.IHyracksDatasetReader; import org.apache.hyracks.api.dataset.ResultSetId; @@ -36,7 +37,6 @@ import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.job.JobFlag; import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobSpecification; -import org.apache.hyracks.client.dataset.HyracksDataset; import org.apache.hyracks.control.nc.resources.memory.FrameManager; import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor; import org.apache.vxquery.compiler.CompilerControlBlock; @@ -52,8 +52,7 @@ import org.apache.vxquery.xmlquery.query.VXQueryCompilationListener; import org.apache.vxquery.xmlquery.query.XMLQueryCompiler; public class TestRunner { - private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern - .compile("org\\.apache\\.vxquery\\.exceptions\\.SystemException: (\\p{javaUpperCase}{4}\\d{4})"); + private static final Pattern EMBEDDED_SYSERROR_PATTERN = Pattern.compile("(\\p{javaUpperCase}{4}\\d{4})"); private XTestOptions opts; private IHyracksClientConnection hcc; @@ -112,6 +111,10 @@ public class TestRunner { FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize()); IFrame frame = new VSizeFrame(resultDisplayFrameMgr); IHyracksDatasetReader reader = hds.createReader(jobId, ccb.getResultSetId()); + // TODO(tillw) remove this loop once the IHyracksDatasetReader reliably returns the correct exception + while (reader.getResultStatus() == DatasetJobRecord.Status.RUNNING) { + Thread.sleep(1); + } IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor(); res.result = ""; while (reader.read(frame) > 0) { @@ -125,10 +128,13 @@ public class TestRunner { while (t.getCause() != null) { t = t.getCause(); } - Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(t.getMessage()); - if (m.find()) { - String eCode = m.group(1); - throw new SystemException(ErrorCode.valueOf(eCode), e); + final String message = t.getMessage(); + if (message != null) { + Matcher m = EMBEDDED_SYSERROR_PATTERN.matcher(message); + if (m.find()) { + String eCode = m.group(1); + throw new SystemException(ErrorCode.valueOf(eCode), e); + } } throw e; } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java index 96cc7cb..1e2dcf6 100644 --- a/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java +++ b/vxquery-xtest/src/test/java/org/apache/vxquery/xtest/AbstractXQueryTest.java @@ -117,15 +117,14 @@ public abstract class AbstractXQueryTest { public static void shutdown() throws IOException { removeFS(); TestClusterUtil.stopCluster(cc, nc); - } public static void removeFS() throws IOException { + dfs.shutdownHDFS(); File tmp = new File(TMP); if (tmp.exists()) { FileUtils.deleteDirectory(tmp); } - dfs.shutdownHDFS(); } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-xtest/src/test/resources/ExpectedTestResults/Simple/data_types.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/Simple/data_types.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/Simple/data_types.txt new file mode 100644 index 0000000..7976dbd --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/Simple/data_types.txt @@ -0,0 +1,15 @@ +10.99 +http://www.example.com/catalog.xml +true +8 +1999-12-31 +1999-12-31T00:00:00 +2.3 +1.5 +0.4 +---30+10:30 +64 +32 +16 +string +01:02:03 http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-xtest/src/test/resources/Queries/XQuery/Simple/data_types.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/Simple/data_types.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/Simple/data_types.xq new file mode 100644 index 0000000..67a24dc --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/Simple/data_types.xq @@ -0,0 +1,32 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(xs:untypedAtomic("10.99"), +xs:anyURI("http://www.example.com/catalog.xml";), +xs:boolean("true"), +xs:byte(8), +xs:date("1999-12-31"), +xs:dateTime("1999-12-31T24:00:00"), +xs:decimal(2.3), +xs:double(1.5), +xs:float(0.4), +xs:gDay("---30+10:30"), +xs:int(64), +xs:long(32), +xs:short(16), +xs:string("string"), +xs:time("01:02:03")) http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-xtest/src/test/resources/cat/SingleQuery.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/cat/SingleQuery.xml b/vxquery-xtest/src/test/resources/cat/SingleQuery.xml index c9bd305..8800339 100644 --- a/vxquery-xtest/src/test/resources/cat/SingleQuery.xml +++ b/vxquery-xtest/src/test/resources/cat/SingleQuery.xml @@ -23,9 +23,19 @@ <query name="add" date="2014-08-18"/> <output-file compare="Text">add.txt</output-file> </test-case> + <test-case name="simple-data-types" FilePath="Simple/" Creator="Preston Carman"> + <description>Adds two numbers.</description> + <query name="data_types" date="2017-04-17"/> + <output-file compare="Text">data_types.txt</output-file> + </test-case> <test-case name="simple-fn-doc" FilePath="Simple/" Creator="Preston Carman"> <description>Adds two numbers.</description> <query name="fn_doc" date="2014-08-18"/> <output-file compare="Text">fn_doc.txt</output-file> </test-case> + <test-case name="simple-list" FilePath="Simple/" Creator="Preston Carman"> + <description>Adds two numbers.</description> + <query name="list" date="2017-04-17"/> + <output-file compare="Text">list.txt</output-file> + </test-case> </test-group>
