VXQUERY-240: Update Hyracks Dependency to version 0.3.0. Primary changes revolve around changes to UTF8StringPointable (storage layout changed) and exception handling.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/2c881020 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/2c881020 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/2c881020 Branch: refs/heads/master Commit: 2c881020c46d260f7b20d0af8439e32dbfaae974 Parents: 9d26b31 Author: Preston Carman <[email protected]> Authored: Sat Apr 29 22:08:46 2017 -0700 Committer: Preston Carman <[email protected]> Committed: Sun Apr 30 21:34:16 2017 -0700 ---------------------------------------------------------------------- pom.xml | 2 +- vxquery-core/pom.xml | 14 + .../algebricks/VXQueryConstantValue.java | 8 +- .../VXQueryExpressionRuntimeProvider.java | 41 ++- .../algebricks/VXQueryMissingWriterFactory.java | 46 +++ .../algebricks/VXQueryNullWriterFactory.java | 47 --- .../VXQueryPrinterFactoryProvider.java | 4 +- .../compiler/rewriter/RewriteRuleset.java | 137 +++---- .../rewriter/VXQueryOptimizationContext.java | 28 +- .../rewriter/rules/AbstractCollectionRule.java | 21 +- .../rewriter/rules/IntroduceCollectionRule.java | 2 +- .../rules/ReplaceSourceMapInDocExpression.java | 37 +- .../accessors/atomic/XSDatePointable.java | 7 +- .../accessors/atomic/XSQNamePointable.java | 17 +- .../accessors/jsonitem/ObjectPointable.java | 10 +- .../accessors/nodes/NodeTreePointable.java | 33 +- .../accessors/nodes/PINodePointable.java | 8 +- .../builders/atomic/StringValueBuilder.java | 4 +- .../builders/atomic/UTF8StringBuilder.java | 51 --- .../atomic/VXQueryUTF8StringBuilder.java | 29 ++ .../builders/nodes/CommentNodeBuilder.java | 5 + .../builders/nodes/DictionaryBuilder.java | 15 +- .../datamodel/builders/nodes/PINodeBuilder.java | 9 + .../builders/nodes/TextNodeBuilder.java | 5 + .../vxquery/datamodel/values/XDMConstants.java | 3 +- .../vxquery/exceptions/SystemException.java | 4 +- .../exceptions/VXQueryDataException.java | 4 +- .../VXQueryFileNotFoundException.java | 4 +- .../exceptions/VXQueryParseException.java | 4 +- .../vxquery/index/IndexDocumentBuilder.java | 8 +- .../metadata/AbstractVXQueryDataSource.java | 21 +- .../vxquery/metadata/QueryResultDataSink.java | 2 +- .../metadata/VXQueryCollectionDataSource.java | 6 +- .../VXQueryCollectionOperatorDescriptor.java | 4 +- .../metadata/VXQueryIndexingDataSource.java | 9 +- .../VXQueryIndexingOperatorDescriptor.java | 10 +- .../metadata/VXQueryMetadataProvider.java | 67 ++-- ...AbstractMaxMinAggregateEvaluatorFactory.java | 24 +- .../AbstractMaxMinScalarEvaluatorFactory.java | 18 +- .../AvgGlobalAggregateEvaluatorFactory.java | 29 +- .../AvgLocalAggregateEvaluatorFactory.java | 30 +- .../FnAvgAggregateEvaluatorFactory.java | 25 +- .../aggregate/FnAvgScalarEvaluatorFactory.java | 17 +- .../FnCountAggregateEvaluatorFactory.java | 24 +- .../FnCountScalarEvaluatorFactory.java | 19 +- .../FnSumAggregateEvaluatorFactory.java | 26 +- .../aggregate/FnSumScalarEvaluatorFactory.java | 17 +- ...bstractArithmeticScalarEvaluatorFactory.java | 13 +- ...ctTaggedValueArgumentAggregateEvaluator.java | 14 +- ...dValueArgumentAggregateEvaluatorFactory.java | 9 +- ...tractTaggedValueArgumentScalarEvaluator.java | 14 +- ...ggedValueArgumentScalarEvaluatorFactory.java | 6 +- ...ctTaggedValueArgumentUnnestingEvaluator.java | 14 +- ...dValueArgumentUnnestingEvaluatorFactory.java | 8 +- .../bool/AndScalarEvaluatorFactory.java | 13 +- .../bool/FnBooleanScalarEvaluatorFactory.java | 25 +- .../bool/FnFalseScalarEvaluatorFactory.java | 13 +- .../bool/FnNotScalarEvaluatorFactory.java | 15 +- .../bool/FnTrueScalarEvaluatorFactory.java | 13 +- .../bool/OrScalarEvaluatorFactory.java | 13 +- .../cast/CastScalarEvaluatorFactory.java | 158 +++----- .../cast/CastToBase64BinaryOperation.java | 10 +- .../functions/cast/CastToBooleanOperation.java | 13 +- .../functions/cast/CastToDecimalOperation.java | 25 +- .../functions/cast/CastToQNameOperation.java | 59 ++- .../functions/cast/CastToStringOperation.java | 365 ++++++++++--------- .../CastableScalarEvaluatorFactory.java | 15 +- ...ctValueComparisonScalarEvaluatorFactory.java | 17 +- ...GeneralComparisonScalarEvaluatorFactory.java | 19 +- .../IfThenElseScalarEvaluatorFactory.java | 37 +- ...ValueFromDateTimeScalarEvaluatorFactory.java | 15 +- ...ValueFromDurationScalarEvaluatorFactory.java | 19 +- ...ateTimeToTimezoneScalarEvaluatorFactory.java | 17 +- ...ustDateToTimezoneScalarEvaluatorFactory.java | 17 +- ...ustTimeToTimezoneScalarEvaluatorFactory.java | 17 +- .../FnCurrentDateScalarEvaluatorFactory.java | 15 +- ...FnCurrentDateTimeScalarEvaluatorFactory.java | 15 +- .../FnCurrentTimeScalarEvaluatorFactory.java | 15 +- .../FnDateTimeScalarEvaluatorFactory.java | 15 +- ...nImplicitTimezoneScalarEvaluatorFactory.java | 15 +- .../error/FnErrorScalarEvaluatorFactory.java | 30 +- .../index/ShowIndexScalarEvaluatorFactory.java | 8 +- .../json/JnDocScalarEvaluatorFactory.java | 4 +- .../json/JnKeysScalarEvaluatorFactory.java | 4 +- .../json/JnMembersScalarEvaluatorFactory.java | 4 +- .../json/JnNullScalarEvaluatorFactory.java | 4 +- .../json/JnParseJsonScalarEvaluatorFactory.java | 17 +- .../KeysOrMembersScalarEvaluatorFactory.java | 4 +- .../LibjnAccumulateScalarEvaluatorFactory.java | 4 +- ...nDescendantArraysScalarEvaluatorFactory.java | 4 +- ...DescendantObjectsScalarEvaluatorFactory.java | 4 +- ...jnDescendantPairsScalarEvaluatorFactory.java | 4 +- .../LibjnFlattenScalarEvaluatorFactory.java | 4 +- .../LibjnIntersectScalarEvaluatorFactory.java | 4 +- .../LibjnProjectScalarEvaluatorFactory.java | 4 +- .../LibjnRemoveKeysScalarEvaluatorFactory.java | 4 +- .../json/LibjnValuesScalarEvaluatorFactory.java | 4 +- .../json/ValueScalarEvaluatorFactory.java | 4 +- .../ArrayConstructorScalarEvaluatorFactory.java | 4 +- .../ObjectConstructorScalarEvaluator.java | 6 +- ...ObjectConstructorScalarEvaluatorFactory.java | 5 +- ...SimpleObjectUnionScalarEvaluatorFactory.java | 5 +- .../misc/FnDataScalarEvaluatorFactory.java | 17 +- .../misc/JnSizeScalarEvaluatorFactory.java | 4 +- ...teNodeConstructorScalarEvaluatorFactory.java | 7 +- ...ntNodeConstructorScalarEvaluatorFactory.java | 7 +- ...ntNodeConstructorScalarEvaluatorFactory.java | 7 +- .../FnDocAvailableScalarEvaluatorFactory.java | 19 +- .../node/FnDocScalarEvaluatorFactory.java | 19 +- .../node/FnNumberScalarEvaluatorFactory.java | 15 +- .../node/IdFromNodeScalarEvaluatorFactory.java | 15 +- .../LocalIdFromNodeScalarEvaluatorFactory.java | 15 +- .../node/OpNodeAfterEvaluatorFactory.java | 7 +- .../node/OpNodeBeforeEvaluatorFactory.java | 7 +- ...PINodeConstructorScalarEvaluatorFactory.java | 7 +- ...xtNodeConstructorScalarEvaluatorFactory.java | 7 +- .../TreeIdFromNodeScalarEvaluatorFactory.java | 15 +- ...NodesAscOrAtomicsScalarEvaluatorFactory.java | 13 +- .../AbstractNumericScalarEvaluatorFactory.java | 17 +- ...FnRoundHalfToEvenScalarEvaluatorFactory.java | 25 +- ...ocalNameFromQNameScalarEvaluatorFactory.java | 17 +- ...spaceUriFromQNameScalarEvaluatorFactory.java | 17 +- ...FnPrefixFromQNameScalarEvaluatorFactory.java | 17 +- .../qname/FnQNameScalarEvaluatorFactory.java | 66 ++-- .../ConcatenateScalarEvaluatorFactory.java | 17 +- .../FnDistinctValuesScalarEvaluatorFactory.java | 19 +- .../sequence/FnEmptyScalarEvaluatorFactory.java | 13 +- .../FnExactlyOneScalarEvaluatorFactory.java | 13 +- .../FnExistsScalarEvaluatorFactory.java | 13 +- .../FnIndexOfScalarEvaluatorFactory.java | 17 +- .../FnInsertBeforeScalarEvaluatorFactory.java | 19 +- .../FnOneOrMoreScalarEvaluatorFactory.java | 13 +- .../FnRemoveScalarEvaluatorFactory.java | 19 +- .../FnReverseScalarEvaluatorFactory.java | 17 +- .../FnSubsequenceScalarEvaluatorFactory.java | 21 +- .../FnUnorderedScalarEvaluatorFactory.java | 13 +- .../FnZeroOrOneScalarEvaluatorFactory.java | 13 +- .../IterateUnnestingEvaluatorFactory.java | 17 +- .../sequence/OpToScalarEvaluatorFactory.java | 17 +- .../SequenceAggregateEvaluatorFactory.java | 29 +- .../FnSerializeScalarEvaluatorFactory.java | 21 +- ...AttributePathStepScalarEvaluatorFactory.java | 7 +- .../step/ChildPathStepScalarEvaluator.java | 18 +- .../ChildPathStepScalarEvaluatorFactory.java | 7 +- .../functions/step/ChildPathStepUnnesting.java | 20 +- .../step/ChildPathStepUnnestingEvaluator.java | 12 +- .../ChildPathStepUnnestingEvaluatorFactory.java | 7 +- ...DescendantOrSelfPathStepScalarEvaluator.java | 18 +- ...antOrSelfPathStepScalarEvaluatorFactory.java | 7 +- .../step/DescendantOrSelfPathStepUnnesting.java | 22 +- ...cendantOrSelfPathStepUnnestingEvaluator.java | 12 +- ...OrSelfPathStepUnnestingEvaluatorFactory.java | 7 +- .../step/DescendantPathStepScalarEvaluator.java | 18 +- ...escendantPathStepScalarEvaluatorFactory.java | 7 +- .../DescendantPathStepUnnestingEvaluator.java | 11 +- ...endantPathStepUnnestingEvaluatorFactory.java | 7 +- .../runtime/functions/step/NodeTestFilter.java | 11 +- .../SelfPathStepScalarEvaluatorFactory.java | 7 +- ...stractCharacterIteratorCopyingEvaluator.java | 26 +- .../AbstractTranscodingCharacterIterator.java | 8 +- .../FnCodepointEqualEvaluatorFactory.java | 15 +- .../FnCodepointsToStringEvaluatorFactory.java | 40 +- .../strings/FnCompareEvaluatorFactory.java | 17 +- .../strings/FnConcatEvaluatorFactory.java | 45 +-- .../strings/FnContainsEvaluatorFactory.java | 19 +- .../strings/FnEndsWithEvaluatorFactory.java | 15 +- .../strings/FnLowerCaseEvaluatorFactory.java | 13 +- .../strings/FnStartsWithEvaluatorFactory.java | 19 +- .../strings/FnStringJoinEvaluatorFactory.java | 37 +- .../strings/FnStringLengthEvaluatorFactory.java | 17 +- .../strings/FnStringScalarEvaluatorFactory.java | 17 +- .../FnStringToCodepointsEvaluatorFactory.java | 19 +- .../FnSubstringAfterEvaluatorFactory.java | 13 +- .../FnSubstringBeforeEvaluatorFactory.java | 13 +- .../strings/FnSubstringEvaluatorFactory.java | 19 +- .../strings/FnUpperCaseEvaluatorFactory.java | 13 +- .../strings/UTF8StringCharacterIterator.java | 10 +- .../trace/FnTraceScalarEvaluatorFactory.java | 4 +- .../type/InstanceOfScalarEvaluatorFactory.java | 13 +- .../type/PromoteScalarEvaluatorFactory.java | 15 +- .../type/TreatScalarEvaluatorFactory.java | 13 +- ...tractNumericUnaryScalarEvaluatorFactory.java | 17 +- .../runtime/functions/util/AtomizeHelper.java | 67 ++-- .../runtime/functions/util/FunctionHelper.java | 124 ++++--- .../vxquery/serializer/XMLSerializer.java | 33 +- .../java/org/apache/vxquery/types/NameTest.java | 6 +- .../types/ProcessingInstructionType.java | 4 +- .../vxquery/xmlparser/SAXContentHandler.java | 64 ++-- .../org/apache/vxquery/xmlparser/XMLParser.java | 15 +- .../query/VXQueryCompilationListener.java | 31 +- .../xmlquery/query/XMLQueryCompiler.java | 42 ++- .../xmlquery/translator/XMLQueryTranslator.java | 6 +- .../xmlquery/query/SimpleXQueryTest.java | 2 +- .../vxquery/cli/VXQueryClusterShutdown.java | 13 +- vxquery-xtest/pom.xml | 21 ++ .../org/apache/vxquery/xtest/TestRunner.java | 20 +- .../vxquery/xtest/AbstractXQueryTest.java | 3 +- .../ExpectedTestResults/Simple/data_types.txt | 15 + .../Queries/XQuery/Simple/data_types.xq | 32 ++ .../src/test/resources/cat/SingleQuery.xml | 10 + 200 files changed, 1968 insertions(+), 1944 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index cc8884f..5f03a49 100644 --- a/pom.xml +++ b/pom.xml @@ -670,7 +670,7 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - <hyracks.version>0.2.17-incubating</hyracks.version> + <hyracks.version>0.3.0</hyracks.version> <apache-rat-plugin.version>0.11</apache-rat-plugin.version> </properties> http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/pom.xml ---------------------------------------------------------------------- diff --git a/vxquery-core/pom.xml b/vxquery-core/pom.xml index 47015a9..38908d7 100644 --- a/vxquery-core/pom.xml +++ b/vxquery-core/pom.xml @@ -182,6 +182,11 @@ <dependency> <groupId>org.apache.hyracks</groupId> + <artifactId>algebricks-rewriter</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.hyracks</groupId> <artifactId>algebricks-data</artifactId> </dependency> @@ -271,50 +276,59 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + <dependency> <artifactId>lucene-core</artifactId> <groupId>org.apache.lucene</groupId> <type>jar</type> <version>5.5.1</version> </dependency> + <dependency> <artifactId>lucene-queryparser</artifactId> <groupId>org.apache.lucene</groupId> <type>jar</type> <version>5.5.1</version> </dependency> + <dependency> <artifactId>lucene-analyzers-common</artifactId> <groupId>org.apache.lucene</groupId> <type>jar</type> <version>5.5.1</version> </dependency> + <dependency> <artifactId>lucene-demo</artifactId> <groupId>org.apache.lucene</groupId> <type>jar</type> <version>5.5.1</version> </dependency> + <dependency> <artifactId>lucene-backward-codecs</artifactId> <groupId>org.apache.lucene</groupId> <type>jar</type> <version>5.5.1</version> </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>2.7.0</version> </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.7.0</version> </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> </dependency> + </dependencies> <reporting> http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryConstantValue.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryConstantValue.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryConstantValue.java index e105546..92b35ba 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryConstantValue.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryConstantValue.java @@ -17,9 +17,8 @@ package org.apache.vxquery.compiler.algebricks; import org.apache.commons.codec.binary.Hex; -import org.apache.vxquery.types.SequenceType; - import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue; +import org.apache.vxquery.types.SequenceType; public class VXQueryConstantValue implements IAlgebricksConstantValue { private final SequenceType type; @@ -61,4 +60,9 @@ public class VXQueryConstantValue implements IAlgebricksConstantValue { .append("])"); return buffer.toString(); } + + @Override + public boolean isMissing() { + return false; + } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryExpressionRuntimeProvider.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryExpressionRuntimeProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryExpressionRuntimeProvider.java index 04d15e7..dcd285c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryExpressionRuntimeProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryExpressionRuntimeProvider.java @@ -19,10 +19,6 @@ package org.apache.vxquery.compiler.algebricks; import java.util.List; import org.apache.commons.lang3.mutable.Mutable; -import org.apache.vxquery.datamodel.values.XDMConstants; -import org.apache.vxquery.exceptions.SystemException; -import org.apache.vxquery.functions.Function; - import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression; @@ -37,15 +33,18 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceE import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext; import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory; -import org.apache.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory; import org.apache.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.algebricks.runtime.base.ISerializedAggregateEvaluatorFactory; import org.apache.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory; -import org.apache.hyracks.algebricks.runtime.evaluators.ConstantEvaluatorFactory; +import org.apache.hyracks.algebricks.runtime.evaluators.ConstantEvalFactory; import org.apache.hyracks.algebricks.runtime.evaluators.TupleFieldEvaluatorFactory; import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.primitive.BooleanPointable; import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.vxquery.datamodel.values.XDMConstants; +import org.apache.vxquery.exceptions.SystemException; +import org.apache.vxquery.functions.Function; public class VXQueryExpressionRuntimeProvider implements IExpressionRuntimeProvider { @Override @@ -53,23 +52,23 @@ public class VXQueryExpressionRuntimeProvider implements IExpressionRuntimeProvi IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { switch (expr.getExpressionTag()) { case CONSTANT: - IAlgebricksConstantValue constantValue = (IAlgebricksConstantValue) ((ConstantExpression) expr) + IAlgebricksConstantValue constantValue = ((ConstantExpression) expr) .getValue(); if (constantValue.isFalse()) { IPointable p = (BooleanPointable) BooleanPointable.FACTORY.createPointable(); XDMConstants.setFalse(p); - return new ConstantEvaluatorFactory(p.getByteArray()); + return new ConstantEvalFactory(p.getByteArray()); } else if (constantValue.isNull()) { IPointable p = (VoidPointable) VoidPointable.FACTORY.createPointable(); XDMConstants.setEmptySequence(p); - return new ConstantEvaluatorFactory(p.getByteArray()); + return new ConstantEvalFactory(p.getByteArray()); } else if (constantValue.isTrue()) { IPointable p = (BooleanPointable) BooleanPointable.FACTORY.createPointable(); XDMConstants.setTrue(p); - return new ConstantEvaluatorFactory(p.getByteArray()); + return new ConstantEvalFactory(p.getByteArray()); } VXQueryConstantValue cv = (VXQueryConstantValue) ((ConstantExpression) expr).getValue(); - return new ConstantEvaluatorFactory(cv.getValue()); + return new ConstantEvalFactory(cv.getValue()); case VARIABLE: VariableReferenceExpression vrExpr = (VariableReferenceExpression) expr; @@ -82,7 +81,11 @@ public class VXQueryExpressionRuntimeProvider implements IExpressionRuntimeProvi IScalarEvaluatorFactory[] argFactories = createArgumentEvaluatorFactories(env, inputSchemas, context, fcExpr.getArguments()); - return fn.createScalarEvaluatorFactory(argFactories); + try { + return fn.createScalarEvaluatorFactory(argFactories); + } catch (SystemException e) { + throw new AlgebricksException(e); + } } throw new UnsupportedOperationException("Cannot create runtime for " + expr.getExpressionTag()); } @@ -106,11 +109,15 @@ public class VXQueryExpressionRuntimeProvider implements IExpressionRuntimeProvi IScalarEvaluatorFactory[] argFactories = createArgumentEvaluatorFactories(env, inputSchemas, context, expr.getArguments()); - return fn.createAggregateEvaluatorFactory(argFactories); + try { + return fn.createAggregateEvaluatorFactory(argFactories); + } catch (SystemException e) { + throw new AlgebricksException(e); + } } @Override - public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory( + public ISerializedAggregateEvaluatorFactory createSerializableAggregateFunctionFactory( AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { return null; @@ -131,6 +138,10 @@ public class VXQueryExpressionRuntimeProvider implements IExpressionRuntimeProvi IScalarEvaluatorFactory[] argFactories = createArgumentEvaluatorFactories(env, inputSchemas, context, expr.getArguments()); - return fn.createUnnestingEvaluatorFactory(argFactories); + try { + return fn.createUnnestingEvaluatorFactory(argFactories); + } catch (SystemException e) { + throw new AlgebricksException(e); + } } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryMissingWriterFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryMissingWriterFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryMissingWriterFactory.java new file mode 100644 index 0000000..4003d79 --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryMissingWriterFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.compiler.algebricks; + +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hyracks.api.dataflow.value.IMissingWriter; +import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.vxquery.datamodel.values.XDMConstants; + +public class VXQueryMissingWriterFactory implements IMissingWriterFactory { + private static final long serialVersionUID = 1L; + + @Override + public IMissingWriter createMissingWriter() { + final VoidPointable vp = (VoidPointable) VoidPointable.FACTORY.createPointable(); + return new IMissingWriter() { + @Override + public void writeMissing(DataOutput out) throws HyracksDataException { + XDMConstants.setEmptySequence(vp); + try { + out.write(vp.getByteArray(), vp.getStartOffset(), vp.getLength()); + } catch (IOException e) { + throw new HyracksDataException(e); + } + } + }; + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryNullWriterFactory.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryNullWriterFactory.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryNullWriterFactory.java deleted file mode 100644 index f0220dd..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryNullWriterFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.vxquery.compiler.algebricks; - -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.vxquery.datamodel.values.XDMConstants; - -import org.apache.hyracks.api.dataflow.value.INullWriter; -import org.apache.hyracks.api.dataflow.value.INullWriterFactory; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.data.std.primitive.VoidPointable; - -public class VXQueryNullWriterFactory implements INullWriterFactory { - private static final long serialVersionUID = 1L; - - @Override - public INullWriter createNullWriter() { - final VoidPointable vp = (VoidPointable) VoidPointable.FACTORY.createPointable(); - return new INullWriter() { - @Override - public void writeNull(DataOutput out) throws HyracksDataException { - XDMConstants.setEmptySequence(vp); - try { - out.write(vp.getByteArray(), vp.getStartOffset(), vp.getLength()); - } catch (IOException e) { - throw new HyracksDataException(e); - } - } - }; - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryPrinterFactoryProvider.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryPrinterFactoryProvider.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryPrinterFactoryProvider.java index 65ef917..11d6204 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryPrinterFactoryProvider.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/algebricks/VXQueryPrinterFactoryProvider.java @@ -16,15 +16,15 @@ */ package org.apache.vxquery.compiler.algebricks; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.data.IPrinterFactory; import org.apache.hyracks.algebricks.data.IPrinterFactoryProvider; +import org.apache.hyracks.api.exceptions.HyracksDataException; public class VXQueryPrinterFactoryProvider implements IPrinterFactoryProvider { public static final IPrinterFactoryProvider INSTANCE = new VXQueryPrinterFactoryProvider(); @Override - public IPrinterFactory getPrinterFactory(Object type) throws AlgebricksException { + public IPrinterFactory getPrinterFactory(Object type) throws HyracksDataException { return new VXQueryPrinterFactory(); } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java index d5909c6..1ee4833 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java @@ -19,33 +19,6 @@ package org.apache.vxquery.compiler.rewriter; import java.util.LinkedList; import java.util.List; -import org.apache.vxquery.compiler.rewriter.rules.ConsolidateAssignAggregateRule; -import org.apache.vxquery.compiler.rewriter.rules.ConsolidateDescandantChild; -import org.apache.vxquery.compiler.rewriter.rules.ConvertAssignToUnnestRule; -import org.apache.vxquery.compiler.rewriter.rules.ConvertFromAlgebricksExpressionsRule; -import org.apache.vxquery.compiler.rewriter.rules.ConvertToAlgebricksExpressionsRule; -import org.apache.vxquery.compiler.rewriter.rules.EliminateSubplanForSingleItemsRule; -import org.apache.vxquery.compiler.rewriter.rules.EliminateUnnestAggregateSequencesRule; -import org.apache.vxquery.compiler.rewriter.rules.EliminateUnnestAggregateSubplanRule; -import org.apache.vxquery.compiler.rewriter.rules.IntroduceCollectionRule; -import org.apache.vxquery.compiler.rewriter.rules.IntroduceIndexingRule; -import org.apache.vxquery.compiler.rewriter.rules.IntroduceTwoStepAggregateRule; -import org.apache.vxquery.compiler.rewriter.rules.PushChildIntoDataScanRule; -import org.apache.vxquery.compiler.rewriter.rules.PushFunctionsOntoEqJoinBranches; -import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantBooleanExpressionsRule; -import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantCastExpressionsRule; -import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantDataExpressionsRule; -import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantPromoteExpressionsRule; -import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantTreatExpressionsRule; -import org.apache.vxquery.compiler.rewriter.rules.RemoveUnusedSortDistinctNodesRule; -import org.apache.vxquery.compiler.rewriter.rules.RemoveUnusedUnnestIterateRule; -import org.apache.vxquery.compiler.rewriter.rules.ReplaceSourceMapInDocExpression; -import org.apache.vxquery.compiler.rewriter.rules.SetCollectionDataSourceRule; -import org.apache.vxquery.compiler.rewriter.rules.SetVariableIdContextRule; -import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.ExtractFunctionsFromJoinConditionRule; -import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.InlineNestedVariablesRule; -import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.MoveFreeVariableOperatorOutOfSubplanRule; - import org.apache.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; import org.apache.hyracks.algebricks.rewriter.rules.BreakSelectIntoConjunctsRule; @@ -55,8 +28,6 @@ import org.apache.hyracks.algebricks.rewriter.rules.ConsolidateAssignsRule; import org.apache.hyracks.algebricks.rewriter.rules.ConsolidateSelectsRule; import org.apache.hyracks.algebricks.rewriter.rules.CopyLimitDownRule; import org.apache.hyracks.algebricks.rewriter.rules.EliminateGroupByEmptyKeyRule; -import org.apache.hyracks.algebricks.rewriter.rules.EliminateSubplanRule; -import org.apache.hyracks.algebricks.rewriter.rules.EliminateSubplanWithInputCardinalityOneRule; import org.apache.hyracks.algebricks.rewriter.rules.EnforceStructuralPropertiesRule; import org.apache.hyracks.algebricks.rewriter.rules.ExtractCommonOperatorsRule; import org.apache.hyracks.algebricks.rewriter.rules.ExtractGbyExpressionsRule; @@ -64,21 +35,18 @@ import org.apache.hyracks.algebricks.rewriter.rules.FactorRedundantGroupAndDecor import org.apache.hyracks.algebricks.rewriter.rules.InferTypesRule; import org.apache.hyracks.algebricks.rewriter.rules.InlineAssignIntoAggregateRule; import org.apache.hyracks.algebricks.rewriter.rules.InlineVariablesRule; -import org.apache.hyracks.algebricks.rewriter.rules.InsertOuterJoinRule; +//import org.apache.hyracks.algebricks.rewriter.rules.InsertOuterJoinRule; import org.apache.hyracks.algebricks.rewriter.rules.IntroJoinInsideSubplanRule; import org.apache.hyracks.algebricks.rewriter.rules.IntroduceAggregateCombinerRule; import org.apache.hyracks.algebricks.rewriter.rules.IntroduceGroupByCombinerRule; -import org.apache.hyracks.algebricks.rewriter.rules.IntroduceGroupByForSubplanRule; import org.apache.hyracks.algebricks.rewriter.rules.IntroduceProjectsRule; import org.apache.hyracks.algebricks.rewriter.rules.IsolateHyracksOperatorsRule; -import org.apache.hyracks.algebricks.rewriter.rules.NestedSubplanToJoinRule; import org.apache.hyracks.algebricks.rewriter.rules.PullSelectOutOfEqJoin; import org.apache.hyracks.algebricks.rewriter.rules.PushMapOperatorDownThroughProductRule; import org.apache.hyracks.algebricks.rewriter.rules.PushProjectDownRule; import org.apache.hyracks.algebricks.rewriter.rules.PushProjectIntoDataSourceScanRule; import org.apache.hyracks.algebricks.rewriter.rules.PushSelectDownRule; import org.apache.hyracks.algebricks.rewriter.rules.PushSelectIntoJoinRule; -import org.apache.hyracks.algebricks.rewriter.rules.PushSubplanIntoGroupByRule; import org.apache.hyracks.algebricks.rewriter.rules.PushSubplanWithAggregateDownThroughProductRule; import org.apache.hyracks.algebricks.rewriter.rules.ReinferAllTypesRule; import org.apache.hyracks.algebricks.rewriter.rules.RemoveRedundantVariablesRule; @@ -86,16 +54,49 @@ import org.apache.hyracks.algebricks.rewriter.rules.RemoveUnusedAssignAndAggrega import org.apache.hyracks.algebricks.rewriter.rules.SetAlgebricksPhysicalOperatorsRule; import org.apache.hyracks.algebricks.rewriter.rules.SetExecutionModeRule; import org.apache.hyracks.algebricks.rewriter.rules.SimpleUnnestToProductRule; -import org.apache.hyracks.algebricks.rewriter.rules.SubplanOutOfGroupRule; +import org.apache.hyracks.algebricks.rewriter.rules.subplan.EliminateSubplanRule; +import org.apache.hyracks.algebricks.rewriter.rules.subplan.EliminateSubplanWithInputCardinalityOneRule; +import org.apache.hyracks.algebricks.rewriter.rules.subplan.NestedSubplanToJoinRule; +import org.apache.hyracks.algebricks.rewriter.rules.subplan.PushSubplanIntoGroupByRule; +import org.apache.hyracks.algebricks.rewriter.rules.subplan.SubplanOutOfGroupRule; +import org.apache.vxquery.compiler.rewriter.rules.ConsolidateAssignAggregateRule; +import org.apache.vxquery.compiler.rewriter.rules.ConsolidateDescandantChild; +import org.apache.vxquery.compiler.rewriter.rules.ConvertAssignToUnnestRule; +import org.apache.vxquery.compiler.rewriter.rules.ConvertFromAlgebricksExpressionsRule; +import org.apache.vxquery.compiler.rewriter.rules.ConvertToAlgebricksExpressionsRule; +import org.apache.vxquery.compiler.rewriter.rules.EliminateSubplanForSingleItemsRule; +import org.apache.vxquery.compiler.rewriter.rules.EliminateUnnestAggregateSequencesRule; +import org.apache.vxquery.compiler.rewriter.rules.EliminateUnnestAggregateSubplanRule; +import org.apache.vxquery.compiler.rewriter.rules.IntroduceCollectionRule; +import org.apache.vxquery.compiler.rewriter.rules.IntroduceIndexingRule; +import org.apache.vxquery.compiler.rewriter.rules.IntroduceTwoStepAggregateRule; +import org.apache.vxquery.compiler.rewriter.rules.PushChildIntoDataScanRule; +import org.apache.vxquery.compiler.rewriter.rules.PushFunctionsOntoEqJoinBranches; +import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantBooleanExpressionsRule; +import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantCastExpressionsRule; +import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantDataExpressionsRule; +import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantPromoteExpressionsRule; +import org.apache.vxquery.compiler.rewriter.rules.RemoveRedundantTreatExpressionsRule; +import org.apache.vxquery.compiler.rewriter.rules.RemoveUnusedSortDistinctNodesRule; +import org.apache.vxquery.compiler.rewriter.rules.RemoveUnusedUnnestIterateRule; +import org.apache.vxquery.compiler.rewriter.rules.ReplaceSourceMapInDocExpression; +import org.apache.vxquery.compiler.rewriter.rules.SetCollectionDataSourceRule; +import org.apache.vxquery.compiler.rewriter.rules.SetVariableIdContextRule; +import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.ExtractFunctionsFromJoinConditionRule; +import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.InlineNestedVariablesRule; +import org.apache.vxquery.compiler.rewriter.rules.algebricksalternatives.MoveFreeVariableOperatorOutOfSubplanRule; public class RewriteRuleset { + RewriteRuleset() { + } + /** * Optimizations specific to XQuery. * * @return List of algebraic rewrite rules. */ - public final static List<IAlgebraicRewriteRule> buildPathStepNormalizationRuleCollection() { - List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildPathStepNormalizationRuleCollection() { + List<IAlgebraicRewriteRule> normalization = new LinkedList<>(); normalization.add(new SetVariableIdContextRule()); // Remove unused functions. @@ -153,8 +154,8 @@ public class RewriteRuleset { * * @return List of algebraic rewrite rules. */ - public final static List<IAlgebraicRewriteRule> buildXQueryNormalizationRuleCollection() { - List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildXQueryNormalizationRuleCollection() { + List<IAlgebraicRewriteRule> normalization = new LinkedList<>(); // Find assign for scalar aggregate function followed by an aggregate operator. normalization.add(new ConsolidateAssignAggregateRule()); @@ -179,8 +180,8 @@ public class RewriteRuleset { * * @return List of algebraic rewrite rules. */ - public final static List<IAlgebraicRewriteRule> buildInlineRedundantExpressionNormalizationRuleCollection() { - List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildInlineRedundantExpressionNormalizationRuleCollection() { + List<IAlgebraicRewriteRule> normalization = new LinkedList<>(); normalization.add(new InlineNestedVariablesRule()); normalization.add(new RemoveRedundantTreatExpressionsRule()); normalization.add(new RemoveRedundantDataExpressionsRule()); @@ -199,8 +200,8 @@ public class RewriteRuleset { * * @return List of algebraic rewrite rules. */ - public final static List<IAlgebraicRewriteRule> buildNestedDataSourceRuleCollection() { - List<IAlgebraicRewriteRule> xquery = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildNestedDataSourceRuleCollection() { + List<IAlgebraicRewriteRule> xquery = new LinkedList<>(); xquery.add(new BreakSelectIntoConjunctsRule()); xquery.add(new SimpleUnnestToProductRule()); xquery.add(new PushMapOperatorDownThroughProductRule()); @@ -213,8 +214,8 @@ public class RewriteRuleset { return xquery; } - public final static List<IAlgebraicRewriteRule> buildTypeInferenceRuleCollection() { - List<IAlgebraicRewriteRule> typeInfer = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildTypeInferenceRuleCollection() { + List<IAlgebraicRewriteRule> typeInfer = new LinkedList<>(); typeInfer.add(new InferTypesRule()); return typeInfer; } @@ -224,8 +225,8 @@ public class RewriteRuleset { * * @return List of algebraic rewrite rules. */ - public final static List<IAlgebraicRewriteRule> buildUnnestingRuleCollection() { - List<IAlgebraicRewriteRule> xquery = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildUnnestingRuleCollection() { + List<IAlgebraicRewriteRule> xquery = new LinkedList<>(); xquery.add(new PushSelectDownRule()); xquery.add(new SimpleUnnestToProductRule()); @@ -235,9 +236,9 @@ public class RewriteRuleset { xquery.add(new IntroJoinInsideSubplanRule()); xquery.add(new PushMapOperatorDownThroughProductRule()); xquery.add(new PushSubplanWithAggregateDownThroughProductRule()); - xquery.add(new IntroduceGroupByForSubplanRule()); + //xquery.add(new IntroduceGroupByForSubplanRule()); xquery.add(new SubplanOutOfGroupRule()); - xquery.add(new InsertOuterJoinRule()); + // xquery.add(new InsertOuterJoinRule()); xquery.add(new ExtractFunctionsFromJoinConditionRule()); xquery.add(new RemoveRedundantVariablesRule()); @@ -253,8 +254,8 @@ public class RewriteRuleset { return xquery; } - public final static List<IAlgebraicRewriteRule> buildNormalizationRuleCollection() { - List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildNormalizationRuleCollection() { + List<IAlgebraicRewriteRule> normalization = new LinkedList<>(); normalization.add(new EliminateSubplanRule()); normalization.add(new BreakSelectIntoConjunctsRule()); normalization.add(new PushSelectIntoJoinRule()); @@ -262,8 +263,8 @@ public class RewriteRuleset { return normalization; } - public final static List<IAlgebraicRewriteRule> buildCondPushDownRuleCollection() { - List<IAlgebraicRewriteRule> condPushDown = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildCondPushDownRuleCollection() { + List<IAlgebraicRewriteRule> condPushDown = new LinkedList<>(); condPushDown.add(new PushSelectDownRule()); condPushDown.add(new InlineVariablesRule()); condPushDown.add(new FactorRedundantGroupAndDecorVarsRule()); @@ -271,34 +272,34 @@ public class RewriteRuleset { return condPushDown; } - public final static List<IAlgebraicRewriteRule> buildJoinInferenceRuleCollection() { - List<IAlgebraicRewriteRule> joinInference = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildJoinInferenceRuleCollection() { + List<IAlgebraicRewriteRule> joinInference = new LinkedList<>(); joinInference.add(new InlineVariablesRule()); joinInference.add(new ComplexJoinInferenceRule()); return joinInference; } - public final static List<IAlgebraicRewriteRule> buildOpPushDownRuleCollection() { - List<IAlgebraicRewriteRule> opPushDown = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildOpPushDownRuleCollection() { + List<IAlgebraicRewriteRule> opPushDown = new LinkedList<>(); opPushDown.add(new PushProjectDownRule()); opPushDown.add(new PushSelectDownRule()); return opPushDown; } - public final static List<IAlgebraicRewriteRule> buildIntroduceProjectRuleCollection() { - List<IAlgebraicRewriteRule> project = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildIntroduceProjectRuleCollection() { + List<IAlgebraicRewriteRule> project = new LinkedList<>(); project.add(new IntroduceProjectsRule()); return project; } - public final static List<IAlgebraicRewriteRule> buildDataExchangeRuleCollection() { - List<IAlgebraicRewriteRule> dataExchange = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildDataExchangeRuleCollection() { + List<IAlgebraicRewriteRule> dataExchange = new LinkedList<>(); dataExchange.add(new SetExecutionModeRule()); return dataExchange; } - public final static List<IAlgebraicRewriteRule> buildConsolidationRuleCollection() { - List<IAlgebraicRewriteRule> consolidation = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildConsolidationRuleCollection() { + List<IAlgebraicRewriteRule> consolidation = new LinkedList<>(); consolidation.add(new ConsolidateSelectsRule()); consolidation.add(new ConsolidateAssignsRule()); consolidation.add(new InlineAssignIntoAggregateRule()); @@ -308,8 +309,8 @@ public class RewriteRuleset { return consolidation; } - public final static List<IAlgebraicRewriteRule> buildPhysicalRewritesAllLevelsRuleCollection() { - List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildPhysicalRewritesAllLevelsRuleCollection() { + List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<>(); physicalPlanRewrites.add(new PullSelectOutOfEqJoin()); physicalPlanRewrites.add(new PushFunctionsOntoEqJoinBranches()); physicalPlanRewrites.add(new SetAlgebricksPhysicalOperatorsRule()); @@ -320,15 +321,15 @@ public class RewriteRuleset { return physicalPlanRewrites; } - public final static List<IAlgebraicRewriteRule> buildPhysicalRewritesTopLevelRuleCollection() { - List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> buildPhysicalRewritesTopLevelRuleCollection() { + List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<>(); physicalPlanRewrites.add(new CopyLimitDownRule()); physicalPlanRewrites.add(new SetExecutionModeRule()); return physicalPlanRewrites; } - public final static List<IAlgebraicRewriteRule> prepareForJobGenRuleCollection() { - List<IAlgebraicRewriteRule> prepareForJobGenRewrites = new LinkedList<IAlgebraicRewriteRule>(); + public static final List<IAlgebraicRewriteRule> prepareForJobGenRuleCollection() { + List<IAlgebraicRewriteRule> prepareForJobGenRewrites = new LinkedList<>(); prepareForJobGenRewrites.add(new ConvertFromAlgebricksExpressionsRule()); prepareForJobGenRewrites .add(new IsolateHyracksOperatorsRule(HeuristicOptimizer.hyraxOperatorsBelowWhichJobGenIsDisabled)); http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java index eef9495..ec31dbc 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java @@ -3,9 +3,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * you may obtain a copy of the License from - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,34 +17,36 @@ package org.apache.vxquery.compiler.rewriter; import java.util.HashMap; import java.util.Map; -import org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.cardinality.Cardinality; -import org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.documentorder.DocumentOrder; -import org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.uniquenodes.UniqueNodes; - +import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; +import org.apache.hyracks.algebricks.core.algebra.expressions.IConflictingTypeResolver; import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer; import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer; import org.apache.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory; -import org.apache.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer; +import org.apache.hyracks.algebricks.core.algebra.expressions.IMissableTypeComputer; import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor; import org.apache.hyracks.algebricks.core.rewriter.base.AlgebricksOptimizationContext; import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig; +import org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.cardinality.Cardinality; +import org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.documentorder.DocumentOrder; +import org.apache.vxquery.compiler.rewriter.rules.propagationpolicies.uniquenodes.UniqueNodes; public class VXQueryOptimizationContext extends AlgebricksOptimizationContext { - private final Map<ILogicalOperator, HashMap<Integer, DocumentOrder>> documentOrderOperatorVariableMap = new HashMap<ILogicalOperator, HashMap<Integer, DocumentOrder>>(); - private final Map<ILogicalOperator, HashMap<Integer, UniqueNodes>> uniqueNodesOperatorVariableMap = new HashMap<ILogicalOperator, HashMap<Integer, UniqueNodes>>(); - private final Map<ILogicalOperator, Cardinality> cardinalityOperatorMap = new HashMap<ILogicalOperator, Cardinality>(); + private final Map<ILogicalOperator, HashMap<Integer, DocumentOrder>> documentOrderOperatorVariableMap = new HashMap<>(); + private final Map<ILogicalOperator, HashMap<Integer, UniqueNodes>> uniqueNodesOperatorVariableMap = new HashMap<>(); + private final Map<ILogicalOperator, Cardinality> cardinalityOperatorMap = new HashMap<>(); private int totalDataSources = 0; private int collectionId = 0; public VXQueryOptimizationContext(int varCounter, IExpressionEvalSizeComputer expressionEvalSizeComputer, IMergeAggregationExpressionFactory mergeAggregationExpressionFactory, - IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer, - PhysicalOptimizationConfig physicalOptimizationConfig, LogicalOperatorPrettyPrintVisitor prettyPrintVisitor) { + IExpressionTypeComputer expressionTypeComputer, IMissableTypeComputer missableTypeComputer, + IConflictingTypeResolver conflictingTypeResovler, PhysicalOptimizationConfig physicalOptimizationConfig, + AlgebricksPartitionConstraint clusterLocations, LogicalOperatorPrettyPrintVisitor prettyPrintVisitor) { super(varCounter, expressionEvalSizeComputer, mergeAggregationExpressionFactory, expressionTypeComputer, - nullableTypeComputer, physicalOptimizationConfig, prettyPrintVisitor); + missableTypeComputer, conflictingTypeResovler, physicalOptimizationConfig, clusterLocations, prettyPrintVisitor); } public void incrementTotalDataSources() { http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java index 2e8a8ac..b62242b 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java @@ -17,9 +17,6 @@ package org.apache.vxquery.compiler.rewriter.rules; import java.io.DataInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.Set; import org.apache.commons.lang3.mutable.Mutable; @@ -79,7 +76,7 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule { AssignOperator assign = (AssignOperator) op2; // Check to see if the expression is a function and fn:Collection. - ILogicalExpression logicalExpression = (ILogicalExpression) assign.getExpressions().get(0).getValue(); + ILogicalExpression logicalExpression = assign.getExpressions().get(0).getValue(); if (logicalExpression.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { return null; } @@ -92,7 +89,7 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule { // Get arguments int size = functionCall.getArguments().size(); if (size > 0) { - String args[] = new String[size]; + String[] args = new String[size]; for (int i = 0; i < size; i++) { args[i] = getArgument(functionCall, opRef, i); } @@ -103,7 +100,7 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule { private String getArgument(AbstractFunctionCallExpression functionCall, Mutable<ILogicalOperator> opRef, int pos) { VXQueryConstantValue constantValue; - ILogicalExpression logicalExpression2 = (ILogicalExpression) functionCall.getArguments().get(pos).getValue(); + ILogicalExpression logicalExpression2 = functionCall.getArguments().get(pos).getValue(); if (logicalExpression2.getExpressionTag() != LogicalExpressionTag.VARIABLE) { return null; } else if (logicalExpression2 == null) { @@ -118,7 +115,7 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule { AssignOperator assign2 = (AssignOperator) op3; // Check to see if the expression is a constant expression and type string. - ILogicalExpression logicalExpression3 = (ILogicalExpression) assign2.getExpressions().get(0).getValue(); + ILogicalExpression logicalExpression3 = assign2.getExpressions().get(0).getValue(); if (logicalExpression3.getExpressionTag() != LogicalExpressionTag.CONSTANT) { return null; } @@ -132,17 +129,9 @@ public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule { } // Constant value is now in a TaggedValuePointable. Convert the value into a java String. tvp.set(constantValue.getValue(), 0, constantValue.getValue().length); - String collectionName = null; if (tvp.getTag() == ValueTag.XS_STRING_TAG) { tvp.getValue(stringp); - try { - bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(), - stringp.getLength() + stringp.getStartOffset())), 0); - collectionName = di.readUTF(); - return collectionName; - } catch (IOException e) { - e.printStackTrace(); - } + return stringp.toString(); } return null; } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java index 42d59aa..20283d8 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java @@ -60,7 +60,7 @@ public class IntroduceCollectionRule extends AbstractCollectionRule { @Override public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) { VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context; - String args[] = getFunctionalArguments(opRef, VXQueryCommons.collectionFunctions); + String[] args = getFunctionalArguments(opRef, VXQueryCommons.collectionFunctions); if (args != null) { String collectionName = args[0]; http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ReplaceSourceMapInDocExpression.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ReplaceSourceMapInDocExpression.java b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ReplaceSourceMapInDocExpression.java index 8f241b3..08766b0 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ReplaceSourceMapInDocExpression.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ReplaceSourceMapInDocExpression.java @@ -23,18 +23,6 @@ import java.io.IOException; import java.util.List; import org.apache.commons.lang3.mutable.Mutable; -import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue; -import org.apache.vxquery.compiler.rewriter.rules.util.ExpressionToolbox; -import org.apache.vxquery.compiler.rewriter.rules.util.OperatorToolbox; -import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; -import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder; -import org.apache.vxquery.datamodel.values.ValueTag; -import org.apache.vxquery.functions.BuiltinFunctions; -import org.apache.vxquery.metadata.VXQueryMetadataProvider; -import org.apache.vxquery.types.BuiltinTypeRegistry; -import org.apache.vxquery.types.Quantifier; -import org.apache.vxquery.types.SequenceType; - import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; @@ -47,27 +35,38 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; +import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue; +import org.apache.vxquery.compiler.rewriter.rules.util.ExpressionToolbox; +import org.apache.vxquery.compiler.rewriter.rules.util.OperatorToolbox; +import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; +import org.apache.vxquery.datamodel.builders.atomic.StringValueBuilder; +import org.apache.vxquery.datamodel.values.ValueTag; +import org.apache.vxquery.functions.BuiltinFunctions; +import org.apache.vxquery.metadata.VXQueryMetadataProvider; +import org.apache.vxquery.types.BuiltinTypeRegistry; +import org.apache.vxquery.types.Quantifier; +import org.apache.vxquery.types.SequenceType; /** * The rule searches for where the function_doc1 function is in the plan in place of XQuery function. * It replaces the string contained in the function with its absolute file path. - * + * * <pre> * Before - * + * * plan__parent * %OPERATOR( $v1 : fn:doc( \@string ) ) * plan__child - * + * * Where xquery_function creates an atomic value. - * - * After - * + * + * After + * * plan__parent * %OPERATOR( $v1 : fn:doc( \@absolute_file_path ) ) ) * plan__child * </pre> - * + * * @author shivanim */ http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDatePointable.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDatePointable.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDatePointable.java index d728927..f635b1d 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDatePointable.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSDatePointable.java @@ -16,23 +16,20 @@ */ package org.apache.vxquery.datamodel.accessors.atomic; -import org.apache.vxquery.datamodel.api.IDate; -import org.apache.vxquery.datamodel.api.ITimezone; - import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.data.std.api.AbstractPointable; import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.api.IPointableFactory; import org.apache.hyracks.data.std.primitive.BytePointable; import org.apache.hyracks.data.std.primitive.ShortPointable; +import org.apache.vxquery.datamodel.api.IDate; +import org.apache.vxquery.datamodel.api.ITimezone; /** * The date is split up into five sections. Due to leap year, we have decided to keep the * storage simple by saving each date section separately. For calculations you can access * YearMonth (months) and DayTime (milliseconds) values. * The date pointable is also used for GDay, GMonth, GYear, GDayMonth and GMonthYear. - * - * @author prestoncarman */ public class XSDatePointable extends AbstractPointable implements IDate, ITimezone { public final static int YEAR_OFFSET = 0; http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSQNamePointable.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSQNamePointable.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSQNamePointable.java index f8e63b6..977101b 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSQNamePointable.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/atomic/XSQNamePointable.java @@ -20,7 +20,7 @@ import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.data.std.api.AbstractPointable; import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.api.IPointableFactory; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; +import org.apache.hyracks.util.string.UTF8StringUtil; /** * The XSQNamePointable holds three strings: URI, Prefix and Local Name. @@ -59,7 +59,8 @@ public class XSQNamePointable extends AbstractPointable { } public static int getUriLength(byte[] bytes, int start) { - return getUriUTFLength(bytes, start) + 2; + int utfLength = getUriUTFLength(bytes, start); + return utfLength + UTF8StringUtil.getNumBytesToStoreLength(utfLength); } public int getUriUTFLength() { @@ -67,7 +68,7 @@ public class XSQNamePointable extends AbstractPointable { } public static int getUriUTFLength(byte[] bytes, int start) { - return UTF8StringPointable.getUTFLength(bytes, start); + return UTF8StringUtil.getUTFLength(bytes, start); } public int getPrefixLength() { @@ -75,7 +76,8 @@ public class XSQNamePointable extends AbstractPointable { } public static int getPrefixLength(byte[] bytes, int start) { - return getPrefixUTFLength(bytes, start) + 2; + int utfLength = getPrefixUTFLength(bytes, start); + return utfLength + UTF8StringUtil.getNumBytesToStoreLength(utfLength); } public int getPrefixUTFLength() { @@ -83,7 +85,7 @@ public class XSQNamePointable extends AbstractPointable { } public static int getPrefixUTFLength(byte[] bytes, int start) { - return UTF8StringPointable.getUTFLength(bytes, start + getUriLength(bytes, start)); + return UTF8StringUtil.getUTFLength(bytes, start + getUriLength(bytes, start)); } public int getLocalNameLength() { @@ -91,7 +93,8 @@ public class XSQNamePointable extends AbstractPointable { } public static int getLocalNameLength(byte[] bytes, int start) { - return getLocalNameUTFLength(bytes, start) + 2; + int utfLength = getLocalNameUTFLength(bytes, start); + return utfLength + UTF8StringUtil.getNumBytesToStoreLength(utfLength); } public int getLocalNameUTFLength() { @@ -99,7 +102,7 @@ public class XSQNamePointable extends AbstractPointable { } public static int getLocalNameUTFLength(byte[] bytes, int start) { - return UTF8StringPointable.getUTFLength(bytes, + return UTF8StringUtil.getUTFLength(bytes, start + getUriLength(bytes, start) + getPrefixLength(bytes, start)); } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/jsonitem/ObjectPointable.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/jsonitem/ObjectPointable.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/jsonitem/ObjectPointable.java index ac96210..5311342 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/jsonitem/ObjectPointable.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/jsonitem/ObjectPointable.java @@ -16,6 +16,8 @@ */ package org.apache.vxquery.datamodel.accessors.jsonitem; +import java.io.IOException; + import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.data.std.api.AbstractPointable; import org.apache.hyracks.data.std.api.IMutableValueStorage; @@ -24,12 +26,11 @@ import org.apache.hyracks.data.std.api.IPointableFactory; import org.apache.hyracks.data.std.primitive.IntegerPointable; import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.util.string.UTF8StringUtil; import org.apache.vxquery.datamodel.builders.sequence.SequenceBuilder; import org.apache.vxquery.datamodel.values.ValueTag; import org.apache.vxquery.runtime.functions.util.FunctionHelper; -import java.io.IOException; - /** * The datamodel of the JSON object is represented in this class: * Byte 1: Value tag of object (109) @@ -65,8 +66,9 @@ public class ObjectPointable extends AbstractPointable { return IntegerPointable.getInteger(bytes, start); } - private static int getKeyLength(byte[] b, int s) { - return UTF8StringPointable.getUTFLength(b, s) + 2; + private static int getKeyLength(byte[] bytes, int start) { + int utfLength = UTF8StringUtil.getUTFLength(bytes, start); + return utfLength + UTF8StringUtil.getNumBytesToStoreLength(utfLength); } private static int getSlotArrayOffset(int start) { http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/NodeTreePointable.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/NodeTreePointable.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/NodeTreePointable.java index db03e28..c0c447c 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/NodeTreePointable.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/NodeTreePointable.java @@ -16,8 +16,6 @@ */ package org.apache.vxquery.datamodel.accessors.nodes; -import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; - import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.data.std.algorithms.BinarySearchAlgorithm; import org.apache.hyracks.data.std.api.AbstractPointable; @@ -28,6 +26,8 @@ import org.apache.hyracks.data.std.primitive.BytePointable; import org.apache.hyracks.data.std.primitive.IntegerPointable; import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.util.string.UTF8StringUtil; +import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; /* * NodeTree { @@ -36,17 +36,17 @@ import org.apache.hyracks.data.std.primitive.VoidPointable; * Dictionary dictionary?; * ElementNode rootNode; * } - * + * * ElementHeader (padded) { * bit nodeIdExists; * bit dictionaryExists; * bit headerTypeExists; * } - * + * * NodeId { * int32 id; * } - * + * * Dictionary { * int32 numberOfItems * int32[numberOfItems] lengthOfItem @@ -97,14 +97,15 @@ public class NodeTreePointable extends AbstractPointable { public int getStart(int index) { int dataAreaStart = getDictionaryDataAreaStartOffset(); int sortedPtrArrayStart = getDictionarySortedPointerArrayOffset(); - int sortedSlotValue = IntegerPointable - .getInteger(bytes, sortedPtrArrayStart + index * SORTED_PTR_SLOT_SIZE); + int sortedSlotValue = IntegerPointable.getInteger(bytes, + sortedPtrArrayStart + index * SORTED_PTR_SLOT_SIZE); return dataAreaStart + sortedSlotValue; } @Override public int getLength(int index) { - return UTF8StringPointable.getUTFLength(bytes, getStart(index)) + 2; + int utfLength = UTF8StringUtil.getUTFLength(bytes, getStart(index)); + return utfLength + UTF8StringUtil.getNumBytesToStoreLength(utfLength); } }; @@ -130,16 +131,18 @@ public class NodeTreePointable extends AbstractPointable { return dictionaryExists() ? IntegerPointable.getInteger(bytes, getDictionaryEntryCountOffset()) : 0; } - public void getString(int idx, IPointable string) { + public void getString(int idx, UTF8StringPointable string) { int nEntries = getDictionaryEntryCount(); if (idx < 0 || idx >= nEntries) { throw new IllegalArgumentException(idx + " not within [0, " + nEntries + ")"); } int dataAreaStart = getDictionaryDataAreaStartOffset(); - int idxSlotValue = idx == 0 ? 0 : IntegerPointable.getInteger(bytes, getDictionaryIndexPointerArrayOffset() - + (idx - 1) * IDX_PTR_SLOT_SIZE); - int strLen = UTF8StringPointable.getUTFLength(bytes, dataAreaStart + idxSlotValue); - string.set(bytes, dataAreaStart + idxSlotValue, strLen + 2); + int idxSlotValue = idx == 0 ? 0 + : IntegerPointable.getInteger(bytes, + getDictionaryIndexPointerArrayOffset() + (idx - 1) * IDX_PTR_SLOT_SIZE); + int strLen = UTF8StringUtil.getUTFLength(bytes, dataAreaStart + idxSlotValue); + int strMetaLen = UTF8StringUtil.getNumBytesToStoreLength(strLen); + string.set(bytes, dataAreaStart + idxSlotValue, strMetaLen + strLen); } public int lookupString(UTF8StringPointable key) { @@ -197,8 +200,8 @@ public class NodeTreePointable extends AbstractPointable { } private int getDictionaryDataAreaStartOffset() { - return getDictionaryIndexPointerArrayOffset() + getDictionaryEntryCount() - * (IDX_PTR_SLOT_SIZE + SORTED_PTR_SLOT_SIZE); + return getDictionaryIndexPointerArrayOffset() + + getDictionaryEntryCount() * (IDX_PTR_SLOT_SIZE + SORTED_PTR_SLOT_SIZE); } private int getRootNodeOffset() { http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/PINodePointable.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/PINodePointable.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/PINodePointable.java index 3ae5126..6aee42b 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/PINodePointable.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/accessors/nodes/PINodePointable.java @@ -19,8 +19,8 @@ package org.apache.vxquery.datamodel.accessors.nodes; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.api.IPointableFactory; -import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.util.string.UTF8StringUtil; /* * PI { @@ -66,7 +66,8 @@ public class PINodePointable extends AbstractNodePointable { } private int getTargetSize(NodeTreePointable nodeTree) { - return UTF8StringPointable.getUTFLength(bytes, getTargetOffset(nodeTree)) + 2; + int utfLength = UTF8StringUtil.getUTFLength(bytes, getTargetOffset(nodeTree)); + return utfLength + UTF8StringUtil.getNumBytesToStoreLength(utfLength); } private int getContentOffset(NodeTreePointable nodeTree) { @@ -74,6 +75,7 @@ public class PINodePointable extends AbstractNodePointable { } private int getContentSize(NodeTreePointable nodeTree) { - return UTF8StringPointable.getUTFLength(bytes, getContentOffset(nodeTree)) + 2; + int utfLength = UTF8StringUtil.getUTFLength(bytes, getContentOffset(nodeTree)); + return utfLength + UTF8StringUtil.getNumBytesToStoreLength(utfLength); } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/StringValueBuilder.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/StringValueBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/StringValueBuilder.java index 215d1b1..453ffe0 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/StringValueBuilder.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/StringValueBuilder.java @@ -19,7 +19,7 @@ package org.apache.vxquery.datamodel.builders.atomic; import java.io.DataOutput; import java.io.IOException; -import org.apache.hyracks.data.std.primitive.UTF8StringWriter; +import org.apache.hyracks.util.string.UTF8StringWriter; public class StringValueBuilder { private final UTF8StringWriter writer; @@ -29,6 +29,6 @@ public class StringValueBuilder { } public void write(CharSequence string, DataOutput out) throws IOException { - writer.writeUTF8String(string, out); + writer.writeUTF8(string, out); } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/UTF8StringBuilder.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/UTF8StringBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/UTF8StringBuilder.java deleted file mode 100644 index a4d301d..0000000 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/UTF8StringBuilder.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.vxquery.datamodel.builders.atomic; - -import java.io.DataOutput; -import java.io.IOException; -import java.io.UTFDataFormatException; - -import org.apache.hyracks.data.std.api.IMutableValueStorage; -import org.apache.hyracks.data.std.primitive.BytePointable; -import org.apache.vxquery.datamodel.builders.base.IBuilder; -import org.apache.vxquery.runtime.functions.util.FunctionHelper; - -public class UTF8StringBuilder implements IBuilder { - private IMutableValueStorage mvs; - private DataOutput out; - - public void reset(IMutableValueStorage mvs) throws IOException { - this.mvs = mvs; - out = mvs.getDataOutput(); - out.write(0); - out.write(0); - } - - public void finish() throws IOException { - int utflen = mvs.getLength() - 2; - BytePointable.setByte(mvs.getByteArray(), 0, (byte) ((utflen >>> 8) & 0xFF)); - BytePointable.setByte(mvs.getByteArray(), 1, (byte) ((utflen >>> 0) & 0xFF)); - } - - public void appendCharArray(char[] ch, int start, int length) throws IOException { - FunctionHelper.writeCharArray(ch, start, length, out); - if (mvs.getLength() > 65535) { - throw new UTFDataFormatException("encoded string too long: " + mvs.getLength() + " bytes"); - } - } -} http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/VXQueryUTF8StringBuilder.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/VXQueryUTF8StringBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/VXQueryUTF8StringBuilder.java new file mode 100644 index 0000000..dcf5bbc --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/atomic/VXQueryUTF8StringBuilder.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.vxquery.datamodel.builders.atomic; + +import java.io.IOException; + +import org.apache.hyracks.data.std.util.UTF8StringBuilder; + +public class VXQueryUTF8StringBuilder extends UTF8StringBuilder { + + public void appendUtf8Bytes(byte[] bytes, int byteStartOffset, int byteLength) throws IOException { + out.write(bytes, byteStartOffset, byteLength); + } + +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/CommentNodeBuilder.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/CommentNodeBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/CommentNodeBuilder.java index d596c3d..68f5d4d 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/CommentNodeBuilder.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/CommentNodeBuilder.java @@ -19,6 +19,7 @@ package org.apache.vxquery.datamodel.builders.nodes; import java.io.IOException; import org.apache.hyracks.data.std.api.IValueReference; +import org.apache.hyracks.data.std.util.GrowableArray; import org.apache.vxquery.datamodel.values.ValueTag; public class CommentNodeBuilder extends AbstractNodeBuilder { @@ -39,4 +40,8 @@ public class CommentNodeBuilder extends AbstractNodeBuilder { public void setValue(IValueReference value) throws IOException { out.write(value.getByteArray(), value.getStartOffset(), value.getLength()); } + + public void setValue(GrowableArray value) throws IOException { + out.write(value.getByteArray(), 0, value.getLength()); + } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java index dae149f..cf13bfe 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/DictionaryBuilder.java @@ -22,14 +22,15 @@ import java.io.IOException; import java.util.Map.Entry; import java.util.TreeMap; -import org.apache.vxquery.util.GrowableIntArray; - import org.apache.hyracks.data.std.algorithms.BinarySearchAlgorithm; import org.apache.hyracks.data.std.collections.api.IValueReferenceVector; import org.apache.hyracks.data.std.primitive.IntegerPointable; import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream; +import org.apache.hyracks.util.string.UTF8StringUtil; +import org.apache.hyracks.util.string.UTF8StringWriter; +import org.apache.vxquery.util.GrowableIntArray; public class DictionaryBuilder { private final GrowableIntArray stringEndOffsets; @@ -46,6 +47,8 @@ public class DictionaryBuilder { private boolean cacheReady; + private final UTF8StringWriter UTF8Writer = new UTF8StringWriter(); + private final IValueReferenceVector sortedStringsVector = new IValueReferenceVector() { @Override public int getStart(int index) { @@ -60,7 +63,8 @@ public class DictionaryBuilder { @Override public int getLength(int index) { - return UTF8StringPointable.getUTFLength(dataBuffer.getByteArray(), getStart(index)) + 2; + int utfLength = UTF8StringUtil.getUTFLength(dataBuffer.getByteArray(), getStart(index)); + return utfLength + UTF8StringUtil.getNumBytesToStoreLength(utfLength); } @Override @@ -77,7 +81,7 @@ public class DictionaryBuilder { dataBuffer = new ByteArrayAccessibleOutputStream(); dataBufferOut = new DataOutputStream(dataBuffer); cache = new ArrayBackedValueStorage(); - hashSlotIndexes = new TreeMap<String, Integer>(); + hashSlotIndexes = new TreeMap<>(); cacheReady = false; } @@ -120,6 +124,7 @@ public class DictionaryBuilder { } } out.write(dataBuffer.getByteArray(), 0, dataBuffer.size()); + // TODO can this value be determined before writing. Could this be append only. IntegerPointable.setInteger(abvs.getByteArray(), sizeOffset, abvs.getLength() - sizeOffset); } @@ -127,7 +132,7 @@ public class DictionaryBuilder { Integer slotIndex = hashSlotIndexes.get(str); if (slotIndex == null) { try { - dataBufferOut.writeUTF(str); + UTF8Writer.writeUTF8(str, dataBufferOut); slotIndex = stringEndOffsets.getSize(); dataBufferOut.writeInt(slotIndex); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/vxquery/blob/2c881020/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/PINodeBuilder.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/PINodeBuilder.java b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/PINodeBuilder.java index 0e62762..05a1a88 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/PINodeBuilder.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/datamodel/builders/nodes/PINodeBuilder.java @@ -19,6 +19,7 @@ package org.apache.vxquery.datamodel.builders.nodes; import java.io.IOException; import org.apache.hyracks.data.std.api.IValueReference; +import org.apache.hyracks.data.std.util.GrowableArray; import org.apache.vxquery.datamodel.values.ValueTag; public class PINodeBuilder extends AbstractNodeBuilder { @@ -43,4 +44,12 @@ public class PINodeBuilder extends AbstractNodeBuilder { public void setContent(IValueReference value) throws IOException { out.write(value.getByteArray(), value.getStartOffset(), value.getLength()); } + + public void setTarget(GrowableArray value) throws IOException { + out.write(value.getByteArray(), 0, value.getLength()); + } + + public void setContent(GrowableArray value) throws IOException { + out.write(value.getByteArray(), 0, value.getLength()); + } }
