Merged Master
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/f90d5873 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/f90d5873 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/f90d5873 Branch: refs/heads/steven/hdfs Commit: f90d5873761de94c64b6d20fcd0216d7f565d972 Parents: 4bd85a1 0ad460a Author: Steven Jacobs <[email protected]> Authored: Mon May 16 16:01:10 2016 -0700 Committer: Steven Jacobs <[email protected]> Committed: Mon May 16 16:01:10 2016 -0700 ---------------------------------------------------------------------- .gitignore | 1 + pom.xml | 140 ++-- src/site/apt/developer_get_started.apt | 8 +- .../apt/development_update_xqts_results.apt | 4 +- src/site/apt/index.apt | 5 +- src/site/apt/user_cluster_installation.apt | 30 +- src/site/apt/user_running_tests.apt | 22 +- .../images/eclipse_xml_editor_screen.png | Bin 0 -> 75948 bytes src/site/resources/images/vxquery_cluster.png | Bin 0 -> 22024 bytes .../images/vxquery_cluster_with_hdfs.png | Bin 0 -> 31459 bytes src/site/resources/images/vxquery_stack.png | Bin 0 -> 13264 bytes src/site/site.xml | 17 +- vxquery-benchmark/pom.xml | 1 - .../noaa-ghcn-daily/other_systems/basex/q00.xq | 32 + .../noaa-ghcn-daily/other_systems/basex/q01.xq | 25 + .../noaa-ghcn-daily/other_systems/basex/q02.xq | 31 + .../noaa-ghcn-daily/other_systems/basex/q03.xq | 25 + .../noaa-ghcn-daily/other_systems/basex/q04.xq | 30 + .../noaa-ghcn-daily/other_systems/basex/q05.xq | 33 + .../noaa-ghcn-daily/other_systems/basex/q06.xq | 30 + .../noaa-ghcn-daily/other_systems/basex/q07.xq | 32 + .../other_systems/basex_index/q00.xq | 32 + .../other_systems/basex_index/q01.xq | 25 + .../other_systems/basex_index/q02.xq | 31 + .../other_systems/basex_index/q03.xq | 25 + .../other_systems/basex_index/q04.xq | 30 + .../other_systems/basex_index/q05.xq | 33 + .../other_systems/basex_index/q06.xq | 30 + .../other_systems/basex_index/q07.xq | 32 + .../basex_index/sensors_database_add.bxs | 2 + .../basex_index/sensors_database_remove.bxs | 1 + .../basex_index/stations_database_add.bxs | 2 + .../basex_index/stations_database_remove.bxs | 1 + .../other_systems/basex_scripts/.basex | 0 .../other_systems/basex_scripts/README | 9 + .../basex_scripts/run_basex_index_tests.sh | 63 ++ .../basex_scripts/run_basex_tests.sh | 45 ++ .../mrql_scripts/run_group_test.sh | 3 +- .../mrql_scripts/run_mrql_tests.sh | 2 +- .../saxon_scripts/run_saxon_tests.sh | 9 +- vxquery-benchmark/src/site/site.xml | 6 +- vxquery-cli/pom.xml | 20 +- .../java/org/apache/vxquery/cli/VXQuery.java | 782 +++++++++---------- vxquery-cli/src/site/site.xml | 6 +- vxquery-core/pom.xml | 16 +- .../vxquery/collations/CodepointCollation.java | 2 +- .../apache/vxquery/collations/Collation.java | 2 +- .../vxquery/compiler/CompilerControlBlock.java | 2 +- .../VXQueryBinaryBooleanInspectorFactory.java | 10 +- .../VXQueryBinaryIntegerInspectorFactory.java | 10 +- .../VXQueryComparatorFactoryProvider.java | 11 +- .../algebricks/VXQueryConstantValue.java | 4 +- .../VXQueryExpressionRuntimeProvider.java | 48 +- .../algebricks/VXQueryGlobalDataFactory.java | 6 +- .../algebricks/VXQueryNullWriterFactory.java | 10 +- .../algebricks/VXQueryPrinterFactory.java | 6 +- .../VXQueryPrinterFactoryProvider.java | 8 +- ...ueryLogicalExpressionPrettyPrintVisitor.java | 30 +- .../compiler/rewriter/RewriteRuleset.java | 99 +-- .../rewriter/VXQueryOptimizationContext.java | 18 +- .../rewriter/rules/AbstractCollectionRule.java | 49 +- ...tractRemoveRedundantTypeExpressionsRule.java | 14 +- .../AbstractUsedVariablesProcessingRule.java | 18 +- .../rules/AbstractVXQueryAggregateRule.java | 6 +- .../rewriter/rules/CollectionFileDomain.java | 2 +- .../rules/ConsolidateAssignAggregateRule.java | 34 +- .../rules/ConsolidateDescandantChild.java | 22 +- .../rewriter/rules/ConsolidateUnnestsRule.java | 20 +- ...tAssignSortDistinctNodesToOperatorsRule.java | 53 +- .../rules/ConvertAssignToAggregateRule.java | 36 +- .../rules/ConvertAssignToUnnestRule.java | 22 +- .../ConvertFromAlgebricksExpressionsRule.java | 24 +- .../ConvertToAlgebricksExpressionsRule.java | 26 +- .../EliminateSubplanForSingleItemsRule.java | 32 +- .../EliminateSubplanForSinglePathsRule.java | 14 +- .../EliminateUnnestAggregateSequencesRule.java | 28 +- .../EliminateUnnestAggregateSubplanRule.java | 30 +- .../rewriter/rules/IntroduceCollectionRule.java | 36 +- .../rules/IntroduceTwoStepAggregateRule.java | 28 +- .../rules/PushChildIntoDataScanRule.java | 24 +- .../rules/PushFunctionsOntoEqJoinBranches.java | 32 +- .../RemoveRedundantBooleanExpressionsRule.java | 2 +- .../RemoveRedundantCastExpressionsRule.java | 2 +- .../RemoveRedundantDataExpressionsRule.java | 2 +- .../RemoveRedundantPromoteExpressionsRule.java | 2 +- .../RemoveRedundantTreatExpressionsRule.java | 2 +- .../RemoveUnusedSortDistinctNodesRule.java | 80 +- .../rules/RemoveUnusedUnnestIterateRule.java | 44 +- .../rules/ReplaceSourceMapInDocExpression.java | 26 +- .../rules/SetCollectionDataSourceRule.java | 15 +- .../rules/SetVariableIdContextRule.java | 14 +- .../ExtractFunctionsFromJoinConditionRule.java | 47 +- .../InlineNestedVariablesRule.java | 26 +- ...oveFreeVariableOperatorOutOfSubplanRule.java | 22 +- .../ConstantPropertyPropagationPolicy.java | 2 +- .../IPropertyPropagationPolicy.java | 2 +- .../InputPropertyPropagationPolicy.java | 2 +- .../cardinality/Cardinality.java | 2 +- .../documentorder/DocumentOrder.java | 2 +- .../DocumentOrderNOPropagationPolicy.java | 2 +- .../DocumentOrderYESPropagationPolicy.java | 2 +- .../uniquenodes/UniqueNodes.java | 2 +- .../UniqueNodesNOPropagationPolicy.java | 2 +- .../UniqueNodesYESPropagationPolicy.java | 2 +- .../rules/util/CardinalityRuleToolbox.java | 22 +- .../rewriter/rules/util/ExpressionToolbox.java | 36 +- .../rewriter/rules/util/OperatorToolbox.java | 22 +- .../context/DefaultDocumentURIResolverImpl.java | 2 +- .../context/DelegatingStaticContextImpl.java | 2 +- .../vxquery/context/DocumentURIResolver.java | 2 +- .../apache/vxquery/context/DynamicContext.java | 6 +- .../vxquery/context/DynamicContextImpl.java | 8 +- .../context/DynamicContextImplFactory.java | 8 +- .../vxquery/context/IDynamicContextFactory.java | 4 +- .../vxquery/context/IStaticContextFactory.java | 2 +- .../vxquery/context/ModuleURIResolver.java | 2 +- .../context/RootStaticContextFactory.java | 2 +- .../vxquery/context/RootStaticContextImpl.java | 2 +- .../apache/vxquery/context/StaticContext.java | 2 +- .../vxquery/context/StaticContextImpl.java | 2 +- .../context/StaticContextImplFactory.java | 2 +- .../vxquery/context/ThinStaticContextImpl.java | 2 +- .../context/ThinStaticContextImplFactory.java | 2 +- .../apache/vxquery/context/XQueryVariable.java | 4 +- .../datamodel/accessors/PointableCache.java | 4 +- .../datamodel/accessors/PointablePool.java | 6 +- .../accessors/PointablePoolFactory.java | 20 +- .../datamodel/accessors/SequencePointable.java | 14 +- .../accessors/TaggedValuePointable.java | 14 +- .../datamodel/accessors/TypedPointables.java | 16 +- .../accessors/atomic/CodedQNamePointable.java | 12 +- .../accessors/atomic/XSBinaryPointable.java | 8 +- .../accessors/atomic/XSDatePointable.java | 14 +- .../accessors/atomic/XSDateTimePointable.java | 16 +- .../accessors/atomic/XSDecimalPointable.java | 20 +- .../accessors/atomic/XSDurationPointable.java | 14 +- .../accessors/atomic/XSQNamePointable.java | 10 +- .../accessors/atomic/XSTimePointable.java | 14 +- .../accessors/nodes/AbstractNodePointable.java | 4 +- .../accessors/nodes/AttributeNodePointable.java | 10 +- .../accessors/nodes/DocumentNodePointable.java | 10 +- .../accessors/nodes/ElementNodePointable.java | 14 +- .../accessors/nodes/NodeTreePointable.java | 22 +- .../accessors/nodes/PINodePointable.java | 12 +- .../nodes/TextOrCommentNodePointable.java | 10 +- .../builders/atomic/StringValueBuilder.java | 4 +- .../builders/atomic/UTF8StringBuilder.java | 6 +- .../builders/base/AbstractBuilder.java | 4 +- .../builders/base/AbstractValueBuilder.java | 2 +- .../datamodel/builders/base/IValueBuilder.java | 2 +- .../builders/nodes/AbstractNodeBuilder.java | 4 +- .../builders/nodes/AttributeNodeBuilder.java | 6 +- .../builders/nodes/CommentNodeBuilder.java | 6 +- .../builders/nodes/DictionaryBuilder.java | 14 +- .../builders/nodes/DocumentNodeBuilder.java | 6 +- .../builders/nodes/ElementNodeBuilder.java | 12 +- .../builders/nodes/NodeSubTreeBuilder.java | 4 +- .../datamodel/builders/nodes/PINodeBuilder.java | 6 +- .../builders/nodes/TextNodeBuilder.java | 6 +- .../builders/sequence/SequenceBuilder.java | 8 +- .../apache/vxquery/datamodel/util/DateTime.java | 44 +- .../vxquery/datamodel/values/ValueTag.java | 2 +- .../vxquery/datamodel/values/XDMConstants.java | 8 +- .../DefaultSystemExceptionFactory.java | 2 +- .../vxquery/exceptions/ErrorReporter.java | 2 +- .../vxquery/exceptions/SystemException.java | 2 +- .../exceptions/SystemExceptionFactory.java | 2 +- .../exceptions/VXQueryDataException.java | 2 +- .../vxquery/exceptions/WarningReporter.java | 2 +- .../vxquery/functions/BuiltinFunction.java | 2 +- .../vxquery/functions/ExternalFunction.java | 2 +- .../org/apache/vxquery/functions/Function.java | 10 +- .../org/apache/vxquery/functions/Operator.java | 2 +- .../org/apache/vxquery/functions/Signature.java | 4 +- .../functions/UserDefinedXQueryFunction.java | 4 +- .../vxquery/metadata/QueryResultDataSink.java | 12 +- .../metadata/QueryResultSetDataSink.java | 12 +- .../metadata/VXQueryCollectionDataSource.java | 16 +- .../VXQueryCollectionOperatorDescriptor.java | 38 +- .../vxquery/metadata/VXQueryIOFileFilter.java | 2 +- .../metadata/VXQueryMetadataProvider.java | 58 +- .../org/apache/vxquery/result/ResultUtils.java | 6 +- .../VXQueryRawBinaryHashFunctionFactory.java | 6 +- ...AbstractMaxMinAggregateEvaluatorFactory.java | 14 +- .../AbstractMaxMinScalarEvaluatorFactory.java | 14 +- .../AvgGlobalAggregateEvaluatorFactory.java | 16 +- .../AvgLocalAggregateEvaluatorFactory.java | 14 +- .../FnAvgAggregateEvaluatorFactory.java | 14 +- .../aggregate/FnAvgScalarEvaluatorFactory.java | 14 +- .../FnCountAggregateEvaluatorFactory.java | 14 +- .../FnCountScalarEvaluatorFactory.java | 14 +- .../FnMaxAggregateEvaluatorFactory.java | 4 +- .../aggregate/FnMaxScalarEvaluatorFactory.java | 4 +- .../FnMinAggregateEvaluatorFactory.java | 4 +- .../aggregate/FnMinScalarEvaluatorFactory.java | 4 +- .../FnSumAggregateEvaluatorFactory.java | 14 +- .../aggregate/FnSumScalarEvaluatorFactory.java | 16 +- .../arithmetic/AbstractArithmeticOperation.java | 10 +- ...bstractArithmeticScalarEvaluatorFactory.java | 12 +- .../functions/arithmetic/AddOperation.java | 12 +- .../arithmetic/AddScalarEvaluatorFactory.java | 4 +- .../functions/arithmetic/DivideOperation.java | 12 +- .../DivideScalarEvaluatorFactory.java | 4 +- .../arithmetic/IntegerDivideOperation.java | 12 +- .../IntegerDivideScalarEvaluatorFactory.java | 4 +- .../functions/arithmetic/ModOperation.java | 12 +- .../arithmetic/ModScalarEvaluatorFactory.java | 4 +- .../functions/arithmetic/MultiplyOperation.java | 12 +- .../MultiplyScalarEvaluatorFactory.java | 4 +- .../functions/arithmetic/SubtractOperation.java | 12 +- .../SubtractScalarEvaluatorFactory.java | 4 +- ...ctTaggedValueArgumentAggregateEvaluator.java | 10 +- ...dValueArgumentAggregateEvaluatorFactory.java | 14 +- ...tractTaggedValueArgumentScalarEvaluator.java | 10 +- ...ggedValueArgumentScalarEvaluatorFactory.java | 10 +- ...ctTaggedValueArgumentUnnestingEvaluator.java | 10 +- ...dValueArgumentUnnestingEvaluatorFactory.java | 14 +- .../bool/AndScalarEvaluatorFactory.java | 16 +- .../bool/FnBooleanScalarEvaluatorFactory.java | 26 +- .../bool/FnFalseScalarEvaluatorFactory.java | 12 +- .../bool/FnNotScalarEvaluatorFactory.java | 14 +- .../bool/FnTrueScalarEvaluatorFactory.java | 12 +- .../bool/OrScalarEvaluatorFactory.java | 16 +- .../functions/cast/AbstractCastToOperation.java | 38 +- .../cast/CastScalarEvaluatorFactory.java | 14 +- .../functions/cast/CastToAnyURIOperation.java | 4 +- .../cast/CastToBase64BinaryOperation.java | 7 +- .../functions/cast/CastToBooleanOperation.java | 12 +- .../functions/cast/CastToByteOperation.java | 20 +- .../cast/CastToDTDurationOperation.java | 8 +- .../functions/cast/CastToDateOperation.java | 4 +- .../functions/cast/CastToDateTimeOperation.java | 4 +- .../functions/cast/CastToDecimalOperation.java | 22 +- .../functions/cast/CastToDoubleOperation.java | 20 +- .../functions/cast/CastToDurationOperation.java | 8 +- .../functions/cast/CastToEntityOperation.java | 2 +- .../functions/cast/CastToFloatOperation.java | 20 +- .../functions/cast/CastToGDayOperation.java | 4 +- .../cast/CastToGMonthDayOperation.java | 4 +- .../functions/cast/CastToGMonthOperation.java | 4 +- .../cast/CastToGYearMonthOperation.java | 4 +- .../functions/cast/CastToGYearOperation.java | 4 +- .../cast/CastToHexBinaryOperation.java | 6 +- .../functions/cast/CastToIDOperation.java | 2 +- .../functions/cast/CastToIDREFOperation.java | 2 +- .../functions/cast/CastToIntOperation.java | 20 +- .../functions/cast/CastToIntegerOperation.java | 20 +- .../functions/cast/CastToLanguageOperation.java | 2 +- .../functions/cast/CastToLongOperation.java | 2 +- .../functions/cast/CastToNCNameOperation.java | 2 +- .../functions/cast/CastToNMTokenOperation.java | 2 +- .../functions/cast/CastToNameOperation.java | 2 +- .../cast/CastToNegativeIntegerOperation.java | 2 +- .../cast/CastToNonNegativeIntegerOperation.java | 2 +- .../cast/CastToNonPositiveIntegerOperation.java | 2 +- .../cast/CastToNormalizedStringOperation.java | 2 +- .../functions/cast/CastToNotationOperation.java | 4 +- .../cast/CastToPositiveIntegerOperation.java | 2 +- .../functions/cast/CastToQNameOperation.java | 6 +- .../functions/cast/CastToShortOperation.java | 20 +- .../functions/cast/CastToStringOperation.java | 22 +- .../functions/cast/CastToTimeOperation.java | 4 +- .../functions/cast/CastToTokenOperation.java | 2 +- .../cast/CastToUnsignedByteOperation.java | 2 +- .../cast/CastToUnsignedIntOperation.java | 2 +- .../cast/CastToUnsignedLongOperation.java | 2 +- .../cast/CastToUnsignedShortOperation.java | 2 +- .../cast/CastToUntypedAtomicOperation.java | 2 +- .../cast/CastToYMDurationOperation.java | 8 +- .../castable/AbstractCastableAsOperation.java | 22 +- .../castable/CastableAsAnyURIOperation.java | 4 +- .../CastableAsBase64BinaryOperation.java | 6 +- .../castable/CastableAsBooleanOperation.java | 14 +- .../castable/CastableAsByteOperation.java | 16 +- .../castable/CastableAsDTDurationOperation.java | 10 +- .../castable/CastableAsDateOperation.java | 6 +- .../castable/CastableAsDateTimeOperation.java | 6 +- .../castable/CastableAsDecimalOperation.java | 14 +- .../castable/CastableAsDoubleOperation.java | 20 +- .../castable/CastableAsDurationOperation.java | 10 +- .../castable/CastableAsFloatOperation.java | 20 +- .../castable/CastableAsGDayOperation.java | 6 +- .../castable/CastableAsGMonthDayOperation.java | 6 +- .../castable/CastableAsGMonthOperation.java | 6 +- .../castable/CastableAsGYearMonthOperation.java | 6 +- .../castable/CastableAsGYearOperation.java | 6 +- .../castable/CastableAsHexBinaryOperation.java | 6 +- .../castable/CastableAsIntOperation.java | 20 +- .../castable/CastableAsIntegerOperation.java | 20 +- .../castable/CastableAsLongOperation.java | 2 +- .../CastableAsNegativeIntegerOperation.java | 2 +- .../CastableAsNonNegativeIntegerOperation.java | 2 +- .../CastableAsNonPositiveIntegerOperation.java | 2 +- .../castable/CastableAsNotationOperation.java | 14 +- .../CastableAsPositiveIntegerOperation.java | 2 +- .../castable/CastableAsQNameOperation.java | 4 +- .../castable/CastableAsShortOperation.java | 18 +- .../castable/CastableAsStringOperation.java | 14 +- .../castable/CastableAsTimeOperation.java | 6 +- .../CastableAsUnsignedByteOperation.java | 2 +- .../CastableAsUnsignedIntOperation.java | 2 +- .../CastableAsUnsignedLongOperation.java | 2 +- .../CastableAsUnsignedShortOperation.java | 2 +- .../CastableAsUntypedAtomicOperation.java | 2 +- .../castable/CastableAsYMDurationOperation.java | 10 +- .../CastableScalarEvaluatorFactory.java | 14 +- .../AbstractDisjunctiveComparisonOperation.java | 14 +- .../AbstractNegatingComparisonOperation.java | 14 +- .../AbstractValueComparisonOperation.java | 14 +- ...ctValueComparisonScalarEvaluatorFactory.java | 16 +- .../comparison/ValueEqComparisonOperation.java | 16 +- ...ValueEqComparisonScalarEvaluatorFactory.java | 4 +- .../comparison/ValueGeComparisonOperation.java | 2 +- ...ValueGeComparisonScalarEvaluatorFactory.java | 4 +- .../comparison/ValueGtComparisonOperation.java | 2 +- ...ValueGtComparisonScalarEvaluatorFactory.java | 4 +- .../comparison/ValueLeComparisonOperation.java | 2 +- ...ValueLeComparisonScalarEvaluatorFactory.java | 4 +- .../comparison/ValueLtComparisonOperation.java | 16 +- ...ValueLtComparisonScalarEvaluatorFactory.java | 4 +- .../comparison/ValueNeComparisonOperation.java | 2 +- ...ValueNeComparisonScalarEvaluatorFactory.java | 4 +- ...GeneralComparisonScalarEvaluatorFactory.java | 18 +- ...neralEqComparisonScalarEvaluatorFactory.java | 4 +- ...neralGeComparisonScalarEvaluatorFactory.java | 4 +- ...neralGtComparisonScalarEvaluatorFactory.java | 4 +- ...neralLeComparisonScalarEvaluatorFactory.java | 4 +- ...neralLtComparisonScalarEvaluatorFactory.java | 4 +- ...neralNeComparisonScalarEvaluatorFactory.java | 4 +- .../IfThenElseScalarEvaluatorFactory.java | 16 +- ...ValueFromDateTimeScalarEvaluatorFactory.java | 12 +- ...ValueFromDurationScalarEvaluatorFactory.java | 16 +- ...ateTimeToTimezoneScalarEvaluatorFactory.java | 14 +- ...ustDateToTimezoneScalarEvaluatorFactory.java | 14 +- ...ustTimeToTimezoneScalarEvaluatorFactory.java | 14 +- .../FnCurrentDateScalarEvaluatorFactory.java | 12 +- ...FnCurrentDateTimeScalarEvaluatorFactory.java | 14 +- .../FnCurrentTimeScalarEvaluatorFactory.java | 14 +- .../FnDateTimeScalarEvaluatorFactory.java | 12 +- .../FnDayFromDateScalarEvaluatorFactory.java | 2 +- ...FnDayFromDateTimeScalarEvaluatorFactory.java | 2 +- ...nDaysFromDurationScalarEvaluatorFactory.java | 2 +- ...HoursFromDateTimeScalarEvaluatorFactory.java | 2 +- ...HoursFromDurationScalarEvaluatorFactory.java | 2 +- .../FnHoursFromTimeScalarEvaluatorFactory.java | 2 +- ...nImplicitTimezoneScalarEvaluatorFactory.java | 12 +- ...nutesFromDateTimeScalarEvaluatorFactory.java | 2 +- ...nutesFromDurationScalarEvaluatorFactory.java | 2 +- ...FnMinutesFromTimeScalarEvaluatorFactory.java | 2 +- .../FnMonthFromDateScalarEvaluatorFactory.java | 2 +- ...MonthFromDateTimeScalarEvaluatorFactory.java | 2 +- ...onthsFromDurationScalarEvaluatorFactory.java | 2 +- ...condsFromDateTimeScalarEvaluatorFactory.java | 2 +- ...condsFromDurationScalarEvaluatorFactory.java | 2 +- ...FnSecondsFromTimeScalarEvaluatorFactory.java | 2 +- ...nTimezoneFromDateScalarEvaluatorFactory.java | 2 +- ...ezoneFromDateTimeScalarEvaluatorFactory.java | 2 +- ...nTimezoneFromTimeScalarEvaluatorFactory.java | 2 +- .../FnYearFromDateScalarEvaluatorFactory.java | 2 +- ...nYearFromDateTimeScalarEvaluatorFactory.java | 2 +- ...YearsFromDurationScalarEvaluatorFactory.java | 2 +- .../error/FnErrorScalarEvaluatorFactory.java | 18 +- .../misc/FnDataScalarEvaluatorFactory.java | 14 +- .../AbstractNodeConstructorScalarEvaluator.java | 12 +- .../AbstractNodePositionalCheckEvaluator.java | 6 +- ...AttributeNodeConstructorScalarEvaluator.java | 10 +- ...teNodeConstructorScalarEvaluatorFactory.java | 10 +- .../CommentNodeConstructorScalarEvaluator.java | 10 +- ...ntNodeConstructorScalarEvaluatorFactory.java | 10 +- .../ElementNodeConstructorScalarEvaluator.java | 14 +- ...ntNodeConstructorScalarEvaluatorFactory.java | 10 +- .../FnDocAvailableScalarEvaluatorFactory.java | 18 +- .../node/FnDocScalarEvaluatorFactory.java | 18 +- .../node/FnNumberScalarEvaluatorFactory.java | 12 +- .../node/IdFromNodeScalarEvaluatorFactory.java | 12 +- .../LocalIdFromNodeScalarEvaluatorFactory.java | 12 +- .../functions/node/OpNodeAfterEvaluator.java | 2 +- .../node/OpNodeAfterEvaluatorFactory.java | 8 +- .../functions/node/OpNodeBeforeEvaluator.java | 2 +- .../node/OpNodeBeforeEvaluatorFactory.java | 8 +- .../node/PINodeConstructorScalarEvaluator.java | 10 +- ...PINodeConstructorScalarEvaluatorFactory.java | 10 +- .../TextNodeConstructorScalarEvaluator.java | 10 +- ...xtNodeConstructorScalarEvaluatorFactory.java | 10 +- .../TreeIdFromNodeScalarEvaluatorFactory.java | 12 +- ...NodesAscOrAtomicsScalarEvaluatorFactory.java | 12 +- .../numeric/AbstractNumericOperation.java | 6 +- .../AbstractNumericScalarEvaluatorFactory.java | 16 +- .../functions/numeric/FnAbsOperation.java | 8 +- .../numeric/FnAbsScalarEvaluatorFactory.java | 4 +- .../functions/numeric/FnCeilingOperation.java | 8 +- .../FnCeilingScalarEvaluatorFactory.java | 4 +- .../functions/numeric/FnFloorOperation.java | 8 +- .../numeric/FnFloorScalarEvaluatorFactory.java | 4 +- ...FnRoundHalfToEvenScalarEvaluatorFactory.java | 26 +- .../functions/numeric/FnRoundOperation.java | 8 +- .../numeric/FnRoundScalarEvaluatorFactory.java | 4 +- ...ocalNameFromQNameScalarEvaluatorFactory.java | 16 +- ...spaceUriFromQNameScalarEvaluatorFactory.java | 16 +- ...FnPrefixFromQNameScalarEvaluatorFactory.java | 16 +- .../qname/FnQNameScalarEvaluatorFactory.java | 16 +- .../ConcatenateScalarEvaluatorFactory.java | 16 +- .../FnDistinctValuesScalarEvaluatorFactory.java | 18 +- .../sequence/FnEmptyScalarEvaluatorFactory.java | 12 +- .../FnExactlyOneScalarEvaluatorFactory.java | 12 +- .../FnExistsScalarEvaluatorFactory.java | 12 +- .../FnIndexOfScalarEvaluatorFactory.java | 16 +- .../FnInsertBeforeScalarEvaluatorFactory.java | 18 +- .../FnOneOrMoreScalarEvaluatorFactory.java | 12 +- .../FnRemoveScalarEvaluatorFactory.java | 18 +- .../FnReverseScalarEvaluatorFactory.java | 16 +- .../FnSubsequenceScalarEvaluatorFactory.java | 24 +- .../FnUnorderedScalarEvaluatorFactory.java | 12 +- .../FnZeroOrOneScalarEvaluatorFactory.java | 12 +- .../IterateUnnestingEvaluatorFactory.java | 14 +- .../sequence/OpToScalarEvaluatorFactory.java | 16 +- .../SequenceAggregateEvaluatorFactory.java | 16 +- ...stractDescendantPathStepScalarEvaluator.java | 12 +- .../step/AbstractForwardAxisPathStep.java | 8 +- .../step/AbstractPathStepScalarEvaluator.java | 10 +- .../AbstractSinglePathStepScalarEvaluator.java | 12 +- .../step/AttributePathStepScalarEvaluator.java | 6 +- ...AttributePathStepScalarEvaluatorFactory.java | 10 +- .../step/ChildPathStepOperatorDescriptor.java | 24 +- .../step/ChildPathStepScalarEvaluator.java | 12 +- .../ChildPathStepScalarEvaluatorFactory.java | 10 +- .../functions/step/ChildPathStepUnnesting.java | 22 +- .../step/ChildPathStepUnnestingEvaluator.java | 10 +- .../ChildPathStepUnnestingEvaluatorFactory.java | 12 +- ...DescendantOrSelfPathStepScalarEvaluator.java | 12 +- ...antOrSelfPathStepScalarEvaluatorFactory.java | 10 +- .../step/DescendantOrSelfPathStepUnnesting.java | 23 +- ...cendantOrSelfPathStepUnnestingEvaluator.java | 10 +- ...OrSelfPathStepUnnestingEvaluatorFactory.java | 12 +- .../step/DescendantPathStepScalarEvaluator.java | 12 +- ...escendantPathStepScalarEvaluatorFactory.java | 10 +- .../DescendantPathStepUnnestingEvaluator.java | 10 +- ...endantPathStepUnnestingEvaluatorFactory.java | 12 +- .../runtime/functions/step/NodeTestFilter.java | 8 +- .../step/SelfPathStepScalarEvaluator.java | 10 +- .../SelfPathStepScalarEvaluatorFactory.java | 10 +- ...stractCharacterIteratorCopyingEvaluator.java | 6 +- ...haracterIteratorCopyingEvaluatorFactory.java | 2 +- .../FnCodepointEqualEvaluatorFactory.java | 12 +- .../FnCodepointsToStringEvaluatorFactory.java | 16 +- .../strings/FnCompareEvaluatorFactory.java | 14 +- .../strings/FnConcatEvaluatorFactory.java | 14 +- .../strings/FnContainsEvaluatorFactory.java | 12 +- .../strings/FnEndsWithEvaluatorFactory.java | 12 +- .../strings/FnLowerCaseEvaluatorFactory.java | 10 +- .../strings/FnStartsWithEvaluatorFactory.java | 12 +- .../strings/FnStringJoinEvaluatorFactory.java | 14 +- .../strings/FnStringLengthEvaluatorFactory.java | 14 +- .../strings/FnStringScalarEvaluatorFactory.java | 14 +- .../FnStringToCodepointsEvaluatorFactory.java | 14 +- .../FnSubstringAfterEvaluatorFactory.java | 10 +- .../FnSubstringBeforeEvaluatorFactory.java | 10 +- .../strings/FnSubstringEvaluatorFactory.java | 16 +- .../strings/FnUpperCaseEvaluatorFactory.java | 10 +- .../strings/UTF8StringCharacterIterator.java | 2 +- .../AbstractTypeScalarEvaluatorFactory.java | 12 +- .../type/InstanceOfScalarEvaluatorFactory.java | 12 +- .../type/PromoteScalarEvaluatorFactory.java | 14 +- .../functions/type/SequenceTypeMatcher.java | 4 +- .../type/TreatScalarEvaluatorFactory.java | 12 +- .../unary/AbstractNumericUnaryOperation.java | 6 +- ...tractNumericUnaryScalarEvaluatorFactory.java | 16 +- .../unary/NumericUnaryMinusOperation.java | 8 +- ...NumericUnaryMinusScalarEvaluatorFactory.java | 4 +- .../unary/NumericUnaryPlusOperation.java | 8 +- .../NumericUnaryPlusScalarEvaluatorFactory.java | 4 +- .../functions/util/ArithmeticHelper.java | 8 +- .../runtime/functions/util/AtomizeHelper.java | 6 +- .../runtime/functions/util/FunctionHelper.java | 47 +- ...XQueryBinaryHashFunctionFactoryProvider.java | 6 +- ...VXQueryBinaryHashFunctionFamilyProvider.java | 8 +- .../vxquery/serializer/XMLSerializer.java | 26 +- .../apache/vxquery/types/AbstractNodeType.java | 4 +- .../org/apache/vxquery/types/AnyItemType.java | 1 + .../org/apache/vxquery/types/AnyNodeType.java | 3 +- .../org/apache/vxquery/types/AnySimpleType.java | 3 +- .../java/org/apache/vxquery/types/AnyType.java | 3 +- .../org/apache/vxquery/types/AtomicType.java | 2 +- .../org/apache/vxquery/types/AttributeType.java | 4 +- .../apache/vxquery/types/BuiltinAtomicType.java | 2 +- .../vxquery/types/BuiltinTypeConstants.java | 2 +- .../apache/vxquery/types/BuiltinTypeQNames.java | 2 +- .../vxquery/types/BuiltinTypeRegistry.java | 2 +- .../org/apache/vxquery/types/CommentType.java | 3 +- .../org/apache/vxquery/types/ComplexType.java | 2 +- .../apache/vxquery/types/DerivationProcess.java | 2 +- .../org/apache/vxquery/types/DocumentType.java | 4 +- .../org/apache/vxquery/types/ElementType.java | 4 +- .../apache/vxquery/types/EmptySequenceType.java | 3 +- .../java/org/apache/vxquery/types/ItemType.java | 2 +- .../java/org/apache/vxquery/types/NameTest.java | 4 +- .../java/org/apache/vxquery/types/NodeKind.java | 2 +- .../java/org/apache/vxquery/types/NodeType.java | 2 +- .../java/org/apache/vxquery/types/NoneType.java | 3 +- .../types/ProcessingInstructionType.java | 6 +- .../org/apache/vxquery/types/SchemaType.java | 2 +- .../org/apache/vxquery/types/SimpleType.java | 2 +- .../java/org/apache/vxquery/types/TextType.java | 3 +- .../org/apache/vxquery/types/TypeHelper.java | 2 +- .../org/apache/vxquery/types/TypeUtils.java | 2 +- .../java/org/apache/vxquery/types/XQType.java | 2 +- .../java/org/apache/vxquery/util/Filter.java | 2 +- .../apache/vxquery/util/GrowableIntArray.java | 2 +- .../org/apache/vxquery/util/SourceLocation.java | 2 +- .../vxquery/xmlparser/ITreeNodeIdProvider.java | 2 +- .../vxquery/xmlparser/SAXContentHandler.java | 77 +- .../vxquery/xmlparser/TreeNodeIdProvider.java | 2 +- .../org/apache/vxquery/xmlparser/XMLParser.java | 29 +- .../apache/vxquery/xmlquery/ast/ASTNode.java | 2 +- .../org/apache/vxquery/xmlquery/ast/ASTTag.java | 2 +- .../vxquery/xmlquery/ast/AnyNodeTestNode.java | 2 +- .../vxquery/xmlquery/ast/AtomicTypeNode.java | 2 +- .../vxquery/xmlquery/ast/AttributeTestNode.java | 2 +- .../vxquery/xmlquery/ast/AxisStepNode.java | 2 +- .../vxquery/xmlquery/ast/BaseUriDeclNode.java | 2 +- .../xmlquery/ast/BoundarySpaceDeclNode.java | 2 +- .../vxquery/xmlquery/ast/CDataSectionNode.java | 2 +- .../vxquery/xmlquery/ast/CaseClauseNode.java | 2 +- .../vxquery/xmlquery/ast/CommentTestNode.java | 2 +- .../ast/ComputedAttributeConstructorNode.java | 2 +- .../ast/ComputedCommentConstructorNode.java | 2 +- .../ast/ComputedDocumentConstructorNode.java | 2 +- .../ast/ComputedElementConstructorNode.java | 2 +- .../xmlquery/ast/ComputedPIConstructorNode.java | 2 +- .../ast/ComputedTextConstructorNode.java | 2 +- .../xmlquery/ast/ConstructionDeclNode.java | 2 +- .../vxquery/xmlquery/ast/ContentCharsNode.java | 2 +- .../xmlquery/ast/ContextItemExprNode.java | 2 +- .../xmlquery/ast/CopyNamespacesDeclNode.java | 2 +- .../ast/DQuotedAttributeContentNode.java | 2 +- .../xmlquery/ast/DefaultCollationDeclNode.java | 2 +- .../ast/DefaultElementNamespaceDeclNode.java | 2 +- .../ast/DefaultFunctionNamespaceDeclNode.java | 2 +- .../ast/DirectAttributeConstructorNode.java | 2 +- .../ast/DirectCommentConstructorNode.java | 2 +- .../ast/DirectElementConstructorNode.java | 2 +- .../xmlquery/ast/DirectPIConstructorNode.java | 2 +- .../vxquery/xmlquery/ast/DocumentTestNode.java | 2 +- .../vxquery/xmlquery/ast/ElementTestNode.java | 2 +- .../xmlquery/ast/EmptyOrderDeclNode.java | 2 +- .../xmlquery/ast/EmptySequenceTypeNode.java | 2 +- .../vxquery/xmlquery/ast/EnclosedExprNode.java | 2 +- .../apache/vxquery/xmlquery/ast/ExprNode.java | 2 +- .../vxquery/xmlquery/ast/ExtensionExprNode.java | 2 +- .../vxquery/xmlquery/ast/FLWORClauseNode.java | 2 +- .../vxquery/xmlquery/ast/FLWORExprNode.java | 2 +- .../vxquery/xmlquery/ast/FTOptionDeclNode.java | 2 +- .../vxquery/xmlquery/ast/FilterExprNode.java | 2 +- .../vxquery/xmlquery/ast/ForClauseNode.java | 2 +- .../vxquery/xmlquery/ast/ForVarDeclNode.java | 2 +- .../vxquery/xmlquery/ast/FunctionDeclNode.java | 2 +- .../vxquery/xmlquery/ast/FunctionExprNode.java | 2 +- .../apache/vxquery/xmlquery/ast/IfExprNode.java | 2 +- .../vxquery/xmlquery/ast/InfixExprNode.java | 2 +- .../vxquery/xmlquery/ast/ItemTestNode.java | 2 +- .../vxquery/xmlquery/ast/ItemTypeNode.java | 2 +- .../vxquery/xmlquery/ast/LetClauseNode.java | 2 +- .../vxquery/xmlquery/ast/LetVarDeclNode.java | 2 +- .../vxquery/xmlquery/ast/LibraryModuleNode.java | 2 +- .../vxquery/xmlquery/ast/LiteralNode.java | 2 +- .../vxquery/xmlquery/ast/MainModuleNode.java | 2 +- .../vxquery/xmlquery/ast/ModuleDeclNode.java | 2 +- .../vxquery/xmlquery/ast/ModuleImportNode.java | 2 +- .../apache/vxquery/xmlquery/ast/ModuleNode.java | 2 +- .../apache/vxquery/xmlquery/ast/NCNameNode.java | 2 +- .../vxquery/xmlquery/ast/NameTestNode.java | 2 +- .../vxquery/xmlquery/ast/NamespaceDeclNode.java | 2 +- .../vxquery/xmlquery/ast/OptionDeclNode.java | 2 +- .../vxquery/xmlquery/ast/OrderSpecNode.java | 2 +- .../vxquery/xmlquery/ast/OrderbyClauseNode.java | 2 +- .../vxquery/xmlquery/ast/OrderedExprNode.java | 2 +- .../xmlquery/ast/OrderingModeDeclNode.java | 2 +- .../apache/vxquery/xmlquery/ast/PITestNode.java | 2 +- .../apache/vxquery/xmlquery/ast/ParamNode.java | 2 +- .../xmlquery/ast/ParenthesizedExprNode.java | 2 +- .../vxquery/xmlquery/ast/PathExprNode.java | 2 +- .../apache/vxquery/xmlquery/ast/PragmaNode.java | 2 +- .../apache/vxquery/xmlquery/ast/PrologNode.java | 2 +- .../apache/vxquery/xmlquery/ast/QNameNode.java | 2 +- .../xmlquery/ast/QuantifiedExprNode.java | 2 +- .../xmlquery/ast/QuantifiedVarDeclNode.java | 2 +- .../vxquery/xmlquery/ast/QueryBodyNode.java | 2 +- .../xmlquery/ast/RelativePathExprNode.java | 2 +- .../ast/SQuotedAttributeContentNode.java | 2 +- .../xmlquery/ast/SchemaAttributeTestNode.java | 2 +- .../xmlquery/ast/SchemaElementTestNode.java | 2 +- .../vxquery/xmlquery/ast/SchemaImportNode.java | 2 +- .../vxquery/xmlquery/ast/SequenceTypeNode.java | 2 +- .../vxquery/xmlquery/ast/SingleTypeNode.java | 2 +- .../vxquery/xmlquery/ast/TextTestNode.java | 2 +- .../xmlquery/ast/TextualNodeContentNode.java | 2 +- .../vxquery/xmlquery/ast/TypeDeclNode.java | 2 +- .../vxquery/xmlquery/ast/TypeExprNode.java | 2 +- .../xmlquery/ast/TypeswitchExprNode.java | 2 +- .../vxquery/xmlquery/ast/UnaryExprNode.java | 2 +- .../vxquery/xmlquery/ast/UnorderedExprNode.java | 2 +- .../vxquery/xmlquery/ast/ValidateExprNode.java | 2 +- .../vxquery/xmlquery/ast/VarDeclNode.java | 2 +- .../apache/vxquery/xmlquery/ast/VarRefNode.java | 2 +- .../vxquery/xmlquery/ast/VersionDeclNode.java | 2 +- .../vxquery/xmlquery/ast/WhereClauseNode.java | 2 +- .../apache/vxquery/xmlquery/query/Module.java | 6 +- .../query/NoopXQueryCompilationListener.java | 2 +- .../vxquery/xmlquery/query/PrologVariable.java | 2 +- .../query/VXQueryCompilationListener.java | 10 +- .../xmlquery/query/XMLQueryCompiler.java | 66 +- .../vxquery/xmlquery/query/XMLQueryParser.java | 2 +- .../xmlquery/query/XMLQueryTypeChecker.java | 2 +- .../query/XQueryCompilationListener.java | 2 +- .../vxquery/xmlquery/query/XQueryConstants.java | 2 +- .../translator/VXQueryPositionWriter.java | 5 +- .../xmlquery/translator/XMLQueryTranslator.java | 178 ++--- .../src/main/xslt/generate-fn-defns.xsl | 6 +- .../src/main/xslt/generate-op-defns.xsl | 6 +- vxquery-core/src/site/site.xml | 6 +- .../xmlquery/query/SimpleXQueryTest.java | 2 +- vxquery-server/pom.xml | 10 +- .../vxquery/cli/VXQueryClusterShutdown.java | 4 +- .../src/main/resources/conf/cluster.properties | 2 +- vxquery-server/src/site/site.xml | 6 +- vxquery-xtest/pom.xml | 12 +- vxquery-xtest/results/xqts.txt | 742 +++++++++--------- .../vxquery/xtest/AbstractTestCaseFactory.java | 2 +- .../apache/vxquery/xtest/ExpectedResult.java | 2 +- .../vxquery/xtest/HTMLFileReporterImpl.java | 2 +- .../vxquery/xtest/LineFileReporterImpl.java | 2 +- .../apache/vxquery/xtest/ResultReporter.java | 2 +- .../vxquery/xtest/ServletReporterImpl.java | 2 +- .../java/org/apache/vxquery/xtest/TestCase.java | 2 +- .../apache/vxquery/xtest/TestCaseFactory.java | 2 +- .../apache/vxquery/xtest/TestCaseResult.java | 2 +- .../org/apache/vxquery/xtest/TestRunner.java | 57 +- .../apache/vxquery/xtest/TestRunnerFactory.java | 2 +- .../vxquery/xtest/XMLFileReporterImpl.java | 2 +- .../java/org/apache/vxquery/xtest/XTest.java | 2 +- .../org/apache/vxquery/xtest/XTestOptions.java | 2 +- .../vxquery/xtest/util/DiskPerformance.java | 215 ----- .../xtest/util/tests/AbstractDiskTest.java | 90 --- .../tests/BufferedParsedCharacterStream.java | 45 -- .../tests/BufferedReaderBufferedStream.java | 48 -- .../xtest/util/tests/BufferedReaderStream.java | 46 -- .../xtest/util/tests/BufferedStream.java | 45 -- .../vxquery/xtest/util/tests/IDiskTest.java | 31 - .../util/tests/ParsedBufferedByteStream.java | 48 -- .../tests/ParsedBufferedCharacterStream.java | 48 -- .../xtest/util/tests/ParsedByteStream.java | 46 -- .../xtest/util/tests/ParsedCharacterStream.java | 47 -- .../xtest/util/tests/ReaderBufferedStream.java | 46 -- .../vxquery/xtest/util/tests/ReaderStream.java | 45 -- .../apache/vxquery/xtest/util/tests/Stream.java | 44 -- vxquery-xtest/src/site/site.xml | 6 +- .../vxquery/xtest/JUnitTestCaseFactory.java | 2 +- 657 files changed, 4349 insertions(+), 4405 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/pom.xml ---------------------------------------------------------------------- diff --cc pom.xml index 8128251,8a97a92..4cabe8e --- a/pom.xml +++ b/pom.xml @@@ -278,25 -295,6 +296,25 @@@ </dependency> <dependency> - <groupId>edu.uci.ics.hyracks</groupId> ++ <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-hdfs-2.x</artifactId> + <version>${hyracks.version}</version> + </dependency> + + <dependency> - <groupId>edu.uci.ics.hyracks</groupId> ++ <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-hdfs-core</artifactId> + <version>${hyracks.version}</version> + <type>jar</type> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>2.7.0</version> + </dependency> + + <dependency> <groupId>ant</groupId> <artifactId>ant-trax</artifactId> <version>1.6.5</version> http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/src/site/site.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-cli/pom.xml ---------------------------------------------------------------------- diff --cc vxquery-cli/pom.xml index 2c4515e,7bf1ff9..c4e11cb --- a/vxquery-cli/pom.xml +++ b/vxquery-cli/pom.xml @@@ -123,22 -123,10 +123,22 @@@ </dependency> <dependency> - <groupId>edu.uci.ics.hyracks</groupId> + <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-dataflow-std</artifactId> </dependency> + + <dependency> - <groupId>edu.uci.ics.hyracks</groupId> ++ <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-hdfs-core</artifactId> + </dependency> + + <dependency> - <groupId>edu.uci.ics.hyracks</groupId> ++ <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-hdfs-2.x</artifactId> + </dependency> </dependencies> + + <reporting> <plugins> http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java ---------------------------------------------------------------------- diff --cc vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java index db95468,a02c65d..a6ef702 --- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java +++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java @@@ -45,401 -63,367 +63,373 @@@ import org.kohsuke.args4j.Argument import org.kohsuke.args4j.CmdLineParser; import org.kohsuke.args4j.Option; - import edu.uci.ics.hyracks.api.client.HyracksConnection; - import edu.uci.ics.hyracks.api.client.IHyracksClientConnection; - import edu.uci.ics.hyracks.api.client.NodeControllerInfo; - import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor; - import edu.uci.ics.hyracks.api.dataset.IHyracksDataset; - import edu.uci.ics.hyracks.api.dataset.IHyracksDatasetReader; - import edu.uci.ics.hyracks.api.dataset.ResultSetId; - import edu.uci.ics.hyracks.api.job.JobFlag; - import edu.uci.ics.hyracks.api.job.JobId; - import edu.uci.ics.hyracks.api.job.JobSpecification; - import edu.uci.ics.hyracks.client.dataset.HyracksDataset; - import edu.uci.ics.hyracks.control.cc.ClusterControllerService; - import edu.uci.ics.hyracks.control.common.controllers.CCConfig; - import edu.uci.ics.hyracks.control.common.controllers.NCConfig; - import edu.uci.ics.hyracks.control.nc.NodeControllerService; - import edu.uci.ics.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor; - public class VXQuery { - private final CmdLineOptions opts; - - private ClusterControllerService cc; - private NodeControllerService[] ncs; - private IHyracksClientConnection hcc; - private IHyracksDataset hds; - - private ResultSetId resultSetId; - private static List<String> timingMessages = new ArrayList<String>(); - private static long sumTiming; - private static long sumSquaredTiming; - private static long minTiming = Long.MAX_VALUE; - private static long maxTiming = Long.MIN_VALUE; - - /** - * Constructor to use command line options passed. - * - * @param opts - * Command line options object - */ - public VXQuery(CmdLineOptions opts) { - this.opts = opts; - } - - /** - * Main method to get command line options and execute query process. - * - * @param args - * @throws Exception - */ - public static void main(String[] args) throws Exception { - Date start = new Date(); - final CmdLineOptions opts = new CmdLineOptions(); - CmdLineParser parser = new CmdLineParser(opts); - - // parse command line options, give error message if no arguments passed - try { - parser.parseArgument(args); - } catch (Exception e) { - parser.printUsage(System.err); - return; - } - if (opts.arguments.isEmpty()) { - parser.printUsage(System.err); - return; - } - VXQuery vxq = new VXQuery(opts); - vxq.execute(); - // if -timing argument passed, show the starting and ending times - if (opts.timing) { - Date end = new Date(); - timingMessage("Execution time: " + (end.getTime() - start.getTime()) + " ms"); - if (opts.repeatExec > opts.timingIgnoreQueries) { - long mean = sumTiming / (opts.repeatExec - opts.timingIgnoreQueries); - double sd = Math - .sqrt(sumSquaredTiming / (opts.repeatExec - new Integer(opts.timingIgnoreQueries).doubleValue()) - - mean * mean); - timingMessage("Average execution time: " + mean + " ms"); - timingMessage("Standard deviation: " + String.format("%.4f", sd)); - timingMessage("Coefficient of variation: " + String.format("%.4f", (sd / mean))); - timingMessage("Minimum execution time: " + minTiming + " ms"); - timingMessage("Maximum execution time: " + maxTiming + " ms"); - } - System.out.println("Timing Summary:"); - for (String time : timingMessages) { - System.out.println(" " + time); - } - } - - } - - /** - * Creates a new Hyracks connection with: the client IP address and port - * provided, if IP address is provided in command line. Otherwise create a - * new virtual cluster with Hyracks nodes. Queries passed are run either - * way. After running queries, if a virtual cluster has been created, it is - * shut down. - * - * @throws Exception - */ - private void execute() throws Exception { - System.setProperty("vxquery.buffer_size", Integer.toString(opts.bufferSize)); - - if (opts.clientNetIpAddress != null) { - hcc = new HyracksConnection(opts.clientNetIpAddress, opts.clientNetPort); - runQueries(); - } else { - if (!opts.compileOnly) { - startLocalHyracks(); - } - try { - runQueries(); - } finally { - if (!opts.compileOnly) { - stopLocalHyracks(); - } - } - } - } - - /** - * Reads the contents of the files passed in the list of arguments to a - * string. If -showquery argument is passed, output the query as string. Run - * the query for the string. - * - * @throws IOException - * @throws SystemException - * @throws Exception - */ - private void runQueries() throws IOException, SystemException, Exception { - Date start = null; - Date end = null; - for (String query : opts.arguments) { - String qStr = slurp(query); - if (opts.showQuery) { - System.err.println(qStr); - } - - VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, - opts.showOET, opts.showRP); - start = opts.timing ? new Date() : null; - XMLQueryCompiler compiler = new XMLQueryCompiler(listener, getNodeList(), opts.frameSize, - opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf); - resultSetId = createResultSetId(); - CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE), - resultSetId, null); - compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel); - // if -timing argument passed, show the starting and ending times - if (opts.timing) { - end = new Date(); - timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms"); - } - if (opts.compileOnly) { - continue; - } - - Module module = compiler.getModule(); - JobSpecification js = module.getHyracksJobSpecification(); - js.setProperty("hdfsConf", opts.hdfsConf); - - DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext()); - js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory())); - OutputStream resultStream = System.out; - if (opts.resultFile != null) { - resultStream = new FileOutputStream(new File(opts.resultFile)); - } - - PrintWriter writer = new PrintWriter(resultStream, true); - // Repeat execution for number of times provided in -repeatexec - // argument - for (int i = 0; i < opts.repeatExec; ++i) { - start = opts.timing ? new Date() : null; - runJob(js, writer); - // if -timing argument passed, show the starting and ending - // times - if (opts.timing) { - end = new Date(); - long currentRun = end.getTime() - start.getTime(); - if ((i + 1) > opts.timingIgnoreQueries) { - sumTiming += currentRun; - sumSquaredTiming += currentRun * currentRun; - if (currentRun < minTiming) { - minTiming = currentRun; - } - if (maxTiming < currentRun) { - maxTiming = currentRun; - } - } - timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms"); - } - } - } - } - - /** - * Get cluster node configuration. - * - * @return Configuration of node controllers as array of Strings. - * @throws Exception - */ - private String[] getNodeList() throws Exception { - Map<String, NodeControllerInfo> nodeControllerInfos = hcc.getNodeControllerInfos(); - String[] nodeList = new String[nodeControllerInfos.size()]; - int index = 0; - for (String node : nodeControllerInfos.keySet()) { - nodeList[index++] = node; - } - return nodeList; - } - - /** - * Creates a Hyracks dataset, if not already existing with the job frame - * size, and 1 reader. Allocates a new buffer of size specified in the frame - * of Hyracks node. Creates new dataset reader with the current job ID and - * result set ID. Outputs the string in buffer for each frame. - * - * @param spec - * JobSpecification object, containing frame size. Current - * specified job. - * @param writer - * Writer for output of job. - * @throws Exception - */ - private void runJob(JobSpecification spec, PrintWriter writer) throws Exception { - int nReaders = 1; - if (hds == null) { - hds = new HyracksDataset(hcc, spec.getFrameSize(), nReaders); - } - - JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME)); - - ByteBuffer buffer = ByteBuffer.allocate(spec.getFrameSize()); - IHyracksDatasetReader reader = hds.createReader(jobId, resultSetId); - IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor(spec.getFrameSize()); - buffer.clear(); - - while (reader.read(buffer) > 0) { - buffer.clear(); - writer.print(ResultUtils.getStringFromBuffer(buffer, frameTupleAccessor)); - writer.flush(); - } - - hcc.waitForCompletion(jobId); - } - - /** - * Create a unique result set id to get the correct query back from the - * cluster. - * - * @return Result Set id generated with current system time. - */ - protected ResultSetId createResultSetId() { - return new ResultSetId(System.nanoTime()); - } - - /** - * Start local virtual cluster with cluster controller node and node - * controller nodes. IP address provided for node controller is localhost. - * Unassigned ports 39000 and 39001 are used for client and cluster port - * respectively. Creates a new Hyracks connection with the IP address and - * client ports. - * - * @throws Exception - */ - public void startLocalHyracks() throws Exception { - CCConfig ccConfig = new CCConfig(); - ccConfig.clientNetIpAddress = "127.0.0.1"; - ccConfig.clientNetPort = 39000; - ccConfig.clusterNetIpAddress = "127.0.0.1"; - ccConfig.clusterNetPort = 39001; - ccConfig.httpPort = 39002; - ccConfig.profileDumpPeriod = 10000; - cc = new ClusterControllerService(ccConfig); - cc.start(); - - ncs = new NodeControllerService[opts.localNodeControllers]; - for (int i = 0; i < ncs.length; i++) { - NCConfig ncConfig = new NCConfig(); - ncConfig.ccHost = "localhost"; - ncConfig.ccPort = 39001; - ncConfig.clusterNetIPAddress = "127.0.0.1"; - ncConfig.dataIPAddress = "127.0.0.1"; - ncConfig.resultIPAddress = "127.0.0.1"; - ncConfig.nodeId = "nc" + (i + 1); - ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString(); - ncs[i] = new NodeControllerService(ncConfig); - ncs[i].start(); - } - - hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); - } - - /** - * Shuts down the virtual cluster, along with all nodes and node execution, - * network and queue managers. - * - * @throws Exception - */ - public void stopLocalHyracks() throws Exception { - for (int i = 0; i < ncs.length; i++) { - ncs[i].stop(); - } - cc.stop(); - } - - /** - * Reads the contents of file given in query into a String. The file is - * always closed. For XML files UTF-8 encoding is used. - * - * @param query - * The query with filename to be processed - * @return UTF-8 formatted query string - * @throws IOException - */ - private static String slurp(String query) throws IOException { - return FileUtils.readFileToString(new File(query), "UTF-8"); - } - - /** - * Save and print out the timing message. - * - * @param message - */ - private static void timingMessage(String message) { - System.out.println(message); - timingMessages.add(message); - } - - /** - * Helper class with fields and methods to handle all command line options - */ - private static class CmdLineOptions { - @Option(name = "-available-processors", usage = "Number of available processors. (default: java's available processors)") - private int availableProcessors = -1; - - @Option(name = "-client-net-ip-address", usage = "IP Address of the ClusterController.") - private String clientNetIpAddress = null; - - @Option(name = "-client-net-port", usage = "Port of the ClusterController. (default: 1098)") - private int clientNetPort = 1098; - - @Option(name = "-local-node-controllers", usage = "Number of local node controllers. (default: 1)") - private int localNodeControllers = 1; - - @Option(name = "-frame-size", usage = "Frame size in bytes. (default: 65,536)") - private int frameSize = 65536; - - @Option(name = "-join-hash-size", usage = "Join hash size in bytes. (default: 67,108,864)") - private long joinHashSize = -1; - - @Option(name = "-maximum-data-size", usage = "Maximum possible data size in bytes. (default: 150,323,855,000)") - private long maximumDataSize = -1; - - @Option(name = "-buffer-size", usage = "Disk read buffer size in bytes.") - private int bufferSize = -1; - - @Option(name = "-O", usage = "Optimization Level. (default: Full Optimization)") - private int optimizationLevel = Integer.MAX_VALUE; - - @Option(name = "-showquery", usage = "Show query string.") - private boolean showQuery; - - @Option(name = "-showast", usage = "Show abstract syntax tree.") - private boolean showAST; - - @Option(name = "-showtet", usage = "Show translated expression tree.") - private boolean showTET; - - @Option(name = "-showoet", usage = "Show optimized expression tree.") - private boolean showOET; - - @Option(name = "-showrp", usage = "Show Runtime plan.") - private boolean showRP; - - @Option(name = "-compileonly", usage = "Compile the query and stop.") - private boolean compileOnly; - - @Option(name = "-repeatexec", usage = "Number of times to repeat execution.") - private int repeatExec = 1; - - @Option(name = "-result-file", usage = "File path to save the query result.") - private String resultFile = null; - - @Option(name = "-timing", usage = "Produce timing information.") - private boolean timing; - - @Option(name = "-timing-ignore-queries", usage = "Ignore the first X number of quereies.") - private int timingIgnoreQueries = 2; - - @Option(name = "-x", usage = "Bind an external variable") - private Map<String, String> bindings = new HashMap<String, String>(); - - @Option(name = "-hdfs-conf", usage = "Directory path to Hadoop configuration files") - private String hdfsConf = null; - - @Argument - private List<String> arguments = new ArrayList<String>(); - } + private final CmdLineOptions opts; + + private ClusterControllerService cc; + private NodeControllerService[] ncs; + private IHyracksClientConnection hcc; + private IHyracksDataset hds; + + private ResultSetId resultSetId; + private static List<String> timingMessages = new ArrayList<String>(); + private static long sumTiming; + private static long sumSquaredTiming; + private static long minTiming = Long.MAX_VALUE; + private static long maxTiming = Long.MIN_VALUE; + + /** + * Constructor to use command line options passed. + * - * @param opts Command line options object ++ * @param opts ++ * Command line options object + */ + public VXQuery(CmdLineOptions opts) { + this.opts = opts; + } + + /** + * Main method to get command line options and execute query process. + * + * @param args + * @throws Exception + */ + public static void main(String[] args) throws Exception { + Date start = new Date(); + final CmdLineOptions opts = new CmdLineOptions(); + CmdLineParser parser = new CmdLineParser(opts); + + // parse command line options, give error message if no arguments passed + try { + parser.parseArgument(args); + } catch (Exception e) { + parser.printUsage(System.err); + return; + } + if (opts.arguments.isEmpty()) { + parser.printUsage(System.err); + return; + } + VXQuery vxq = new VXQuery(opts); + vxq.execute(); + // if -timing argument passed, show the starting and ending times + if (opts.timing) { + Date end = new Date(); + timingMessage("Execution time: " + (end.getTime() - start.getTime()) + " ms"); + if (opts.repeatExec > opts.timingIgnoreQueries) { + long mean = sumTiming / (opts.repeatExec - opts.timingIgnoreQueries); + double sd = Math + .sqrt(sumSquaredTiming / (opts.repeatExec - new Integer(opts.timingIgnoreQueries).doubleValue()) + - mean * mean); + timingMessage("Average execution time: " + mean + " ms"); + timingMessage("Standard deviation: " + String.format("%.4f", sd)); + timingMessage("Coefficient of variation: " + String.format("%.4f", (sd / mean))); + timingMessage("Minimum execution time: " + minTiming + " ms"); + timingMessage("Maximum execution time: " + maxTiming + " ms"); + } + System.out.println("Timing Summary:"); + for (String time : timingMessages) { + System.out.println(" " + time); + } + } + + } + + /** + * Creates a new Hyracks connection with: the client IP address and port provided, if IP address is provided in command line. Otherwise create a new virtual + * cluster with Hyracks nodes. Queries passed are run either way. After running queries, if a virtual cluster has been created, it is shut down. + * + * @throws Exception + */ + private void execute() throws Exception { + System.setProperty("vxquery.buffer_size", Integer.toString(opts.bufferSize)); + + if (opts.clientNetIpAddress != null) { + hcc = new HyracksConnection(opts.clientNetIpAddress, opts.clientNetPort); + runQueries(); + } else { + if (!opts.compileOnly) { + startLocalHyracks(); + } + try { + runQueries(); + } finally { + if (!opts.compileOnly) { + stopLocalHyracks(); + } + } + } + } + + /** + * Reads the contents of the files passed in the list of arguments to a string. If -showquery argument is passed, output the query as string. Run the query + * for the string. + * + * @throws IOException + * @throws SystemException + * @throws Exception + */ + private void runQueries() throws IOException, SystemException, Exception { + Date start = null; + Date end = null; + for (String query : opts.arguments) { + String qStr = slurp(query); + if (opts.showQuery) { + System.err.println(qStr); + } + + VXQueryCompilationListener listener = new VXQueryCompilationListener(opts.showAST, opts.showTET, + opts.showOET, opts.showRP); + + start = opts.timing ? new Date() : null; + XMLQueryCompiler compiler = new XMLQueryCompiler(listener, getNodeList(), opts.frameSize, - opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize); ++ opts.availableProcessors, opts.joinHashSize, opts.maximumDataSize, opts.hdfsConf); + resultSetId = createResultSetId(); + CompilerControlBlock ccb = new CompilerControlBlock(new StaticContextImpl(RootStaticContextImpl.INSTANCE), + resultSetId, null); + compiler.compile(query, new StringReader(qStr), ccb, opts.optimizationLevel); + // if -timing argument passed, show the starting and ending times + if (opts.timing) { + end = new Date(); + timingMessage("Compile time: " + (end.getTime() - start.getTime()) + " ms"); + } + if (opts.compileOnly) { + continue; + } + + Module module = compiler.getModule(); + JobSpecification js = module.getHyracksJobSpecification(); + + DynamicContext dCtx = new DynamicContextImpl(module.getModuleContext()); + js.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory())); + + OutputStream resultStream = System.out; + if (opts.resultFile != null) { + resultStream = new FileOutputStream(new File(opts.resultFile)); + } + + PrintWriter writer = new PrintWriter(resultStream, true); + // Repeat execution for number of times provided in -repeatexec argument + for (int i = 0; i < opts.repeatExec; ++i) { + start = opts.timing ? new Date() : null; + runJob(js, writer); + // if -timing argument passed, show the starting and ending times + if (opts.timing) { + end = new Date(); + long currentRun = end.getTime() - start.getTime(); + if ((i + 1) > opts.timingIgnoreQueries) { + sumTiming += currentRun; + sumSquaredTiming += currentRun * currentRun; + if (currentRun < minTiming) { + minTiming = currentRun; + } + if (maxTiming < currentRun) { + maxTiming = currentRun; + } + } + timingMessage("Job (" + (i + 1) + ") execution time: " + currentRun + " ms"); + } + } + } + } + + /** + * Get cluster node configuration. + * + * @return Configuration of node controllers as array of Strings. + * @throws Exception + */ + private String[] getNodeList() throws Exception { + if (hcc != null) { + Map<String, NodeControllerInfo> nodeControllerInfos = hcc.getNodeControllerInfos(); + String[] nodeList = new String[nodeControllerInfos.size()]; + int index = 0; + for (String node : nodeControllerInfos.keySet()) { + nodeList[index++] = node; + } + return nodeList; + } + return new String[0]; + } + + /** + * Creates a Hyracks dataset, if not already existing with the job frame size, and 1 reader. Allocates a new buffer of size specified in the frame of Hyracks + * node. Creates new dataset reader with the current job ID and result set ID. Outputs the string in buffer for each frame. + * - * @param spec JobSpecification object, containing frame size. Current specified job. - * @param writer Writer for output of job. ++ * @param spec ++ * JobSpecification object, containing frame size. Current specified job. ++ * @param writer ++ * Writer for output of job. + * @throws Exception + */ + private void runJob(JobSpecification spec, PrintWriter writer) throws Exception { + int nReaders = 1; + if (hds == null) { + hds = new HyracksDataset(hcc, spec.getFrameSize(), nReaders); + } + + JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME)); + + FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize()); + IFrame frame = new VSizeFrame(resultDisplayFrameMgr); + IHyracksDatasetReader reader = hds.createReader(jobId, resultSetId); + IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor(); + + while (reader.read(frame) > 0) { + writer.print(ResultUtils.getStringFromBuffer(frame.getBuffer(), frameTupleAccessor)); + writer.flush(); + frame.getBuffer().clear(); + } + + hcc.waitForCompletion(jobId); + } + + /** + * Create a unique result set id to get the correct query back from the cluster. + * + * @return Result Set id generated with current system time. + */ + protected ResultSetId createResultSetId() { + return new ResultSetId(System.nanoTime()); + } + + /** + * Start local virtual cluster with cluster controller node and node controller nodes. IP address provided for node controller is localhost. Unassigned ports + * 39000 and 39001 are used for client and cluster port respectively. Creates a new Hyracks connection with the IP address and client ports. + * + * @throws Exception + */ + public void startLocalHyracks() throws Exception { + CCConfig ccConfig = new CCConfig(); + ccConfig.clientNetIpAddress = "127.0.0.1"; + ccConfig.clientNetPort = 39000; + ccConfig.clusterNetIpAddress = "127.0.0.1"; + ccConfig.clusterNetPort = 39001; + ccConfig.httpPort = 39002; + ccConfig.profileDumpPeriod = 10000; + cc = new ClusterControllerService(ccConfig); + cc.start(); + + ncs = new NodeControllerService[opts.localNodeControllers]; + for (int i = 0; i < ncs.length; i++) { + NCConfig ncConfig = new NCConfig(); + ncConfig.ccHost = "localhost"; + ncConfig.ccPort = 39001; + ncConfig.clusterNetIPAddress = "127.0.0.1"; + ncConfig.dataIPAddress = "127.0.0.1"; + ncConfig.resultIPAddress = "127.0.0.1"; + ncConfig.nodeId = "nc" + (i + 1); + ncConfig.ioDevices = Files.createTempDirectory(ncConfig.nodeId).toString(); + ncs[i] = new NodeControllerService(ncConfig); + ncs[i].start(); + } + + hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort); + } + + /** + * Shuts down the virtual cluster, along with all nodes and node execution, network and queue managers. + * + * @throws Exception + */ + public void stopLocalHyracks() throws Exception { + for (int i = 0; i < ncs.length; i++) { + ncs[i].stop(); + } + cc.stop(); + } + + /** + * Reads the contents of file given in query into a String. The file is always closed. For XML files UTF-8 encoding is used. + * - * @param query The query with filename to be processed ++ * @param query ++ * The query with filename to be processed + * @return UTF-8 formatted query string + * @throws IOException + */ + private static String slurp(String query) throws IOException { + return FileUtils.readFileToString(new File(query), "UTF-8"); + } + + /** + * Save and print out the timing message. + * + * @param message + */ + private static void timingMessage(String message) { + System.out.println(message); + timingMessages.add(message); + } + + /** + * Helper class with fields and methods to handle all command line options + */ + private static class CmdLineOptions { - @Option(name = "-available-processors", - usage = "Number of available processors. (default: java's available processors)") ++ @Option(name = "-available-processors", usage = "Number of available processors. (default: java's available processors)") + private int availableProcessors = -1; + + @Option(name = "-client-net-ip-address", usage = "IP Address of the ClusterController.") + private String clientNetIpAddress = null; + + @Option(name = "-client-net-port", usage = "Port of the ClusterController. (default: 1098)") + private int clientNetPort = 1098; + + @Option(name = "-local-node-controllers", usage = "Number of local node controllers. (default: 1)") + private int localNodeControllers = 1; + + @Option(name = "-frame-size", usage = "Frame size in bytes. (default: 65,536)") + private int frameSize = 65536; + + @Option(name = "-join-hash-size", usage = "Join hash size in bytes. (default: 67,108,864)") + private long joinHashSize = -1; + + @Option(name = "-maximum-data-size", usage = "Maximum possible data size in bytes. (default: 150,323,855,000)") + private long maximumDataSize = -1; + + @Option(name = "-buffer-size", usage = "Disk read buffer size in bytes.") + private int bufferSize = -1; + + @Option(name = "-O", usage = "Optimization Level. (default: Full Optimization)") + private int optimizationLevel = Integer.MAX_VALUE; + + @Option(name = "-showquery", usage = "Show query string.") + private boolean showQuery; + + @Option(name = "-showast", usage = "Show abstract syntax tree.") + private boolean showAST; + + @Option(name = "-showtet", usage = "Show translated expression tree.") + private boolean showTET; + + @Option(name = "-showoet", usage = "Show optimized expression tree.") + private boolean showOET; + + @Option(name = "-showrp", usage = "Show Runtime plan.") + private boolean showRP; + + @Option(name = "-compileonly", usage = "Compile the query and stop.") + private boolean compileOnly; + + @Option(name = "-repeatexec", usage = "Number of times to repeat execution.") + private int repeatExec = 1; + + @Option(name = "-result-file", usage = "File path to save the query result.") + private String resultFile = null; + + @Option(name = "-timing", usage = "Produce timing information.") + private boolean timing; + + @Option(name = "-timing-ignore-queries", usage = "Ignore the first X number of quereies.") + private int timingIgnoreQueries = 2; + + @Option(name = "-x", usage = "Bind an external variable") + private Map<String, String> bindings = new HashMap<String, String>(); + ++ @Option(name = "-hdfs-conf", usage = "Directory path to Hadoop configuration files") ++ private String hdfsConf = null; ++ + @Argument + private List<String> arguments = new ArrayList<String>(); + } } http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-core/pom.xml ---------------------------------------------------------------------- diff --cc vxquery-core/pom.xml index 3c409c8,59cc987..38b8030 --- a/vxquery-core/pom.xml +++ b/vxquery-core/pom.xml @@@ -209,17 -209,6 +209,17 @@@ </dependency> <dependency> - <groupId>edu.uci.ics.hyracks</groupId> ++ <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-hdfs-2.x</artifactId> + </dependency> + + <dependency> - <groupId>edu.uci.ics.hyracks</groupId> ++ <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-hdfs-core</artifactId> + <type>jar</type> + </dependency> + + <dependency> <groupId>ant</groupId> <artifactId>ant-trax</artifactId> <scope>provided</scope> http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java ---------------------------------------------------------------------- diff --cc vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java index 0695e00,53011d2..dd9965b --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/AbstractCollectionRule.java @@@ -22,31 -22,31 +22,30 @@@ import java.nio.ByteBuffer import java.util.Arrays; import org.apache.commons.lang3.mutable.Mutable; -import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue; -import org.apache.vxquery.compiler.rewriter.rules.util.OperatorToolbox; -import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; -import org.apache.vxquery.datamodel.values.ValueTag; -import org.apache.vxquery.functions.BuiltinFunctions; -import org.apache.vxquery.types.BuiltinTypeRegistry; -import org.apache.vxquery.types.Quantifier; -import org.apache.vxquery.types.SequenceType; - + import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; + import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; + import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; + import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; + import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; + import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; + import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; + import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression; + import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; + import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; + import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; + import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; + import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; + import org.apache.hyracks.data.std.primitive.UTF8StringPointable; + import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; +import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue; +import org.apache.vxquery.compiler.rewriter.rules.util.OperatorToolbox; +import org.apache.vxquery.datamodel.accessors.TaggedValuePointable; +import org.apache.vxquery.datamodel.values.ValueTag; +import org.apache.vxquery.functions.BuiltinFunctions; +import org.apache.vxquery.types.BuiltinTypeRegistry; +import org.apache.vxquery.types.Quantifier; +import org.apache.vxquery.types.SequenceType; - import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException; - import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression; - import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator; - import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext; - import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; - import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; - import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; - import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression; - import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; - import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; - import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator; - import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; - import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; - import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable; - import edu.uci.ics.hyracks.dataflow.common.comm.util.ByteBufferInputStream; - public abstract class AbstractCollectionRule implements IAlgebraicRewriteRule { final ByteBufferInputStream bbis = new ByteBufferInputStream(); final DataInputStream di = new DataInputStream(bbis); @@@ -54,9 -54,15 +53,14 @@@ final TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable(); /** - * Get the constant value for the collection. Return null for not a collection. + * Get the arguments for the collection and collection-with-tag. Return null for not a collection. + * + * @param opRef + * Logical operator + * @return collection name */ - protected String getCollectionName(Mutable<ILogicalOperator> opRef) { - VXQueryConstantValue constantValue; + protected String[] getCollectionName(Mutable<ILogicalOperator> opRef) throws AlgebricksException { + AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.UNNEST) { return null; @@@ -76,10 -82,7 +80,10 @@@ return null; } AbstractFunctionCallExpression functionCall = (AbstractFunctionCallExpression) logicalExpression; - if (!functionCall.getFunctionIdentifier().equals( - BuiltinFunctions.FN_COLLECTION_WITH_TAG_2.getFunctionIdentifier()) - && !functionCall.getFunctionIdentifier().equals( - BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier())) { - if (!functionCall.getFunctionIdentifier().equals(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier())) { ++ if (!functionCall.getFunctionIdentifier() ++ .equals(BuiltinFunctions.FN_COLLECTION_WITH_TAG_2.getFunctionIdentifier()) ++ && !functionCall.getFunctionIdentifier() ++ .equals(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier())) { return null; } @@@ -130,11 -118,9 +134,10 @@@ if (tvp.getTag() == ValueTag.XS_STRING_TAG) { tvp.getValue(stringp); try { - bbis.setByteBuffer( - ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(), - stringp.getLength() + stringp.getStartOffset())), 0); + bbis.setByteBuffer(ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(), + stringp.getLength() + stringp.getStartOffset())), 0); collectionName = di.readUTF(); + return collectionName; } catch (IOException e) { e.printStackTrace(); } http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java ---------------------------------------------------------------------- diff --cc vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java index e1e0442,cc857a1..8ed8bb1 --- a/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/IntroduceCollectionRule.java @@@ -62,12 -62,11 +62,12 @@@ import org.apache.hyracks.algebricks.co */ public class IntroduceCollectionRule extends AbstractCollectionRule { @Override - public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { + public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) { VXQueryOptimizationContext vxqueryContext = (VXQueryOptimizationContext) context; - String collectionName = getCollectionName(opRef); + String args[] = getCollectionName(opRef); - if (collectionName != null) { + if (args != null) { + String collectionName = args[0]; // Build the new operator and update the query plan. int collectionId = vxqueryContext.newCollectionId(); VXQueryCollectionDataSource ds = VXQueryCollectionDataSource.create(collectionId, collectionName, http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java ---------------------------------------------------------------------- diff --cc vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index 6b57641,d5966b8..e2f705e --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@@ -30,42 -23,26 +30,61 @@@ import java.util.List import java.util.logging.Level; import java.util.logging.Logger; +import javax.xml.parsers.ParserConfigurationException; + import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.TrueFileFilter; ++<<<<<<< HEAD +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.input.FileSplit; ++======= + import org.apache.hyracks.api.comm.IFrame; + import org.apache.hyracks.api.comm.IFrameFieldAppender; + import org.apache.hyracks.api.comm.VSizeFrame; + import org.apache.hyracks.api.context.IHyracksTaskContext; + import org.apache.hyracks.api.dataflow.IOperatorNodePushable; + import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; + import org.apache.hyracks.api.dataflow.value.RecordDescriptor; + import org.apache.hyracks.api.exceptions.HyracksDataException; + import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; + import org.apache.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender; + import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; + import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; + import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; ++>>>>>>> master import org.apache.vxquery.context.DynamicContext; +import org.apache.vxquery.hdfs2.HDFSFunctions; import org.apache.vxquery.xmlparser.ITreeNodeIdProvider; import org.apache.vxquery.xmlparser.TreeNodeIdProvider; import org.apache.vxquery.xmlparser.XMLParser; +import org.xml.sax.SAXException; + ++<<<<<<< HEAD +import edu.uci.ics.hyracks.api.context.IHyracksTaskContext; +import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable; +import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider; +import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor; +import edu.uci.ics.hyracks.api.exceptions.HyracksDataException; +import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry; +import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor; +import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender; +import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils; +import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; +import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; +import edu.uci.ics.hyracks.hdfs.ContextFactory; +import edu.uci.ics.hyracks.hdfs2.dataflow.FileSplitsFactory; ++======= ++>>>>>>> master public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor { private static final long serialVersionUID = 1L; private short dataSourceId; @@@ -118,26 -89,20 +137,30 @@@ public void nextFrame(ByteBuffer buffer) throws HyracksDataException { fta.reset(buffer); String collectionModifiedName = collectionName.replace("${nodeId}", nodeId); - File collectionDirectory = new File(collectionModifiedName); - - // Go through each tuple. - if (collectionDirectory.isDirectory()) { - for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) { - @SuppressWarnings("unchecked") - Iterator<File> it = FileUtils.iterateFiles(collectionDirectory, new VXQueryIOFileFilter(), - TrueFileFilter.INSTANCE); - while (it.hasNext()) { - File xmlDocument = it.next(); - if (LOGGER.isLoggable(Level.FINE)) { - LOGGER.fine("Starting to read XML document: " + xmlDocument.getAbsolutePath()); + if (!collectionModifiedName.contains("hdfs:/")) { + File collectionDirectory = new File(collectionModifiedName); + //check if directory is in the local file system + if (collectionDirectory.exists()) { + // Go through each tuple. + if (collectionDirectory.isDirectory()) { + for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) { + Iterator<File> it = FileUtils.iterateFiles(collectionDirectory, + new VXQueryIOFileFilter(), TrueFileFilter.INSTANCE); + while (it.hasNext()) { + File xmlDocument = it.next(); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.fine("Starting to read XML document: " + xmlDocument.getAbsolutePath()); + } + parser.parseElements(xmlDocument, writer, fta, tupleIndex); + } } ++<<<<<<< HEAD + } else { + throw new HyracksDataException("Invalid directory parameter (" + nodeId + ":" + + collectionDirectory.getAbsolutePath() + ") passed to collection."); ++======= + parser.parseElements(xmlDocument, writer, tupleIndex); ++>>>>>>> master } } } else { http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/f90d5873/vxquery-core/src/main/java/org/apache/vxquery/runtime/functions/util/FunctionHelper.java ----------------------------------------------------------------------
