[SYSTEMML-2496] Skip MapReduce tests as it is in maintenance mode - This commit makes hybrid_spark as default runtime for junit tests to increase the coverage of our spark backend. - The FrameConverterTest and parfor tests are kept as is and can be modified in subsequent commits. - The MapReduce tests can be turned on using TEST_MR_BACKEND flag in AutomatedTestBase class. - We only disable HADOOP runtime and not HYBRID runtime. We can disable latter in subsequent commit.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/95bf8cfe Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/95bf8cfe Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/95bf8cfe Branch: refs/heads/master Commit: 95bf8cfe6c0ae4f45b460a03c397203b7b354fc5 Parents: 7907c0e Author: Niketan Pansare <npan...@us.ibm.com> Authored: Sun Oct 14 12:14:19 2018 -0700 Committer: Niketan Pansare <npan...@us.ibm.com> Committed: Sun Oct 14 12:14:19 2018 -0700 ---------------------------------------------------------------------- .../test/integration/AutomatedTestBase.java | 161 ++++++- .../applications/ApplyTransformTest.java | 258 +++++------ .../integration/applications/ArimaTest.java | 3 + .../integration/applications/CsplineCGTest.java | 3 + .../integration/applications/CsplineDSTest.java | 3 + .../test/integration/applications/GLMTest.java | 3 + .../test/integration/applications/GNMFTest.java | 3 + .../test/integration/applications/HITSTest.java | 2 + .../test/integration/applications/ID3Test.java | 8 +- .../integration/applications/L2SVMTest.java | 3 + .../applications/LinearRegressionTest.java | 3 + .../applications/MDABivariateStatsTest.java | 3 + .../applications/MultiClassSVMTest.java | 3 + .../applications/NaiveBayesParforTest.java | 3 + .../applications/NaiveBayesTest.java | 3 + .../integration/applications/PageRankTest.java | 3 + .../integration/applications/WelchTTest.java | 3 + .../BivariateCategoricalCategoricallTest.java | 9 + .../BivariateOrdinalOrdinalTest.java | 6 + .../BivariateScaleCategoricalTest.java | 5 + .../BivariateScaleScaleTest.java | 8 +- .../descriptivestats/OrderStatisticsTest.java | 3 + .../UnivariateCategoricalTest.java | 6 + .../descriptivestats/UnivariateStatsBase.java | 10 +- .../dml/ScalableDecompositionTest.java | 14 +- .../parfor/ParForBivariateStatsTest.java | 5 +- .../parfor/ParForCVMulticlassSVMTest.java | 5 +- .../parfor/ParForCorrelationTest.java | 8 +- .../parfor/ParForCorrelationTestLarge.java | 4 +- .../parfor/ParForNaiveBayesTest.java | 12 +- .../applications/parfor/ParForSampleTest.java | 28 +- .../parfor/ParForUnivariateStatsTest.java | 4 +- .../conversion/RDDConverterUtilsExtTest.java | 9 +- .../functions/aggregate/AggregateInfTest.java | 7 +- .../functions/aggregate/AggregateNaNTest.java | 11 +- .../functions/aggregate/ColStdDevsTest.java | 21 +- .../functions/aggregate/ColSumTest.java | 6 + .../functions/aggregate/ColSumsSqTest.java | 28 +- .../functions/aggregate/ColVariancesTest.java | 28 +- .../functions/aggregate/FullAggregateTest.java | 17 +- .../aggregate/FullColAggregateTest.java | 12 +- .../FullGroupedAggregateMatrixTest.java | 13 +- .../aggregate/FullGroupedAggregateTest.java | 15 +- .../aggregate/FullRowAggregateTest.java | 13 +- .../functions/aggregate/LengthTest.java | 6 + .../functions/aggregate/MaxTest.java | 6 + .../functions/aggregate/MinTest.java | 6 + .../functions/aggregate/NColTest.java | 6 + .../functions/aggregate/NRowTest.java | 6 + .../functions/aggregate/ProdTest.java | 6 + .../aggregate/PushdownSumBinaryTest.java | 17 +- .../aggregate/RowColProdsAggregateTest.java | 14 +- .../functions/aggregate/RowStdDevsTest.java | 21 +- .../functions/aggregate/RowSumTest.java | 6 + .../functions/aggregate/RowSumsSqTest.java | 28 +- .../functions/aggregate/RowVariancesTest.java | 27 +- .../functions/aggregate/StdDevTest.java | 21 +- .../functions/aggregate/SumSqTest.java | 27 +- .../functions/aggregate/SumTest.java | 6 + .../functions/aggregate/TraceTest.java | 6 + .../functions/aggregate/VarianceTest.java | 20 +- .../functions/append/AppendChainTest.java | 8 +- .../functions/append/AppendMatrixTest.java | 14 +- .../functions/append/AppendVectorTest.java | 11 +- .../functions/append/RBindCBindMatrixTest.java | 15 +- .../functions/append/StringAppendTest.java | 15 +- .../binary/matrix/BinUaggChainTest.java | 15 +- .../binary/matrix/CentralMomentTest.java | 15 +- .../functions/binary/matrix/CovarianceTest.java | 14 +- .../matrix/DiagMatrixMultiplicationTest.java | 13 +- .../ElementwiseAdditionMultiplicationTest.java | 9 + .../binary/matrix/ElementwiseAdditionTest.java | 33 ++ .../matrix/ElementwiseBitwLogicalTest.java | 16 +- .../binary/matrix/ElementwiseDivisionTest.java | 32 ++ .../binary/matrix/ElementwiseLogicalTest.java | 13 +- .../binary/matrix/ElementwiseModulusTest.java | 30 ++ .../matrix/ElementwiseMultiplicationTest.java | 27 ++ .../matrix/ElementwiseSubtractionTest.java | 33 ++ .../binary/matrix/MapMultChainTest.java | 20 +- .../binary/matrix/MapMultLimitTest.java | 11 +- .../binary/matrix/MatrixMultiplicationTest.java | 15 + .../binary/matrix/MatrixVectorTest.java | 7 +- .../binary/matrix/OuterProductTest.java | 21 +- .../functions/binary/matrix/QuantileTest.java | 16 +- .../binary/matrix/ScalarAdditionTest.java | 14 + .../binary/matrix/ScalarDivisionTest.java | 18 + .../binary/matrix/ScalarModulusTest.java | 18 + .../binary/matrix/ScalarMultiplicationTest.java | 12 + .../binary/matrix/ScalarSubtractionTest.java | 15 + .../TransposeMatrixMultiplicationTest.java | 13 +- .../binary/matrix/UaggOuterChainTest.java | 20 +- .../UltraSparseMRMatrixMultiplicationTest.java | 15 +- .../ZipMMSparkMatrixMultiplicationTest.java | 13 +- .../FullMatrixMatrixCellwiseOperationTest.java | 13 +- ...ullMatrixVectorColCellwiseOperationTest.java | 13 +- ...ullMatrixVectorRowCellwiseOperationTest.java | 13 +- .../FullMinus1MultTest.java | 19 +- .../FullSortedOuterCompareTest.java | 13 +- ...ectorVectorCellwiseCompareOperationTest.java | 14 +- .../FullVectorVectorCellwiseOperationTest.java | 13 +- ...FullDistributedMatrixMultiplicationTest.java | 15 +- .../FullIntegerDivisionTest.java | 5 +- .../FullLogicalMatrixTest.java | 18 +- .../FullLogicalScalarLeftTest.java | 11 +- .../FullLogicalScalarRightTest.java | 11 +- .../FullMatrixMultiplicationTest.java | 19 +- ...lMatrixMultiplicationTransposeSelf2Test.java | 17 +- ...llMatrixMultiplicationTransposeSelfTest.java | 9 +- ...FullMatrixMultiplicationUltraSparseTest.java | 12 +- .../FullMinMaxComparisonTest.java | 7 +- .../binary/matrix_full_other/FullPowerTest.java | 7 +- .../MatrixMultShortLhsTest.java | 5 +- .../functions/binary/scalar/AdditionTest.java | 15 + .../functions/binary/scalar/AndTest.java | 3 + .../functions/binary/scalar/DivisionTest.java | 30 ++ .../functions/binary/scalar/EqualTest.java | 3 + .../binary/scalar/FullStringComparisonTest.java | 10 +- .../binary/scalar/GreaterThanOrEqualTest.java | 3 + .../binary/scalar/GreaterThanTest.java | 3 + .../binary/scalar/LessThanOrEqualTest.java | 3 + .../functions/binary/scalar/LessThanTest.java | 3 + .../functions/binary/scalar/LogarithmTest.java | 3 + .../functions/binary/scalar/LogicalTest.java | 13 +- .../functions/binary/scalar/ModulusTest.java | 30 +- .../binary/scalar/MultiplicationTest.java | 12 + .../functions/binary/scalar/OrTest.java | 3 + .../functions/binary/scalar/PowerTest.java | 12 + .../binary/scalar/SubtractionTest.java | 30 ++ .../functions/binary/scalar/XorTest.java | 8 +- .../functions/blocks/VariableTest.java | 9 + .../integration/functions/blocks/WhileTest.java | 9 +- .../caching/CachingPWriteExportTest.java | 8 +- .../functions/codegen/APICodegenTest.java | 6 +- .../functions/codegen/CPlanComparisonTest.java | 93 ++-- .../codegen/CPlanVectorPrimitivesTest.java | 12 +- .../functions/codegen/CellwiseTmplTest.java | 39 +- .../codegen/CompressedCellwiseTest.java | 21 +- .../codegen/CompressedMultiAggregateTest.java | 20 +- .../codegen/CompressedOuterProductTest.java | 20 +- .../CompressedRowAggregateLargeTest.java | 22 +- .../codegen/CompressedRowAggregateTest.java | 20 +- .../functions/codegen/DAGCellwiseTmplTest.java | 21 +- .../functions/codegen/MiscPatternTest.java | 24 +- .../functions/codegen/MultiAggTmplTest.java | 17 +- .../functions/codegen/OuterProdTmplTest.java | 26 +- .../functions/codegen/RowAggTmplTest.java | 42 +- .../codegen/RowConv2DOperationsTest.java | 22 +- .../codegen/RowVectorComparisonTest.java | 17 +- .../functions/codegen/SparseSideInputTest.java | 18 +- .../functions/codegen/SumProductChainTest.java | 15 +- .../codegenalg/AlgorithmAutoEncoder.java | 18 +- .../functions/codegenalg/AlgorithmDatagen.java | 16 +- .../functions/codegenalg/AlgorithmGLM.java | 17 +- .../functions/codegenalg/AlgorithmKMeans.java | 21 +- .../functions/codegenalg/AlgorithmL2SVM.java | 21 +- .../functions/codegenalg/AlgorithmLinregCG.java | 19 +- .../functions/codegenalg/AlgorithmMLogreg.java | 20 +- .../functions/codegenalg/AlgorithmMSVM.java | 17 +- .../functions/codegenalg/AlgorithmPNMF.java | 19 +- .../functions/codegenalg/AlgorithmPageRank.java | 20 +- .../codegenalg/AlgorithmStepwiseRegression.java | 18 +- .../compress/BasicCompressionTest.java | 5 +- .../functions/compress/BasicGetValueTest.java | 5 +- .../compress/BasicMatrixAppendTest.java | 7 +- .../compress/BasicMatrixCentralMomentTest.java | 5 +- .../compress/BasicMatrixMultChainTest.java | 5 +- .../compress/BasicMatrixQuantileTest.java | 5 +- .../BasicMatrixTransposeSelfMultTest.java | 5 +- .../compress/BasicMatrixVectorMultTest.java | 5 +- .../BasicScalarOperationsSparseUnsafeTest.java | 5 +- .../compress/BasicScalarOperationsTest.java | 5 +- .../BasicTransposeSelfLeftMatrixMultTest.java | 5 +- .../compress/BasicUnaryAggregateTest.java | 5 +- .../compress/BasicVectorMatrixMultTest.java | 5 +- .../functions/compress/CompressedL2SVM.java | 18 +- .../functions/compress/CompressedLinregCG.java | 17 +- .../compress/CompressedSerializationTest.java | 5 +- .../compress/LargeCompressionTest.java | 5 +- .../compress/LargeMatrixMatrixMultTest.java | 5 +- .../compress/LargeMatrixVectorMultTest.java | 5 +- .../compress/LargeParMatrixVectorMultTest.java | 5 +- .../compress/LargeParUnaryAggregateTest.java | 5 +- .../compress/LargeVectorMatrixMultTest.java | 5 +- .../functions/compress/ParCompressionTest.java | 5 +- .../compress/ParMatrixMultChainTest.java | 5 +- .../compress/ParMatrixVectorMultTest.java | 5 +- .../ParTransposeSelfLeftMatrixMultTest.java | 5 +- .../compress/ParUnaryAggregateTest.java | 5 +- .../compress/ParVectorMatrixMultTest.java | 5 +- .../functions/data/FullReblockTest.java | 14 +- .../data/FullStringInitializeTest.java | 11 +- .../functions/data/MatrixMarketFormatTest.java | 13 +- .../functions/data/RandRuntimePlatformTest.java | 34 +- .../integration/functions/data/RandTest1.java | 5 + .../integration/functions/data/RandTest2.java | 3 + .../integration/functions/data/RandTest3.java | 3 + .../integration/functions/data/RandTest4.java | 3 + .../integration/functions/data/RandTest5.java | 15 +- .../functions/data/RandVarMinMaxTest.java | 13 +- .../functions/data/RandVarSeedTest.java | 13 +- .../integration/functions/data/ReadMMTest.java | 34 +- .../integration/functions/data/ReblockTest.java | 3 + .../integration/functions/data/SampleTest.java | 3 + .../functions/data/SequenceTest.java | 52 ++- .../functions/data/VariableTest.java | 2 + .../integration/functions/data/WriteMMTest.java | 15 +- .../functions/data/WriteReadZeroDimsTest.java | 17 +- .../integration/functions/data/WriteTest.java | 7 + .../functions/estim/OpBindChainTest.java | 3 + .../integration/functions/estim/OpBindTest.java | 3 + .../functions/estim/OpElemWChainTest.java | 3 + .../functions/estim/OpElemWTest.java | 3 + .../functions/estim/OpSingleTest.java | 3 + .../functions/estim/OuterProductTest.java | 3 + .../functions/estim/SelfProductTest.java | 3 + .../estim/SquaredProductChainTest.java | 3 + .../functions/estim/SquaredProductTest.java | 3 + .../functions/external/DynProjectTest.java | 4 +- .../functions/external/DynReadWriteTest.java | 3 + .../functions/external/EvalFunctionTest.java | 6 +- .../external/FunctionExpressionsTest.java | 10 +- .../functions/external/OrderTest.java | 8 +- .../functions/frame/FrameAppendDistTest.java | 7 +- .../functions/frame/FrameAppendTest.java | 8 +- .../functions/frame/FrameCastingTest.java | 8 +- .../functions/frame/FrameConverterTest.java | 11 +- .../functions/frame/FrameCopyTest.java | 10 +- .../functions/frame/FrameEvictionTest.java | 2 + .../functions/frame/FrameFunctionTest.java | 16 +- .../functions/frame/FrameGetSetTest.java | 8 +- .../functions/frame/FrameIndexingDistTest.java | 16 +- .../functions/frame/FrameIndexingTest.java | 8 +- .../functions/frame/FrameMatrixCastingTest.java | 15 +- .../functions/frame/FrameMatrixReblockTest.java | 13 +- .../functions/frame/FrameMatrixWriteTest.java | 13 +- .../functions/frame/FrameMetaReadWriteTest.java | 18 +- .../functions/frame/FrameReadWriteTest.java | 6 +- .../frame/FrameScalarCastingIntegratedTest.java | 14 +- .../functions/frame/FrameScalarCastingTest.java | 6 +- .../functions/frame/FrameSchemaReadTest.java | 6 +- .../functions/frame/FrameSerializationTest.java | 8 +- .../frame/ParforFrameIntermediateTest.java | 13 +- .../indexing/IndexRangeBlockAlignmentTest.java | 33 +- .../indexing/Jdk7IssueRightIndexingTest.java | 14 +- .../indexing/LeftIndexingScalarTest.java | 14 +- .../indexing/LeftIndexingSparseDenseTest.java | 19 +- .../indexing/LeftIndexingSparseSparseTest.java | 17 +- .../functions/indexing/LeftIndexingTest.java | 17 +- .../indexing/LeftIndexingUpdateInPlaceTest.java | 6 +- .../indexing/PyDMLImplicitSlicingBounds.java | 3 + .../indexing/RightIndexingMatrixTest.java | 13 +- .../indexing/RightIndexingVectorTest.java | 13 +- .../indexing/RowBatchRightIndexingTest.java | 13 +- .../UnboundedScalarRightIndexingTest.java | 12 +- .../functions/io/FullDynWriteTest.java | 13 +- .../test/integration/functions/io/IOTest1.java | 3 + .../test/integration/functions/io/IOTest2.java | 3 + .../test/integration/functions/io/IOTest3.java | 3 + .../test/integration/functions/io/IOTest4.java | 3 + .../test/integration/functions/io/IOTest5.java | 3 + .../integration/functions/io/ScalarIOTest.java | 92 ++-- .../functions/io/SeqParReadTest.java | 2 + .../functions/io/binary/SerializeTest.java | 8 +- .../functions/io/csv/CSVParametersTest.java | 77 +++- .../functions/io/csv/FormatChangeTest.java | 4 + .../functions/io/csv/ReadCSVTest.java | 10 +- .../functions/io/csv/WriteCSVTest.java | 5 +- .../io/matrixmarket/FormatChangeTest.java | 6 + .../functions/io/matrixmarket/ReadMMTest.java | 5 +- .../functions/jmlc/FrameCastingTest.java | 5 +- .../functions/jmlc/FrameDecodeTest.java | 5 +- .../functions/jmlc/FrameEncodeTest.java | 5 +- .../functions/jmlc/FrameIndexingAppendTest.java | 5 +- .../functions/jmlc/FrameLeftIndexingTest.java | 5 +- .../functions/jmlc/FrameReadMetaTest.java | 5 +- .../functions/jmlc/FrameTransformTest.java | 5 +- .../jmlc/JMLCClonedPreparedScriptTest.java | 3 +- .../functions/jmlc/JMLCInputOutputTest.java | 13 +- .../functions/jmlc/JMLCInputStreamReadTest.java | 3 + .../jmlc/JMLCParfor2ForCompileTest.java | 5 +- .../functions/jmlc/MulticlassSVMScoreTest.java | 3 + .../functions/jmlc/NamespaceFunctionTest.java | 3 +- .../functions/jmlc/ReuseModelVariablesTest.java | 5 +- .../functions/misc/AssertExpressionTest.java | 3 + .../functions/misc/ConditionalValidateTest.java | 2 + .../functions/misc/DataTypeCastingTest.java | 5 +- .../functions/misc/DataTypeChangeTest.java | 8 +- .../functions/misc/ExistsVariableTest.java | 5 +- .../misc/FunctionInExpressionTest.java | 6 +- .../functions/misc/FunctionInliningTest.java | 20 +- .../functions/misc/FunctionNamespaceTest.java | 25 +- .../functions/misc/FunctionNotFoundTest.java | 3 + .../functions/misc/FunctionPotpourriTest.java | 6 +- .../functions/misc/FunctionReturnTest.java | 3 + ...antFoldingScalarVariablePropagationTest.java | 6 +- .../misc/IPADeadCodeEliminationTest.java | 7 +- .../functions/misc/IPAFunctionInliningTest.java | 9 +- .../misc/IPALiteralReplacementTest.java | 6 +- .../functions/misc/IPANnzPropagationTest.java | 6 +- .../functions/misc/IPAScalarRecursionTest.java | 3 + .../misc/IPAScalarVariablePropagationTest.java | 9 +- .../functions/misc/IPAUnknownRecursionTest.java | 4 +- .../test/integration/functions/misc/IfTest.java | 3 + .../misc/InvalidBuiltinFunctionCallTest.java | 3 + .../misc/InvalidFunctionAssignmentTest.java | 3 + .../misc/InvalidFunctionSignatureTest.java | 3 + .../functions/misc/ListAndStructTest.java | 9 +- .../functions/misc/LongOverflowTest.java | 2 + .../misc/NegativeLoopIncrementsTest.java | 4 +- .../functions/misc/NrowNcolStringTest.java | 2 + .../misc/NrowNcolUnknownCSVReadTest.java | 6 +- .../functions/misc/OuterTableExpandTest.java | 15 +- .../functions/misc/PrintExpressionTest.java | 3 + .../functions/misc/PrintMatrixTest.java | 3 + .../functions/misc/ReadAfterWriteTest.java | 2 + .../misc/RewriteBinaryMV2OuterTest.java | 5 +- .../misc/RewriteCSETransposeScalarTest.java | 8 +- .../misc/RewriteCTableToRExpandTest.java | 14 +- ...ElementwiseMultChainOptimizationAllTest.java | 15 +- ...iteElementwiseMultChainOptimizationTest.java | 15 +- .../misc/RewriteEliminateAggregatesTest.java | 6 +- .../functions/misc/RewriteFoldMinMaxTest.java | 17 +- .../functions/misc/RewriteFoldRCBindTest.java | 17 +- .../misc/RewriteFuseBinaryOpChainTest.java | 21 +- .../functions/misc/RewriteFusedRandTest.java | 12 +- .../RewriteHoistingLoopInvariantOpsTest.java | 17 +- .../functions/misc/RewriteIfElseTest.java | 17 +- .../misc/RewriteIndexingVectorizationTest.java | 10 +- .../misc/RewriteLoopVectorization.java | 5 +- .../misc/RewriteMatrixMultChainOptTest.java | 15 +- .../functions/misc/RewriteMergeBlocksTest.java | 6 +- .../functions/misc/RewriteNNIssueTest.java | 3 + .../misc/RewritePushdownSumBinaryMult.java | 5 +- .../misc/RewritePushdownSumOnBinaryTest.java | 10 +- .../functions/misc/RewritePushdownUaggTest.java | 9 +- .../misc/RewriteRemoveComparisonChainsTest.java | 9 +- .../RewriteSimplifyRowColSumMVMultTest.java | 9 +- .../misc/RewriteSlicedMatrixMultTest.java | 6 +- .../functions/misc/ScalarAssignmentTest.java | 3 + .../functions/misc/ScalarFunctionTest.java | 6 +- .../misc/ScalarMatrixUnaryBinaryTermTest.java | 5 +- .../misc/ScalarToMatrixInLoopTest.java | 4 +- .../functions/misc/SetWorkingDirTest.java | 6 +- .../functions/misc/SizePropagationTest.java | 15 +- .../functions/misc/ToStringTest.java | 9 +- .../misc/UnivariateStatsBasicTest.java | 7 +- .../misc/ValueTypeAutoCastingTest.java | 2 + .../functions/misc/ValueTypeCastingTest.java | 10 +- .../misc/ValueTypeMatrixScalarBuiltinTest.java | 6 +- .../functions/misc/ZeroRowsColsMatrixTest.java | 12 +- .../DataFrameMatrixConversionTest.java | 9 +- .../DataFrameRowFrameConversionTest.java | 5 +- .../DataFrameVectorFrameConversionTest.java | 5 +- .../mlcontext/DataFrameVectorScriptTest.java | 6 +- .../functions/mlcontext/FrameTest.java | 2 + .../functions/mlcontext/GNMFTest.java | 15 +- .../functions/nary/NaryCBindTest.java | 15 +- .../functions/nary/NaryMinMaxTest.java | 12 +- .../functions/nary/NaryRBindTest.java | 15 +- .../paramserv/BaseDataPartitionerTest.java | 13 + .../paramserv/LocalDataPartitionerTest.java | 33 +- .../paramserv/ParamservSparkNNTest.java | 6 +- .../functions/paramserv/SerializationTest.java | 28 +- .../paramserv/SparkDataPartitionerTest.java | 5 +- .../functions/parfor/ForLoopPredicateTest.java | 4 +- .../ParForAccumulatorResultMergeTest.java | 1 + .../parfor/ParForAdversarialLiteralsTest.java | 3 +- .../ParForBlockwiseDataPartitioningTest.java | 1 + .../ParForColwiseDataPartitioningTest.java | 1 + .../parfor/ParForDataPartitionExecuteTest.java | 1 + .../ParForDataPartitionLeftIndexingTest.java | 3 +- .../parfor/ParForDependencyAnalysisTest.java | 2 + .../parfor/ParForFunctionSerializationTest.java | 3 +- .../parfor/ParForListResultVarsTest.java | 3 +- .../ParForMultipleDataPartitioningTest.java | 3 +- .../parfor/ParForNaNResultMergeTest.java | 3 +- .../ParForParallelRemoteResultMergeTest.java | 3 +- .../parfor/ParForRepeatedOptimizationTest.java | 1 + .../ParForReplaceThreadIDRecompileTest.java | 2 + .../ParForRowwiseDataPartitioningTest.java | 1 + .../parfor/ParForRulebasedOptimizerTest.java | 3 +- .../ParForSerialRemoteResultMergeTest.java | 1 + .../piggybacking/PiggybackingTest1.java | 18 +- .../piggybacking/PiggybackingTest2.java | 43 +- .../quaternary/RewritesWeightedSigmoidTest.java | 19 +- .../quaternary/WeightedCrossEntropyTest.java | 19 +- .../quaternary/WeightedDivMatrixMultTest.java | 17 +- .../quaternary/WeightedSigmoidTest.java | 17 +- .../quaternary/WeightedSquaredLossTest.java | 19 +- .../quaternary/WeightedUnaryMatrixMultTest.java | 19 +- .../functions/recompile/BranchRemovalTest.java | 4 +- .../recompile/CSVReadUnknownSizeTest.java | 6 +- .../recompile/FunctionRecompileTest.java | 8 +- .../IPAAssignConstantPropagationTest.java | 4 +- .../recompile/IPAComplexAppendTest.java | 3 + .../recompile/IPAConstantPropagationTest.java | 4 +- ...IPAPropagationSizeMultipleFunctionsTest.java | 3 + .../LiteralReplaceCastScalarReadTest.java | 12 +- .../recompile/MultipleReadsIPATest.java | 5 +- .../recompile/PredicateRecompileTest.java | 18 +- .../recompile/RandJobRecompileTest.java | 6 +- .../functions/recompile/RandRecompileTest.java | 10 +- .../recompile/RandSizeExpressionEvalTest.java | 10 +- .../recompile/ReblockRecompileTest.java | 10 +- .../RecursiveFunctionRecompileTest.java | 10 +- .../recompile/RemoveEmptyPotpourriTest.java | 11 +- .../recompile/RemoveEmptyRecompileTest.java | 13 +- .../RewriteComplexMapMultChainTest.java | 5 +- .../SparsityFunctionRecompileTest.java | 14 +- .../recompile/SparsityRecompileTest.java | 14 +- .../functions/reorg/DiagV2MTest.java | 8 +- .../functions/reorg/FullOrderTest.java | 20 +- .../functions/reorg/FullReverseTest.java | 19 +- .../functions/reorg/FullTransposeTest.java | 14 +- .../functions/reorg/MatrixReshapeTest.java | 13 +- .../reorg/MultipleOrderByColsTest.java | 15 +- .../functions/reorg/VectorReshapeTest.java | 12 +- .../functions/sparse/SparseBlockAlignment.java | 12 +- .../functions/sparse/SparseBlockAppendSort.java | 12 +- .../functions/sparse/SparseBlockDelete.java | 14 +- .../sparse/SparseBlockGetFirstIndex.java | 12 +- .../functions/sparse/SparseBlockGetSet.java | 14 +- .../functions/sparse/SparseBlockIndexRange.java | 14 +- .../functions/sparse/SparseBlockIterator.java | 14 +- .../sparse/SparseBlockMemEstimate.java | 7 +- .../functions/sparse/SparseBlockMerge.java | 10 +- .../functions/sparse/SparseBlockScan.java | 14 +- .../functions/sparse/SparseBlockSize.java | 14 +- .../tensor/Conv2DBackwardDataTest.java | 17 +- .../functions/tensor/Conv2DBackwardTest.java | 11 +- .../functions/tensor/Conv2DTest.java | 11 +- .../functions/tensor/PoolBackwardTest.java | 12 +- .../integration/functions/tensor/PoolTest.java | 12 +- .../functions/tensor/ReluBackwardTest.java | 16 +- .../ternary/ABATernaryAggregateTest.java | 16 +- .../ternary/CTableMatrixIgnoreZerosTest.java | 20 +- .../functions/ternary/CTableSequenceTest.java | 18 +- .../ternary/CentralMomentWeightsTest.java | 15 +- .../ternary/CovarianceWeightsTest.java | 15 +- .../functions/ternary/FullIfElseTest.java | 12 +- .../functions/ternary/QuantileWeightsTest.java | 13 +- .../functions/ternary/TableOutputTest.java | 21 +- .../functions/ternary/TernaryAggregateTest.java | 16 +- .../transform/FrameCSVReadWriteTest.java | 11 +- .../TransformApplyEmptyRecodeMapTest.java | 10 +- .../TransformCSVFrameEncodeDecodeTest.java | 14 +- .../TransformCSVFrameEncodeReadTest.java | 11 +- .../transform/TransformEncodeDecodeTest.java | 11 +- .../TransformFrameEncodeApplySubsetTest.java | 16 +- .../TransformFrameEncodeApplyTest.java | 14 +- .../TransformFrameEncodeColmapTest.java | 11 +- .../TransformFrameEncodeDecodeTest.java | 14 +- .../TransformFrameEncodeDecodeTokenTest.java | 14 +- .../functions/unary/matrix/ACosTest.java | 6 + .../functions/unary/matrix/ASinTest.java | 6 + .../functions/unary/matrix/ATanTest.java | 6 + .../functions/unary/matrix/AbsTest.java | 9 +- .../unary/matrix/CastAsScalarTest.java | 2 + .../functions/unary/matrix/CholeskyTest.java | 11 +- .../functions/unary/matrix/CosTest.java | 6 + .../functions/unary/matrix/DiagTest.java | 4 + .../unary/matrix/EigenFactorizeTest.java | 8 +- .../unary/matrix/ExtractTriangularTest.java | 13 +- .../functions/unary/matrix/FullCummaxTest.java | 17 +- .../functions/unary/matrix/FullCumminTest.java | 17 +- .../functions/unary/matrix/FullCumprodTest.java | 17 +- .../functions/unary/matrix/FullCumsumTest.java | 17 +- .../unary/matrix/FullCumsumprodTest.java | 14 +- .../unary/matrix/FullSelectPosTest.java | 17 +- .../functions/unary/matrix/FullSignTest.java | 19 +- .../functions/unary/matrix/IQMTest.java | 10 +- .../functions/unary/matrix/LUFactorizeTest.java | 8 +- .../unary/matrix/MLUnaryBuiltinTest.java | 18 +- .../unary/matrix/MatrixInverseTest.java | 8 +- .../functions/unary/matrix/MinusTest.java | 12 +- .../functions/unary/matrix/NegationTest.java | 12 +- .../functions/unary/matrix/PrintTest.java | 3 + .../functions/unary/matrix/QRSolverTest.java | 8 +- .../unary/matrix/RemoveEmptySelTest.java | 22 +- .../functions/unary/matrix/RemoveEmptyTest.java | 13 +- .../functions/unary/matrix/ReplaceTest.java | 17 +- .../functions/unary/matrix/RoundTest.java | 8 +- .../unary/matrix/SVDFactorizeTest.java | 10 +- .../functions/unary/matrix/SinTest.java | 12 +- .../functions/unary/matrix/SqrtTest.java | 12 + .../functions/unary/matrix/TanTest.java | 9 +- .../functions/unary/matrix/TransposeTest.java | 3 + .../functions/unary/scalar/ACosTest.java | 12 + .../functions/unary/scalar/ASinTest.java | 9 + .../functions/unary/scalar/ATanTest.java | 9 + .../functions/unary/scalar/AbsTest.java | 12 + .../functions/unary/scalar/BooleanTest.java | 3 + .../functions/unary/scalar/CosTest.java | 12 + .../functions/unary/scalar/ExponentTest.java | 17 + .../unary/scalar/FullDistributionTest.java | 13 +- .../functions/unary/scalar/NotTest.java | 3 + .../functions/unary/scalar/PrintTest.java | 18 + .../functions/unary/scalar/RoundTest.java | 3 + .../functions/unary/scalar/SinTest.java | 9 + .../functions/unary/scalar/SqrtTest.java | 15 + .../functions/unary/scalar/StopTest.java | 3 + .../functions/unary/scalar/StopTest2.java | 2 + .../functions/unary/scalar/StopTestCtrlStr.java | 5 +- .../functions/unary/scalar/TanTest.java | 11 + .../updateinplace/UpdateInPlaceTest.java | 2 + .../functions/vect/AutoVectorizationTest.java | 3 + .../vect/LeftIndexingChainUpdateTest.java | 3 + .../mlcontext/MLContextFrameTest.java | 185 ++++---- .../mlcontext/MLContextOptLevelTest.java | 3 +- .../mlcontext/MLContextOutputBlocksizeTest.java | 5 +- .../integration/mlcontext/MLContextTest.java | 441 +++++++++---------- .../MLContextUnivariateStatisticsTest.java | 15 +- .../org/apache/sysml/test/utils/TestUtils.java | 12 +- 513 files changed, 3563 insertions(+), 3126 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java index 703a850..2135f45 100644 --- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java @@ -19,11 +19,6 @@ package org.apache.sysml.test.integration; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -42,6 +37,7 @@ import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.lops.Lop; +import org.apache.sysml.lops.LopProperties.ExecType; import org.apache.sysml.parser.DataExpression; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; @@ -65,6 +61,7 @@ import org.apache.wink.json4j.JSONObject; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.internal.ArrayComparisonFailure; /** @@ -91,6 +88,16 @@ public abstract class AutomatedTestBase return super.toString().toLowerCase(); } } + + // Since MR backend is in the maintenance mode, the MR tests can be skipped to reduce the time + // taken for running the entire test suite. This will also help continuous integration process. + public static final boolean TEST_MR_BACKEND = false; + public boolean shouldSkipTest() { + if(rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK || rtplatform == RUNTIME_PLATFORM.SPARK) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + // Let's skip first HADOOP tests. In the subsequent commits, we can visit HYBRID + return !TEST_MR_BACKEND && rtplatform == RUNTIME_PLATFORM.HADOOP; + } public static final boolean EXCEPTION_EXPECTED = true; public static final boolean EXCEPTION_NOT_EXPECTED = false; @@ -193,7 +200,34 @@ public abstract class AutomatedTestBase * Also set DMLScript.USE_LOCAL_SPARK_CONFIG to true for running the test * suite in spark mode */ - protected static RUNTIME_PLATFORM rtplatform = RUNTIME_PLATFORM.HYBRID; + protected RUNTIME_PLATFORM rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; + + protected RUNTIME_PLATFORM setRuntimePlatform(RUNTIME_PLATFORM platform) { + RUNTIME_PLATFORM platformOld = rtplatform; + if(platform == RUNTIME_PLATFORM.SPARK || platform == RUNTIME_PLATFORM.HYBRID_SPARK) { + DMLScript.USE_LOCAL_SPARK_CONFIG = true; // Always use local config for junit tests + } + rtplatform = platform; + return platformOld; + } + + protected RUNTIME_PLATFORM setRuntimePlatform(ExecType et) { + RUNTIME_PLATFORM platformOld = rtplatform; + switch (et) { + case MR: + rtplatform = RUNTIME_PLATFORM.HADOOP; + break; + case SPARK: { + rtplatform = RUNTIME_PLATFORM.SPARK; + DMLScript.USE_LOCAL_SPARK_CONFIG = true; // Always use local config for junit tests + break; + } + default: + rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; + break; + } + return platformOld; + } protected static final boolean DEBUG = false; @@ -705,7 +739,7 @@ public abstract class AutomatedTestBase expectedFiles.add(baseDirectory + EXPECTED_DIR + cacheDir + name); } - protected static HashMap<CellIndex, Double> readDMLMatrixFromHDFS(String fileName) { + protected HashMap<CellIndex, Double> readDMLMatrixFromHDFS(String fileName) { return TestUtils.readDMLMatrixFromHDFS(baseDirectory + OUTPUT_DIR + fileName); } @@ -714,12 +748,12 @@ public abstract class AutomatedTestBase return TestUtils.readRMatrixFromFS(baseDirectory + EXPECTED_DIR + cacheDir + fileName); } - protected static HashMap<CellIndex, Double> readDMLScalarFromHDFS(String fileName) { + protected HashMap<CellIndex, Double> readDMLScalarFromHDFS(String fileName) { return TestUtils.readDMLScalarFromHDFS(baseDirectory + OUTPUT_DIR + fileName); } - protected static FrameBlock readDMLFrameFromHDFS(String fileName, InputInfo iinfo) throws IOException { + protected FrameBlock readDMLFrameFromHDFS(String fileName, InputInfo iinfo) throws IOException { //read frame data from hdfs String strFrameFileName = baseDirectory + OUTPUT_DIR + fileName; FrameReader reader = FrameReaderFactory.createFrameReader(iinfo); @@ -729,7 +763,7 @@ public abstract class AutomatedTestBase } - protected static FrameBlock readDMLFrameFromHDFS(String fileName, InputInfo iinfo, MatrixCharacteristics md) throws IOException { + protected FrameBlock readDMLFrameFromHDFS(String fileName, InputInfo iinfo, MatrixCharacteristics md) throws IOException { //read frame data from hdfs String strFrameFileName = baseDirectory + OUTPUT_DIR + fileName; FrameReader reader = FrameReaderFactory.createFrameReader(iinfo); @@ -737,7 +771,7 @@ public abstract class AutomatedTestBase return reader.readFrameFromHDFS(strFrameFileName, md.getRows(), md.getCols()); } - protected static FrameBlock readRFrameFromHDFS(String fileName, InputInfo iinfo, MatrixCharacteristics md) throws IOException { + protected FrameBlock readRFrameFromHDFS(String fileName, InputInfo iinfo, MatrixCharacteristics md) throws IOException { //read frame data from hdfs String strFrameFileName = baseDirectory + EXPECTED_DIR + fileName; @@ -755,10 +789,10 @@ public abstract class AutomatedTestBase return TestUtils.readRScalarFromFS(baseDirectory + EXPECTED_DIR + cacheDir + fileName); } - public static void checkDMLMetaDataFile(String fileName, MatrixCharacteristics mc) { + public void checkDMLMetaDataFile(String fileName, MatrixCharacteristics mc) { MatrixCharacteristics rmc = readDMLMetaDataFile(fileName); - Assert.assertEquals(mc.getRows(), rmc.getRows()); - Assert.assertEquals(mc.getCols(), rmc.getCols()); + assertEquals(mc.getRows(), rmc.getRows()); + assertEquals(mc.getCols(), rmc.getCols()); } public static MatrixCharacteristics readDMLMetaDataFile(String fileName) @@ -920,13 +954,107 @@ public abstract class AutomatedTestBase */ protected void runRScript() { runRScript(false); + } + + public void assertNotEquals(Object expected, Object actual) { + Assert.assertNotEquals(expected, actual); + } + + public void assertNotEquals(double expected, double actual) { + Assert.assertNotEquals(expected, actual); + } + + public void assertNotEquals(int expected, int actual) { + Assert.assertNotEquals(expected, actual); + } + + public void assertNotEquals(long expected, long actual) { + Assert.assertNotEquals(expected, actual); + } + + public void assertNotEquals(String message, double expected, double actual) { + Assert.assertNotEquals(message, expected, actual); + } + + public void assertNotEquals(String message, Long expected, Long actual) { + Assert.assertNotEquals(message, expected, actual); + } + + public void assertNotEquals(String message, long expected, long actual) { + Assert.assertNotEquals(message, expected, actual); + } + + public void assertArrayEquals(double[] expecteds, + double[] actuals, double delta) throws ArrayComparisonFailure { + Assert.assertArrayEquals(expecteds, actuals, delta); + } + + public void assertArrayEquals(double[] expecteds, + double[] actuals, int delta) throws ArrayComparisonFailure { + Assert.assertArrayEquals(expecteds, actuals, delta); + } + + public void assertEquals(double expected, double actual, double delta) { + Assert.assertEquals(expected, actual, delta); + } + + public void assertEquals(String message, double expected, double actual) { + Assert.assertEquals(message, expected, actual); + } + + public void assertEquals(String message, long expected, long actual) { + Assert.assertEquals(message, expected, actual); + } + + public void assertEquals(int expected, int actual) { + Assert.assertEquals(expected, actual); + } + + public void assertEquals(Object expected, Object actual) { + Assert.assertEquals(expected, actual); + } + + public void assertEquals(Integer expected, Long actual) { + Assert.assertEquals((long)expected, (long)actual); + } + + public void assertEquals(Long expected, Integer actual) { + Assert.assertEquals((long)expected, (long)actual); + } + + public void assertEquals(boolean expected, boolean actual) { + Assert.assertEquals(expected, actual); + } + + public void assertFalse(boolean condition) { + Assert.assertFalse(condition); + } + + public void assertFalse(String message, boolean condition) { + Assert.assertFalse(message, condition); + } + + public void assertTrue(String message, boolean condition) { + Assert.assertTrue(message, condition); + } + public void assertTrue(boolean condition) { + Assert.assertTrue(condition); + } + + public void assertEquals(String message, Object expected, + Object actual) { + Assert.assertEquals(message, expected, actual); + } + + public void fail(String message) { + Assert.fail(message); } + /** * Runs an R script in the old or the new way */ protected void runRScript(boolean newWay) { - String executionFile = sourceDirectory + selectedTest + ".R"; // *** HACK ALERT *** HACK ALERT *** HACK ALERT *** @@ -1148,7 +1276,6 @@ public abstract class AutomatedTestBase * -1 there is no limit. */ protected void runTest(boolean newWay, boolean exceptionExpected, Class<?> expectedException, String errMessage, int maxMRJobs) { - String executionFile = sourceDirectory + selectedTest + ".dml"; if( !newWay ) { @@ -1420,7 +1547,7 @@ public abstract class AutomatedTestBase // Skip MapReduce-related checks when running in Spark mode. return; } - + assertEquals("Unexpected number of compiled MR jobs.", expectedNumCompiled, Statistics.getNoOfCompiledMRJobs()); } http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java b/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java index 2ee2544..b96ce6e 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/ApplyTransformTest.java @@ -28,152 +28,154 @@ import java.util.List; import java.util.Map; import org.junit.runners.Parameterized.Parameters; - import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysml.test.integration.AutomatedTestBase; public abstract class ApplyTransformTest extends AutomatedTestBase{ - + protected final static String TEST_DIR = "applications/apply-transform/"; protected final static String TEST_NAME = "apply-transform"; protected String TEST_CLASS_DIR = TEST_DIR + ApplyTransformTest.class.getSimpleName() + "/"; - + protected String X, missing_value_maps, binning_maps, dummy_coding_maps, normalization_maps; - + public ApplyTransformTest(String X, - String missing_value_maps, - String binning_maps, - String dummy_coding_maps, - String normalization_maps) { + String missing_value_maps, + String binning_maps, + String dummy_coding_maps, + String normalization_maps) { this.X = X; this.missing_value_maps = missing_value_maps; this.binning_maps = binning_maps; this.dummy_coding_maps = dummy_coding_maps; this.normalization_maps = normalization_maps; } - + @Parameters - public static Collection<Object[]> data() { - Object[][] data = new Object[][] { - {"newX.mtx", "missing_value_map.mtx", "bindefns.mtx", "dummy_code_maps.mtx", "normalization_maps.mtx"}, - {"newX.mtx", "missing_value_map.mtx", " ", " ", " "}, - {"newX.mtx", "missing_value_map.mtx", " ", " ", "normalization_maps.mtx"}, - {"newX.mtx", "missing_value_map.mtx", "bindefns.mtx", " ", "normalization_maps.mtx"}, - {"newX_nomissing.mtx", " ", "bindefns.mtx", " ", " "}, - {"newX_nomissing.mtx", " ", "bindefns.mtx", "dummy_code_maps.mtx", " "}, - {"newX_nomissing.mtx", " ", " ", " ", "normalization_maps.mtx"} + public static Collection<Object[]> data() { + Object[][] data = new Object[][] { + {"newX.mtx", "missing_value_map.mtx", "bindefns.mtx", "dummy_code_maps.mtx", "normalization_maps.mtx"}, + {"newX.mtx", "missing_value_map.mtx", " ", " ", " "}, + {"newX.mtx", "missing_value_map.mtx", " ", " ", "normalization_maps.mtx"}, + {"newX.mtx", "missing_value_map.mtx", "bindefns.mtx", " ", "normalization_maps.mtx"}, + {"newX_nomissing.mtx", " ", "bindefns.mtx", " ", " "}, + {"newX_nomissing.mtx", " ", "bindefns.mtx", "dummy_code_maps.mtx", " "}, + {"newX_nomissing.mtx", " ", " ", " ", "normalization_maps.mtx"} }; - return Arrays.asList(data); - } + return Arrays.asList(data); + } - @Override - public void setUp() { - addTestConfiguration(TEST_CLASS_DIR, TEST_NAME); - } + @Override + public void setUp() { + addTestConfiguration(TEST_CLASS_DIR, TEST_NAME); + } - protected void testApplyTransform(ScriptType scriptType) { - System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + X + ", " + missing_value_maps - + ", " + binning_maps + ", " + dummy_coding_maps + ", " + normalization_maps + "} ------------"); - this.scriptType = scriptType; - - getAndLoadTestConfiguration(TEST_NAME); - - List<String> proArgs = new ArrayList<String>(); - if (scriptType == ScriptType.PYDML) { + protected void testApplyTransform(ScriptType scriptType) { + if(shouldSkipTest()) + return; + + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + X + ", " + missing_value_maps + + ", " + binning_maps + ", " + dummy_coding_maps + ", " + normalization_maps + "} ------------"); + this.scriptType = scriptType; + + getAndLoadTestConfiguration(TEST_NAME); + + List<String> proArgs = new ArrayList<String>(); + if (scriptType == ScriptType.PYDML) { proArgs.add("-python"); - } - proArgs.add("-stats"); - proArgs.add("-nvargs"); - proArgs.add("X=" + sourceDirectory + X); - proArgs.add("missing_value_maps=" + (missing_value_maps.equals(" ") ? " " : sourceDirectory + missing_value_maps)); - proArgs.add("bin_defns=" + (binning_maps.equals(" ") ? " " : sourceDirectory + binning_maps)); - proArgs.add("dummy_code_maps=" + (dummy_coding_maps.equals(" ") ? " " : sourceDirectory + dummy_coding_maps)); - proArgs.add("normalization_maps=" + (normalization_maps.equals(" ") ? " " : sourceDirectory + normalization_maps)); - proArgs.add("transformed_X=" + output("transformed_X.mtx")); - proArgs.add("Log=" + output("log.csv")); - programArgs = proArgs.toArray(new String[proArgs.size()]); - - fullDMLScriptName = getScript(); - - runTest(true, EXCEPTION_NOT_EXPECTED, null, -1); - - HashMap<CellIndex, Double> XDML= readDMLMatrixFromHDFS("transformed_X.mtx"); - - Iterator<Map.Entry<CellIndex, Double>> iter = XDML.entrySet().iterator(); - while(iter.hasNext()){ - Map.Entry<CellIndex, Double> elt = iter.next(); - int row = elt.getKey().row; - int col = elt.getKey().column; - double val = elt.getValue(); - - System.out.println("[" + row + "," + col + "]->" + val); - } - - boolean success = true; - - if(missing_value_maps != " " && normalization_maps != " "){ - CellIndex cell; - if(dummy_coding_maps != " ") cell = new CellIndex(3,3); - else cell = new CellIndex(3,2); - - if(XDML.containsKey(cell)){ - double val = XDML.get(cell).doubleValue(); - success = success && (Math.abs(val) < 0.0000001); - } - }else if(missing_value_maps != " "){ - CellIndex cell; - if(dummy_coding_maps != " ") cell = new CellIndex(3,3); - else cell = new CellIndex(3,2); - - if(XDML.containsKey(cell)){ - double val = XDML.get(cell).doubleValue(); - success = success && (Math.abs(-0.2/3 - val) < 0.0000001); - }else success = false; - }else if(normalization_maps != " "){ - CellIndex cell; - if(dummy_coding_maps != " ") cell = new CellIndex(3,3); - else cell = new CellIndex(3,2); - - if(XDML.containsKey(cell)){ - double val = XDML.get(cell).doubleValue(); - success = success && (Math.abs(0.2/3 - val) < 0.0000001); - }else success = false; - }else{ - CellIndex cell; - if(dummy_coding_maps != " ") cell = new CellIndex(3,3); - else cell = new CellIndex(3,2); - - if(XDML.containsKey(cell)){ - double val = XDML.get(cell).doubleValue(); - success = success && (Math.abs(val) < 0.0000001); - } - } - - if(binning_maps != " "){ - CellIndex cell1, cell2, cell3, cell4; - if(dummy_coding_maps != " "){ - cell1 = new CellIndex(1,1); - cell2 = new CellIndex(2,1); - cell3 = new CellIndex(3,2); - cell4 = new CellIndex(4,2); - }else{ - cell1 = new CellIndex(1,1); - cell2 = new CellIndex(2,1); - cell3 = new CellIndex(3,1); - cell4 = new CellIndex(4,1); - } - - if(!XDML.containsKey(cell1)) success = false; - else success = success && (XDML.get(cell1).doubleValue() == 1); - - if(!XDML.containsKey(cell2)) success = false; - else success = success && (XDML.get(cell2).doubleValue() == 1); - - if(!XDML.containsKey(cell3)) success = false; - else success = success && (dummy_coding_maps != " ") ? (XDML.get(cell3).doubleValue() == 1) : (XDML.get(cell3).doubleValue() == 2); - - if(!XDML.containsKey(cell4)) success = false; - else success = success && (dummy_coding_maps != " ") ? (XDML.get(cell4).doubleValue() == 1) : (XDML.get(cell4).doubleValue() == 2); - } - } + } + proArgs.add("-stats"); + proArgs.add("-nvargs"); + proArgs.add("X=" + sourceDirectory + X); + proArgs.add("missing_value_maps=" + (missing_value_maps.equals(" ") ? " " : sourceDirectory + missing_value_maps)); + proArgs.add("bin_defns=" + (binning_maps.equals(" ") ? " " : sourceDirectory + binning_maps)); + proArgs.add("dummy_code_maps=" + (dummy_coding_maps.equals(" ") ? " " : sourceDirectory + dummy_coding_maps)); + proArgs.add("normalization_maps=" + (normalization_maps.equals(" ") ? " " : sourceDirectory + normalization_maps)); + proArgs.add("transformed_X=" + output("transformed_X.mtx")); + proArgs.add("Log=" + output("log.csv")); + programArgs = proArgs.toArray(new String[proArgs.size()]); + + fullDMLScriptName = getScript(); + + runTest(true, EXCEPTION_NOT_EXPECTED, null, -1); + + HashMap<CellIndex, Double> XDML= readDMLMatrixFromHDFS("transformed_X.mtx"); + + Iterator<Map.Entry<CellIndex, Double>> iter = XDML.entrySet().iterator(); + while(iter.hasNext()){ + Map.Entry<CellIndex, Double> elt = iter.next(); + int row = elt.getKey().row; + int col = elt.getKey().column; + double val = elt.getValue(); + + System.out.println("[" + row + "," + col + "]->" + val); + } + + boolean success = true; + + if(missing_value_maps != " " && normalization_maps != " "){ + CellIndex cell; + if(dummy_coding_maps != " ") cell = new CellIndex(3,3); + else cell = new CellIndex(3,2); + + if(XDML.containsKey(cell)){ + double val = XDML.get(cell).doubleValue(); + success = success && (Math.abs(val) < 0.0000001); + } + }else if(missing_value_maps != " "){ + CellIndex cell; + if(dummy_coding_maps != " ") cell = new CellIndex(3,3); + else cell = new CellIndex(3,2); + + if(XDML.containsKey(cell)){ + double val = XDML.get(cell).doubleValue(); + success = success && (Math.abs(-0.2/3 - val) < 0.0000001); + }else success = false; + }else if(normalization_maps != " "){ + CellIndex cell; + if(dummy_coding_maps != " ") cell = new CellIndex(3,3); + else cell = new CellIndex(3,2); + + if(XDML.containsKey(cell)){ + double val = XDML.get(cell).doubleValue(); + success = success && (Math.abs(0.2/3 - val) < 0.0000001); + }else success = false; + }else{ + CellIndex cell; + if(dummy_coding_maps != " ") cell = new CellIndex(3,3); + else cell = new CellIndex(3,2); + + if(XDML.containsKey(cell)){ + double val = XDML.get(cell).doubleValue(); + success = success && (Math.abs(val) < 0.0000001); + } + } + + if(binning_maps != " "){ + CellIndex cell1, cell2, cell3, cell4; + if(dummy_coding_maps != " "){ + cell1 = new CellIndex(1,1); + cell2 = new CellIndex(2,1); + cell3 = new CellIndex(3,2); + cell4 = new CellIndex(4,2); + }else{ + cell1 = new CellIndex(1,1); + cell2 = new CellIndex(2,1); + cell3 = new CellIndex(3,1); + cell4 = new CellIndex(4,1); + } + + if(!XDML.containsKey(cell1)) success = false; + else success = success && (XDML.get(cell1).doubleValue() == 1); + + if(!XDML.containsKey(cell2)) success = false; + else success = success && (XDML.get(cell2).doubleValue() == 1); + + if(!XDML.containsKey(cell3)) success = false; + else success = success && (dummy_coding_maps != " ") ? (XDML.get(cell3).doubleValue() == 1) : (XDML.get(cell3).doubleValue() == 2); + + if(!XDML.containsKey(cell4)) success = false; + else success = success && (dummy_coding_maps != " ") ? (XDML.get(cell4).doubleValue() == 1) : (XDML.get(cell4).doubleValue() == 2); + } + } } http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java b/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java index 149c0b1..37792d6 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/ArimaTest.java @@ -71,6 +71,9 @@ public abstract class ArimaTest extends AutomatedTestBase { } protected void testArima(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME1 + " " + scriptType + " TEST WITH {" + max_func_invoc + ", " + p + ", " + http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java b/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java index bbe644f..7ecfdf0 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/CsplineCGTest.java @@ -54,6 +54,9 @@ public abstract class CsplineCGTest extends AutomatedTestBase { } protected void testCsplineCG(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + numRecords + ", " + numDim + "} ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java b/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java index da905a1..a1505ad 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/CsplineDSTest.java @@ -61,6 +61,9 @@ public abstract class CsplineDSTest extends AutomatedTestBase { protected void testCsplineDS(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + numRecords + ", " + numDim + "} ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java b/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java index 26f89b6..4f26a2b 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java @@ -197,6 +197,9 @@ public abstract class GLMTest extends AutomatedTestBase protected void testGLM(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + numRecords + ", " + numFeatures + ", " + http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java b/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java index 51efb93..5244238 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/GNMFTest.java @@ -58,6 +58,9 @@ public abstract class GNMFTest extends AutomatedTestBase } protected void testGNMF(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST {" + m + ", " + n + ", " + k + "} ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java b/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java index 56560e5..32d84e9 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/HITSTest.java @@ -40,6 +40,8 @@ public abstract class HITSTest extends AutomatedTestBase } protected void testHits(ScriptType scriptType) { + if(shouldSkipTest()) + return; System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java b/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java index a158f52..e6433e8 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/ID3Test.java @@ -25,7 +25,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; -import org.junit.Assert; import org.junit.runners.Parameterized.Parameters; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; @@ -63,6 +62,9 @@ public abstract class ID3Test extends AutomatedTestBase protected void testID3(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST {" + numRecords + ", " + numFeatures + "} ------------"); this.scriptType = scriptType; @@ -101,9 +103,9 @@ public abstract class ID3Test extends AutomatedTestBase runRScript(true); //check also num actually executed jobs - if(AutomatedTestBase.rtplatform != RUNTIME_PLATFORM.SPARK) { + if(rtplatform != RUNTIME_PLATFORM.SPARK) { long actualMR = Statistics.getNoOfExecutedMRJobs(); - Assert.assertEquals("Wrong number of executed jobs: expected 0 but executed "+actualMR+".", 0, actualMR); + assertEquals("Wrong number of executed jobs: expected 0 but executed "+actualMR+".", 0, actualMR); } //compare results http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java b/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java index eae542c..c451355 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/L2SVMTest.java @@ -64,6 +64,9 @@ public abstract class L2SVMTest extends AutomatedTestBase protected void testL2SVM(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + numRecords + ", " + numFeatures + ", " + sparsity + ", " + intercept + "} ------------"); http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java b/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java index 8012e30..0bf0557 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/LinearRegressionTest.java @@ -63,6 +63,9 @@ public abstract class LinearRegressionTest extends AutomatedTestBase { } protected void testLinearRegression(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + numRecords + ", " + numFeatures + ", " + sparsity + "} ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java b/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java index f635241..936d604 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/MDABivariateStatsTest.java @@ -62,6 +62,9 @@ public abstract class MDABivariateStatsTest extends AutomatedTestBase } protected void testMDABivariateStats(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + n + ", " + m + ", " + label_index + ", " + label_measurement_level + "} ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java b/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java index b8dc20c..11f6602 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/MultiClassSVMTest.java @@ -76,6 +76,9 @@ public abstract class MultiClassSVMTest extends AutomatedTestBase protected void testMultiClassSVM( ScriptType scriptType ) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST WITH {" + _numRecords + ", " + _numFeatures + ", " + http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesParforTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesParforTest.java b/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesParforTest.java index 7387e16..d28e73a 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesParforTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesParforTest.java @@ -70,6 +70,9 @@ public abstract class NaiveBayesParforTest extends AutomatedTestBase{ protected void testNaiveBayes(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST {" + numRecords + ", " + numFeatures + ", " + numClasses + ", " + sparsity + "} ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java b/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java index 8030cb9..adaa547 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/NaiveBayesTest.java @@ -70,6 +70,9 @@ public abstract class NaiveBayesTest extends AutomatedTestBase{ protected void testNaiveBayes(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST {" + numRecords + ", " + numFeatures + ", " + numClasses + ", " + sparsity + "} ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java b/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java index 9db5602..0baef99 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/PageRankTest.java @@ -55,6 +55,9 @@ public abstract class PageRankTest extends AutomatedTestBase { } protected void testPageRank(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST {" + numRows + ", " + numCols + "} ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java b/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java index 0d58648..181daee 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/WelchTTest.java @@ -61,6 +61,9 @@ public abstract class WelchTTest extends AutomatedTestBase { } protected void testWelchTTest(ScriptType scriptType) { + if(shouldSkipTest()) + return; + System.out.println("------------ BEGIN " + TEST_NAME + " " + scriptType + " TEST {" + numAttr + ", " + numPosSamples + ", " + numNegSamples + "} ------------"); this.scriptType = scriptType; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java index 2430d52..7cb9f76 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateCategoricalCategoricallTest.java @@ -69,6 +69,9 @@ public class BivariateCategoricalCategoricallTest extends AutomatedTestBase @Test public void testCategoricalCategorical() { + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration(TEST_NOMINAL_NOMINAL); config.addVariable("rows", rows); loadTestConfiguration(config); @@ -123,6 +126,9 @@ public class BivariateCategoricalCategoricallTest extends AutomatedTestBase @Test public void testCategoricalCategoricalWithWeights() { + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration(TEST_NOMINAL_NOMINAL_WEIGHTS); config.addVariable("rows", rows); loadTestConfiguration(config); @@ -181,6 +187,9 @@ public class BivariateCategoricalCategoricallTest extends AutomatedTestBase @Test public void testOddsRatio() { + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration(TEST_ODDS_RATIO); config.addVariable("rows", rows); loadTestConfiguration(config); http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java index 45d8966..490573f 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateOrdinalOrdinalTest.java @@ -55,6 +55,9 @@ public class BivariateOrdinalOrdinalTest extends AutomatedTestBase @Test public void testOrdinalOrdinal() { + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration(TEST_ORDINAL_ORDINAL); config.addVariable("rows", rows); loadTestConfiguration(config); @@ -99,6 +102,9 @@ public class BivariateOrdinalOrdinalTest extends AutomatedTestBase @Test public void testOrdinalOrdinalWithWeights() { + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration(TEST_ORDINAL_ORDINAL_WEIGHTS); config.addVariable("rows", rows); loadTestConfiguration(config); http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java index 431be84..9aa3401 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleCategoricalTest.java @@ -58,6 +58,8 @@ public class BivariateScaleCategoricalTest extends AutomatedTestBase @Test public void testScaleCategorical() { + if(shouldSkipTest()) + return; TestConfiguration config = getTestConfiguration(TEST_SCALE_NOMINAL); config.addVariable("rows", rows); @@ -105,6 +107,9 @@ public class BivariateScaleCategoricalTest extends AutomatedTestBase @Test public void testScaleCategoricalWithWeights() { + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration(TEST_SCALE_NOMINAL_WEIGHTS); config.addVariable("rows", rows); loadTestConfiguration(config); http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java index 7c1bbc8..ffeddee 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/BivariateScaleScaleTest.java @@ -55,7 +55,9 @@ public class BivariateScaleScaleTest extends AutomatedTestBase @Test public void testPearsonR() { - + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration(TEST_SCALE_SCALE); config.addVariable("rows", rows); loadTestConfiguration(config); @@ -105,7 +107,9 @@ public class BivariateScaleScaleTest extends AutomatedTestBase @Test public void testPearsonRWithWeights() { - + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration(TEST_SCALE_SCALE_WEIGHTS); config.addVariable("rows", rows); loadTestConfiguration(config); http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java index 393ca36..a2db13b 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/OrderStatisticsTest.java @@ -120,6 +120,9 @@ public class OrderStatisticsTest extends AutomatedTestBase @Test public void testQuantile() { + if(shouldSkipTest()) + return; + int rows1 = 10; int rows2 = 5; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java index 9d960d7..e06be77 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateCategoricalTest.java @@ -40,6 +40,9 @@ public class UnivariateCategoricalTest extends UnivariateStatsBase @Test public void testCategoricalWithR() { + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration("Categorical"); config.addVariable("rows1", rows1); loadTestConfiguration(config); @@ -92,6 +95,9 @@ public class UnivariateCategoricalTest extends UnivariateStatsBase @Test public void testWeightedCategoricalWithR() { + if(shouldSkipTest()) + return; + TestConfiguration config = getTestConfiguration("WeightedCategoricalTest"); config.addVariable("rows1", rows1); loadTestConfiguration(config); http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java index 5f2b6aa..720145c 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/descriptivestats/UnivariateStatsBase.java @@ -138,8 +138,9 @@ public abstract class UnivariateStatsBase extends AutomatedTestBase { * backend platform to test */ protected void testScaleWithR(SIZE sz, RANGE rng, SPARSITY sp, RUNTIME_PLATFORM rt) { - RUNTIME_PLATFORM oldrt = rtplatform; - rtplatform = rt; + RUNTIME_PLATFORM oldrt = setRuntimePlatform(rt); + if(shouldSkipTest()) + return; try { TestConfiguration config = getTestConfiguration("Scale"); @@ -215,8 +216,9 @@ public abstract class UnivariateStatsBase extends AutomatedTestBase { protected void testWeightedScaleWithR(SIZE sz, RANGE rng, SPARSITY sp, RUNTIME_PLATFORM rt) { - RUNTIME_PLATFORM oldrt = rtplatform; - rtplatform = rt; + RUNTIME_PLATFORM oldrt = setRuntimePlatform(rt); + if(shouldSkipTest()) + return; try { TestConfiguration config = getTestConfiguration("WeightedScaleTest"); http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java index 2fd1e7d..fee028b 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java @@ -160,16 +160,14 @@ public class ScalableDecompositionTest extends AutomatedTestBase private void runKMeansTest(String testname, DecompType type, boolean rewrites, ExecType instType) { + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + RUNTIME_PLATFORM platformOld = setRuntimePlatform(instType); + if(shouldSkipTest()) + return; + boolean oldFlag1 = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; boolean oldFlag2 = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; - RUNTIME_PLATFORM platformOld = rtplatform; - switch( instType ){ - case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; - default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break; - } - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK ) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; + try { http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java index bad8589..e1d2b7f 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForBivariateStatsTest.java @@ -22,7 +22,6 @@ package org.apache.sysml.test.integration.applications.parfor; import java.util.HashMap; import org.junit.Test; - import org.apache.sysml.hops.Hop; import org.apache.sysml.lops.LopProperties.ExecType; import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode; @@ -93,6 +92,10 @@ public class ParForBivariateStatsTest extends AutomatedTestBase */ private void runParForBivariateStatsTest( boolean parallel, PExecMode outer, PExecMode inner, ExecType instType ) { + setRuntimePlatform(instType); + if(shouldSkipTest()) + return; + //inst exec type, influenced via rows int rows = -1; if( instType == ExecType.CP ) http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java index fe53897..59e3be7 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCVMulticlassSVMTest.java @@ -22,7 +22,6 @@ package org.apache.sysml.test.integration.applications.parfor; import java.util.HashMap; import org.junit.Test; - import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysml.test.integration.AutomatedTestBase; @@ -119,6 +118,10 @@ public class ParForCVMulticlassSVMTest extends AutomatedTestBase */ private void runParForMulticlassSVMTest( int scriptNum, boolean sparse ) { + if(shouldSkipTest()) + return; + + TestConfiguration config = getTestConfiguration(TEST_NAME); config.addVariable("rows", rows); config.addVariable("cols", cols); http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java index a7243f5..e55449e 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java @@ -22,7 +22,6 @@ package org.apache.sysml.test.integration.applications.parfor; import java.util.HashMap; import org.junit.Test; - import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.lops.LopProperties.ExecType; @@ -136,9 +135,10 @@ public class ParForCorrelationTest extends AutomatedTestBase */ private void runParForCorrelationTest( boolean parallel, PExecMode outer, PExecMode inner, ExecType instType, boolean profile, boolean debug, boolean statistics ) { - //inst exec type, influenced via rows - RUNTIME_PLATFORM oldPlatform = rtplatform; - rtplatform = (instType==ExecType.MR)? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID; + RUNTIME_PLATFORM oldPlatform = setRuntimePlatform(instType); + if(shouldSkipTest()) + return; + int cols = (instType==ExecType.MR)? cols2 : cols1; //script http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java index 8c08e7f..acc5e08 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTestLarge.java @@ -22,7 +22,6 @@ package org.apache.sysml.test.integration.applications.parfor; import java.util.HashMap; import org.junit.Test; - import org.apache.sysml.hops.Hop; import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; @@ -94,6 +93,9 @@ public class ParForCorrelationTestLarge extends AutomatedTestBase */ private void runParForCorrelationTest( PExecMode outer, PExecMode inner ) { + if(shouldSkipTest()) + return; + //script int scriptNum = -1; if( inner == PExecMode.REMOTE_MR ) scriptNum=2; http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java index 7f4a7aa..6b4d6ef 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForNaiveBayesTest.java @@ -22,7 +22,6 @@ package org.apache.sysml.test.integration.applications.parfor; import java.util.HashMap; import org.junit.Test; - import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.lops.LopProperties.ExecType; import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode; @@ -148,11 +147,12 @@ public class ParForNaiveBayesTest extends AutomatedTestBase */ private void runParForNaiveBayesTest( PExecMode outer, ExecType instType, boolean smallMem, boolean sparse ) { - int cols = (instType==ExecType.MR)? cols2 : cols1; + RUNTIME_PLATFORM oldPlatform = setRuntimePlatform(instType); + if(shouldSkipTest()) + return; - //inst exec type, influenced via rows - RUNTIME_PLATFORM oldPlatform = rtplatform; - rtplatform = (instType==ExecType.MR)? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID; + + int cols = (instType==ExecType.MR)? cols2 : cols1; //determine the script int scriptNum = -1; @@ -212,4 +212,4 @@ public class ParForNaiveBayesTest extends AutomatedTestBase } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/systemml/blob/95bf8cfe/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForSampleTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForSampleTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForSampleTest.java index 5b36acd..3dfee68 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForSampleTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForSampleTest.java @@ -23,7 +23,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map.Entry; -import org.junit.Assert; import org.junit.Test; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; @@ -97,18 +96,11 @@ public class ParForSampleTest extends AutomatedTestBase @SuppressWarnings({ "unchecked" }) private void runParForSampleTest( boolean sparse, ExecType et ) { - RUNTIME_PLATFORM platformOld = rtplatform; - switch( et ){ - case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; - case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; - default: rtplatform = RUNTIME_PLATFORM.HYBRID; break; - } - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK ) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - + RUNTIME_PLATFORM platformOld = setRuntimePlatform(et); + if(shouldSkipTest()) + return; + try { //invocation arguments @@ -136,11 +128,11 @@ public class ParForSampleTest extends AutomatedTestBase MatrixCharacteristics B2mc = readDMLMetaDataFile("B2"); //compare meta data - Assert.assertEquals(new Long(rows), new Long(B1mc.getRows()+B2mc.getRows())); //join full coverage rows - Assert.assertEquals(new Long(cols), new Long(B1mc.getCols())); //full coverage cols - Assert.assertEquals(new Long(cols), new Long(B2mc.getCols())); //full coverage cols - Assert.assertNotEquals(new Long(rows), new Long(B1mc.getRows())); //no sample contains all rows - Assert.assertNotEquals(new Long(rows), new Long(B2mc.getRows())); //no sample contains all rows + assertEquals(new Long(rows), new Long(B1mc.getRows()+B2mc.getRows())); //join full coverage rows + assertEquals(new Long(cols), new Long(B1mc.getCols())); //full coverage cols + assertEquals(new Long(cols), new Long(B2mc.getCols())); //full coverage cols + assertNotEquals(new Long(rows), new Long(B1mc.getRows())); //no sample contains all rows + assertNotEquals(new Long(rows), new Long(B2mc.getRows())); //no sample contains all rows //compare data HashSet<Integer> probe = new HashSet<Integer>(rows); @@ -150,7 +142,7 @@ public class ParForSampleTest extends AutomatedTestBase for( Entry<CellIndex,Double> e : B.entrySet() ) if( e.getKey().column == 1 ) { boolean flag = probe.remove(e.getValue().intValue()); - Assert.assertTrue("Wrong return value for "+e.getKey()+": "+e.getValue(), flag); + assertTrue("Wrong return value for "+e.getKey()+": "+e.getValue(), flag); } } finally
