Merge pull request #380 from myui/jira/HIVEMALL-12 jira/hivemall-12
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/28909548 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/28909548 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/28909548 Branch: refs/heads/master Commit: 289095482f5a932472f1e86f30f7658a12b50753 Parents: 9a6d1c0 0268609 Author: Makoto YUI <[email protected]> Authored: Sat Nov 12 15:04:24 2016 +0900 Committer: GitHub <[email protected]> Committed: Sat Nov 12 15:04:24 2016 +0900 ---------------------------------------------------------------------- DISCLAIMER | 1 + NOTICE | 36 ++- README.md | 23 +- bin/build.sh | 18 ++ bin/build_site.sh | 18 ++ bin/build_xgboost.sh | 27 +-- bin/format_header.sh | 44 ++++ bin/maven_central_release.sh | 18 ++ bin/mixserv_cluster.sh | 27 +-- bin/mixserv_daemon.sh | 27 +-- bin/start_mixserv.sh | 27 +-- bin/stop_mixserv.sh | 27 +-- core/pom.xml | 42 ++-- .../main/java/hivemall/HivemallConstants.java | 28 +-- .../main/java/hivemall/HivemallVersionUDF.java | 28 +-- .../src/main/java/hivemall/LearnerBaseUDTF.java | 28 +-- core/src/main/java/hivemall/UDFWithOptions.java | 28 +-- .../src/main/java/hivemall/UDTFWithOptions.java | 28 +-- .../java/hivemall/anomaly/ChangeFinder1D.java | 27 +-- .../java/hivemall/anomaly/ChangeFinder2D.java | 27 +-- .../java/hivemall/anomaly/ChangeFinderUDF.java | 27 +-- core/src/main/java/hivemall/anomaly/SDAR1D.java | 27 +-- core/src/main/java/hivemall/anomaly/SDAR2D.java | 27 +-- .../hivemall/classifier/AROWClassifierUDTF.java | 28 +-- .../hivemall/classifier/AdaGradRDAUDTF.java | 28 +-- .../classifier/BinaryOnlineClassifierUDTF.java | 28 +-- .../classifier/ConfidenceWeightedUDTF.java | 28 +-- .../classifier/PassiveAggressiveUDTF.java | 28 +-- .../hivemall/classifier/PerceptronUDTF.java | 28 +-- .../classifier/SoftConfideceWeightedUDTF.java | 28 +-- .../MulticlassAROWClassifierUDTF.java | 28 +-- .../MulticlassConfidenceWeightedUDTF.java | 28 +-- .../MulticlassOnlineClassifierUDTF.java | 28 +-- .../MulticlassPassiveAggressiveUDTF.java | 28 +-- .../multiclass/MulticlassPerceptronUDTF.java | 28 +-- .../MulticlassSoftConfidenceWeightedUDTF.java | 28 +-- .../java/hivemall/common/ConversionState.java | 28 +-- .../main/java/hivemall/common/EtaEstimator.java | 28 +-- .../java/hivemall/common/LossFunctions.java | 28 +-- .../java/hivemall/common/OnlineVariance.java | 28 +-- .../hivemall/common/RandomizedAmplifier.java | 28 +-- .../java/hivemall/common/ReservoirSampler.java | 28 +-- .../LogisticRegressionDataGeneratorUDTF.java | 28 +-- .../hivemall/ensemble/ArgminKLDistanceUDAF.java | 28 +-- .../main/java/hivemall/ensemble/MaxRowUDAF.java | 43 ++-- .../hivemall/ensemble/MaxValueLabelUDAF.java | 28 +-- .../hivemall/ensemble/bagging/VotedAvgUDAF.java | 28 +-- .../ensemble/bagging/WeightVotedAvgUDAF.java | 28 +-- .../main/java/hivemall/evaluation/AUCUDAF.java | 27 +-- .../evaluation/BinaryResponsesMeasures.java | 28 +-- .../java/hivemall/evaluation/FMeasureUDAF.java | 28 +-- .../evaluation/GradedResponsesMeasures.java | 28 +-- .../evaluation/LogarithmicLossUDAF.java | 27 +-- .../main/java/hivemall/evaluation/MAPUDAF.java | 27 +-- .../main/java/hivemall/evaluation/MRRUDAF.java | 27 +-- .../evaluation/MeanAbsoluteErrorUDAF.java | 28 +-- .../evaluation/MeanSquaredErrorUDAF.java | 28 +-- .../main/java/hivemall/evaluation/NDCGUDAF.java | 27 +-- .../java/hivemall/evaluation/PrecisionUDAF.java | 27 +-- .../main/java/hivemall/evaluation/R2UDAF.java | 28 +-- .../java/hivemall/evaluation/RecallUDAF.java | 27 +-- .../evaluation/RootMeanSquaredErrorUDAF.java | 28 +-- core/src/main/java/hivemall/fm/Entry.java | 28 +-- .../main/java/hivemall/fm/FFMPredictUDF.java | 28 +-- .../java/hivemall/fm/FFMPredictionModel.java | 28 +-- .../hivemall/fm/FFMStringFeatureMapModel.java | 28 +-- .../src/main/java/hivemall/fm/FMArrayModel.java | 28 +-- .../java/hivemall/fm/FMHyperParameters.java | 27 +-- .../java/hivemall/fm/FMIntFeatureMapModel.java | 28 +-- .../java/hivemall/fm/FMPredictGenericUDAF.java | 28 +-- .../hivemall/fm/FMStringFeatureMapModel.java | 28 +-- .../hivemall/fm/FactorizationMachineModel.java | 28 +-- .../hivemall/fm/FactorizationMachineUDTF.java | 28 +-- core/src/main/java/hivemall/fm/Feature.java | 28 +-- .../fm/FieldAwareFactorizationMachineModel.java | 28 +-- .../fm/FieldAwareFactorizationMachineUDTF.java | 28 +-- core/src/main/java/hivemall/fm/IntFeature.java | 28 +-- .../main/java/hivemall/fm/StringFeature.java | 28 +-- .../main/java/hivemall/ftvec/AddBiasUDF.java | 28 +-- .../java/hivemall/ftvec/AddFeatureIndexUDF.java | 28 +-- .../java/hivemall/ftvec/ExtractFeatureUDF.java | 28 +-- .../java/hivemall/ftvec/ExtractWeightUDF.java | 28 +-- .../java/hivemall/ftvec/FeatureIndexUDF.java | 28 +-- .../main/java/hivemall/ftvec/FeatureUDF.java | 27 +-- .../java/hivemall/ftvec/SortByFeatureUDF.java | 28 +-- .../hivemall/ftvec/amplify/AmplifierUDTF.java | 28 +-- .../ftvec/amplify/RandomAmplifierUDTF.java | 28 +-- .../hivemall/ftvec/binning/BuildBinsUDAF.java | 28 +-- .../ftvec/binning/FeatureBinningUDF.java | 27 +-- .../ftvec/binning/NumericHistogram.java | 27 +-- .../ftvec/conv/ConvertToDenseModelUDAF.java | 28 +-- .../ftvec/conv/QuantifyColumnsUDTF.java | 28 +-- .../hivemall/ftvec/conv/ToDenseFeaturesUDF.java | 27 +-- .../ftvec/conv/ToSparseFeaturesUDF.java | 27 +-- .../ftvec/hashing/ArrayHashValuesUDF.java | 28 +-- .../hashing/ArrayPrefixedHashValuesUDF.java | 28 +-- .../ftvec/hashing/FeatureHashingUDF.java | 27 +-- .../hivemall/ftvec/hashing/MurmurHash3UDF.java | 28 +-- .../java/hivemall/ftvec/hashing/Sha1UDF.java | 28 +-- .../ftvec/pairing/PolynomialFeaturesUDF.java | 28 +-- .../ftvec/pairing/PoweredFeaturesUDF.java | 28 +-- .../hivemall/ftvec/ranking/BprSamplingUDTF.java | 28 +-- .../ftvec/ranking/ItemPairsSamplingUDTF.java | 28 +-- .../ranking/PerEventPositiveOnlyFeedback.java | 28 +-- .../ftvec/ranking/PopulateNotInUDTF.java | 28 +-- .../ftvec/ranking/PositiveOnlyFeedback.java | 28 +-- .../ftvec/scaling/L2NormalizationUDF.java | 28 +-- .../java/hivemall/ftvec/scaling/RescaleUDF.java | 28 +-- .../java/hivemall/ftvec/scaling/ZScoreUDF.java | 28 +-- .../hivemall/ftvec/text/TermFrequencyUDAF.java | 28 +-- .../hivemall/ftvec/trans/BinarizeLabelUDTF.java | 27 +-- .../ftvec/trans/CategoricalFeaturesUDF.java | 28 +-- .../hivemall/ftvec/trans/FFMFeaturesUDF.java | 28 +-- .../hivemall/ftvec/trans/IndexedFeatures.java | 27 +-- .../ftvec/trans/OnehotEncodingUDAF.java | 27 +-- .../ftvec/trans/QuantifiedFeaturesUDTF.java | 28 +-- .../ftvec/trans/QuantitativeFeaturesUDF.java | 28 +-- .../ftvec/trans/VectorizeFeaturesUDF.java | 28 +-- .../knn/distance/AngularDistanceUDF.java | 28 +-- .../knn/distance/CosineDistanceUDF.java | 28 +-- .../knn/distance/EuclidDistanceUDF.java | 28 +-- .../knn/distance/HammingDistanceUDF.java | 28 +-- .../knn/distance/JaccardDistanceUDF.java | 28 +-- .../hivemall/knn/distance/KLDivergenceUDF.java | 28 +-- .../knn/distance/ManhattanDistanceUDF.java | 28 +-- .../knn/distance/MinkowskiDistanceUDF.java | 28 +-- .../java/hivemall/knn/distance/PopcountUDF.java | 28 +-- .../main/java/hivemall/knn/lsh/MinHashUDTF.java | 28 +-- .../java/hivemall/knn/lsh/MinHashesUDF.java | 28 +-- .../java/hivemall/knn/lsh/bBitMinHashUDF.java | 28 +-- .../knn/similarity/AngularSimilarityUDF.java | 18 ++ .../knn/similarity/CosineSimilarityUDF.java | 28 +-- .../knn/similarity/Distance2SimilarityUDF.java | 28 +-- .../knn/similarity/EuclidSimilarity.java | 28 +-- .../knn/similarity/JaccardIndexUDF.java | 28 +-- .../java/hivemall/mf/BPRMFPredictionUDF.java | 28 +-- .../hivemall/mf/BPRMatrixFactorizationUDTF.java | 28 +-- .../main/java/hivemall/mf/FactorizedModel.java | 28 +-- .../main/java/hivemall/mf/MFPredictionUDF.java | 28 +-- .../mf/MatrixFactorizationAdaGradUDTF.java | 28 +-- .../hivemall/mf/MatrixFactorizationSGDUDTF.java | 28 +-- .../mf/OnlineMatrixFactorizationUDTF.java | 28 +-- core/src/main/java/hivemall/mf/Rating.java | 28 +-- .../main/java/hivemall/mf/RatingInitilizer.java | 28 +-- core/src/main/java/hivemall/mix/MixEnv.java | 28 +-- core/src/main/java/hivemall/mix/MixMessage.java | 28 +-- .../java/hivemall/mix/MixMessageDecoder.java | 28 +-- .../java/hivemall/mix/MixMessageEncoder.java | 28 +-- core/src/main/java/hivemall/mix/MixedModel.java | 28 +-- .../src/main/java/hivemall/mix/MixedWeight.java | 28 +-- core/src/main/java/hivemall/mix/NodeInfo.java | 28 +-- .../java/hivemall/mix/client/MixClient.java | 28 +-- .../hivemall/mix/client/MixClientHandler.java | 28 +-- .../mix/client/MixClientInitializer.java | 28 +-- .../hivemall/mix/client/MixRequestRouter.java | 28 +-- .../hivemall/model/AbstractPredictionModel.java | 28 +-- .../main/java/hivemall/model/DenseModel.java | 28 +-- .../main/java/hivemall/model/FeatureValue.java | 28 +-- .../main/java/hivemall/model/IWeightValue.java | 28 +-- core/src/main/java/hivemall/model/Margin.java | 28 +-- .../java/hivemall/model/ModelUpdateHandler.java | 28 +-- .../java/hivemall/model/PredictionModel.java | 28 +-- .../java/hivemall/model/PredictionResult.java | 28 +-- .../model/SpaceEfficientDenseModel.java | 28 +-- .../main/java/hivemall/model/SparseModel.java | 28 +-- .../model/SynchronizedModelWrapper.java | 28 +-- .../main/java/hivemall/model/WeightValue.java | 28 +-- .../hivemall/model/WeightValueWithClock.java | 28 +-- .../hivemall/regression/AROWRegressionUDTF.java | 28 +-- .../java/hivemall/regression/AdaDeltaUDTF.java | 28 +-- .../java/hivemall/regression/AdaGradUDTF.java | 28 +-- .../java/hivemall/regression/LogressUDTF.java | 28 +-- .../PassiveAggressiveRegressionUDTF.java | 28 +-- .../hivemall/regression/RegressionBaseUDTF.java | 28 +-- .../src/main/java/hivemall/smile/ModelType.java | 27 +-- .../smile/classification/DecisionTree.java | 27 +-- .../GradientTreeBoostingClassifierUDTF.java | 28 +-- .../RandomForestClassifierUDTF.java | 28 +-- .../java/hivemall/smile/data/Attribute.java | 28 +-- .../regression/RandomForestRegressionUDTF.java | 28 +-- .../smile/regression/RegressionTree.java | 27 +-- .../smile/tools/GuessAttributesUDF.java | 28 +-- .../smile/tools/RandomForestEnsembleUDAF.java | 28 +-- .../hivemall/smile/tools/TreePredictUDF.java | 27 +-- .../hivemall/smile/utils/SmileExtUtils.java | 27 +-- .../hivemall/smile/utils/SmileTaskExecutor.java | 27 +-- .../main/java/hivemall/smile/vm/Operation.java | 27 +-- .../java/hivemall/smile/vm/StackMachine.java | 27 +-- .../hivemall/smile/vm/VMRuntimeException.java | 27 +-- .../java/hivemall/tools/ConvertLabelUDF.java | 28 +-- .../main/java/hivemall/tools/EachTopKUDTF.java | 28 +-- .../java/hivemall/tools/GenerateSeriesUDTF.java | 28 +-- .../java/hivemall/tools/RankSequenceUDF.java | 28 +-- .../tools/array/AllocFloatArrayUDF.java | 28 +-- .../tools/array/ArrayAvgGenericUDAF.java | 28 +-- .../hivemall/tools/array/ArrayConcatUDF.java | 28 +-- .../hivemall/tools/array/ArrayIntersectUDF.java | 28 +-- .../hivemall/tools/array/ArrayRemoveUDF.java | 28 +-- .../java/hivemall/tools/array/ArraySumUDAF.java | 28 +-- .../hivemall/tools/array/CollectAllUDAF.java | 28 +-- .../tools/array/SortAndUniqArrayUDF.java | 28 +-- .../tools/array/SubarrayEndWithUDF.java | 28 +-- .../tools/array/SubarrayStartWithUDF.java | 28 +-- .../java/hivemall/tools/array/SubarrayUDF.java | 28 +-- .../hivemall/tools/array/ToStringArrayUDF.java | 28 +-- .../hivemall/tools/bits/BitsCollectUDAF.java | 28 +-- .../java/hivemall/tools/bits/BitsORUDF.java | 28 +-- .../java/hivemall/tools/bits/ToBitsUDF.java | 18 ++ .../java/hivemall/tools/bits/UnBitsUDF.java | 18 ++ .../hivemall/tools/compress/DeflateUDF.java | 28 +-- .../hivemall/tools/compress/InflateUDF.java | 28 +-- .../java/hivemall/tools/map/MapGetSumUDF.java | 28 +-- .../java/hivemall/tools/map/MapTailNUDF.java | 28 +-- .../main/java/hivemall/tools/map/UDAFToMap.java | 28 +-- .../hivemall/tools/map/UDAFToOrderedMap.java | 28 +-- .../tools/mapred/DistributedCacheLookupUDF.java | 28 +-- .../hivemall/tools/mapred/JobConfGetsUDF.java | 28 +-- .../java/hivemall/tools/mapred/JobIdUDF.java | 28 +-- .../java/hivemall/tools/mapred/RowIdUDF.java | 28 +-- .../java/hivemall/tools/mapred/TaskIdUDF.java | 28 +-- .../hivemall/tools/math/SigmoidGenericUDF.java | 28 +-- .../java/hivemall/tools/text/Base91UDF.java | 27 +-- .../tools/text/NormalizeUnicodeUDF.java | 28 +-- .../java/hivemall/tools/text/SplitWordsUDF.java | 28 +-- .../java/hivemall/tools/text/StopwordUDF.java | 28 +-- .../java/hivemall/tools/text/TokenizeUDF.java | 28 +-- .../java/hivemall/tools/text/Unbase91UDF.java | 27 +-- .../main/java/hivemall/utils/HivemallUtils.java | 28 +-- .../hivemall/utils/buffer/DynamicByteArray.java | 30 +-- .../java/hivemall/utils/buffer/HeapBuffer.java | 27 +-- .../main/java/hivemall/utils/codec/Base91.java | 27 +-- .../hivemall/utils/codec/CompressionCodec.java | 28 +-- .../java/hivemall/utils/codec/DeflateCodec.java | 28 +-- .../hivemall/utils/codec/VariableByteCodec.java | 27 +-- .../hivemall/utils/codec/ZigZagLEB128Codec.java | 27 +-- .../utils/collections/BoundedPriorityQueue.java | 28 +-- .../utils/collections/DoubleArray3D.java | 28 +-- .../utils/collections/DoubleArrayList.java | 27 +-- .../utils/collections/DoubleRingBuffer.java | 27 +-- .../utils/collections/FixedIntArray.java | 27 +-- .../utils/collections/IMapIterator.java | 28 +-- .../hivemall/utils/collections/IndexedSet.java | 28 +-- .../collections/Int2FloatOpenHashTable.java | 28 +-- .../utils/collections/Int2IntOpenHashTable.java | 28 +-- .../collections/Int2LongOpenHashTable.java | 28 +-- .../hivemall/utils/collections/IntArray.java | 27 +-- .../utils/collections/IntArrayList.java | 27 +-- .../utils/collections/IntOpenHashMap.java | 28 +-- .../utils/collections/IntOpenHashTable.java | 28 +-- .../java/hivemall/utils/collections/LRUMap.java | 28 +-- .../hivemall/utils/collections/OpenHashMap.java | 28 +-- .../utils/collections/OpenHashTable.java | 28 +-- .../hivemall/utils/collections/RingBuffer.java | 27 +-- .../utils/collections/SparseIntArray.java | 27 +-- .../utils/concurrent/ExecutorFactory.java | 27 +-- .../utils/concurrent/NamedThreadFactory.java | 27 +-- .../utils/datetime/DateTimeFormatter.java | 28 +-- .../java/hivemall/utils/datetime/StopWatch.java | 28 +-- .../java/hivemall/utils/hadoop/HadoopUtils.java | 28 +-- .../java/hivemall/utils/hadoop/HiveUtils.java | 28 +-- .../main/java/hivemall/utils/hadoop/Text2.java | 27 +-- .../main/java/hivemall/utils/hadoop/Text3.java | 27 +-- .../hivemall/utils/hadoop/WritableUtils.java | 28 +-- .../hivemall/utils/hashing/HashFunction.java | 28 +-- .../utils/hashing/HashFunctionFactory.java | 28 +-- .../hivemall/utils/hashing/MurmurHash3.java | 28 +-- .../utils/hashing/MurmurHash3Function.java | 28 +-- .../hivemall/utils/io/ASCII85InputStream.java | 229 ------------------- .../hivemall/utils/io/ASCII85OutputStream.java | 150 ------------ .../hivemall/utils/io/Base91InputStream.java | 27 +-- .../hivemall/utils/io/Base91OutputStream.java | 27 +-- .../utils/io/CompressionStreamFactory.java | 28 +-- .../hivemall/utils/io/DeflaterOutputStream.java | 27 +-- .../utils/io/FastByteArrayInputStream.java | 28 +-- .../utils/io/FastByteArrayOutputStream.java | 28 +-- .../io/FastMultiByteArrayOutputStream.java | 28 +-- .../main/java/hivemall/utils/io/FileUtils.java | 28 +-- .../utils/io/FinishableOutputStream.java | 28 +-- .../utils/io/FinishableOutputStreamAdapter.java | 27 +-- .../main/java/hivemall/utils/io/IOUtils.java | 28 +-- .../main/java/hivemall/utils/io/NIOUtils.java | 28 +-- .../java/hivemall/utils/io/NioFixedSegment.java | 28 +-- .../main/java/hivemall/utils/io/NioSegment.java | 28 +-- .../hivemall/utils/io/NioStatefullSegment.java | 28 +-- .../main/java/hivemall/utils/io/Segments.java | 28 +-- .../java/hivemall/utils/lang/ArrayUtils.java | 28 +-- .../main/java/hivemall/utils/lang/BitUtils.java | 28 +-- .../hivemall/utils/lang/CommandLineUtils.java | 28 +-- .../main/java/hivemall/utils/lang/Copyable.java | 28 +-- .../main/java/hivemall/utils/lang/Counter.java | 28 +-- .../hivemall/utils/lang/FloatAccumulator.java | 28 +-- .../java/hivemall/utils/lang/HalfFloat.java | 28 +-- .../java/hivemall/utils/lang/Identifier.java | 28 +-- .../java/hivemall/utils/lang/NumberUtils.java | 28 +-- .../java/hivemall/utils/lang/ObjectUtils.java | 28 +-- .../java/hivemall/utils/lang/Preconditions.java | 27 +-- .../java/hivemall/utils/lang/Primitives.java | 28 +-- .../hivemall/utils/lang/PrivilegedAccessor.java | 28 +-- .../java/hivemall/utils/lang/RandomUtils.java | 27 +-- .../main/java/hivemall/utils/lang/SizeOf.java | 28 +-- .../java/hivemall/utils/lang/StringUtils.java | 28 +-- .../java/hivemall/utils/lang/UnsafeUtils.java | 27 +-- .../utils/lang/mutable/MutableDouble.java | 28 +-- .../utils/lang/mutable/MutableFloat.java | 28 +-- .../hivemall/utils/lang/mutable/MutableInt.java | 28 +-- .../utils/lang/mutable/MutableLong.java | 28 +-- .../src/main/java/hivemall/utils/lock/Lock.java | 28 +-- .../main/java/hivemall/utils/lock/TTASLock.java | 28 +-- .../java/hivemall/utils/math/MathUtils.java | 28 +-- .../java/hivemall/utils/math/MatrixUtils.java | 27 +-- .../main/java/hivemall/utils/math/Primes.java | 28 +-- .../java/hivemall/utils/math/StatsUtils.java | 28 +-- .../main/java/hivemall/utils/net/NetUtils.java | 28 +-- .../hive/ql/exec/MapredContextAccessor.java | 28 +-- .../hivemall/anomaly/ChangeFinder1DTest.java | 27 +-- .../hivemall/anomaly/ChangeFinder2DTest.java | 27 +-- .../classifier/PassiveAggressiveUDTFTest.java | 28 +-- .../hivemall/classifier/PerceptronUDTFTest.java | 28 +-- .../hivemall/common/OnlineVarianceTest.java | 28 +-- .../common/RandomizedAmplifierTest.java | 28 +-- .../evaluation/BinaryResponsesMeasuresTest.java | 27 +-- .../evaluation/GradedResponsesMeasuresTest.java | 27 +-- .../test/java/hivemall/fm/ArrayModelTest.java | 28 +-- .../hivemall/fm/FFMPredictionModelTest.java | 27 +-- .../fm/FactorizationMachineUDTFTest.java | 18 ++ core/src/test/java/hivemall/fm/FeatureTest.java | 27 +-- .../FieldAwareFactorizationMachineUDTFTest.java | 28 +-- .../hivemall/fm/IntFeatureMapModelTest.java | 28 +-- .../hivemall/fm/StringFeatureMapModelTest.java | 28 +-- .../hivemall/ftvec/ExtractWeightUDFTest.java | 27 +-- .../java/hivemall/ftvec/FeatureUDFTest.java | 27 +-- .../ftvec/hashing/FeatureHashingUDFTest.java | 28 +-- .../ftvec/hashing/MurmurHash3UDFTest.java | 28 +-- .../hivemall/ftvec/hashing/Sha1UDFTest.java | 28 +-- .../pairing/PolynomialFeaturesUDFTest.java | 28 +-- .../ftvec/pairing/PoweredFeaturesUDFTest.java | 28 +-- .../ftvec/scaling/L2NormalizationUDFTest.java | 28 +-- .../ftvec/trans/TestBinarizeLabelUDTF.java | 27 +-- .../ftvec/trans/VectorizeFeaturesUDFTest.java | 28 +-- .../knn/distance/EuclidDistanceUDFTest.java | 28 +-- .../java/hivemall/knn/lsh/MinHashUDFTest.java | 28 +-- .../hivemall/knn/lsh/bBitMinHashUDFTest.java | 28 +-- .../knn/similarity/CosineSimilarityUDFTest.java | 28 +-- .../mf/BPRMatrixFactorizationUDTFTest.java | 28 +-- .../mf/MatrixFactorizationAdaGradUDTFTest.java | 28 +-- .../mf/MatrixFactorizationSGDUDTFTest.java | 28 +-- .../mix/client/MixRequestRouterTest.java | 28 +-- .../java/hivemall/model/FeatureValueTest.java | 28 +-- .../model/SpaceEfficientDenseModelTest.java | 28 +-- .../hivemall/regression/AdaGradUDTFTest.java | 28 +-- .../smile/classification/DecisionTreeTest.java | 28 +-- .../RandomForestClassifierUDTFTest.java | 28 +-- .../smile/regression/RegressionTreeTest.java | 28 +-- .../smile/tools/TreePredictUDFTest.java | 28 +-- .../hivemall/smile/vm/StackMachineTest.java | 28 +-- .../tools/array/ToStringArrayUDFTest.java | 28 +-- .../java/hivemall/utils/ArrayUtilsTest.java | 28 +-- .../test/java/hivemall/utils/BitUtilsTest.java | 28 +-- .../hivemall/utils/buffer/HeapBufferTest.java | 27 +-- .../java/hivemall/utils/codec/Base91Test.java | 27 +-- .../hivemall/utils/codec/DeflateCodecTest.java | 28 +-- .../utils/codec/ZigZagLEB128CodecTest.java | 18 ++ .../utils/collections/DoubleArray3DTest.java | 27 +-- .../utils/collections/DoubleRingBufferTest.java | 27 +-- .../collections/Int2FloatOpenHashMapTest.java | 28 +-- .../collections/Int2LongOpenHashMapTest.java | 28 +-- .../utils/collections/IntOpenHashMapTest.java | 28 +-- .../utils/collections/IntOpenHashTableTest.java | 28 +-- .../utils/collections/OpenHashMapTest.java | 28 +-- .../utils/collections/OpenHashTableTest.java | 28 +-- .../utils/collections/SparseIntArrayTest.java | 18 ++ .../hivemall/utils/hadoop/HadoopUtilsTest.java | 28 +-- .../java/hivemall/utils/hadoop/Text2Test.java | 28 +-- .../hivemall/utils/hashing/MurmurHash3Test.java | 28 +-- .../utils/io/Base91OutputStreamTest.java | 27 +-- .../java/hivemall/utils/lang/HalfFloatTest.java | 28 +-- .../hivemall/utils/lang/NumberUtilsTest.java | 28 +-- .../hivemall/utils/math/MatrixUtilsTest.java | 27 +-- docs/gitbook/README.md | 19 ++ docs/gitbook/SUMMARY.md | 19 ++ docs/gitbook/anomaly/lof.md | 19 ++ docs/gitbook/binaryclass/a9a.md | 19 ++ docs/gitbook/binaryclass/a9a_dataset.md | 19 ++ docs/gitbook/binaryclass/a9a_lr.md | 19 ++ docs/gitbook/binaryclass/a9a_minibatch.md | 19 ++ docs/gitbook/binaryclass/kdd2010a.md | 19 ++ docs/gitbook/binaryclass/kdd2010a_dataset.md | 19 ++ docs/gitbook/binaryclass/kdd2010a_scw.md | 19 ++ docs/gitbook/binaryclass/kdd2010b.md | 19 ++ docs/gitbook/binaryclass/kdd2010b_arow.md | 19 ++ docs/gitbook/binaryclass/kdd2010b_dataset.md | 19 ++ docs/gitbook/binaryclass/news20.md | 19 ++ docs/gitbook/binaryclass/news20_adagrad.md | 19 ++ docs/gitbook/binaryclass/news20_dataset.md | 19 ++ docs/gitbook/binaryclass/news20_pa.md | 19 ++ docs/gitbook/binaryclass/news20_scw.md | 19 ++ docs/gitbook/binaryclass/webspam.md | 19 ++ docs/gitbook/binaryclass/webspam_dataset.md | 19 ++ docs/gitbook/binaryclass/webspam_scw.md | 19 ++ docs/gitbook/eval/datagen.md | 19 ++ docs/gitbook/eval/lr_datagen.md | 19 ++ docs/gitbook/eval/stat_eval.md | 19 ++ docs/gitbook/ft_engineering/ft_trans.md | 19 ++ docs/gitbook/ft_engineering/hashing.md | 19 ++ docs/gitbook/ft_engineering/quantify.md | 19 ++ docs/gitbook/ft_engineering/scaling.md | 19 ++ docs/gitbook/ft_engineering/tfidf.md | 19 ++ docs/gitbook/ft_engineering/vectorizer.md | 19 ++ docs/gitbook/getting_started/README.md | 19 ++ docs/gitbook/getting_started/input-format.md | 19 ++ docs/gitbook/getting_started/installation.md | 19 ++ .../getting_started/permanent-functions.md | 19 ++ docs/gitbook/misc/generic_funcs.md | 19 ++ docs/gitbook/misc/tokenizer.md | 19 ++ docs/gitbook/misc/topk.md | 19 ++ docs/gitbook/multiclass/iris.md | 19 ++ docs/gitbook/multiclass/iris_dataset.md | 19 ++ docs/gitbook/multiclass/iris_randomforest.md | 19 ++ docs/gitbook/multiclass/iris_scw.md | 19 ++ docs/gitbook/multiclass/news20.md | 19 ++ docs/gitbook/multiclass/news20_dataset.md | 19 ++ docs/gitbook/multiclass/news20_ensemble.md | 19 ++ .../multiclass/news20_one-vs-the-rest.md | 19 ++ .../news20_one-vs-the-rest_dataset.md | 19 ++ docs/gitbook/multiclass/news20_pa.md | 19 ++ docs/gitbook/multiclass/news20_scw.md | 19 ++ docs/gitbook/recommend/cf.md | 19 ++ docs/gitbook/recommend/item_based_cf.md | 19 ++ docs/gitbook/recommend/movielens.md | 19 ++ docs/gitbook/recommend/movielens_cv.md | 19 ++ docs/gitbook/recommend/movielens_dataset.md | 19 ++ docs/gitbook/recommend/movielens_fm.md | 19 ++ docs/gitbook/recommend/movielens_mf.md | 19 ++ docs/gitbook/recommend/news20.md | 19 ++ docs/gitbook/recommend/news20_bbit_minhash.md | 19 ++ docs/gitbook/recommend/news20_jaccard.md | 19 ++ docs/gitbook/recommend/news20_knn.md | 19 ++ docs/gitbook/regression/e2006.md | 19 ++ docs/gitbook/regression/e2006_arow.md | 19 ++ docs/gitbook/regression/e2006_dataset.md | 19 ++ docs/gitbook/regression/kddcup12tr2.md | 19 ++ docs/gitbook/regression/kddcup12tr2_adagrad.md | 19 ++ docs/gitbook/regression/kddcup12tr2_dataset.md | 19 ++ docs/gitbook/regression/kddcup12tr2_lr.md | 19 ++ .../regression/kddcup12tr2_lr_amplify.md | 19 ++ docs/gitbook/tips/README.md | 19 ++ docs/gitbook/tips/addbias.md | 19 ++ docs/gitbook/tips/emr.md | 19 ++ docs/gitbook/tips/ensemble_learning.md | 19 ++ docs/gitbook/tips/general_tips.md | 19 ++ docs/gitbook/tips/hadoop_tuning.md | 19 ++ docs/gitbook/tips/mixserver.md | 19 ++ docs/gitbook/tips/rand_amplify.md | 19 ++ docs/gitbook/tips/rowid.md | 19 ++ docs/gitbook/tips/rt_prediction.md | 19 ++ docs/gitbook/troubleshooting/README.md | 19 ++ docs/gitbook/troubleshooting/asterisk.md | 19 ++ .../troubleshooting/mapjoin_classcastex.md | 19 ++ .../troubleshooting/mapjoin_task_error.md | 19 ++ docs/gitbook/troubleshooting/num_mappers.md | 19 ++ docs/gitbook/troubleshooting/oom.md | 19 ++ mixserv/pom.xml | 42 ++-- .../hivemall/mix/metrics/MetricsRegistry.java | 28 +-- .../hivemall/mix/metrics/MixServerMetrics.java | 28 +-- .../mix/metrics/MixServerMetricsMBean.java | 28 +-- .../hivemall/mix/metrics/ThroughputCounter.java | 28 +-- .../java/hivemall/mix/server/MixServer.java | 28 +-- .../hivemall/mix/server/MixServerHandler.java | 28 +-- .../mix/server/MixServerInitializer.java | 28 +-- .../hivemall/mix/store/PartialArgminKLD.java | 28 +-- .../java/hivemall/mix/store/PartialAverage.java | 28 +-- .../java/hivemall/mix/store/PartialResult.java | 28 +-- .../java/hivemall/mix/store/SessionObject.java | 28 +-- .../java/hivemall/mix/store/SessionStore.java | 28 +-- .../mix/server/MixServerHandlerTest.java | 28 +-- .../java/hivemall/mix/server/MixServerTest.java | 28 +-- .../hivemall/mix/server/PartialResultTest.java | 28 +-- .../java/hivemall/test/HivemallTestBase.java | 28 +-- nlp/pom.xml | 42 ++-- .../hivemall/nlp/tokenizer/KuromojiUDF.java | 28 +-- .../hivemall/nlp/tokenizer/KuromojiUDFTest.java | 28 +-- pom.xml | 68 +++++- resources/header-definition.xml | 11 + resources/license-header.txt | 28 +-- .../org/apache/spark/sql/hive/HiveShim.scala | 27 +-- spark/spark-1.6/pom.xml | 19 ++ .../hivemall/tools/RegressionDatagen.scala | 27 +-- .../apache/spark/sql/hive/GroupedDataEx.scala | 27 +-- .../org/apache/spark/sql/hive/HivemallOps.scala | 27 +-- .../apache/spark/sql/hive/HivemallUtils.scala | 27 +-- .../hivemall/mix/server/MixServerSuite.scala | 27 +-- .../hivemall/tools/RegressionDatagenSuite.scala | 27 +-- .../scala/org/apache/spark/SparkFunSuite.scala | 27 +-- .../ml/feature/HivemallLabeledPointSuite.scala | 27 +-- .../scala/org/apache/spark/sql/QueryTest.scala | 27 +-- .../spark/sql/catalyst/plans/PlanTest.scala | 27 +-- .../apache/spark/sql/hive/HiveUdfSuite.scala | 27 +-- .../spark/sql/hive/HivemallOpsSuite.scala | 27 +-- .../spark/sql/hive/ModelMixingSuite.scala | 27 +-- .../spark/streaming/HivemallOpsSuite.scala | 27 +-- .../apache/spark/test/HivemallQueryTest.scala | 27 +-- .../scala/org/apache/spark/test/TestUtils.scala | 27 +-- .../org/apache/spark/sql/hive/HiveShim.scala | 27 +-- spark/spark-2.0/pom.xml | 19 ++ .../java/hivemall/xgboost/XGBoostOptions.scala | 27 +-- .../XGBoostBinaryClassifierUDTFWrapper.java | 27 +-- .../XGBoostMulticlassClassifierUDTFWrapper.java | 27 +-- .../main/java/hivemall/xgboost/package.scala | 29 ++- .../XGBoostRegressionUDTFWrapper.java | 27 +-- .../hivemall/tools/RegressionDatagen.scala | 27 +-- .../sql/catalyst/expressions/EachTopK.scala | 27 +-- .../spark/sql/hive/HivemallGroupedDataset.scala | 27 +-- .../org/apache/spark/sql/hive/HivemallOps.scala | 27 +-- .../apache/spark/sql/hive/HivemallUtils.scala | 27 +-- .../sql/hive/source/XGBoostFileFormat.scala | 27 +-- .../hivemall/mix/server/MixServerSuite.scala | 27 +-- .../hivemall/tools/RegressionDatagenSuite.scala | 27 +-- .../scala/org/apache/spark/SparkFunSuite.scala | 27 +-- .../ml/feature/HivemallLabeledPointSuite.scala | 27 +-- .../scala/org/apache/spark/sql/QueryTest.scala | 27 +-- .../spark/sql/catalyst/plans/PlanTest.scala | 27 +-- .../apache/spark/sql/hive/HiveUdfSuite.scala | 27 +-- .../spark/sql/hive/HivemallOpsSuite.scala | 27 +-- .../spark/sql/hive/ModelMixingSuite.scala | 27 +-- .../apache/spark/sql/hive/XGBoostSuite.scala | 27 +-- .../sql/hive/benchmark/MiscBenchmark.scala | 27 +-- .../spark/sql/hive/test/TestHiveSingleton.scala | 27 +-- .../streaming/HivemallFeatureOpsSuite.scala | 27 +-- .../spark/test/HivemallFeatureQueryTest.scala | 27 +-- .../scala/org/apache/spark/test/TestUtils.scala | 27 +-- .../org/apache/spark/test/VectorQueryTest.scala | 27 +-- spark/spark-common/pom.xml | 19 ++ spark/spark-common/scalastyle-config.xml | 60 ++--- ...isticRegressionDataGeneratorUDTFWrapper.java | 27 +-- .../java/hivemall/ftvec/AddBiasUDFWrapper.java | 27 +-- .../ftvec/AddFeatureIndexUDFWrapper.java | 27 +-- .../ftvec/ExtractFeatureUDFWrapper.java | 27 +-- .../hivemall/ftvec/ExtractWeightUDFWrapper.java | 27 +-- .../hivemall/ftvec/SortByFeatureUDFWrapper.java | 27 +-- .../scaling/L2NormalizationUDFWrapper.java | 27 +-- .../hivemall/knn/lsh/MinHashesUDFWrapper.java | 27 +-- .../hivemall/tools/mapred/RowIdUDFWrapper.java | 27 +-- .../main/scala/hivemall/HivemallException.scala | 27 +-- .../spark/ml/feature/HivemallLabeledPoint.scala | 27 +-- .../spark/streaming/HivemallStreamingOps.scala | 27 +-- src/site/markdown/404.md | 19 ++ src/site/markdown/contributing.md | 19 ++ src/site/markdown/download.md | 19 ++ src/site/markdown/faq.md | 19 ++ src/site/markdown/overview.md | 33 ++- src/site/markdown/poweredby.md | 19 ++ src/site/markdown/repository.md | 33 ++- src/site/markdown/userguide.md | 19 ++ xgboost/pom.xml | 39 ++-- .../java/hivemall/xgboost/NativeLibLoader.java | 28 +-- .../hivemall/xgboost/XGBoostPredictUDTF.java | 28 +-- .../main/java/hivemall/xgboost/XGBoostUDTF.java | 28 +-- .../java/hivemall/xgboost/XGBoostUtils.java | 28 +-- .../XGBoostBinaryClassifierUDTF.java | 28 +-- .../XGBoostMulticlassClassifierUDTF.java | 28 +-- .../regression/XGBoostRegressionUDTF.java | 28 +-- .../tools/XGBoostMulticlassPredictUDTF.java | 28 +-- .../xgboost/tools/XGBoostPredictUDTF.java | 28 +-- 563 files changed, 8484 insertions(+), 6681 deletions(-) ----------------------------------------------------------------------
