Updated userguide for spark and anomaly detection
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/commit/2d3fdeb2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/tree/2d3fdeb2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/diff/2d3fdeb2 Branch: refs/heads/asf-site Commit: 2d3fdeb2c9506f14d98ead6b0c4a63fe53bfee1a Parents: 990fed4 Author: myui <yuin...@gmail.com> Authored: Thu Feb 23 22:10:41 2017 +0900 Committer: myui <yuin...@gmail.com> Committed: Thu Feb 23 22:10:41 2017 +0900 ---------------------------------------------------------------------- userguide/anomaly/changefinder.html | 2145 ++++++++++++++++++ userguide/anomaly/lof.html | 116 +- userguide/anomaly/sst.html | 2131 +++++++++++++++++ userguide/binaryclass/a9a.html | 112 +- userguide/binaryclass/a9a_dataset.html | 112 +- userguide/binaryclass/a9a_lr.html | 112 +- userguide/binaryclass/a9a_minibatch.html | 112 +- userguide/binaryclass/kdd2010a.html | 112 +- userguide/binaryclass/kdd2010a_dataset.html | 112 +- userguide/binaryclass/kdd2010a_scw.html | 112 +- userguide/binaryclass/kdd2010b.html | 112 +- userguide/binaryclass/kdd2010b_arow.html | 112 +- userguide/binaryclass/kdd2010b_dataset.html | 112 +- userguide/binaryclass/news20.html | 112 +- userguide/binaryclass/news20_adagrad.html | 112 +- userguide/binaryclass/news20_dataset.html | 112 +- userguide/binaryclass/news20_pa.html | 112 +- userguide/binaryclass/news20_scw.html | 112 +- userguide/binaryclass/titanic_rf.html | 112 +- userguide/binaryclass/webspam.html | 112 +- userguide/binaryclass/webspam_dataset.html | 112 +- userguide/binaryclass/webspam_scw.html | 112 +- userguide/eval/datagen.html | 112 +- userguide/eval/lr_datagen.html | 112 +- userguide/eval/stat_eval.html | 112 +- userguide/ft_engineering/feature_selection.html | 112 +- userguide/ft_engineering/ft_trans.html | 112 +- userguide/ft_engineering/hashing.html | 112 +- userguide/ft_engineering/quantify.html | 112 +- userguide/ft_engineering/scaling.html | 112 +- userguide/ft_engineering/tfidf.html | 112 +- userguide/ft_engineering/vectorizer.html | 112 +- userguide/getting_started/index.html | 112 +- userguide/getting_started/input-format.html | 112 +- userguide/getting_started/installation.html | 112 +- .../getting_started/permanent-functions.html | 112 +- userguide/index.html | 112 +- userguide/misc/generic_funcs.html | 112 +- userguide/misc/tokenizer.html | 112 +- userguide/misc/topk.html | 112 +- userguide/multiclass/iris.html | 112 +- userguide/multiclass/iris_dataset.html | 112 +- userguide/multiclass/iris_randomforest.html | 112 +- userguide/multiclass/iris_scw.html | 112 +- userguide/multiclass/news20.html | 112 +- userguide/multiclass/news20_dataset.html | 112 +- userguide/multiclass/news20_ensemble.html | 112 +- .../multiclass/news20_one-vs-the-rest.html | 112 +- .../news20_one-vs-the-rest_dataset.html | 112 +- userguide/multiclass/news20_pa.html | 112 +- userguide/multiclass/news20_scw.html | 112 +- userguide/recommend/cf.html | 112 +- userguide/recommend/item_based_cf.html | 112 +- userguide/recommend/movielens.html | 112 +- userguide/recommend/movielens_cv.html | 112 +- userguide/recommend/movielens_dataset.html | 112 +- userguide/recommend/movielens_fm.html | 112 +- userguide/recommend/movielens_mf.html | 112 +- userguide/recommend/news20.html | 112 +- userguide/recommend/news20_bbit_minhash.html | 112 +- userguide/recommend/news20_jaccard.html | 112 +- userguide/recommend/news20_knn.html | 112 +- userguide/regression/e2006.html | 112 +- userguide/regression/e2006_arow.html | 112 +- userguide/regression/e2006_dataset.html | 112 +- userguide/regression/kddcup12tr2.html | 112 +- userguide/regression/kddcup12tr2_adagrad.html | 112 +- userguide/regression/kddcup12tr2_dataset.html | 112 +- userguide/regression/kddcup12tr2_lr.html | 112 +- .../regression/kddcup12tr2_lr_amplify.html | 112 +- userguide/search_index.json | 2 +- userguide/sitemap.xml | 5 + userguide/spark/binaryclass/a9a_df.html | 2014 ++++++++++++++++ userguide/spark/binaryclass/index.html | 1954 ++++++++++++++++ userguide/spark/getting_started/README.md | 20 + userguide/spark/getting_started/installation.md | 49 + userguide/spark/misc/misc.html | 134 +- userguide/spark/misc/topk_join.html | 464 ++-- userguide/spark/regression/e2006_df.html | 2018 ++++++++++++++++ userguide/spark/regression/index.md | 18 + userguide/tips/addbias.html | 112 +- userguide/tips/emr.html | 112 +- userguide/tips/ensemble_learning.html | 112 +- userguide/tips/general_tips.html | 112 +- userguide/tips/hadoop_tuning.html | 112 +- userguide/tips/index.html | 112 +- userguide/tips/mixserver.html | 112 +- userguide/tips/rand_amplify.html | 112 +- userguide/tips/rowid.html | 125 +- userguide/tips/rt_prediction.html | 130 +- userguide/troubleshooting/asterisk.html | 112 +- userguide/troubleshooting/index.html | 112 +- .../troubleshooting/mapjoin_classcastex.html | 112 +- .../troubleshooting/mapjoin_task_error.html | 112 +- userguide/troubleshooting/num_mappers.html | 112 +- userguide/troubleshooting/oom.html | 112 +- 96 files changed, 19743 insertions(+), 654 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/2d3fdeb2/userguide/anomaly/changefinder.html ---------------------------------------------------------------------- diff --git a/userguide/anomaly/changefinder.html b/userguide/anomaly/changefinder.html new file mode 100644 index 0000000..104a33d --- /dev/null +++ b/userguide/anomaly/changefinder.html @@ -0,0 +1,2145 @@ + +<!DOCTYPE HTML> +<html lang="" > + <head> + <meta charset="UTF-8"> + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <title>ChangeFinder: Detecting Outlier and Change-Point Simultaneously · Hivemall User Manual</title> + <meta http-equiv="X-UA-Compatible" content="IE=edge" /> + <meta name="description" content=""> + <meta name="generator" content="GitBook 3.2.2"> + + + + + <link rel="stylesheet" href="../gitbook/style.css"> + + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-splitter/splitter.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-etoc/plugin.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-callouts/plugin.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-toggle-chapters/toggle.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-codeblock-filename/block.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-multipart/multipart.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-katex/katex.min.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-emphasize/plugin.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-theme-api/theme-api.css"> + + + + + + + + + + + + + + + + + + + + + + + + <meta name="HandheldFriendly" content="true"/> + <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"> + <meta name="apple-mobile-web-app-capable" content="yes"> + <meta name="apple-mobile-web-app-status-bar-style" content="black"> + <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png"> + <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon"> + + + <link rel="next" href="../spark/binaryclass/" /> + + + <link rel="prev" href="sst.html" /> + + + </head> + <body> + +<div class="book"> + <div class="book-summary"> + + +<div id="book-search-input" role="search"> + <input type="text" placeholder="Type to search" /> +</div> + + + <nav role="navigation"> + + + +<ul class="summary"> + + + + + <li> + <a href="http://hivemall.incubator.apache.org/"; target="_blank" class="custom-link"><i class="fa fa-home"></i> Home</a> + </li> + + + + + <li class="divider"></li> + + + + + <li class="header">TABLE OF CONTENTS</li> + + + + <li class="chapter " data-level="1.1" data-path="../"> + + <a href="../"> + + + <b>1.1.</b> + + Introduction + + </a> + + + + </li> + + <li class="chapter " data-level="1.2" data-path="../getting_started/"> + + <a href="../getting_started/"> + + + <b>1.2.</b> + + Getting Started + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.2.1" data-path="../getting_started/installation.html"> + + <a href="../getting_started/installation.html"> + + + <b>1.2.1.</b> + + Installation + + </a> + + + + </li> + + <li class="chapter " data-level="1.2.2" data-path="../getting_started/permanent-functions.html"> + + <a href="../getting_started/permanent-functions.html"> + + + <b>1.2.2.</b> + + Install as permanent functions + + </a> + + + + </li> + + <li class="chapter " data-level="1.2.3" data-path="../getting_started/input-format.html"> + + <a href="../getting_started/input-format.html"> + + + <b>1.2.3.</b> + + Input Format + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="1.3" data-path="../tips/"> + + <a href="../tips/"> + + + <b>1.3.</b> + + Tips for Effective Hivemall + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.3.1" data-path="../tips/addbias.html"> + + <a href="../tips/addbias.html"> + + + <b>1.3.1.</b> + + Explicit addBias() for better prediction + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.2" data-path="../tips/rand_amplify.html"> + + <a href="../tips/rand_amplify.html"> + + + <b>1.3.2.</b> + + Use rand_amplify() to better prediction results + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.3" data-path="../tips/rt_prediction.html"> + + <a href="../tips/rt_prediction.html"> + + + <b>1.3.3.</b> + + Real-time Prediction on RDBMS + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.4" data-path="../tips/ensemble_learning.html"> + + <a href="../tips/ensemble_learning.html"> + + + <b>1.3.4.</b> + + Ensemble learning for stable prediction + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.5" data-path="../tips/mixserver.html"> + + <a href="../tips/mixserver.html"> + + + <b>1.3.5.</b> + + Mixing models for a better prediction convergence (MIX server) + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.6" data-path="../tips/emr.html"> + + <a href="../tips/emr.html"> + + + <b>1.3.6.</b> + + Run Hivemall on Amazon Elastic MapReduce + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="1.4" data-path="../tips/general_tips.html"> + + <a href="../tips/general_tips.html"> + + + <b>1.4.</b> + + General Hive/Hadoop tips + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.4.1" data-path="../tips/rowid.html"> + + <a href="../tips/rowid.html"> + + + <b>1.4.1.</b> + + Adding rowid for each row + + </a> + + + + </li> + + <li class="chapter " data-level="1.4.2" data-path="../tips/hadoop_tuning.html"> + + <a href="../tips/hadoop_tuning.html"> + + + <b>1.4.2.</b> + + Hadoop tuning for Hivemall + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="1.5" data-path="../troubleshooting/"> + + <a href="../troubleshooting/"> + + + <b>1.5.</b> + + Troubleshooting + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.5.1" data-path="../troubleshooting/oom.html"> + + <a href="../troubleshooting/oom.html"> + + + <b>1.5.1.</b> + + OutOfMemoryError in training + + </a> + + + + </li> + + <li class="chapter " data-level="1.5.2" data-path="../troubleshooting/mapjoin_task_error.html"> + + <a href="../troubleshooting/mapjoin_task_error.html"> + + + <b>1.5.2.</b> + + SemanticException Generate Map Join Task Error: Cannot serialize object + + </a> + + + + </li> + + <li class="chapter " data-level="1.5.3" data-path="../troubleshooting/asterisk.html"> + + <a href="../troubleshooting/asterisk.html"> + + + <b>1.5.3.</b> + + Asterisk argument for UDTF does not work + + </a> + + + + </li> + + <li class="chapter " data-level="1.5.4" data-path="../troubleshooting/num_mappers.html"> + + <a href="../troubleshooting/num_mappers.html"> + + + <b>1.5.4.</b> + + The number of mappers is less than input splits in Hadoop 2.x + + </a> + + + + </li> + + <li class="chapter " data-level="1.5.5" data-path="../troubleshooting/mapjoin_classcastex.html"> + + <a href="../troubleshooting/mapjoin_classcastex.html"> + + + <b>1.5.5.</b> + + Map-side Join causes ClassCastException on Tez + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part II - Generic Features</li> + + + + <li class="chapter " data-level="2.1" data-path="../misc/generic_funcs.html"> + + <a href="../misc/generic_funcs.html"> + + + <b>2.1.</b> + + List of generic Hivemall functions + + </a> + + + + </li> + + <li class="chapter " data-level="2.2" data-path="../misc/topk.html"> + + <a href="../misc/topk.html"> + + + <b>2.2.</b> + + Efficient Top-K query processing + + </a> + + + + </li> + + <li class="chapter " data-level="2.3" data-path="../misc/tokenizer.html"> + + <a href="../misc/tokenizer.html"> + + + <b>2.3.</b> + + English/Japanese Text Tokenizer + + </a> + + + + </li> + + + + + <li class="header">Part III - Feature Engineering</li> + + + + <li class="chapter " data-level="3.1" data-path="../ft_engineering/scaling.html"> + + <a href="../ft_engineering/scaling.html"> + + + <b>3.1.</b> + + Feature Scaling + + </a> + + + + </li> + + <li class="chapter " data-level="3.2" data-path="../ft_engineering/hashing.html"> + + <a href="../ft_engineering/hashing.html"> + + + <b>3.2.</b> + + Feature Hashing + + </a> + + + + </li> + + <li class="chapter " data-level="3.3" data-path="../ft_engineering/tfidf.html"> + + <a href="../ft_engineering/tfidf.html"> + + + <b>3.3.</b> + + TF-IDF calculation + + </a> + + + + </li> + + <li class="chapter " data-level="3.4" data-path="../ft_engineering/ft_trans.html"> + + <a href="../ft_engineering/ft_trans.html"> + + + <b>3.4.</b> + + FEATURE TRANSFORMATION + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="3.4.1" data-path="../ft_engineering/vectorizer.html"> + + <a href="../ft_engineering/vectorizer.html"> + + + <b>3.4.1.</b> + + Vectorize Features + + </a> + + + + </li> + + <li class="chapter " data-level="3.4.2" data-path="../ft_engineering/quantify.html"> + + <a href="../ft_engineering/quantify.html"> + + + <b>3.4.2.</b> + + Quantify non-number features + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="3.5" data-path="../ft_engineering/feature_selection.html"> + + <a href="../ft_engineering/feature_selection.html"> + + + <b>3.5.</b> + + Feature selection + + </a> + + + + </li> + + + + + <li class="header">Part IV - Evaluation</li> + + + + <li class="chapter " data-level="4.1" data-path="../eval/stat_eval.html"> + + <a href="../eval/stat_eval.html"> + + + <b>4.1.</b> + + Statistical evaluation of a prediction model + + </a> + + + + </li> + + <li class="chapter " data-level="4.2" data-path="../eval/datagen.html"> + + <a href="../eval/datagen.html"> + + + <b>4.2.</b> + + Data Generation + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="4.2.1" data-path="../eval/lr_datagen.html"> + + <a href="../eval/lr_datagen.html"> + + + <b>4.2.1.</b> + + Logistic Regression data generation + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part V - Binary classification</li> + + + + <li class="chapter " data-level="5.1" data-path="../binaryclass/a9a.html"> + + <a href="../binaryclass/a9a.html"> + + + <b>5.1.</b> + + a9a Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.1.1" data-path="../binaryclass/a9a_dataset.html"> + + <a href="../binaryclass/a9a_dataset.html"> + + + <b>5.1.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.1.2" data-path="../binaryclass/a9a_lr.html"> + + <a href="../binaryclass/a9a_lr.html"> + + + <b>5.1.2.</b> + + Logistic Regression + + </a> + + + + </li> + + <li class="chapter " data-level="5.1.3" data-path="../binaryclass/a9a_minibatch.html"> + + <a href="../binaryclass/a9a_minibatch.html"> + + + <b>5.1.3.</b> + + Mini-batch Gradient Descent + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.2" data-path="../binaryclass/news20.html"> + + <a href="../binaryclass/news20.html"> + + + <b>5.2.</b> + + News20 Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.2.1" data-path="../binaryclass/news20_dataset.html"> + + <a href="../binaryclass/news20_dataset.html"> + + + <b>5.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.2.2" data-path="../binaryclass/news20_pa.html"> + + <a href="../binaryclass/news20_pa.html"> + + + <b>5.2.2.</b> + + Perceptron, Passive Aggressive + + </a> + + + + </li> + + <li class="chapter " data-level="5.2.3" data-path="../binaryclass/news20_scw.html"> + + <a href="../binaryclass/news20_scw.html"> + + + <b>5.2.3.</b> + + CW, AROW, SCW + + </a> + + + + </li> + + <li class="chapter " data-level="5.2.4" data-path="../binaryclass/news20_adagrad.html"> + + <a href="../binaryclass/news20_adagrad.html"> + + + <b>5.2.4.</b> + + AdaGradRDA, AdaGrad, AdaDelta + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.3" data-path="../binaryclass/kdd2010a.html"> + + <a href="../binaryclass/kdd2010a.html"> + + + <b>5.3.</b> + + KDD2010a Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.3.1" data-path="../binaryclass/kdd2010a_dataset.html"> + + <a href="../binaryclass/kdd2010a_dataset.html"> + + + <b>5.3.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.3.2" data-path="../binaryclass/kdd2010a_scw.html"> + + <a href="../binaryclass/kdd2010a_scw.html"> + + + <b>5.3.2.</b> + + PA, CW, AROW, SCW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.4" data-path="../binaryclass/kdd2010b.html"> + + <a href="../binaryclass/kdd2010b.html"> + + + <b>5.4.</b> + + KDD2010b Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.4.1" data-path="../binaryclass/kdd2010b_dataset.html"> + + <a href="../binaryclass/kdd2010b_dataset.html"> + + + <b>5.4.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.4.2" data-path="../binaryclass/kdd2010b_arow.html"> + + <a href="../binaryclass/kdd2010b_arow.html"> + + + <b>5.4.2.</b> + + AROW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.5" data-path="../binaryclass/webspam.html"> + + <a href="../binaryclass/webspam.html"> + + + <b>5.5.</b> + + Webspam Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.5.1" data-path="../binaryclass/webspam_dataset.html"> + + <a href="../binaryclass/webspam_dataset.html"> + + + <b>5.5.1.</b> + + Data pareparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.5.2" data-path="../binaryclass/webspam_scw.html"> + + <a href="../binaryclass/webspam_scw.html"> + + + <b>5.5.2.</b> + + PA1, AROW, SCW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.6" data-path="../binaryclass/titanic_rf.html"> + + <a href="../binaryclass/titanic_rf.html"> + + + <b>5.6.</b> + + Kaggle Titanic Tutorial + + </a> + + + + </li> + + + + + <li class="header">Part VI - Multiclass classification</li> + + + + <li class="chapter " data-level="6.1" data-path="../multiclass/news20.html"> + + <a href="../multiclass/news20.html"> + + + <b>6.1.</b> + + News20 Multiclass Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="6.1.1" data-path="../multiclass/news20_dataset.html"> + + <a href="../multiclass/news20_dataset.html"> + + + <b>6.1.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.2" data-path="../multiclass/news20_one-vs-the-rest_dataset.html"> + + <a href="../multiclass/news20_one-vs-the-rest_dataset.html"> + + + <b>6.1.2.</b> + + Data preparation for one-vs-the-rest classifiers + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.3" data-path="../multiclass/news20_pa.html"> + + <a href="../multiclass/news20_pa.html"> + + + <b>6.1.3.</b> + + PA + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.4" data-path="../multiclass/news20_scw.html"> + + <a href="../multiclass/news20_scw.html"> + + + <b>6.1.4.</b> + + CW, AROW, SCW + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.5" data-path="../multiclass/news20_ensemble.html"> + + <a href="../multiclass/news20_ensemble.html"> + + + <b>6.1.5.</b> + + Ensemble learning + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.6" data-path="../multiclass/news20_one-vs-the-rest.html"> + + <a href="../multiclass/news20_one-vs-the-rest.html"> + + + <b>6.1.6.</b> + + one-vs-the-rest classifier + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="6.2" data-path="../multiclass/iris.html"> + + <a href="../multiclass/iris.html"> + + + <b>6.2.</b> + + Iris Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="6.2.1" data-path="../multiclass/iris_dataset.html"> + + <a href="../multiclass/iris_dataset.html"> + + + <b>6.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="6.2.2" data-path="../multiclass/iris_scw.html"> + + <a href="../multiclass/iris_scw.html"> + + + <b>6.2.2.</b> + + SCW + + </a> + + + + </li> + + <li class="chapter " data-level="6.2.3" data-path="../multiclass/iris_randomforest.html"> + + <a href="../multiclass/iris_randomforest.html"> + + + <b>6.2.3.</b> + + RandomForest + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part VII - Regression</li> + + + + <li class="chapter " data-level="7.1" data-path="../regression/e2006.html"> + + <a href="../regression/e2006.html"> + + + <b>7.1.</b> + + E2006-tfidf regression Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="7.1.1" data-path="../regression/e2006_dataset.html"> + + <a href="../regression/e2006_dataset.html"> + + + <b>7.1.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="7.1.2" data-path="../regression/e2006_arow.html"> + + <a href="../regression/e2006_arow.html"> + + + <b>7.1.2.</b> + + Passive Aggressive, AROW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="7.2" data-path="../regression/kddcup12tr2.html"> + + <a href="../regression/kddcup12tr2.html"> + + + <b>7.2.</b> + + KDDCup 2012 track 2 CTR prediction Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="7.2.1" data-path="../regression/kddcup12tr2_dataset.html"> + + <a href="../regression/kddcup12tr2_dataset.html"> + + + <b>7.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="7.2.2" data-path="../regression/kddcup12tr2_lr.html"> + + <a href="../regression/kddcup12tr2_lr.html"> + + + <b>7.2.2.</b> + + Logistic Regression, Passive Aggressive + + </a> + + + + </li> + + <li class="chapter " data-level="7.2.3" data-path="../regression/kddcup12tr2_lr_amplify.html"> + + <a href="../regression/kddcup12tr2_lr_amplify.html"> + + + <b>7.2.3.</b> + + Logistic Regression with Amplifier + + </a> + + + + </li> + + <li class="chapter " data-level="7.2.4" data-path="../regression/kddcup12tr2_adagrad.html"> + + <a href="../regression/kddcup12tr2_adagrad.html"> + + + <b>7.2.4.</b> + + AdaGrad, AdaDelta + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part VIII - Recommendation</li> + + + + <li class="chapter " data-level="8.1" data-path="../recommend/cf.html"> + + <a href="../recommend/cf.html"> + + + <b>8.1.</b> + + Collaborative Filtering + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="8.1.1" data-path="../recommend/item_based_cf.html"> + + <a href="../recommend/item_based_cf.html"> + + + <b>8.1.1.</b> + + Item-based Collaborative Filtering + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="8.2" data-path="../recommend/news20.html"> + + <a href="../recommend/news20.html"> + + + <b>8.2.</b> + + News20 related article recommendation Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="8.2.1" data-path="../multiclass/news20_dataset.html"> + + <a href="../multiclass/news20_dataset.html"> + + + <b>8.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.2" data-path="../recommend/news20_jaccard.html"> + + <a href="../recommend/news20_jaccard.html"> + + + <b>8.2.2.</b> + + LSH/Minhash and Jaccard Similarity + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.3" data-path="../recommend/news20_knn.html"> + + <a href="../recommend/news20_knn.html"> + + + <b>8.2.3.</b> + + LSH/Minhash and Brute-Force Search + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.4" data-path="../recommend/news20_bbit_minhash.html"> + + <a href="../recommend/news20_bbit_minhash.html"> + + + <b>8.2.4.</b> + + kNN search using b-Bits Minhash + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="8.3" data-path="../recommend/movielens.html"> + + <a href="../recommend/movielens.html"> + + + <b>8.3.</b> + + MovieLens movie recommendation Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="8.3.1" data-path="../recommend/movielens_dataset.html"> + + <a href="../recommend/movielens_dataset.html"> + + + <b>8.3.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="8.3.2" data-path="../recommend/movielens_mf.html"> + + <a href="../recommend/movielens_mf.html"> + + + <b>8.3.2.</b> + + Matrix Factorization + + </a> + + + + </li> + + <li class="chapter " data-level="8.3.3" data-path="../recommend/movielens_fm.html"> + + <a href="../recommend/movielens_fm.html"> + + + <b>8.3.3.</b> + + Factorization Machine + + </a> + + + + </li> + + <li class="chapter " data-level="8.3.4" data-path="../recommend/movielens_cv.html"> + + <a href="../recommend/movielens_cv.html"> + + + <b>8.3.4.</b> + + 10-fold Cross Validation (Matrix Factorization) + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part IX - Anomaly Detection</li> + + + + <li class="chapter " data-level="9.1" data-path="lof.html"> + + <a href="lof.html"> + + + <b>9.1.</b> + + Outlier Detection using Local Outlier Factor (LOF) + + </a> + + + + </li> + + <li class="chapter " data-level="9.2" data-path="sst.html"> + + <a href="sst.html"> + + + <b>9.2.</b> + + Change-Point Detection using Singular Spectrum Transformation (SST) + + </a> + + + + </li> + + <li class="chapter active" data-level="9.3" data-path="changefinder.html"> + + <a href="changefinder.html"> + + + <b>9.3.</b> + + ChangeFinder: Detecting Outlier and Change-Point Simultaneously + + </a> + + + + </li> + + + + + <li class="header">Part X - Hivemall on Spark</li> + + + + <li class="chapter " data-level="10.1" data-path="../spark/binaryclass/"> + + <a href="../spark/binaryclass/"> + + + <b>10.1.</b> + + Binary Classification + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="10.1.1" data-path="../spark/binaryclass/a9a_df.html"> + + <a href="../spark/binaryclass/a9a_df.html"> + + + <b>10.1.1.</b> + + a9a Tutorial for DataFrame + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="10.2" data-path="../spark/binaryclass/"> + + <a href="../spark/binaryclass/"> + + + <b>10.2.</b> + + Regression + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="10.2.1" data-path="../spark/regression/e2006_df.html"> + + <a href="../spark/regression/e2006_df.html"> + + + <b>10.2.1.</b> + + E2006-tfidf regression Tutorial for DataFrame + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="10.3" data-path="../spark/misc/misc.html"> + + <a href="../spark/misc/misc.html"> + + + <b>10.3.</b> + + Generic features + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="10.3.1" data-path="../spark/misc/topk_join.html"> + + <a href="../spark/misc/topk_join.html"> + + + <b>10.3.1.</b> + + Top-k Join processing + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part X - External References</li> + + + + <li class="chapter " data-level="11.1" > + + <a target="_blank" href="https://github.com/maropu/hivemall-spark";> + + + <b>11.1.</b> + + Hivemall on Apache Spark + + </a> + + + + </li> + + <li class="chapter " data-level="11.2" > + + <a target="_blank" href="https://github.com/daijyc/hivemall/wiki/PigHome";> + + + <b>11.2.</b> + + Hivemall on Apache Pig + + </a> + + + + </li> + + + + + <li class="divider"></li> + + <li> + <a href="https://www.gitbook.com"; target="blank" class="gitbook-link"> + Published with GitBook + </a> + </li> +</ul> + + + </nav> + + + </div> + + <div class="book-body"> + + <div class="body-inner"> + + + +<div class="book-header" role="navigation"> + + + <!-- Title --> + <h1> + <i class="fa fa-circle-o-notch fa-spin"></i> + <a href=".." >ChangeFinder: Detecting Outlier and Change-Point Simultaneously</a> + </h1> +</div> + + + + + <div class="page-wrapper" tabindex="-1" role="main"> + <div class="page-inner"> + +<div id="book-search-results"> + <div class="search-noresults"> + + <section class="normal markdown-section"> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<p>In a context of anomaly detection, there are two types of anomalies, <strong><em>outlier</em></strong> and <strong><em>change-point</em></strong>, as discussed in <a href="sst.html#outlier-vs-change-point">this section</a>. Hivemall has two functions which respectively detect outliers and change-points; the former is <a href="lof.html">Local Outlier Detection</a>, and the latter is <a href="sst.html">Singular Spectrum Transformation</a>.</p> +<p>In some cases, we might want to detect outlier and change-point simultaneously in order to figure out characteristics of a time series both in a local and global scale. <strong>ChangeFinder</strong> is an anomaly detection technique which enables us to detect both of outliers and change-points in a single framework. A key reference for the technique is:</p> +<ul> +<li>K. Yamanishi and J. Takeuchi. <a href="http://dl.acm.org/citation.cfm?id=775148"; target="_blank">A Unifying Framework for Detecting Outliers and Change Points from Non-Stationary Time Series Data</a>. KDD'02.</li> +</ul> +<!-- toc --><div id="toc" class="toc"> + +<ul> +<li><a href="#outlier-and-change-point-detection-using-changefinder">Outlier and Change-Point Detection using ChangeFinder</a></li> +<li><a href="#changefinder-for-multi-dimensional-data">ChangeFinder for Multi-Dimensional Data</a><ul> +<li><a href="#data-preparation">Data preparation</a></li> +<li><a href="#detecting-outliers-and-change-points-of-the-5-dimensional-data">Detecting outliers and change-points of the 5-dimensional data</a></li> +</ul> +</li> +</ul> + +</div><!-- tocstop --> +<h1 id="outlier-and-change-point-detection-using-changefinder">Outlier and Change-Point Detection using ChangeFinder</h1> +<p>By using Twitter's time series data we prepared in <a href="sst.html#data-preparation">this section</a>, let us try to use ChangeFinder on Hivemall.</p> +<pre><code>use twitter; +</code></pre><p>A function <code>changefinder()</code> can be used in a very similar way to <code>sst()</code>, a UDF for <a href="sst.html">Singular Spectrum Transformation</a>. The following query detects outliers and change-points with different thresholds:</p> +<pre><code class="lang-sql"><span class="hljs-keyword">SELECT</span> + <span class="hljs-keyword">num</span>, + changefinder(<span class="hljs-keyword">value</span>, <span class="hljs-string">"-outlier_threshold 0.03 -changepoint_threshold 0.0035"</span>) <span class="hljs-keyword">AS</span> <span class="hljs-keyword">result</span> +<span class="hljs-keyword">FROM</span> + timeseries +<span class="hljs-keyword">ORDER</span> <span class="hljs-keyword">BY</span> <span class="hljs-keyword">num</span> <span class="hljs-keyword">ASC</span> +; +</code></pre> +<p>As a consequence, finding outliers and change-points in the data points should be easy:</p> +<table> +<thead> +<tr> +<th style="text-align:center">num</th> +<th style="text-align:left">result</th> +</tr> +</thead> +<tbody> +<tr> +<td style="text-align:center">...</td> +<td style="text-align:left">...</td> +</tr> +<tr> +<td style="text-align:center">16</td> +<td style="text-align:left">{"outlier_score":0.051287243859365894,"changepoint_score":0.003292139657059704,"is_outlier":true,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">17</td> +<td style="text-align:left">{"outlier_score":0.03994335565212781,"changepoint_score":0.003484242549446824,"is_outlier":true,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">18</td> +<td style="text-align:left">{"outlier_score":0.9153515196592132,"changepoint_score":0.0036439645550477373,"is_outlier":true,"is_changepoint":true}</td> +</tr> +<tr> +<td style="text-align:center">19</td> +<td style="text-align:left">{"outlier_score":0.03940593403992665,"changepoint_score":0.0035825157392152134,"is_outlier":true,"is_changepoint":true}</td> +</tr> +<tr> +<td style="text-align:center">20</td> +<td style="text-align:left">{"outlier_score":0.27172093630215555,"changepoint_score":0.003542822324886785,"is_outlier":true,"is_changepoint":true}</td> +</tr> +<tr> +<td style="text-align:center">21</td> +<td style="text-align:left">{"outlier_score":0.006784031454620809,"changepoint_score":0.0035029441620275975,"is_outlier":false,"is_changepoint":true}</td> +</tr> +<tr> +<td style="text-align:center">22</td> +<td style="text-align:left">{"outlier_score":0.011838969816513334,"changepoint_score":0.003519599336202336,"is_outlier":false,"is_changepoint":true}</td> +</tr> +<tr> +<td style="text-align:center">23</td> +<td style="text-align:left">{"outlier_score":0.09609857927656007,"changepoint_score":0.003478729798944702,"is_outlier":true,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">24</td> +<td style="text-align:left">{"outlier_score":0.23927000145081978,"changepoint_score":0.0034338476757061237,"is_outlier":true,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">25</td> +<td style="text-align:left">{"outlier_score":0.04645945042821564,"changepoint_score":0.0034052091926036914,"is_outlier":true,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">...</td> +<td style="text-align:left">...</td> +</tr> +</tbody> +</table> +<h1 id="changefinder-for-multi-dimensional-data">ChangeFinder for Multi-Dimensional Data</h1> +<p>ChangeFinder additionally supports multi-dimensional data. Let us try this on synthetic data.</p> +<h2 id="data-preparation">Data preparation</h2> +<p>You first need to get synthetic 5-dimensional data from <a href="https://github.com/apache/incubator-hivemall/blob/master/core/src/test/resources/hivemall/anomaly/synthetic5d.t.gz?raw=true"; target="_blank">HERE</a> and uncompress to a <code>synthetic5d.t</code> file:</p> +<pre><code>$ head synthetic5d.t +0#71.45185411564131#54.456141290891466#71.78932846605129#76.73002575911214#81.71265594077099 +1#58.374230566196786#57.9798651697631#75.65793151143754#73.76101930504493#69.50315805346253 +2#66.3595943896099#52.866595973073295#76.7987325026338#78.95890786682095#74.67527753118893 +3#58.242560151043236#52.449574430621226#73.20383710416358#77.81502394558085#76.59077723631032 +4#55.89878019680371#52.69611781315756#75.02482987204824#74.11154526135637#75.86881583921179 +5#56.93554246767561#56.55687136423391#74.4056583421317#73.82419594611444#71.3017150863033 +6#65.55704393868689#52.136347983404974#71.14213602046532#72.87394198561904#73.40278960429114 +7#56.65735280596217#57.293605941063035#75.36713340281246#80.70254745535183#75.32423746923857 +8#61.22095211566127#53.47603728473668#77.48215321523912#80.7760107465893#74.43951386292905 +9#52.47574856682803#52.03250504263378#77.59550963025158#76.16623830860391#76.98394610743863 +</code></pre><p>The first column indicates a dummy timestamp, and the following four columns are values in each dimension. </p> +<p>Second, the following Hive operations create a Hive table for the data:</p> +<pre><code>create database synthetic; +use synthetic; +</code></pre><pre><code class="lang-sql"><span class="hljs-keyword">CREATE</span> <span class="hljs-keyword">EXTERNAL</span> <span class="hljs-keyword">TABLE</span> synthetic5d ( + <span class="hljs-keyword">num</span> <span class="hljs-built_in">INT</span>, + value1 <span class="hljs-keyword">DOUBLE</span>, + value2 <span class="hljs-keyword">DOUBLE</span>, + value3 <span class="hljs-keyword">DOUBLE</span>, + value4 <span class="hljs-keyword">DOUBLE</span>, + value5 <span class="hljs-keyword">DOUBLE</span> +) <span class="hljs-keyword">ROW</span> <span class="hljs-keyword">FORMAT</span> <span class="hljs-keyword">DELIMITED</span> +<span class="hljs-keyword">FIELDS</span> <span class="hljs-keyword">TERMINATED</span> <span class="hljs-keyword">BY</span> <span class="hljs-string">'#'</span> +<span class="hljs-keyword">STORED</span> <span class="hljs-keyword">AS</span> TEXTFILE +LOCATION <span class="hljs-string">'/dataset/synthetic/synthetic5d'</span>; +</code></pre> +<p>Finally, you can load the synthetic data to the table by:</p> +<pre><code>$ hadoop fs -put synthetic5d.t /dataset/synthetic/synthetic5d +</code></pre><h2 id="detecting-outliers-and-change-points-of-the-5-dimensional-data">Detecting outliers and change-points of the 5-dimensional data</h2> +<p>Using <code>changefinder()</code> for multi-dimensional data requires us to pass the first argument as an array. In our case, the data is 5-dimensional, so the first argument should be an array with 5 elements. Except for that point, basic usage of the function is same as the previous 1-dimensional example:</p> +<pre><code class="lang-sql"><span class="hljs-keyword">SELECT</span> + <span class="hljs-keyword">num</span>, + changefinder(<span class="hljs-built_in">array</span>(value1, value2, value3, value4, value5), + <span class="hljs-string">"-outlier_threshold 0.015 -changepoint_threshold 0.0045"</span>) <span class="hljs-keyword">AS</span> <span class="hljs-keyword">result</span> +<span class="hljs-keyword">FROM</span> + synthetic5d +<span class="hljs-keyword">ORDER</span> <span class="hljs-keyword">BY</span> <span class="hljs-keyword">num</span> <span class="hljs-keyword">ASC</span> +; +</code></pre> +<p>Output might be:</p> +<table> +<thead> +<tr> +<th style="text-align:center">num</th> +<th style="text-align:left">result</th> +</tr> +</thead> +<tbody> +<tr> +<td style="text-align:center">...</td> +<td style="text-align:left">...</td> +</tr> +<tr> +<td style="text-align:center">90</td> +<td style="text-align:left">{"outlier_score":0.014014718350674471,"changepoint_score":0.004520174906936474,"is_outlier":false,"is_changepoint":true}</td> +</tr> +<tr> +<td style="text-align:center">91</td> +<td style="text-align:left">{"outlier_score":0.013145554693405614,"changepoint_score":0.004480713237042799,"is_outlier":false,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">92</td> +<td style="text-align:left">{"outlier_score":0.011631759675989617,"changepoint_score":0.004442031415725316,"is_outlier":false,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">93</td> +<td style="text-align:left">{"outlier_score":0.012140065235943798,"changepoint_score":0.004404170732687428,"is_outlier":false,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">94</td> +<td style="text-align:left">{"outlier_score":0.012555903663657997,"changepoint_score":0.0043670553008087355,"is_outlier":false,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">95</td> +<td style="text-align:left">{"outlier_score":0.013503247137325314,"changepoint_score":0.0043306667027628466,"is_outlier":false,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">96</td> +<td style="text-align:left">{"outlier_score":0.013896893553710932,"changepoint_score":0.004294969164345527,"is_outlier":false,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">97</td> +<td style="text-align:left">{"outlier_score":0.01322874844578159,"changepoint_score":0.004259994590721001,"is_outlier":false,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">98</td> +<td style="text-align:left">{"outlier_score":0.019383618511936707,"changepoint_score":0.004225604978710543,"is_outlier":true,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">99</td> +<td style="text-align:left">{"outlier_score":0.01121758589038846,"changepoint_score":0.004191881992962213,"is_outlier":false,"is_changepoint":false}</td> +</tr> +<tr> +<td style="text-align:center">...</td> +<td style="text-align:left">...</td> +</tr> +</tbody> +</table> +<p><div id="page-footer"><hr><!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<p><sub><font color="gray"> +Apache Hivemall is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. +</font></sub></p> +</div></p> + + + </section> + + </div> + <div class="search-results"> + <div class="has-results"> + + <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1> + <ul class="search-results-list"></ul> + + </div> + <div class="no-results"> + + <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1> + + </div> + </div> +</div> + + </div> + </div> + + </div> + + + + + </div> + + <script> + var gitbook = gitbook || []; + gitbook.push(function() { + gitbook.page.hasChanged({"page":{"title":"ChangeFinder: Detecting Outlier and Change-Point Simultaneously","level":"9.3","depth":1,"next":{"title":"Binary Classification","level":"10.1","depth":1,"path":"spark/binaryclass/index.md","ref":"spark/binaryclass/index.md","articles":[{"title":"a9a Tutorial for DataFrame","level":"10.1.1","depth":2,"path":"spark/binaryclass/a9a_df.md","ref":"spark/binaryclass/a9a_df.md","articles":[]}]},"previous":{"title":"Change-Point Detection using Singular Spectrum Transformation (SST)","level":"9.2","depth":1,"path":"anomaly/sst.md","ref":"anomaly/sst.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"s tyles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles": {"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"anomaly/changefinder.md","mtime":"2017-02-08T09:17:59.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-02-23T13:06:39.809Z"},"basePath":"..","book":{"language":""}}); + }); + </script> +</div> + + + <script src="../gitbook/gitbook.js"></script> + <script src="../gitbook/theme.js"></script> + + + <script src="../gitbook/gitbook-plugin-edit-link/plugin.js"></script> + + + + <script src="../gitbook/gitbook-plugin-github/plugin.js"></script> + + + + <script src="../gitbook/gitbook-plugin-splitter/splitter.js"></script> + + + + <script src="../gitbook/gitbook-plugin-etoc/plugin.js"></script> + + + + <script src="../gitbook/gitbook-plugin-toggle-chapters/toggle.js"></script> + + + + <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/3.1.1/anchor.min.js";></script> + + + + <script src="../gitbook/gitbook-plugin-anchorjs/anchor-style.js"></script> + + + + <script src="../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.js"></script> + + + + <script src="../gitbook/gitbook-plugin-search/search-engine.js"></script> + + + + <script src="../gitbook/gitbook-plugin-search/search.js"></script> + + + + <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script> + + + + <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script> + + + + <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script> + + + + <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script> + + + + <script src="../gitbook/gitbook-plugin-theme-api/theme-api.js"></script> + + + + </body> +</html> + http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/2d3fdeb2/userguide/anomaly/lof.html ---------------------------------------------------------------------- diff --git a/userguide/anomaly/lof.html b/userguide/anomaly/lof.html index de855c2..5801ada 100644 --- a/userguide/anomaly/lof.html +++ b/userguide/anomaly/lof.html @@ -97,7 +97,7 @@ <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon"> - <link rel="next" href="../spark/misc/misc.html" /> + <link rel="next" href="sst.html" /> <link rel="prev" href="../recommend/movielens_cv.html" /> @@ -1579,6 +1579,36 @@ </li> + <li class="chapter " data-level="9.2" data-path="sst.html"> + + <a href="sst.html"> + + + <b>9.2.</b> + + Change-Point Detection using Singular Spectrum Transformation (SST) + + </a> + + + + </li> + + <li class="chapter " data-level="9.3" data-path="changefinder.html"> + + <a href="changefinder.html"> + + + <b>9.3.</b> + + ChangeFinder: Detecting Outlier and Change-Point Simultaneously + + </a> + + + + </li> + @@ -1586,13 +1616,85 @@ - <li class="chapter " data-level="10.1" data-path="../spark/misc/misc.html"> + <li class="chapter " data-level="10.1" data-path="../spark/binaryclass/"> - <a href="../spark/misc/misc.html"> + <a href="../spark/binaryclass/"> <b>10.1.</b> + Binary Classification + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="10.1.1" data-path="../spark/binaryclass/a9a_df.html"> + + <a href="../spark/binaryclass/a9a_df.html"> + + + <b>10.1.1.</b> + + a9a Tutorial for DataFrame + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="10.2" data-path="../spark/binaryclass/"> + + <a href="../spark/binaryclass/"> + + + <b>10.2.</b> + + Regression + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="10.2.1" data-path="../spark/regression/e2006_df.html"> + + <a href="../spark/regression/e2006_df.html"> + + + <b>10.2.1.</b> + + E2006-tfidf regression Tutorial for DataFrame + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="10.3" data-path="../spark/misc/misc.html"> + + <a href="../spark/misc/misc.html"> + + + <b>10.3.</b> + Generic features </a> @@ -1602,12 +1704,12 @@ <ul class="articles"> - <li class="chapter " data-level="10.1.1" data-path="../spark/misc/topk_join.html"> + <li class="chapter " data-level="10.3.1" data-path="../spark/misc/topk_join.html"> <a href="../spark/misc/topk_join.html"> - <b>10.1.1.</b> + <b>10.3.1.</b> Top-k Join processing @@ -1722,7 +1824,7 @@ specific language governing permissions and limitations under the License. --> -<p>This article introduce how to find outliers using <a href="http://en.wikipedia.org/wiki/Local_outlier_factor"; target="_blank">Local Outlier Detection (LOF)</a> on Hivemall.</p> +<p>This article introduces how to find outliers using <a href="http://en.wikipedia.org/wiki/Local_outlier_factor"; target="_blank">Local Outlier Detection (LOF)</a> on Hivemall.</p> <!-- toc --><div id="toc" class="toc"> <ul> @@ -1946,7 +2048,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda <script> var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"Outlier Detection using Local Outlier Factor (LOF)","level":"9.1","depth":1,"next":{"title":"Generic features","level":"10.1","depth":1,"path":"spark/misc/misc.md","ref":"spark/misc/misc.md","articles":[{"title":"Top-k Join processing","level":"10.1.1","depth":2,"path":"spark/misc/topk_join.md","ref":"spark/misc/topk_join.md","articles":[]}]},"previous":{"title":"10-fold Cross Validation (Matrix Factorization)","level":"8.3.4","depth":2,"path":"recommend/movielens_cv.md","ref":"recommend/movielens_cv.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css" },"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styl es/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"anomaly/lof.md","mtime":"2016-12-02T08:02:42.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-02-02T05:07:49.882Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"Outlier Detection using Local Outlier Factor (LOF)","level":"9.1","depth":1,"next":{"title":"Change-Point Detection using Singular Spectrum Transformation (SST)","level":"9.2","depth":1,"path":"anomaly/sst.md","ref":"anomaly/sst.md","articles":[]},"previous":{"title":"10-fold Cross Validation (Matrix Factorization)","level":"8.3.4","depth":2,"path":"recommend/movielens_cv.md","ref":"recommend/movielens_cv.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"maxdepth":3,"mindepth":1,"notoc":true}," github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"http://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print": "styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"http://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"anomaly/lof.md","mtime":"2017-02-08T09:17:59.000Z","type":"markdown"},"gitbook":{"version":"3.2.2","time":"2017-02-23T13:06:39.809Z"},"basePath":"..","book":{"language":""}}); }); </script> </div>
