http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/fea73243/userguide/eval/rank.html ---------------------------------------------------------------------- diff --git a/userguide/eval/rank.html b/userguide/eval/rank.html new file mode 100644 index 0000000..9dc9ce6 --- /dev/null +++ b/userguide/eval/rank.html @@ -0,0 +1,2418 @@ + +<!DOCTYPE HTML> +<html lang="" > + <head> + <meta charset="UTF-8"> + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <title>Ranking Measures · Hivemall User Manual</title> + <meta http-equiv="X-UA-Compatible" content="IE=edge" /> + <meta name="description" content=""> + <meta name="generator" content="GitBook 3.2.2"> + + + + + <link rel="stylesheet" href="../gitbook/style.css"> + + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-splitter/splitter.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-etoc/plugin.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-callouts/plugin.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-toggle-chapters/toggle.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-codeblock-filename/block.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-multipart/multipart.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-katex/katex.min.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-emphasize/plugin.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-theme-api/theme-api.css"> + + + + + + + + + + + + + + + + + + + + + + + + <meta name="HandheldFriendly" content="true"/> + <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"> + <meta name="apple-mobile-web-app-capable" content="yes"> + <meta name="apple-mobile-web-app-status-bar-style" content="black"> + <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png"> + <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon"> + + + <link rel="next" href="datagen.html" /> + + + <link rel="prev" href="auc.html" /> + + + </head> + <body> + +<div class="book"> + <div class="book-summary"> + + +<div id="book-search-input" role="search"> + <input type="text" placeholder="Type to search" /> +</div> + + + <nav role="navigation"> + + + +<ul class="summary"> + + + + + <li> + <a href="http://hivemall.incubator.apache.org/"; target="_blank" class="custom-link"><i class="fa fa-home"></i> Home</a> + </li> + + + + + <li class="divider"></li> + + + + + <li class="header">TABLE OF CONTENTS</li> + + + + <li class="chapter " data-level="1.1" data-path="../"> + + <a href="../"> + + + <b>1.1.</b> + + Introduction + + </a> + + + + </li> + + <li class="chapter " data-level="1.2" data-path="../getting_started/"> + + <a href="../getting_started/"> + + + <b>1.2.</b> + + Getting Started + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.2.1" data-path="../getting_started/installation.html"> + + <a href="../getting_started/installation.html"> + + + <b>1.2.1.</b> + + Installation + + </a> + + + + </li> + + <li class="chapter " data-level="1.2.2" data-path="../getting_started/permanent-functions.html"> + + <a href="../getting_started/permanent-functions.html"> + + + <b>1.2.2.</b> + + Install as permanent functions + + </a> + + + + </li> + + <li class="chapter " data-level="1.2.3" data-path="../getting_started/input-format.html"> + + <a href="../getting_started/input-format.html"> + + + <b>1.2.3.</b> + + Input Format + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="1.3" data-path="../tips/"> + + <a href="../tips/"> + + + <b>1.3.</b> + + Tips for Effective Hivemall + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.3.1" data-path="../tips/addbias.html"> + + <a href="../tips/addbias.html"> + + + <b>1.3.1.</b> + + Explicit addBias() for better prediction + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.2" data-path="../tips/rand_amplify.html"> + + <a href="../tips/rand_amplify.html"> + + + <b>1.3.2.</b> + + Use rand_amplify() to better prediction results + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.3" data-path="../tips/rt_prediction.html"> + + <a href="../tips/rt_prediction.html"> + + + <b>1.3.3.</b> + + Real-time Prediction on RDBMS + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.4" data-path="../tips/ensemble_learning.html"> + + <a href="../tips/ensemble_learning.html"> + + + <b>1.3.4.</b> + + Ensemble learning for stable prediction + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.5" data-path="../tips/mixserver.html"> + + <a href="../tips/mixserver.html"> + + + <b>1.3.5.</b> + + Mixing models for a better prediction convergence (MIX server) + + </a> + + + + </li> + + <li class="chapter " data-level="1.3.6" data-path="../tips/emr.html"> + + <a href="../tips/emr.html"> + + + <b>1.3.6.</b> + + Run Hivemall on Amazon Elastic MapReduce + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="1.4" data-path="../tips/general_tips.html"> + + <a href="../tips/general_tips.html"> + + + <b>1.4.</b> + + General Hive/Hadoop tips + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.4.1" data-path="../tips/rowid.html"> + + <a href="../tips/rowid.html"> + + + <b>1.4.1.</b> + + Adding rowid for each row + + </a> + + + + </li> + + <li class="chapter " data-level="1.4.2" data-path="../tips/hadoop_tuning.html"> + + <a href="../tips/hadoop_tuning.html"> + + + <b>1.4.2.</b> + + Hadoop tuning for Hivemall + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="1.5" data-path="../troubleshooting/"> + + <a href="../troubleshooting/"> + + + <b>1.5.</b> + + Troubleshooting + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.5.1" data-path="../troubleshooting/oom.html"> + + <a href="../troubleshooting/oom.html"> + + + <b>1.5.1.</b> + + OutOfMemoryError in training + + </a> + + + + </li> + + <li class="chapter " data-level="1.5.2" data-path="../troubleshooting/mapjoin_task_error.html"> + + <a href="../troubleshooting/mapjoin_task_error.html"> + + + <b>1.5.2.</b> + + SemanticException Generate Map Join Task Error: Cannot serialize object + + </a> + + + + </li> + + <li class="chapter " data-level="1.5.3" data-path="../troubleshooting/asterisk.html"> + + <a href="../troubleshooting/asterisk.html"> + + + <b>1.5.3.</b> + + Asterisk argument for UDTF does not work + + </a> + + + + </li> + + <li class="chapter " data-level="1.5.4" data-path="../troubleshooting/num_mappers.html"> + + <a href="../troubleshooting/num_mappers.html"> + + + <b>1.5.4.</b> + + The number of mappers is less than input splits in Hadoop 2.x + + </a> + + + + </li> + + <li class="chapter " data-level="1.5.5" data-path="../troubleshooting/mapjoin_classcastex.html"> + + <a href="../troubleshooting/mapjoin_classcastex.html"> + + + <b>1.5.5.</b> + + Map-side Join causes ClassCastException on Tez + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part II - Generic Features</li> + + + + <li class="chapter " data-level="2.1" data-path="../misc/generic_funcs.html"> + + <a href="../misc/generic_funcs.html"> + + + <b>2.1.</b> + + List of generic Hivemall functions + + </a> + + + + </li> + + <li class="chapter " data-level="2.2" data-path="../misc/topk.html"> + + <a href="../misc/topk.html"> + + + <b>2.2.</b> + + Efficient Top-K query processing + + </a> + + + + </li> + + <li class="chapter " data-level="2.3" data-path="../misc/tokenizer.html"> + + <a href="../misc/tokenizer.html"> + + + <b>2.3.</b> + + English/Japanese Text Tokenizer + + </a> + + + + </li> + + + + + <li class="header">Part III - Feature Engineering</li> + + + + <li class="chapter " data-level="3.1" data-path="../ft_engineering/scaling.html"> + + <a href="../ft_engineering/scaling.html"> + + + <b>3.1.</b> + + Feature Scaling + + </a> + + + + </li> + + <li class="chapter " data-level="3.2" data-path="../ft_engineering/hashing.html"> + + <a href="../ft_engineering/hashing.html"> + + + <b>3.2.</b> + + Feature Hashing + + </a> + + + + </li> + + <li class="chapter " data-level="3.3" data-path="../ft_engineering/tfidf.html"> + + <a href="../ft_engineering/tfidf.html"> + + + <b>3.3.</b> + + TF-IDF calculation + + </a> + + + + </li> + + <li class="chapter " data-level="3.4" data-path="../ft_engineering/ft_trans.html"> + + <a href="../ft_engineering/ft_trans.html"> + + + <b>3.4.</b> + + FEATURE TRANSFORMATION + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="3.4.1" data-path="../ft_engineering/vectorizer.html"> + + <a href="../ft_engineering/vectorizer.html"> + + + <b>3.4.1.</b> + + Vectorize Features + + </a> + + + + </li> + + <li class="chapter " data-level="3.4.2" data-path="../ft_engineering/quantify.html"> + + <a href="../ft_engineering/quantify.html"> + + + <b>3.4.2.</b> + + Quantify non-number features + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="3.5" data-path="../ft_engineering/feature_selection.html"> + + <a href="../ft_engineering/feature_selection.html"> + + + <b>3.5.</b> + + Feature selection + + </a> + + + + </li> + + + + + <li class="header">Part IV - Evaluation</li> + + + + <li class="chapter " data-level="4.1" data-path="stat_eval.html"> + + <a href="stat_eval.html"> + + + <b>4.1.</b> + + Statistical evaluation of a prediction model + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="4.1.1" data-path="auc.html"> + + <a href="auc.html"> + + + <b>4.1.1.</b> + + Area Under the ROC Curve + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter active" data-level="4.2" data-path="rank.html"> + + <a href="rank.html"> + + + <b>4.2.</b> + + Ranking Measures + + </a> + + + + </li> + + <li class="chapter " data-level="4.3" data-path="datagen.html"> + + <a href="datagen.html"> + + + <b>4.3.</b> + + Data Generation + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="4.3.1" data-path="lr_datagen.html"> + + <a href="lr_datagen.html"> + + + <b>4.3.1.</b> + + Logistic Regression data generation + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part V - Binary classification</li> + + + + <li class="chapter " data-level="5.1" data-path="../binaryclass/a9a.html"> + + <a href="../binaryclass/a9a.html"> + + + <b>5.1.</b> + + a9a Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.1.1" data-path="../binaryclass/a9a_dataset.html"> + + <a href="../binaryclass/a9a_dataset.html"> + + + <b>5.1.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.1.2" data-path="../binaryclass/a9a_lr.html"> + + <a href="../binaryclass/a9a_lr.html"> + + + <b>5.1.2.</b> + + Logistic Regression + + </a> + + + + </li> + + <li class="chapter " data-level="5.1.3" data-path="../binaryclass/a9a_minibatch.html"> + + <a href="../binaryclass/a9a_minibatch.html"> + + + <b>5.1.3.</b> + + Mini-batch Gradient Descent + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.2" data-path="../binaryclass/news20.html"> + + <a href="../binaryclass/news20.html"> + + + <b>5.2.</b> + + News20 Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.2.1" data-path="../binaryclass/news20_dataset.html"> + + <a href="../binaryclass/news20_dataset.html"> + + + <b>5.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.2.2" data-path="../binaryclass/news20_pa.html"> + + <a href="../binaryclass/news20_pa.html"> + + + <b>5.2.2.</b> + + Perceptron, Passive Aggressive + + </a> + + + + </li> + + <li class="chapter " data-level="5.2.3" data-path="../binaryclass/news20_scw.html"> + + <a href="../binaryclass/news20_scw.html"> + + + <b>5.2.3.</b> + + CW, AROW, SCW + + </a> + + + + </li> + + <li class="chapter " data-level="5.2.4" data-path="../binaryclass/news20_adagrad.html"> + + <a href="../binaryclass/news20_adagrad.html"> + + + <b>5.2.4.</b> + + AdaGradRDA, AdaGrad, AdaDelta + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.3" data-path="../binaryclass/kdd2010a.html"> + + <a href="../binaryclass/kdd2010a.html"> + + + <b>5.3.</b> + + KDD2010a Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.3.1" data-path="../binaryclass/kdd2010a_dataset.html"> + + <a href="../binaryclass/kdd2010a_dataset.html"> + + + <b>5.3.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.3.2" data-path="../binaryclass/kdd2010a_scw.html"> + + <a href="../binaryclass/kdd2010a_scw.html"> + + + <b>5.3.2.</b> + + PA, CW, AROW, SCW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.4" data-path="../binaryclass/kdd2010b.html"> + + <a href="../binaryclass/kdd2010b.html"> + + + <b>5.4.</b> + + KDD2010b Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.4.1" data-path="../binaryclass/kdd2010b_dataset.html"> + + <a href="../binaryclass/kdd2010b_dataset.html"> + + + <b>5.4.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.4.2" data-path="../binaryclass/kdd2010b_arow.html"> + + <a href="../binaryclass/kdd2010b_arow.html"> + + + <b>5.4.2.</b> + + AROW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.5" data-path="../binaryclass/webspam.html"> + + <a href="../binaryclass/webspam.html"> + + + <b>5.5.</b> + + Webspam Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="5.5.1" data-path="../binaryclass/webspam_dataset.html"> + + <a href="../binaryclass/webspam_dataset.html"> + + + <b>5.5.1.</b> + + Data pareparation + + </a> + + + + </li> + + <li class="chapter " data-level="5.5.2" data-path="../binaryclass/webspam_scw.html"> + + <a href="../binaryclass/webspam_scw.html"> + + + <b>5.5.2.</b> + + PA1, AROW, SCW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="5.6" data-path="../binaryclass/titanic_rf.html"> + + <a href="../binaryclass/titanic_rf.html"> + + + <b>5.6.</b> + + Kaggle Titanic Tutorial + + </a> + + + + </li> + + + + + <li class="header">Part VI - Multiclass classification</li> + + + + <li class="chapter " data-level="6.1" data-path="../multiclass/news20.html"> + + <a href="../multiclass/news20.html"> + + + <b>6.1.</b> + + News20 Multiclass Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="6.1.1" data-path="../multiclass/news20_dataset.html"> + + <a href="../multiclass/news20_dataset.html"> + + + <b>6.1.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.2" data-path="../multiclass/news20_one-vs-the-rest_dataset.html"> + + <a href="../multiclass/news20_one-vs-the-rest_dataset.html"> + + + <b>6.1.2.</b> + + Data preparation for one-vs-the-rest classifiers + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.3" data-path="../multiclass/news20_pa.html"> + + <a href="../multiclass/news20_pa.html"> + + + <b>6.1.3.</b> + + PA + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.4" data-path="../multiclass/news20_scw.html"> + + <a href="../multiclass/news20_scw.html"> + + + <b>6.1.4.</b> + + CW, AROW, SCW + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.5" data-path="../multiclass/news20_ensemble.html"> + + <a href="../multiclass/news20_ensemble.html"> + + + <b>6.1.5.</b> + + Ensemble learning + + </a> + + + + </li> + + <li class="chapter " data-level="6.1.6" data-path="../multiclass/news20_one-vs-the-rest.html"> + + <a href="../multiclass/news20_one-vs-the-rest.html"> + + + <b>6.1.6.</b> + + one-vs-the-rest classifier + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="6.2" data-path="../multiclass/iris.html"> + + <a href="../multiclass/iris.html"> + + + <b>6.2.</b> + + Iris Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="6.2.1" data-path="../multiclass/iris_dataset.html"> + + <a href="../multiclass/iris_dataset.html"> + + + <b>6.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="6.2.2" data-path="../multiclass/iris_scw.html"> + + <a href="../multiclass/iris_scw.html"> + + + <b>6.2.2.</b> + + SCW + + </a> + + + + </li> + + <li class="chapter " data-level="6.2.3" data-path="../multiclass/iris_randomforest.html"> + + <a href="../multiclass/iris_randomforest.html"> + + + <b>6.2.3.</b> + + RandomForest + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part VII - Regression</li> + + + + <li class="chapter " data-level="7.1" data-path="../regression/e2006.html"> + + <a href="../regression/e2006.html"> + + + <b>7.1.</b> + + E2006-tfidf regression Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="7.1.1" data-path="../regression/e2006_dataset.html"> + + <a href="../regression/e2006_dataset.html"> + + + <b>7.1.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="7.1.2" data-path="../regression/e2006_arow.html"> + + <a href="../regression/e2006_arow.html"> + + + <b>7.1.2.</b> + + Passive Aggressive, AROW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="7.2" data-path="../regression/kddcup12tr2.html"> + + <a href="../regression/kddcup12tr2.html"> + + + <b>7.2.</b> + + KDDCup 2012 track 2 CTR prediction Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="7.2.1" data-path="../regression/kddcup12tr2_dataset.html"> + + <a href="../regression/kddcup12tr2_dataset.html"> + + + <b>7.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="7.2.2" data-path="../regression/kddcup12tr2_lr.html"> + + <a href="../regression/kddcup12tr2_lr.html"> + + + <b>7.2.2.</b> + + Logistic Regression, Passive Aggressive + + </a> + + + + </li> + + <li class="chapter " data-level="7.2.3" data-path="../regression/kddcup12tr2_lr_amplify.html"> + + <a href="../regression/kddcup12tr2_lr_amplify.html"> + + + <b>7.2.3.</b> + + Logistic Regression with Amplifier + + </a> + + + + </li> + + <li class="chapter " data-level="7.2.4" data-path="../regression/kddcup12tr2_adagrad.html"> + + <a href="../regression/kddcup12tr2_adagrad.html"> + + + <b>7.2.4.</b> + + AdaGrad, AdaDelta + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part VIII - Recommendation</li> + + + + <li class="chapter " data-level="8.1" data-path="../recommend/cf.html"> + + <a href="../recommend/cf.html"> + + + <b>8.1.</b> + + Collaborative Filtering + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="8.1.1" data-path="../recommend/item_based_cf.html"> + + <a href="../recommend/item_based_cf.html"> + + + <b>8.1.1.</b> + + Item-based Collaborative Filtering + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="8.2" data-path="../recommend/news20.html"> + + <a href="../recommend/news20.html"> + + + <b>8.2.</b> + + News20 related article recommendation Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="8.2.1" data-path="../multiclass/news20_dataset.html"> + + <a href="../multiclass/news20_dataset.html"> + + + <b>8.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.2" data-path="../recommend/news20_jaccard.html"> + + <a href="../recommend/news20_jaccard.html"> + + + <b>8.2.2.</b> + + LSH/Minhash and Jaccard Similarity + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.3" data-path="../recommend/news20_knn.html"> + + <a href="../recommend/news20_knn.html"> + + + <b>8.2.3.</b> + + LSH/Minhash and Brute-Force Search + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.4" data-path="../recommend/news20_bbit_minhash.html"> + + <a href="../recommend/news20_bbit_minhash.html"> + + + <b>8.2.4.</b> + + kNN search using b-Bits Minhash + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="8.3" data-path="../recommend/movielens.html"> + + <a href="../recommend/movielens.html"> + + + <b>8.3.</b> + + MovieLens movie recommendation Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="8.3.1" data-path="../recommend/movielens_dataset.html"> + + <a href="../recommend/movielens_dataset.html"> + + + <b>8.3.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="8.3.2" data-path="../recommend/movielens_mf.html"> + + <a href="../recommend/movielens_mf.html"> + + + <b>8.3.2.</b> + + Matrix Factorization + + </a> + + + + </li> + + <li class="chapter " data-level="8.3.3" data-path="../recommend/movielens_fm.html"> + + <a href="../recommend/movielens_fm.html"> + + + <b>8.3.3.</b> + + Factorization Machine + + </a> + + + + </li> + + <li class="chapter " data-level="8.3.4" data-path="../recommend/movielens_cv.html"> + + <a href="../recommend/movielens_cv.html"> + + + <b>8.3.4.</b> + + 10-fold Cross Validation (Matrix Factorization) + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part IX - Anomaly Detection</li> + + + + <li class="chapter " data-level="9.1" data-path="../anomaly/lof.html"> + + <a href="../anomaly/lof.html"> + + + <b>9.1.</b> + + Outlier Detection using Local Outlier Factor (LOF) + + </a> + + + + </li> + + <li class="chapter " data-level="9.2" data-path="../anomaly/sst.html"> + + <a href="../anomaly/sst.html"> + + + <b>9.2.</b> + + Change-Point Detection using Singular Spectrum Transformation (SST) + + </a> + + + + </li> + + <li class="chapter " data-level="9.3" data-path="../anomaly/changefinder.html"> + + <a href="../anomaly/changefinder.html"> + + + <b>9.3.</b> + + ChangeFinder: Detecting Outlier and Change-Point Simultaneously + + </a> + + + + </li> + + + + + <li class="header">Part X - Hivemall on Spark</li> + + + + <li class="chapter " data-level="10.1" data-path="../spark/binaryclass/"> + + <a href="../spark/binaryclass/"> + + + <b>10.1.</b> + + Binary Classification + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="10.1.1" data-path="../spark/binaryclass/a9a_df.html"> + + <a href="../spark/binaryclass/a9a_df.html"> + + + <b>10.1.1.</b> + + a9a Tutorial for DataFrame + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="10.2" data-path="../spark/binaryclass/"> + + <a href="../spark/binaryclass/"> + + + <b>10.2.</b> + + Regression + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="10.2.1" data-path="../spark/regression/e2006_df.html"> + + <a href="../spark/regression/e2006_df.html"> + + + <b>10.2.1.</b> + + E2006-tfidf regression Tutorial for DataFrame + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="10.3" data-path="../spark/misc/misc.html"> + + <a href="../spark/misc/misc.html"> + + + <b>10.3.</b> + + Generic features + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="10.3.1" data-path="../spark/misc/topk_join.html"> + + <a href="../spark/misc/topk_join.html"> + + + <b>10.3.1.</b> + + Top-k Join processing + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part X - External References</li> + + + + <li class="chapter " data-level="11.1" > + + <a target="_blank" href="https://github.com/maropu/hivemall-spark";> + + + <b>11.1.</b> + + Hivemall on Apache Spark + + </a> + + + + </li> + + <li class="chapter " data-level="11.2" > + + <a target="_blank" href="https://github.com/daijyc/hivemall/wiki/PigHome";> + + + <b>11.2.</b> + + Hivemall on Apache Pig + + </a> + + + + </li> + + + + + <li class="divider"></li> + + <li> + <a href="https://www.gitbook.com"; target="blank" class="gitbook-link"> + Published with GitBook + </a> + </li> +</ul> + + + </nav> + + + </div> + + <div class="book-body"> + + <div class="body-inner"> + + + +<div class="book-header" role="navigation"> + + + <!-- Title --> + <h1> + <i class="fa fa-circle-o-notch fa-spin"></i> + <a href=".." >Ranking Measures</a> + </h1> +</div> + + + + + <div class="page-wrapper" tabindex="-1" role="main"> + <div class="page-inner"> + +<div id="book-search-results"> + <div class="search-noresults"> + + <section class="normal markdown-section"> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<!-- toc --><div id="toc" class="toc"> + +<ul> +<li><a href="#ranking-problems">Ranking Problems</a></li> +<li><a href="#binary-response-measures">Binary Response Measures</a><ul> +<li><a href="#recall-at-k">Recall-At-k</a></li> +<li><a href="#precision-at-k">Precision-At-k</a></li> +<li><a href="#mean-average-precision-map">Mean Average Precision (MAP)</a></li> +<li><a href="#area-under-the-roc-curve-auc">Area Under the ROC Curve (AUC)</a></li> +<li><a href="#mean-reciprocal-rank-mrr">Mean Reciprocal Rank (MRR)</a></li> +<li><a href="#normalized-discounted-cumulative-gain-ndcg">Normalized Discounted Cumulative Gain (NDCG)</a></li> +</ul> +</li> +<li><a href="#graded-response-measures">Graded Response Measures</a></li> +</ul> + +</div><!-- tocstop --> +<h1 id="ranking-problems">Ranking Problems</h1> +<p>Practical machine learning applications such as information retrieval and recommendation internally solve ranking problem which generates and returns a ranked list of items. Hivemall provides a way to tackle the problems as follows:</p> +<ul> +<li><a href="../misc/topk.html">Efficient top-k query processing</a></li> +<li><a href="../recommend/item_based_cf.html">Recommendation based on item-based collaborative filtering</a></li> +</ul> +<p>This page focuses on evaluation of the results from such ranking problems.</p> +<h1 id="binary-response-measures">Binary Response Measures</h1> +<p>In a context of ranking problem, <strong>binary response</strong> means that binary labels are assigned to items, and positive items are considered as <em>truth</em> observations.</p> +<p>In a <code>dummy_truth</code> table, we assume that there are three users (<code>userid = 1, 2, 3</code>) who have exactly same three truth ranked items (<code>itemid = 1, 2, 4</code>) chosen from existing six items:</p> +<table> +<thead> +<tr> +<th style="text-align:center">userid</th> +<th style="text-align:center">itemid</th> +</tr> +</thead> +<tbody> +<tr> +<td style="text-align:center">1</td> +<td style="text-align:center">1</td> +</tr> +<tr> +<td style="text-align:center">1</td> +<td style="text-align:center">2</td> +</tr> +<tr> +<td style="text-align:center">1</td> +<td style="text-align:center">4</td> +</tr> +<tr> +<td style="text-align:center">2</td> +<td style="text-align:center">1</td> +</tr> +<tr> +<td style="text-align:center">2</td> +<td style="text-align:center">2</td> +</tr> +<tr> +<td style="text-align:center">2</td> +<td style="text-align:center">4</td> +</tr> +<tr> +<td style="text-align:center">3</td> +<td style="text-align:center">1</td> +</tr> +<tr> +<td style="text-align:center">3</td> +<td style="text-align:center">2</td> +</tr> +<tr> +<td style="text-align:center">3</td> +<td style="text-align:center">4</td> +</tr> +</tbody> +</table> +<p>Additionally, here is a <code>dummy_rec</code> table we obtained as a result of prediction:</p> +<table> +<thead> +<tr> +<th style="text-align:center">userid</th> +<th style="text-align:center">itemid</th> +<th style="text-align:center">score</th> +</tr> +</thead> +<tbody> +<tr> +<td style="text-align:center">1</td> +<td style="text-align:center">1</td> +<td style="text-align:center">10.0</td> +</tr> +<tr> +<td style="text-align:center">1</td> +<td style="text-align:center">3</td> +<td style="text-align:center">8.0</td> +</tr> +<tr> +<td style="text-align:center">1</td> +<td style="text-align:center">2</td> +<td style="text-align:center">6.0</td> +</tr> +<tr> +<td style="text-align:center">1</td> +<td style="text-align:center">6</td> +<td style="text-align:center">2.0</td> +</tr> +<tr> +<td style="text-align:center">2</td> +<td style="text-align:center">1</td> +<td style="text-align:center">10.0</td> +</tr> +<tr> +<td style="text-align:center">2</td> +<td style="text-align:center">3</td> +<td style="text-align:center">8.0</td> +</tr> +<tr> +<td style="text-align:center">2</td> +<td style="text-align:center">2</td> +<td style="text-align:center">6.0</td> +</tr> +<tr> +<td style="text-align:center">2</td> +<td style="text-align:center">6</td> +<td style="text-align:center">2.0</td> +</tr> +<tr> +<td style="text-align:center">3</td> +<td style="text-align:center">1</td> +<td style="text-align:center">10.0</td> +</tr> +<tr> +<td style="text-align:center">3</td> +<td style="text-align:center">3</td> +<td style="text-align:center">8.0</td> +</tr> +<tr> +<td style="text-align:center">3</td> +<td style="text-align:center">2</td> +<td style="text-align:center">6.0</td> +</tr> +<tr> +<td style="text-align:center">3</td> +<td style="text-align:center">6</td> +<td style="text-align:center">2.0</td> +</tr> +</tbody> +</table> +<p>How can we compare <code>dummy_rec</code> with <code>dummy_truth</code> to figure out the accuracy of <code>dummy_rec</code>?</p> +<p>To be more precise, in case we built a recommender system, let a target user <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mi>u</mi><mo>∈</mo><mrow><mi mathvariant="script">U</mi></mrow></mrow><annotation encoding="application/x-tex">u \in \mathcal{U}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.68333em;"></span><span class="strut bottom" style="height:0.72243em;vertical-align:-0.0391em;"></span><span class="base textstyle uncramped"><span class="mord mathit">u</span><span class="mrel">∈</span><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.09931em;">U</span></span></span></span></span>, set of all items <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mrow><mi mathvariant="script">I</mi></mrow></mrow><annotation encoding="application/x-tex">\mathcal{I}</annotation></semantics></math></span><span class="katex -html" aria-hidden="true"><span class="strut" style="height:0.68333em;"></span><span class="strut bottom" style="height:0.68333em;vertical-align:0em;"></span><span class="base textstyle uncramped"><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span></span></span></span>, ordered set of top-k recommended items <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msub><mi>I</mi><mi>k</mi></msub><mo>(</mo><mi>u</mi><mo>)</mo><mo>⊂</mo><mrow><mi mathvariant="script">I</mi></mrow></mrow><annotation encoding="application/x-tex">I_k(u) \subset \mathcal{I}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom" style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord"><span class="mord mathit" style="margin-right:0.07847em;">I</span><span class="vlist"><sp an style="top:0.15em;margin-right:0.05em;margin-left:-0.07847em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mopen">(</span><span class="mord mathit">u</span><span class="mclose">)</span><span class="mrel">⊂</span><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span></span></span></span>, and set of truth items <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msubsup><mrow><mi mathvariant="script">I</mi></mrow><mi>u</mi><mo>+</mo></msubsup></mrow><annotation encoding="application/x-tex">\mathcal{I}^+_u</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.771331em;"></span><span class="strut bottom" style="height:1.0183309999999999em;vertical-align:-0.247em;"></span><span class="base textstyle uncramped"><span class=""><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span><span class="vlist"><span style="top:0.247em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit">u</span></span></span><span style="top:-0.363em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle uncramped"><span class="mord">+</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</ span></span></span></span></span></span>. Hence, when we launch top-2 recommendation for the above tables, <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mrow><mi mathvariant="script">U</mi></mrow><mo>=</mo><mo>{</mo><mn>1</mn><mo separator="true">,</mo><mn>2</mn><mo separator="true">,</mo><mn>3</mn><mo>}</mo></mrow><annotation encoding="application/x-tex">\mathcal{U} = \{1, 2, 3\}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom" style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.09931em;">U</span></span><span class="mrel">=</span><span class="mopen">{</span><span class="mord mathrm">1</span><span class="mpunct">,</span><span class="mord mathrm">2</span><span class="mpunct">,</span><span class="mord mathrm">3</span><span class= "mclose">}</span></span></span></span>, <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mrow><mi mathvariant="script">I</mi></mrow><mo>=</mo><mo>{</mo><mn>1</mn><mo separator="true">,</mo><mn>2</mn><mo separator="true">,</mo><mn>3</mn><mo separator="true">,</mo><mn>4</mn><mo separator="true">,</mo><mn>5</mn><mo separator="true">,</mo><mn>6</mn><mo>}</mo></mrow><annotation encoding="application/x-tex">\mathcal{I} = \{1, 2, 3, 4, 5, 6\}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom" style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span><span class="mrel">=</span><span class="mopen">{</span><span class="mord mathrm">1</span><span class="mpunct">,</span><span class="mord mathrm">2</span><span class="mpunct">, </span><span class="mord mathrm">3</span><span class="mpunct">,</span><span class="mord mathrm">4</span><span class="mpunct">,</span><span class="mord mathrm">5</span><span class="mpunct">,</span><span class="mord mathrm">6</span><span class="mclose">}</span></span></span></span> and <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msub><mi>I</mi><mn>2</mn></msub><mo>(</mo><mi>u</mi><mo>)</mo><mo>=</mo><mo>{</mo><mn>1</mn><mo separator="true">,</mo><mn>3</mn><mo>}</mo></mrow><annotation encoding="application/x-tex">I_2(u) = \{1, 3\}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom" style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord"><span class="mord mathit" style="margin-right:0.07847em;">I</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.07847em;"><span class="f ontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathrm">2</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mopen">(</span><span class="mord mathit">u</span><span class="mclose">)</span><span class="mrel">=</span><span class="mopen">{</span><span class="mord mathrm">1</span><span class="mpunct">,</span><span class="mord mathrm">3</span><span class="mclose">}</span></span></span></span> which consists of two highest-scored items, and <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msubsup><mrow><mi mathvariant="script">I</mi></mrow><mi>u</mi><mo>+</mo></msubsup><mo>=</mo><mo>{</mo><mn>1</mn><mo separator="true">,</mo><mn>2</mn><mo separator="true">,</mo><mn>4</mn><mo>}</mo></mrow><annotation encoding="application/x-tex"> \mathcal{I}^+_u = \{1, 2, 4\}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.771331em;"></span><span class="strut bottom" style="height:1.021331em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class=""><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span><span class="vlist"><span style="top:0.247em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit">u</span></span></span><span style="top:-0.363em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle uncramped"><span class="mord">+</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5 "><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mrel">=</span><span class="mopen">{</span><span class="mord mathrm">1</span><span class="mpunct">,</span><span class="mord mathrm">2</span><span class="mpunct">,</span><span class="mord mathrm">4</span><span class="mclose">}</span></span></span></span>.</p> +<p>Evaluation of the ordered sets can be done by the following query:</p> +<pre><code class="lang-sql">with truth as ( + <span class="hljs-keyword">select</span> userid, collect_set(itemid) <span class="hljs-keyword">as</span> truth + <span class="hljs-keyword">from</span> dummy_truth + <span class="hljs-keyword">group</span> <span class="hljs-keyword">by</span> userid +), +rec <span class="hljs-keyword">as</span> ( + <span class="hljs-keyword">select</span> + userid, + map_values(to_ordered_map(score, itemid, <span class="hljs-literal">true</span>)) <span class="hljs-keyword">as</span> rec, + <span class="hljs-keyword">cast</span>(<span class="hljs-keyword">count</span>(itemid) <span class="hljs-keyword">as</span> <span class="hljs-built_in">int</span>) <span class="hljs-keyword">as</span> max_k + <span class="hljs-keyword">from</span> dummy_rec + <span class="hljs-keyword">group</span> <span class="hljs-keyword">by</span> userid +) +<span class="hljs-keyword">select</span> + <span class="hljs-comment">-- rec = [1,3,2,6], truth = [1,2,4] for each user</span> + + <span class="hljs-comment">-- Recall@k</span> + recall(t1.rec, t2.truth, t1.max_k) <span class="hljs-keyword">as</span> recall, + recall(t1.rec, t2.truth, <span class="hljs-number">2</span>) <span class="hljs-keyword">as</span> recall_at_2, + + <span class="hljs-comment">-- Precision@k</span> + <span class="hljs-keyword">precision</span>(t1.rec, t2.truth, t1.max_k) <span class="hljs-keyword">as</span> <span class="hljs-keyword">precision</span>, + <span class="hljs-keyword">precision</span>(t1.rec, t2.truth, <span class="hljs-number">2</span>) <span class="hljs-keyword">as</span> precision_at_2, + + <span class="hljs-comment">-- MAP</span> + average_precision(t1.rec, t2.truth, t1.max_k) <span class="hljs-keyword">as</span> average_precision, + average_precision(t1.rec, t2.truth, <span class="hljs-number">2</span>) <span class="hljs-keyword">as</span> average_precision_at_2, + + <span class="hljs-comment">-- AUC</span> + auc(t1.rec, t2.truth, t1.max_k) <span class="hljs-keyword">as</span> auc, + auc(t1.rec, t2.truth, <span class="hljs-number">2</span>) <span class="hljs-keyword">as</span> auc_at_2, + + <span class="hljs-comment">-- MRR</span> + mrr(t1.rec, t2.truth, t1.max_k) <span class="hljs-keyword">as</span> mrr, + mrr(t1.rec, t2.truth, <span class="hljs-number">2</span>) <span class="hljs-keyword">as</span> mrr_at_2, + + <span class="hljs-comment">-- NDCG</span> + ndcg(t1.rec, t2.truth, t1.max_k) <span class="hljs-keyword">as</span> ndcg, + ndcg(t1.rec, t2.truth, <span class="hljs-number">2</span>) <span class="hljs-keyword">as</span> ndcg_at_2 +<span class="hljs-keyword">from</span> rec t1 +<span class="hljs-keyword">join</span> truth t2 <span class="hljs-keyword">on</span> (t1.userid = t2.userid) +; +</code></pre> +<p>We have six different measures, and outputs will be:</p> +<table> +<thead> +<tr> +<th style="text-align:center">Ranking measure</th> +<th style="text-align:left">top-4 (max_k)</th> +<th style="text-align:left">top-2</th> +</tr> +</thead> +<tbody> +<tr> +<td style="text-align:center">Recall</td> +<td style="text-align:left">0.6666666666666666</td> +<td style="text-align:left">0.3333333333333333</td> +</tr> +<tr> +<td style="text-align:center">Precision</td> +<td style="text-align:left">0.5</td> +<td style="text-align:left">0.5</td> +</tr> +<tr> +<td style="text-align:center">MAP</td> +<td style="text-align:left">0.5555555555555555</td> +<td style="text-align:left">0.3333333333333333</td> +</tr> +<tr> +<td style="text-align:center">AUC</td> +<td style="text-align:left">0.75</td> +<td style="text-align:left">1.0</td> +</tr> +<tr> +<td style="text-align:center">MRR</td> +<td style="text-align:left">1.0</td> +<td style="text-align:left">1.0</td> +</tr> +<tr> +<td style="text-align:center">NDCG</td> +<td style="text-align:left">0.7039180890341349</td> +<td style="text-align:left">0.6131471927654585</td> +</tr> +</tbody> +</table> +<p>Here, we introduce the six measures for evaluation of ranked list of items. Importantly, each metric has a different concept behind formulation, and the accuracy measured by the metrics shows different values even for the exactly same input as demonstrated above. Thus, evaluation using multiple ranking measures is more convincing, and it should be easy in Hivemall.</p> +<h2 id="recall-at-k">Recall-At-k</h2> +<p><strong>Recall-at-k (Recall@k)</strong> indicates coverage of truth samples as a result of top-k recommendation. The value is computed by the following equation: +<span class="katex-display"><span class="katex"><span class="katex-mathml"><math><semantics><mrow><mrow><mi mathvariant="normal">R</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">@</mi></mrow><mi>k</mi><mo>=</mo><mfrac><mrow><mi mathvariant="normal">∣</mi><msubsup><mrow><mi mathvariant="script">I</mi></mrow><mi>u</mi><mo>+</mo></msubsup><mo>∩</mo><msub><mi>I</mi><mi>k</mi></msub><mo>(</mo><mi>u</mi><mo>)</mo><mi mathvariant="normal">∣</mi></mrow><mrow><mi mathvariant="normal">∣</mi><msubsup><mrow><mi mathvariant="script">I</mi></mrow><mi>u</mi><mo>+</mo></msubsup><mi mathvariant="normal">∣</mi></mrow></mfrac><mi mathvariant="normal">.</mi></mrow><annotation encoding="application/x-tex"> +\mathrm{Recall@}k = \frac{|\mathcal{I}^+_u \cap I_k(u)|}{|\mathcal{I}^+_u|}. +</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:1.448331em;"></span><span class="strut bottom" style="height:2.384331em;vertical-align:-0.936em;"></span><span class="base displaystyle textstyle uncramped"><span class="mord displaystyle textstyle uncramped"><span class="mord mathrm">R</span><span class="mord mathrm">e</span><span class="mord mathrm">c</span><span class="mord mathrm">a</span><span class="mord mathrm">l</span><span class="mord mathrm">l</span><span class="mord mathrm">@</span></span><span class="mord mathit" style="margin-right:0.03148em;">k</span><span class="mrel">=</span><span class="mord reset-textstyle displaystyle textstyle uncramped"><span class="sizing reset-size5 size5 reset-textstyle textstyle uncramped nulldelimiter"></span><span class="mfrac"><span class="vlist"><span style="top:0.686em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><s pan class="reset-textstyle textstyle cramped"><span class="mord textstyle cramped"><span class="mord mathrm">∣</span><span class=""><span class="mord textstyle cramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span><span class="vlist"><span style="top:0.247em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit">u</span></span></span><span style="top:-0.29733em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord">+</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mord mathrm">∣</span></span></span></span><span style="top:-0.230 0000000000001em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle textstyle uncramped frac-line"></span></span><span style="top:-0.677em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle textstyle uncramped"><span class="mord textstyle uncramped"><span class="mord mathrm">∣</span><span class=""><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span><span class="vlist"><span style="top:0.247em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit">u</span></span></span><span style="top:-0.363em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle uncramped"><span class="mord">+</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mbin">∩</span><span class="mord"><span class="mord mathit" style="margin-right:0.07847em;">I</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.07847em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mopen">(</span><span class="mord mathit">u</span><span class="mclose">)</span><span class="mord mathrm">∣</span></span></span></span><span c lass="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="sizing reset-size5 size5 reset-textstyle textstyle uncramped nulldelimiter"></span></span><span class="mord mathrm">.</span></span></span></span></span> +Here, <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mi mathvariant="normal">∣</mi><msubsup><mrow><mi mathvariant="script">I</mi></mrow><mi>u</mi><mo>+</mo></msubsup><mo>∩</mo><msub><mi>I</mi><mi>k</mi></msub><mo>(</mo><mi>u</mi><mo>)</mo><mi mathvariant="normal">∣</mi></mrow><annotation encoding="application/x-tex">|\mathcal{I}^+_u \cap I_k(u)|</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.771331em;"></span><span class="strut bottom" style="height:1.021331em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord mathrm">∣</span><span class=""><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span><span class="vlist"><span style="top:0.247em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span cla ss="reset-textstyle scriptstyle cramped"><span class="mord mathit">u</span></span></span><span style="top:-0.363em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle uncramped"><span class="mord">+</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mbin">∩</span><span class="mord"><span class="mord mathit" style="margin-right:0.07847em;">I</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.07847em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size 5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mopen">(</span><span class="mord mathit">u</span><span class="mclose">)</span><span class="mord mathrm">∣</span></span></span></span> is the number of true positives. If <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msub><mi>I</mi><mn>2</mn></msub><mo>(</mo><mi>u</mi><mo>)</mo><mo>=</mo><mo>{</mo><mn>1</mn><mo separator="true">,</mo><mn>3</mn><mo>}</mo></mrow><annotation encoding="application/x-tex">I_2(u) = \{1, 3\}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom" style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord"><span class="mord mathit" style="margin-right:0.07847em;">I</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.07847em;"><span class="fontsize-ensu rer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathrm">2</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mopen">(</span><span class="mord mathit">u</span><span class="mclose">)</span><span class="mrel">=</span><span class="mopen">{</span><span class="mord mathrm">1</span><span class="mpunct">,</span><span class="mord mathrm">3</span><span class="mclose">}</span></span></span></span> and <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msubsup><mrow><mi mathvariant="script">I</mi></mrow><mi>u</mi><mo>+</mo></msubsup><mo>=</mo><mo>{</mo><mn>1</mn><mo separator="true">,</mo><mn>2</mn><mo separator="true">,</mo><mn>4</mn><mo>}</mo></mrow><annotation encoding="application/x-tex">\mathcal{I}^+_u = \{1, 2, 4\}</annotation></semantics></ math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.771331em;"></span><span class="strut bottom" style="height:1.021331em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class=""><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span><span class="vlist"><span style="top:0.247em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit">u</span></span></span><span style="top:-0.363em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle uncramped"><span class="mord">+</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>&#x 200B;</span></span></span><span class="mrel">=</span><span class="mopen">{</span><span class="mord mathrm">1</span><span class="mpunct">,</span><span class="mord mathrm">2</span><span class="mpunct">,</span><span class="mord mathrm">4</span><span class="mclose">}</span></span></span></span>, <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mrow><mi mathvariant="normal">R</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">@</mi></mrow><mn>2</mn><mo>=</mo><mn>1</mn><mi mathvariant="normal">/</mi><mn>3</mn><mo>≈</mo><mn>0</mn><mi mathvariant="normal">.</mi><mn>3</mn><mn>3</mn><mn>3</mn></mrow><annotation encoding="application/x-tex">\mathrm{Recall@}2 = 1 / 3 \approx 0.333</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom " style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord textstyle uncramped"><span class="mord mathrm">R</span><span class="mord mathrm">e</span><span class="mord mathrm">c</span><span class="mord mathrm">a</span><span class="mord mathrm">l</span><span class="mord mathrm">l</span><span class="mord mathrm">@</span></span><span class="mord mathrm">2</span><span class="mrel">=</span><span class="mord mathrm">1</span><span class="mord mathrm">/</span><span class="mord mathrm">3</span><span class="mrel">≈</span><span class="mord mathrm">0</span><span class="mord mathrm">.</span><span class="mord mathrm">3</span><span class="mord mathrm">3</span><span class="mord mathrm">3</span></span></span></span>.</p> +<h2 id="precision-at-k">Precision-At-k</h2> +<p>Unlike Recall@k, <strong>Precision-at-k (Precision@k)</strong> evaluates correctness of a top-k recommendation list <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msub><mi>I</mi><mi>k</mi></msub><mo>(</mo><mi>u</mi><mo>)</mo></mrow><annotation encoding="application/x-tex">I_k(u)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom" style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord"><span class="mord mathit" style="margin-right:0.07847em;">I</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.07847em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span></span></span><span class="baseline-fix"><span class ="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mopen">(</span><span class="mord mathit">u</span><span class="mclose">)</span></span></span></span> according to the portion of true positives in the list as: +<span class="katex-display"><span class="katex"><span class="katex-mathml"><math><semantics><mrow><mrow><mi mathvariant="normal">P</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">@</mi></mrow><mi>k</mi><mo>=</mo><mfrac><mrow><mi mathvariant="normal">∣</mi><msubsup><mrow><mi mathvariant="script">I</mi></mrow><mi>u</mi><mo>+</mo></msubsup><mo>∩</mo><msub><mi>I</mi><mi>k</mi></msub><mo>(</mo><mi>u</mi><mo>)</mo><mi mathvariant="normal">∣</mi></mrow><mrow><mi mathvariant="normal">∣</mi><msub><mi>I</mi><mi>k</mi></msub><mo>(</mo><mi>u</mi><mo>)</mo><mi mathvariant="normal">∣</mi></mrow></mfrac><mi mathvariant="normal">.</mi></mrow><annotation encoding="application/x-tex"> +\mathrm{Precision@}k = \frac{|\mathcal{I}^+_u \cap I_k(u)|}{|I_k(u)|}. +</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:1.448331em;"></span><span class="strut bottom" style="height:2.384331em;vertical-align:-0.936em;"></span><span class="base displaystyle textstyle uncramped"><span class="mord displaystyle textstyle uncramped"><span class="mord mathrm">P</span><span class="mord mathrm">r</span><span class="mord mathrm">e</span><span class="mord mathrm">c</span><span class="mord mathrm">i</span><span class="mord mathrm">s</span><span class="mord mathrm">i</span><span class="mord mathrm">o</span><span class="mord mathrm">n</span><span class="mord mathrm">@</span></span><span class="mord mathit" style="margin-right:0.03148em;">k</span><span class="mrel">=</span><span class="mord reset-textstyle displaystyle textstyle uncramped"><span class="sizing reset-size5 size5 reset-textstyle textstyle uncramped nulldelimiter"></span><span class="mfrac"><span class="vlist"><span style="top:0.686em;"> <span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle textstyle cramped"><span class="mord textstyle cramped"><span class="mord mathrm">∣</span><span class="mord"><span class="mord mathit" style="margin-right:0.07847em;">I</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.07847em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mopen">(</span><span class="mord mathit">u</span><span class="mclose">)</span><span class="mord mathrm">∣</span></span></span></span><span style="top:-0.2300000000000001em;"><span class="font size-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle textstyle uncramped frac-line"></span></span><span style="top:-0.677em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle textstyle uncramped"><span class="mord textstyle uncramped"><span class="mord mathrm">∣</span><span class=""><span class="mord textstyle uncramped"><span class="mord mathcal" style="margin-right:0.07382em;">I</span></span><span class="vlist"><span style="top:0.247em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit">u</span></span></span><span style="top:-0.363em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle uncramped"><span class="mord">+</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mbin">∩</span><span class="mord"><span class="mord mathit" style="margin-right:0.07847em;">I</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.07847em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathit" style="margin-right:0.03148em;">k</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mopen">(</span><span class="mord mathit">u</span><span class="mclose">)</span><span class="mord mathrm">∣</span></span></span></span><span class="baseline-fix"><span class="fo ntsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="sizing reset-size5 size5 reset-textstyle textstyle uncramped nulldelimiter"></span></span><span class="mord mathrm">.</span></span></span></span></span> +In other words, Precision@k means how much the recommendation list covers true pairs. Here, <span class="katex"><span class="katex-mathml"><math><semantics><mrow><mrow><mi mathvariant="normal">P</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">@</mi></mrow><mn>2</mn><mo>=</mo><mn>1</mn><mi mathvariant="normal">/</mi><mn>2</mn><mo>=</mo><mn>0</mn><mi mathvariant="normal">.</mi><mn>5</mn></mrow><annotation encoding="application/x-tex">\mathrm{Precision@}2 = 1 / 2 = 0.5</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom" style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord textstyle uncramped"><span clas s="mord mathrm">P</span><span class="mord mathrm">r</span><span class="mord mathrm">e</span><span class="mord mathrm">c</span><span class="mord mathrm">i</span><span class="mord mathrm">s</span><span class="mord mathrm">i</span><span class="mord mathrm">o</span><span class="mord mathrm">n</span><span class="mord mathrm">@</span></span><span class="mord mathrm">2</span><span class="mrel">=</span><span class="mord mathrm">1</span><span class="mord mathrm">/</span><span class="mord mathrm">2</span><span class="mrel">=</span><span class="mord mathrm">0</span><span class="mord mathrm">.</span><span class="mord mathrm">5</span></span></span></span> where <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msub><mi>I</mi><mn>2</mn></msub><mo>(</mo><mi>u</mi><mo>)</mo><mo>=</mo><mo>{</mo><mn>1</mn><mo separator="true">,</mo><mn>3</mn><mo>}</mo></mrow><annotation encoding="application/x-tex">I_2(u) = \{1, 3\}</annotation></semantics></math></span><span class="katex-html" a ria-hidden="true"><span class="strut" style="height:0.75em;"></span><span class="strut bottom" style="height:1em;vertical-align:-0.25em;"></span><span class="base textstyle uncramped"><span class="mord"><span class="mord mathit" style="margin-right:0.07847em;">I</span><span class="vlist"><span style="top:0.15em;margin-right:0.05em;margin-left:-0.07847em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle cramped"><span class="mord mathrm">2</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mopen">(</span><span class="mord mathit">u</span><span class="mclose">)</span><span class="mrel">=</span><span class="mopen">{</span><span class="mord mathrm">1</span><span class="mpunct">,</span><span class="mord mathrm">3</span><span class="mclose">}</span></span></span ></span> and <span class="katex"><span >class="katex-mathml"><math><semantics><mrow><msubsup><mrow><mi >mathvariant="script">I</mi></mrow><mi>u</mi><mo>+</mo></msubsup><mo>=</mo><mo>{</mo><mn>1</mn><mo > separator="true">,</mo><mn>2</mn><mo >separator="true">,</mo><mn>4</mn><mo>}</mo></mrow><annotation >encoding="application/x-tex">\mathcal{I}^+_u = \{1, 2, >4\}</annotation></semantics></math></span><span class="katex-html" >aria-hidden="true"><span class="strut" >style="height:0.771331em;"></span><span class="strut bottom" >style="height:1.021331em;vertical-align:-0.25em;"></span><span class="base >textstyle uncramped"><span class=""><span class="mord textstyle >uncramped"><span class="mord mathcal" >style="margin-right:0.07382em;">I</span></span><span class="vlist"><span >style="top:0.247em;margin-right:0.05em;"><span class="fontsize-ensurer >reset-size5 size5"><span style="font-size:0em;">​</span></span><span >class="reset-textstyle scriptstyle cramped"><span class="mord >mathit">u</span>< /span></span><span style="top:-0.363em;margin-right:0.05em;"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span><span class="reset-textstyle scriptstyle uncramped"><span class="mord">+</span></span></span><span class="baseline-fix"><span class="fontsize-ensurer reset-size5 size5"><span style="font-size:0em;">​</span></span>​</span></span></span><span class="mrel">=</span><span class="mopen">{</span><span class="mord mathrm">1</span><span class="mpunct">,</span><span class="mord mathrm">2</span><span class="mpunct">,</span><span class="mord mathrm">4</span><span class="mclose">}</span></span></span></span>.</p> +<h2 id="mean-average-precision-map">Mean Average Precision (MAP)</h2> +<p>While the original Precision@k provides a score for a fixed-length recommendation list <span class="katex"><span class="katex-mathml"><math><semantics><mrow><msub><mi>I</mi><mi>k</mi></msub><mo>(</mo><mi>u</mi><mo>)</mo></mrow><annotation
<TRUNCATED>
