http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/b8a847fa/userguide/misc/funcs.html ---------------------------------------------------------------------- diff --git a/userguide/misc/funcs.html b/userguide/misc/funcs.html new file mode 100644 index 0000000..37f5250 --- /dev/null +++ b/userguide/misc/funcs.html @@ -0,0 +1,2799 @@ + +<!DOCTYPE HTML> +<html lang="" > + <head> + <meta charset="UTF-8"> + <meta content="text/html; charset=utf-8" http-equiv="Content-Type"> + <title>List of Functions · Hivemall User Manual</title> + <meta http-equiv="X-UA-Compatible" content="IE=edge" /> + <meta name="description" content=""> + <meta name="generator" content="GitBook 3.2.3"> + + + + + <link rel="stylesheet" href="../gitbook/style.css"> + + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-splitter/splitter.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-etoc/plugin.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-callouts/plugin.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-toggle-chapters/toggle.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-codeblock-filename/block.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-expandable-chapters/expandable-chapters.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-multipart/multipart.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-katex/katex.min.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-emphasize/plugin.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css"> + + + + <link rel="stylesheet" href="../gitbook/gitbook-plugin-theme-api/theme-api.css"> + + + + + + + + + + + + + + + + + + + + + + + + <meta name="HandheldFriendly" content="true"/> + <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no"> + <meta name="apple-mobile-web-app-capable" content="yes"> + <meta name="apple-mobile-web-app-status-bar-style" content="black"> + <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png"> + <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon"> + + + <link rel="next" href="../tips/" /> + + + <link rel="prev" href="../getting_started/input-format.html" /> + + + </head> + <body> + +<div class="book"> + <div class="book-summary"> + + +<div id="book-search-input" role="search"> + <input type="text" placeholder="Type to search" /> +</div> + + + <nav role="navigation"> + + + +<ul class="summary"> + + + + + <li> + <a href="http://hivemall.incubator.apache.org/"; target="_blank" class="custom-link"><i class="fa fa-home"></i> Home</a> + </li> + + + + + <li class="divider"></li> + + + + + <li class="header">TABLE OF CONTENTS</li> + + + + <li class="chapter " data-level="1.1" data-path="../"> + + <a href="../"> + + + <b>1.1.</b> + + Introduction + + </a> + + + + </li> + + <li class="chapter " data-level="1.2" data-path="../getting_started/"> + + <a href="../getting_started/"> + + + <b>1.2.</b> + + Getting Started + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.2.1" data-path="../getting_started/installation.html"> + + <a href="../getting_started/installation.html"> + + + <b>1.2.1.</b> + + Installation + + </a> + + + + </li> + + <li class="chapter " data-level="1.2.2" data-path="../getting_started/permanent-functions.html"> + + <a href="../getting_started/permanent-functions.html"> + + + <b>1.2.2.</b> + + Install as permanent functions + + </a> + + + + </li> + + <li class="chapter " data-level="1.2.3" data-path="../getting_started/input-format.html"> + + <a href="../getting_started/input-format.html"> + + + <b>1.2.3.</b> + + Input Format + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter active" data-level="1.3" data-path="funcs.html"> + + <a href="funcs.html"> + + + <b>1.3.</b> + + List of Functions + + </a> + + + + </li> + + <li class="chapter " data-level="1.4" data-path="../tips/"> + + <a href="../tips/"> + + + <b>1.4.</b> + + Tips for Effective Hivemall + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.4.1" data-path="../tips/addbias.html"> + + <a href="../tips/addbias.html"> + + + <b>1.4.1.</b> + + Explicit add_bias() for better prediction + + </a> + + + + </li> + + <li class="chapter " data-level="1.4.2" data-path="../tips/rand_amplify.html"> + + <a href="../tips/rand_amplify.html"> + + + <b>1.4.2.</b> + + Use rand_amplify() to better prediction results + + </a> + + + + </li> + + <li class="chapter " data-level="1.4.3" data-path="../tips/rt_prediction.html"> + + <a href="../tips/rt_prediction.html"> + + + <b>1.4.3.</b> + + Real-time prediction on RDBMS + + </a> + + + + </li> + + <li class="chapter " data-level="1.4.4" data-path="../tips/ensemble_learning.html"> + + <a href="../tips/ensemble_learning.html"> + + + <b>1.4.4.</b> + + Ensemble learning for stable prediction + + </a> + + + + </li> + + <li class="chapter " data-level="1.4.5" data-path="../tips/mixserver.html"> + + <a href="../tips/mixserver.html"> + + + <b>1.4.5.</b> + + Mixing models for a better prediction convergence (MIX server) + + </a> + + + + </li> + + <li class="chapter " data-level="1.4.6" data-path="../tips/emr.html"> + + <a href="../tips/emr.html"> + + + <b>1.4.6.</b> + + Run Hivemall on Amazon Elastic MapReduce + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="1.5" data-path="../tips/general_tips.html"> + + <a href="../tips/general_tips.html"> + + + <b>1.5.</b> + + General Hive/Hadoop Tips + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.5.1" data-path="../tips/rowid.html"> + + <a href="../tips/rowid.html"> + + + <b>1.5.1.</b> + + Adding rowid for each row + + </a> + + + + </li> + + <li class="chapter " data-level="1.5.2" data-path="../tips/hadoop_tuning.html"> + + <a href="../tips/hadoop_tuning.html"> + + + <b>1.5.2.</b> + + Hadoop tuning for Hivemall + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="1.6" data-path="../troubleshooting/"> + + <a href="../troubleshooting/"> + + + <b>1.6.</b> + + Troubleshooting + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="1.6.1" data-path="../troubleshooting/oom.html"> + + <a href="../troubleshooting/oom.html"> + + + <b>1.6.1.</b> + + OutOfMemoryError in training + + </a> + + + + </li> + + <li class="chapter " data-level="1.6.2" data-path="../troubleshooting/mapjoin_task_error.html"> + + <a href="../troubleshooting/mapjoin_task_error.html"> + + + <b>1.6.2.</b> + + SemanticException generate map join task error: Cannot serialize object + + </a> + + + + </li> + + <li class="chapter " data-level="1.6.3" data-path="../troubleshooting/asterisk.html"> + + <a href="../troubleshooting/asterisk.html"> + + + <b>1.6.3.</b> + + Asterisk argument for UDTF does not work + + </a> + + + + </li> + + <li class="chapter " data-level="1.6.4" data-path="../troubleshooting/num_mappers.html"> + + <a href="../troubleshooting/num_mappers.html"> + + + <b>1.6.4.</b> + + The number of mappers is less than input splits in Hadoop 2.x + + </a> + + + + </li> + + <li class="chapter " data-level="1.6.5" data-path="../troubleshooting/mapjoin_classcastex.html"> + + <a href="../troubleshooting/mapjoin_classcastex.html"> + + + <b>1.6.5.</b> + + Map-side join causes ClassCastException on Tez + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part II - Generic Features</li> + + + + <li class="chapter " data-level="2.1" data-path="generic_funcs.html"> + + <a href="generic_funcs.html"> + + + <b>2.1.</b> + + List of Generic Hivemall Functions + + </a> + + + + </li> + + <li class="chapter " data-level="2.2" data-path="topk.html"> + + <a href="topk.html"> + + + <b>2.2.</b> + + Efficient Top-K Query Processing + + </a> + + + + </li> + + <li class="chapter " data-level="2.3" data-path="tokenizer.html"> + + <a href="tokenizer.html"> + + + <b>2.3.</b> + + Text Tokenizer + + </a> + + + + </li> + + <li class="chapter " data-level="2.4" data-path="approx.html"> + + <a href="approx.html"> + + + <b>2.4.</b> + + Approximate Aggregate Functions + + </a> + + + + </li> + + + + + <li class="header">Part III - Feature Engineering</li> + + + + <li class="chapter " data-level="3.1" data-path="../ft_engineering/scaling.html"> + + <a href="../ft_engineering/scaling.html"> + + + <b>3.1.</b> + + Feature Scaling + + </a> + + + + </li> + + <li class="chapter " data-level="3.2" data-path="../ft_engineering/hashing.html"> + + <a href="../ft_engineering/hashing.html"> + + + <b>3.2.</b> + + Feature Hashing + + </a> + + + + </li> + + <li class="chapter " data-level="3.3" data-path="../ft_engineering/selection.html"> + + <a href="../ft_engineering/selection.html"> + + + <b>3.3.</b> + + Feature Selection + + </a> + + + + </li> + + <li class="chapter " data-level="3.4" data-path="../ft_engineering/binning.html"> + + <a href="../ft_engineering/binning.html"> + + + <b>3.4.</b> + + Feature Binning + + </a> + + + + </li> + + <li class="chapter " data-level="3.5" data-path="../ft_engineering/pairing.html"> + + <a href="../ft_engineering/pairing.html"> + + + <b>3.5.</b> + + Feature Paring + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="3.5.1" data-path="../ft_engineering/polynomial.html"> + + <a href="../ft_engineering/polynomial.html"> + + + <b>3.5.1.</b> + + Polynomial features + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="3.6" data-path="../ft_engineering/ft_trans.html"> + + <a href="../ft_engineering/ft_trans.html"> + + + <b>3.6.</b> + + Feature Transformation + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="3.6.1" data-path="../ft_engineering/vectorization.html"> + + <a href="../ft_engineering/vectorization.html"> + + + <b>3.6.1.</b> + + Feature vectorization + + </a> + + + + </li> + + <li class="chapter " data-level="3.6.2" data-path="../ft_engineering/quantify.html"> + + <a href="../ft_engineering/quantify.html"> + + + <b>3.6.2.</b> + + Quantify non-number features + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="3.7" data-path="../ft_engineering/tfidf.html"> + + <a href="../ft_engineering/tfidf.html"> + + + <b>3.7.</b> + + TF-IDF Calculation + + </a> + + + + </li> + + + + + <li class="header">Part IV - Evaluation</li> + + + + <li class="chapter " data-level="4.1" data-path="../eval/binary_classification_measures.html"> + + <a href="../eval/binary_classification_measures.html"> + + + <b>4.1.</b> + + Binary Classification Metrics + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="4.1.1" data-path="../eval/auc.html"> + + <a href="../eval/auc.html"> + + + <b>4.1.1.</b> + + Area under the ROC curve + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="4.2" data-path="../eval/multilabel_classification_measures.html"> + + <a href="../eval/multilabel_classification_measures.html"> + + + <b>4.2.</b> + + Multi-label Classification Metrics + + </a> + + + + </li> + + <li class="chapter " data-level="4.3" data-path="../eval/regression.html"> + + <a href="../eval/regression.html"> + + + <b>4.3.</b> + + Regression Metrics + + </a> + + + + </li> + + <li class="chapter " data-level="4.4" data-path="../eval/rank.html"> + + <a href="../eval/rank.html"> + + + <b>4.4.</b> + + Ranking Measures + + </a> + + + + </li> + + <li class="chapter " data-level="4.5" data-path="../eval/datagen.html"> + + <a href="../eval/datagen.html"> + + + <b>4.5.</b> + + Data Generation + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="4.5.1" data-path="../eval/lr_datagen.html"> + + <a href="../eval/lr_datagen.html"> + + + <b>4.5.1.</b> + + Logistic Regression data generation + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part V - Supervised Learning</li> + + + + <li class="chapter " data-level="5.1" data-path="prediction.html"> + + <a href="prediction.html"> + + + <b>5.1.</b> + + How Prediction Works + + </a> + + + + </li> + + + + + <li class="header">Part VI - Binary Classification</li> + + + + <li class="chapter " data-level="6.1" data-path="../binaryclass/general.html"> + + <a href="../binaryclass/general.html"> + + + <b>6.1.</b> + + Binary Classification + + </a> + + + + </li> + + <li class="chapter " data-level="6.2" data-path="../binaryclass/a9a.html"> + + <a href="../binaryclass/a9a.html"> + + + <b>6.2.</b> + + a9a Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="6.2.1" data-path="../binaryclass/a9a_dataset.html"> + + <a href="../binaryclass/a9a_dataset.html"> + + + <b>6.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="6.2.2" data-path="../binaryclass/a9a_lr.html"> + + <a href="../binaryclass/a9a_lr.html"> + + + <b>6.2.2.</b> + + Logistic Regression + + </a> + + + + </li> + + <li class="chapter " data-level="6.2.3" data-path="../binaryclass/a9a_minibatch.html"> + + <a href="../binaryclass/a9a_minibatch.html"> + + + <b>6.2.3.</b> + + Mini-batch gradient descent + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="6.3" data-path="../binaryclass/news20.html"> + + <a href="../binaryclass/news20.html"> + + + <b>6.3.</b> + + News20 Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="6.3.1" data-path="../binaryclass/news20_dataset.html"> + + <a href="../binaryclass/news20_dataset.html"> + + + <b>6.3.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="6.3.2" data-path="../binaryclass/news20_pa.html"> + + <a href="../binaryclass/news20_pa.html"> + + + <b>6.3.2.</b> + + Perceptron, Passive Aggressive + + </a> + + + + </li> + + <li class="chapter " data-level="6.3.3" data-path="../binaryclass/news20_scw.html"> + + <a href="../binaryclass/news20_scw.html"> + + + <b>6.3.3.</b> + + CW, AROW, SCW + + </a> + + + + </li> + + <li class="chapter " data-level="6.3.4" data-path="../binaryclass/news20_adagrad.html"> + + <a href="../binaryclass/news20_adagrad.html"> + + + <b>6.3.4.</b> + + AdaGradRDA, AdaGrad, AdaDelta + + </a> + + + + </li> + + <li class="chapter " data-level="6.3.5" data-path="../binaryclass/news20_rf.html"> + + <a href="../binaryclass/news20_rf.html"> + + + <b>6.3.5.</b> + + Random Forest + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="6.4" data-path="../binaryclass/kdd2010a.html"> + + <a href="../binaryclass/kdd2010a.html"> + + + <b>6.4.</b> + + KDD2010a Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="6.4.1" data-path="../binaryclass/kdd2010a_dataset.html"> + + <a href="../binaryclass/kdd2010a_dataset.html"> + + + <b>6.4.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="6.4.2" data-path="../binaryclass/kdd2010a_scw.html"> + + <a href="../binaryclass/kdd2010a_scw.html"> + + + <b>6.4.2.</b> + + PA, CW, AROW, SCW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="6.5" data-path="../binaryclass/kdd2010b.html"> + + <a href="../binaryclass/kdd2010b.html"> + + + <b>6.5.</b> + + KDD2010b Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="6.5.1" data-path="../binaryclass/kdd2010b_dataset.html"> + + <a href="../binaryclass/kdd2010b_dataset.html"> + + + <b>6.5.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="6.5.2" data-path="../binaryclass/kdd2010b_arow.html"> + + <a href="../binaryclass/kdd2010b_arow.html"> + + + <b>6.5.2.</b> + + AROW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="6.6" data-path="../binaryclass/webspam.html"> + + <a href="../binaryclass/webspam.html"> + + + <b>6.6.</b> + + Webspam Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="6.6.1" data-path="../binaryclass/webspam_dataset.html"> + + <a href="../binaryclass/webspam_dataset.html"> + + + <b>6.6.1.</b> + + Data pareparation + + </a> + + + + </li> + + <li class="chapter " data-level="6.6.2" data-path="../binaryclass/webspam_scw.html"> + + <a href="../binaryclass/webspam_scw.html"> + + + <b>6.6.2.</b> + + PA1, AROW, SCW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="6.7" data-path="../binaryclass/titanic_rf.html"> + + <a href="../binaryclass/titanic_rf.html"> + + + <b>6.7.</b> + + Kaggle Titanic Tutorial + + </a> + + + + </li> + + + + + <li class="header">Part VII - Multiclass Classification</li> + + + + <li class="chapter " data-level="7.1" data-path="../multiclass/news20.html"> + + <a href="../multiclass/news20.html"> + + + <b>7.1.</b> + + News20 Multiclass Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="7.1.1" data-path="../multiclass/news20_dataset.html"> + + <a href="../multiclass/news20_dataset.html"> + + + <b>7.1.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="7.1.2" data-path="../multiclass/news20_one-vs-the-rest_dataset.html"> + + <a href="../multiclass/news20_one-vs-the-rest_dataset.html"> + + + <b>7.1.2.</b> + + Data preparation for one-vs-the-rest classifiers + + </a> + + + + </li> + + <li class="chapter " data-level="7.1.3" data-path="../multiclass/news20_pa.html"> + + <a href="../multiclass/news20_pa.html"> + + + <b>7.1.3.</b> + + PA + + </a> + + + + </li> + + <li class="chapter " data-level="7.1.4" data-path="../multiclass/news20_scw.html"> + + <a href="../multiclass/news20_scw.html"> + + + <b>7.1.4.</b> + + CW, AROW, SCW + + </a> + + + + </li> + + <li class="chapter " data-level="7.1.5" data-path="../multiclass/news20_ensemble.html"> + + <a href="../multiclass/news20_ensemble.html"> + + + <b>7.1.5.</b> + + Ensemble learning + + </a> + + + + </li> + + <li class="chapter " data-level="7.1.6" data-path="../multiclass/news20_one-vs-the-rest.html"> + + <a href="../multiclass/news20_one-vs-the-rest.html"> + + + <b>7.1.6.</b> + + one-vs-the-rest classifier + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="7.2" data-path="../multiclass/iris.html"> + + <a href="../multiclass/iris.html"> + + + <b>7.2.</b> + + Iris Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="7.2.1" data-path="../multiclass/iris_dataset.html"> + + <a href="../multiclass/iris_dataset.html"> + + + <b>7.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="7.2.2" data-path="../multiclass/iris_scw.html"> + + <a href="../multiclass/iris_scw.html"> + + + <b>7.2.2.</b> + + SCW + + </a> + + + + </li> + + <li class="chapter " data-level="7.2.3" data-path="../multiclass/iris_randomforest.html"> + + <a href="../multiclass/iris_randomforest.html"> + + + <b>7.2.3.</b> + + Random Forest + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part VIII - Regression</li> + + + + <li class="chapter " data-level="8.1" data-path="../regression/general.html"> + + <a href="../regression/general.html"> + + + <b>8.1.</b> + + Regression + + </a> + + + + </li> + + <li class="chapter " data-level="8.2" data-path="../regression/e2006.html"> + + <a href="../regression/e2006.html"> + + + <b>8.2.</b> + + E2006-tfidf Regression Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="8.2.1" data-path="../regression/e2006_dataset.html"> + + <a href="../regression/e2006_dataset.html"> + + + <b>8.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.2" data-path="../regression/e2006_arow.html"> + + <a href="../regression/e2006_arow.html"> + + + <b>8.2.2.</b> + + Passive Aggressive, AROW + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="8.3" data-path="../regression/kddcup12tr2.html"> + + <a href="../regression/kddcup12tr2.html"> + + + <b>8.3.</b> + + KDDCup 2012 Track 2 CTR Prediction Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="8.3.1" data-path="../regression/kddcup12tr2_dataset.html"> + + <a href="../regression/kddcup12tr2_dataset.html"> + + + <b>8.3.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="8.3.2" data-path="../regression/kddcup12tr2_lr.html"> + + <a href="../regression/kddcup12tr2_lr.html"> + + + <b>8.3.2.</b> + + Logistic Regression, Passive Aggressive + + </a> + + + + </li> + + <li class="chapter " data-level="8.3.3" data-path="../regression/kddcup12tr2_lr_amplify.html"> + + <a href="../regression/kddcup12tr2_lr_amplify.html"> + + + <b>8.3.3.</b> + + Logistic Regression with amplifier + + </a> + + + + </li> + + <li class="chapter " data-level="8.3.4" data-path="../regression/kddcup12tr2_adagrad.html"> + + <a href="../regression/kddcup12tr2_adagrad.html"> + + + <b>8.3.4.</b> + + AdaGrad, AdaDelta + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part IX - Recommendation</li> + + + + <li class="chapter " data-level="9.1" data-path="../recommend/cf.html"> + + <a href="../recommend/cf.html"> + + + <b>9.1.</b> + + Collaborative Filtering + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="9.1.1" data-path="../recommend/item_based_cf.html"> + + <a href="../recommend/item_based_cf.html"> + + + <b>9.1.1.</b> + + Item-based collaborative filtering + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="9.2" data-path="../recommend/news20.html"> + + <a href="../recommend/news20.html"> + + + <b>9.2.</b> + + News20 Related Article Recommendation Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="9.2.1" data-path="../multiclass/news20_dataset.html"> + + <a href="../multiclass/news20_dataset.html"> + + + <b>9.2.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="9.2.2" data-path="../recommend/news20_jaccard.html"> + + <a href="../recommend/news20_jaccard.html"> + + + <b>9.2.2.</b> + + LSH/MinHash and Jaccard similarity + + </a> + + + + </li> + + <li class="chapter " data-level="9.2.3" data-path="../recommend/news20_knn.html"> + + <a href="../recommend/news20_knn.html"> + + + <b>9.2.3.</b> + + LSH/MinHash and brute-force search + + </a> + + + + </li> + + <li class="chapter " data-level="9.2.4" data-path="../recommend/news20_bbit_minhash.html"> + + <a href="../recommend/news20_bbit_minhash.html"> + + + <b>9.2.4.</b> + + kNN search using b-Bits MinHash + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="9.3" data-path="../recommend/movielens.html"> + + <a href="../recommend/movielens.html"> + + + <b>9.3.</b> + + MovieLens Movie Recommendation Tutorial + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="9.3.1" data-path="../recommend/movielens_dataset.html"> + + <a href="../recommend/movielens_dataset.html"> + + + <b>9.3.1.</b> + + Data preparation + + </a> + + + + </li> + + <li class="chapter " data-level="9.3.2" data-path="../recommend/movielens_cf.html"> + + <a href="../recommend/movielens_cf.html"> + + + <b>9.3.2.</b> + + Item-based collaborative filtering + + </a> + + + + </li> + + <li class="chapter " data-level="9.3.3" data-path="../recommend/movielens_mf.html"> + + <a href="../recommend/movielens_mf.html"> + + + <b>9.3.3.</b> + + Matrix Factorization + + </a> + + + + </li> + + <li class="chapter " data-level="9.3.4" data-path="../recommend/movielens_fm.html"> + + <a href="../recommend/movielens_fm.html"> + + + <b>9.3.4.</b> + + Factorization Machine + + </a> + + + + </li> + + <li class="chapter " data-level="9.3.5" data-path="../recommend/movielens_slim.html"> + + <a href="../recommend/movielens_slim.html"> + + + <b>9.3.5.</b> + + SLIM for fast top-k recommendation + + </a> + + + + </li> + + <li class="chapter " data-level="9.3.6" data-path="../recommend/movielens_cv.html"> + + <a href="../recommend/movielens_cv.html"> + + + <b>9.3.6.</b> + + 10-fold cross validation (Matrix Factorization) + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part X - Anomaly Detection</li> + + + + <li class="chapter " data-level="10.1" data-path="../anomaly/lof.html"> + + <a href="../anomaly/lof.html"> + + + <b>10.1.</b> + + Outlier Detection using Local Outlier Factor (LOF) + + </a> + + + + </li> + + <li class="chapter " data-level="10.2" data-path="../anomaly/sst.html"> + + <a href="../anomaly/sst.html"> + + + <b>10.2.</b> + + Change-Point Detection using Singular Spectrum Transformation (SST) + + </a> + + + + </li> + + <li class="chapter " data-level="10.3" data-path="../anomaly/changefinder.html"> + + <a href="../anomaly/changefinder.html"> + + + <b>10.3.</b> + + ChangeFinder: Detecting Outlier and Change-Point Simultaneously + + </a> + + + + </li> + + + + + <li class="header">Part XI - Clustering</li> + + + + <li class="chapter " data-level="11.1" data-path="../clustering/lda.html"> + + <a href="../clustering/lda.html"> + + + <b>11.1.</b> + + Latent Dirichlet Allocation + + </a> + + + + </li> + + <li class="chapter " data-level="11.2" data-path="../clustering/plsa.html"> + + <a href="../clustering/plsa.html"> + + + <b>11.2.</b> + + Probabilistic Latent Semantic Analysis + + </a> + + + + </li> + + + + + <li class="header">Part XII - GeoSpatial Functions</li> + + + + <li class="chapter " data-level="12.1" data-path="../geospatial/latlon.html"> + + <a href="../geospatial/latlon.html"> + + + <b>12.1.</b> + + Lat/Lon functions + + </a> + + + + </li> + + + + + <li class="header">Part XIII - Hivemall on Spark</li> + + + + <li class="chapter " data-level="13.1" data-path="../spark/getting_started/"> + + <a href="../spark/getting_started/"> + + + <b>13.1.</b> + + Getting Started + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="13.1.1" data-path="../spark/getting_started/installation.html"> + + <a href="../spark/getting_started/installation.html"> + + + <b>13.1.1.</b> + + Installation + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="13.2" data-path="../spark/binaryclass/"> + + <a href="../spark/binaryclass/"> + + + <b>13.2.</b> + + Binary Classification + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="13.2.1" data-path="../spark/binaryclass/a9a_df.html"> + + <a href="../spark/binaryclass/a9a_df.html"> + + + <b>13.2.1.</b> + + a9a tutorial for DataFrame + + </a> + + + + </li> + + <li class="chapter " data-level="13.2.2" data-path="../spark/binaryclass/a9a_sql.html"> + + <a href="../spark/binaryclass/a9a_sql.html"> + + + <b>13.2.2.</b> + + a9a tutorial for SQL + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="13.3" data-path="../spark/binaryclass/"> + + <a href="../spark/binaryclass/"> + + + <b>13.3.</b> + + Regression + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="13.3.1" data-path="../spark/regression/e2006_df.html"> + + <a href="../spark/regression/e2006_df.html"> + + + <b>13.3.1.</b> + + E2006-tfidf regression tutorial for DataFrame + + </a> + + + + </li> + + <li class="chapter " data-level="13.3.2" data-path="../spark/regression/e2006_sql.html"> + + <a href="../spark/regression/e2006_sql.html"> + + + <b>13.3.2.</b> + + E2006-tfidf regression tutorial for SQL + + </a> + + + + </li> + + + </ul> + + </li> + + <li class="chapter " data-level="13.4" data-path="../spark/misc/misc.html"> + + <a href="../spark/misc/misc.html"> + + + <b>13.4.</b> + + Generic features + + </a> + + + + <ul class="articles"> + + + <li class="chapter " data-level="13.4.1" data-path="../spark/misc/topk_join.html"> + + <a href="../spark/misc/topk_join.html"> + + + <b>13.4.1.</b> + + Top-k join processing + + </a> + + + + </li> + + <li class="chapter " data-level="13.4.2" data-path="../spark/misc/functions.html"> + + <a href="../spark/misc/functions.html"> + + + <b>13.4.2.</b> + + Other utility functions + + </a> + + + + </li> + + + </ul> + + </li> + + + + + <li class="header">Part XIV - Hivemall on Docker</li> + + + + <li class="chapter " data-level="14.1" data-path="../docker/getting_started.html"> + + <a href="../docker/getting_started.html"> + + + <b>14.1.</b> + + Getting Started + + </a> + + + + </li> + + + + + <li class="header">Part XIV - External References</li> + + + + <li class="chapter " data-level="15.1" > + + <a target="_blank" href="https://github.com/daijyc/hivemall/wiki/PigHome";> + + + <b>15.1.</b> + + Hivemall on Apache Pig + + </a> + + + + </li> + + + + + <li class="divider"></li> + + <li> + <a href="https://www.gitbook.com"; target="blank" class="gitbook-link"> + Published with GitBook + </a> + </li> +</ul> + + + </nav> + + + </div> + + <div class="book-body"> + + <div class="body-inner"> + + + +<div class="book-header" role="navigation"> + + + <!-- Title --> + <h1> + <i class="fa fa-circle-o-notch fa-spin"></i> + <a href=".." >List of Functions</a> + </h1> +</div> + + + + + <div class="page-wrapper" tabindex="-1" role="main"> + <div class="page-inner"> + +<div id="book-search-results"> + <div class="search-noresults"> + + <section class="normal markdown-section"> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<p>This page describes a list of Hivemall functions. See also a <a href="generic_funcs.html">list of generic Hivemall functions</a> for more general-purpose functions such as array and map UDFs.</p> +<!-- toc --><div id="toc" class="toc"> + +<ul> +<li><a href="#regression">Regression</a></li> +<li><a href="#classification">Classification</a><ul> +<li><a href="#binary-classification">Binary classification</a></li> +<li><a href="#multiclass-classification">Multiclass classification</a></li> +</ul> +</li> +<li><a href="#matrix-factorization">Matrix factorization</a></li> +<li><a href="#factorization-machines">Factorization machines</a></li> +<li><a href="#recommendation">Recommendation</a></li> +<li><a href="#anomaly-detection">Anomaly detection</a></li> +<li><a href="#topic-modeling">Topic modeling</a></li> +<li><a href="#preprocessing">Preprocessing</a><ul> +<li><a href="#feature-creation">Feature creation</a></li> +<li><a href="#data-amplification">Data amplification</a></li> +<li><a href="#feature-binning">Feature binning</a></li> +<li><a href="#feature-format-conversion">Feature format conversion</a></li> +<li><a href="#feature-hashing">Feature hashing</a></li> +<li><a href="#feature-paring">Feature paring</a></li> +<li><a href="#ranking">Ranking</a></li> +<li><a href="#feature-scaling">Feature scaling</a></li> +<li><a href="#feature-selection">Feature selection</a></li> +<li><a href="#feature-transformation-and-vectorization">Feature transformation and vectorization</a></li> +</ul> +</li> +<li><a href="#geospatial-functions">Geospatial functions</a></li> +<li><a href="#distance-measures">Distance measures</a></li> +<li><a href="#locality-sensitive-hashing">Locality-sensitive hashing</a></li> +<li><a href="#similarity-measures">Similarity measures</a></li> +<li><a href="#evaluation">Evaluation</a></li> +<li><a href="#sketching">Sketching</a></li> +<li><a href="#ensemble-learning">Ensemble learning</a><ul> +<li><a href="#utils">Utils</a></li> +<li><a href="#bagging">Bagging</a></li> +</ul> +</li> +<li><a href="#dicision-trees-and-randomforest">Dicision trees and RandomForest</a></li> +<li><a href="#xgboost">XGBoost</a></li> +<li><a href="#others">Others</a></li> +</ul> + +</div><!-- tocstop --> +<h1 id="regression">Regression</h1> +<ul> +<li><p><code>train_arow_regr(array<int|bigint|string> features, float target [, constant string options])</code> - Returns a relation consists of <{int|bigint|string} feature, float weight, float covar></p> +</li> +<li><p><code>train_arowe2_regr(array<int|bigint|string> features, float target [, constant string options])</code> - Returns a relation consists of <{int|bigint|string} feature, float weight, float covar></p> +</li> +<li><p><code>train_arowe_regr(array<int|bigint|string> features, float target [, constant string options])</code> - Returns a relation consists of <{int|bigint|string} feature, float weight, float covar></p> +</li> +<li><p><code>train_pa1_regr(array<int|bigint|string> features, float target [, constant string options])</code> - Returns a relation consists of <{int|bigint|string} feature, float weight></p> +</li> +<li><p><code>train_pa1a_regr(array<int|bigint|string> features, float target [, constant string options])</code> - Returns a relation consists of <{int|bigint|string} feature, float weight></p> +</li> +<li><p><code>train_pa2_regr(array<int|bigint|string> features, float target [, constant string options])</code> - Returns a relation consists of <{int|bigint|string} feature, float weight></p> +</li> +<li><p><code>train_pa2a_regr(array<int|bigint|string> features, float target [, constant string options])</code> - Returns a relation consists of <{int|bigint|string} feature, float weight></p> +</li> +<li><p><code>train_regressor(list<string|int|bigint> features, double label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight></p> +<pre><code>Build a prediction model by a generic regressor +</code></pre></li> +</ul> +<h1 id="classification">Classification</h1> +<h2 id="binary-classification">Binary classification</h2> +<ul> +<li><p><code>kpa_predict(@Nonnull double xh, @Nonnull double xk, @Nullable float w0, @Nonnull float w1, @Nonnull float w2, @Nullable float w3)</code> - Returns a prediction value in Double</p> +</li> +<li><p><code>train_arow(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight, float covar></p> +<pre><code>Build a prediction model by Adaptive Regularization of Weight Vectors (AROW) binary classifier +</code></pre></li> +<li><p><code>train_arowh(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight, float covar></p> +<pre><code>Build a prediction model by AROW binary classifier using hinge loss +</code></pre></li> +<li><p><code>train_classifier(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight></p> +<pre><code>Build a prediction model by a generic classifier +</code></pre></li> +<li><p><code>train_cw(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight, float covar></p> +<pre><code>Build a prediction model by Confidence-Weighted (CW) binary classifier +</code></pre></li> +<li><p><code>train_kpa(array<string|int|bigint> features, int label [, const string options])</code> - returns a relation <h int, hk int, float w0, float w1, float w2, float w3></p> +</li> +<li><p><code>train_pa(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight></p> +<pre><code>Build a prediction model by Passive-Aggressive (PA) binary classifier +</code></pre></li> +<li><p><code>train_pa1(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight></p> +<pre><code>Build a prediction model by Passive-Aggressive 1 (PA-1) binary classifier +</code></pre></li> +<li><p><code>train_pa2(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight></p> +<pre><code>Build a prediction model by Passive-Aggressive 2 (PA-2) binary classifier +</code></pre></li> +<li><p><code>train_perceptron(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight></p> +<pre><code>Build a prediction model by Perceptron binary classifier +</code></pre></li> +<li><p><code>train_scw(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight, float covar></p> +<pre><code>Build a prediction model by Soft Confidence-Weighted (SCW-1) binary classifier +</code></pre></li> +<li><p><code>train_scw2(list<string|int|bigint> features, int label [, const string options])</code> - Returns a relation consists of <string|int|bigint feature, float weight, float covar></p> +<pre><code>Build a prediction model by Soft Confidence-Weighted 2 (SCW-2) binary classifier +</code></pre></li> +</ul> +<h2 id="multiclass-classification">Multiclass classification</h2> +<ul> +<li><p><code>train_multiclass_arow(list<string|int|bigint> features, {int|string} label [, const string options])</code> - Returns a relation consists of <{int|string} label, {string|int|bigint} feature, float weight, float covar></p> +<pre><code>Build a prediction model by Adaptive Regularization of Weight Vectors (AROW) multiclass classifier +</code></pre></li> +<li><p><code>train_multiclass_arowh(list<string|int|bigint> features, int|string label [, const string options])</code> - Returns a relation consists of <int|string label, string|int|bigint feature, float weight, float covar></p> +<pre><code>Build a prediction model by Adaptive Regularization of Weight Vectors (AROW) multiclass classifier using hinge loss +</code></pre></li> +<li><p><code>train_multiclass_cw(list<string|int|bigint> features, {int|string} label [, const string options])</code> - Returns a relation consists of <{int|string} label, {string|int|bigint} feature, float weight, float covar></p> +<pre><code>Build a prediction model by Confidence-Weighted (CW) multiclass classifier +</code></pre></li> +<li><p><code>train_multiclass_pa(list<string|int|bigint> features, {int|string} label [, const string options])</code> - Returns a relation consists of <{int|string} label, {string|int|bigint} feature, float weight></p> +<pre><code>Build a prediction model by Passive-Aggressive (PA) multiclass classifier +</code></pre></li> +<li><p><code>train_multiclass_pa1(list<string|int|bigint> features, {int|string} label [, const string options])</code> - Returns a relation consists of <{int|string} label, {string|int|bigint} feature, float weight></p> +<pre><code>Build a prediction model by Passive-Aggressive 1 (PA-1) multiclass classifier +</code></pre></li> +<li><p><code>train_multiclass_pa2(list<string|int|bigint> features, {int|string} label [, const string options])</code> - Returns a relation consists of <{int|string} label, {string|int|bigint} feature, float weight></p> +<pre><code>Build a prediction model by Passive-Aggressive 2 (PA-2) multiclass classifier +</code></pre></li> +<li><p><code>train_multiclass_perceptron(list<string|int|bigint> features, {int|string} label [, const string options])</code> - Returns a relation consists of <{int|string} label, {string|int|bigint} feature, float weight></p> +<pre><code>Build a prediction model by Perceptron multiclass classifier +</code></pre></li> +<li><p><code>train_multiclass_scw(list<string|int|bigint> features, {int|string} label [, const string options])</code> - Returns a relation consists of <{int|string} label, {string|int|bigint} feature, float weight, float covar></p> +<pre><code>Build a prediction model by Soft Confidence-Weighted (SCW-1) multiclass classifier +</code></pre></li> +<li><p><code>train_multiclass_scw2(list<string|int|bigint> features, {int|string} label [, const string options])</code> - Returns a relation consists of <{int|string} label, {string|int|bigint} feature, float weight, float covar></p> +<pre><code>Build a prediction model by Soft Confidence-Weighted 2 (SCW-2) multiclass classifier +</code></pre></li> +</ul> +<h1 id="matrix-factorization">Matrix factorization</h1> +<ul> +<li><p><code>bprmf_predict(List<Float> Pu, List<Float> Qi[, double Bi])</code> - Returns the prediction value</p> +</li> +<li><p><code>mf_predict(List<Float> Pu, List<Float> Qi[, double Bu, double Bi[, double mu]])</code> - Returns the prediction value</p> +</li> +<li><p><code>train_bprmf(INT user, INT posItem, INT negItem [, String options])</code> - Returns a relation <INT i, FLOAT Pi, FLOAT Qi [, FLOAT Bi]></p> +</li> +<li><p><code>train_mf_adagrad(INT user, INT item, FLOAT rating [, CONSTANT STRING options])</code> - Returns a relation consists of <int idx, array<float> Pu, array<float> Qi [, float Bu, float Bi [, float mu]]></p> +</li> +<li><p><code>train_mf_sgd(INT user, INT item, FLOAT rating [, CONSTANT STRING options])</code> - Returns a relation consists of <int idx, array<float> Pu, array<float> Qi [, float Bu, float Bi [, float mu]]></p> +</li> +</ul> +<h1 id="factorization-machines">Factorization machines</h1> +<ul> +<li><p><code>ffm_predict(float Wi, array<float> Vifj, array<float> Vjfi, float Xi, float Xj)</code> - Returns a prediction value in Double</p> +</li> +<li><p><code>fm_predict(Float Wj, array<float> Vjf, float Xj)</code> - Returns a prediction value in Double</p> +</li> +<li><p><code>train_ffm(array<string> x, double y [, const string options])</code> - Returns a prediction model</p> +</li> +<li><p><code>train_fm(array<string> x, double y [, const string options])</code> - Returns a prediction model</p> +</li> +</ul> +<h1 id="recommendation">Recommendation</h1> +<ul> +<li><code>train_slim( int i, map<int, double> r_i, map<int, map<int, double>> topKRatesOfI, int j, map<int, double> r_j [, constant string options])</code> - Returns row index, column index and non-zero weight value of prediction model</li> +</ul> +<h1 id="anomaly-detection">Anomaly detection</h1> +<ul> +<li><p><code>changefinder(double|array<double> x [, const string options])</code> - Returns outlier/change-point scores and decisions using ChangeFinder. It will return a tuple <double outlier_score, double changepoint_score [, boolean is_anomaly [, boolean is_changepoint]]</p> +</li> +<li><p><code>sst(double|array<double> x [, const string options])</code> - Returns change-point scores and decisions using Singular Spectrum Transformation (SST). It will return a tuple <double changepoint_score [, boolean is_changepoint]></p> +</li> +</ul> +<h1 id="topic-modeling">Topic modeling</h1> +<ul> +<li><p><code>lda_predict(string word, float value, int label, float lambda[, const string options])</code> - Returns a list which consists of <int label, float prob></p> +</li> +<li><p><code>plsa_predict(string word, float value, int label, float prob[, const string options])</code> - Returns a list which consists of <int label, float prob></p> +</li> +<li><p><code>train_lda(array<string> words[, const string options])</code> - Returns a relation consists of <int topic, string word, float score></p> +</li> +<li><p><code>train_plsa(array<string> words[, const string options])</code> - Returns a relation consists of <int topic, string word, float score></p> +</li> +</ul> +<h1 id="preprocessing">Preprocessing</h1> +<h2 id="feature-creation">Feature creation</h2> +<ul> +<li><p><code>add_bias(feature_vector in array<string>)</code> - Returns features with a bias in array<string></p> +</li> +<li><p><code>add_feature_index(ARRAY[DOUBLE]: dense feature vector)</code> - Returns a feature vector with feature indices</p> +</li> +<li><p><code>extract_feature(feature_vector in array<string>)</code> - Returns features in array<string></p> +</li> +<li><p><code>extract_weight(feature_vector in array<string>)</code> - Returns the weights of features in array<string></p> +</li> +<li><p><code>feature(<string|int|long|short|byte> feature, <number> value)</code> - Returns a feature string</p> +</li> +<li><p><code>feature_index(feature_vector in array<string>)</code> - Returns feature indices in array<index></p> +</li> +<li><p><code>sort_by_feature(map in map<int,float>)</code> - Returns a sorted map</p> +</li> +</ul> +<h2 id="data-amplification">Data amplification</h2> +<ul> +<li><p><code>amplify(const int xtimes, *)</code> - amplify the input records x-times</p> +</li> +<li><p><code>rand_amplify(const int xtimes [, const string options], *)</code> - amplify the input records x-times in map-side</p> +</li> +</ul> +<h2 id="feature-binning">Feature binning</h2> +<ul> +<li><p><code>build_bins(number weight, const int num_of_bins[, const boolean auto_shrink = false])</code> - Return quantiles representing bins: array<double></p> +</li> +<li><p><code>feature_binning(array<features::string> features, const map<string, array<number>> quantiles_map)</code> / _FUNC(number weight, const array<number> quantiles) - Returns binned features as an array<features::string> / bin ID as int</p> +</li> +</ul> +<h2 id="feature-format-conversion">Feature format conversion</h2> +<ul> +<li><p><code>conv2dense(int feature, float weight, int nDims)</code> - Return a dense model in array<float></p> +</li> +<li><p><code>quantify(boolean outout, col1, col2, ...)</code> - Returns an identified features</p> +</li> +<li><p><code>to_dense_features(array<string> feature_vector, int dimensions)</code> - Returns a dense feature in array<float></p> +</li> +<li><p><code>to_sparse_features(array<float> feature_vector)</code> - Returns a sparse feature in array<string></p> +</li> +</ul> +<h2 id="feature-hashing">Feature hashing</h2> +<ul> +<li><p><code>array_hash_values(array<string> values, [string prefix [, int numFeatures], boolean useIndexAsPrefix])</code> returns hash values in array<int></p> +</li> +<li><p><code>feature_hashing(array<string> features [, const string options])</code> - returns a hashed feature vector in array<string></p> +</li> +<li><p><code>mhash(string word)</code> returns a murmurhash3 INT value starting from 1</p> +</li> +<li><p><code>prefixed_hash_values(array<string> values, string prefix [, boolean useIndexAsPrefix])</code> returns array<string> that each element has the specified prefix</p> +</li> +<li><p><code>sha1(string word [, int numFeatures])</code> returns a SHA-1 value</p> +</li> +</ul> +<h2 id="feature-paring">Feature paring</h2> +<ul> +<li><p><code>feature_pairs(feature_vector in array<string>, [, const string options])</code> - Returns a relation <string i, string j, double xi, double xj></p> +</li> +<li><p><code>polynomial_features(feature_vector in array<string>)</code> - Returns a feature vectorhaving polynominal feature space</p> +</li> +<li><p><code>powered_features(feature_vector in array<string>, int degree [, boolean truncate])</code> - Returns a feature vector having a powered feature space</p> +</li> +</ul> +<h2 id="ranking">Ranking</h2> +<ul> +<li><p><code>bpr_sampling(int userId, List<int> posItems [, const string options])</code>- Returns a relation consists of <int userId, int itemId></p> +</li> +<li><p><code>item_pairs_sampling(array<int|long> pos_items, const int max_item_id [, const string options])</code>- Returns a relation consists of <int pos_item_id, int neg_item_id></p> +</li> +<li><p><code>populate_not_in(list items, const int max_item_id [, const string options])</code>- Returns a relation consists of <int item> that item does not exist in the given items</p> +</li> +</ul> +<h2 id="feature-scaling">Feature scaling</h2> +<ul> +<li><p><code>l1_normalize(ftvec string)</code> - Returned a L1 normalized value</p> +</li> +<li><p><code>l2_normalize(ftvec string)</code> - Returned a L2 normalized value</p> +</li> +<li><p><code>rescale(value, min, max)</code> - Returns rescaled value by min-max normalization</p> +</li> +<li><p><code>zscore(value, mean, stddev)</code> - Returns a standard score (zscore)</p> +</li> +</ul> +<h2 id="feature-selection">Feature selection</h2> +<ul> +<li><p><code>chi2(array<array<number>> observed, array<array<number>> expected)</code> - Returns chi2_val and p_val of each columns as <array<double>, array<double>></p> +</li> +<li><p><code>snr(array<number> features, array<int> one-hot class label)</code> - Returns Signal Noise Ratio for each feature as array<double></p> +</li> +</ul> +<h2 id="feature-transformation-and-vectorization">Feature transformation and vectorization</h2> +<ul> +<li><p><code>add_field_indices(array<string> features)</code> - Returns arrays of string that field indices (<field>:<feature>)* are argumented</p> +</li> +<li><p><code>binarize_label(int/long positive, int/long negative, ...)</code> - Returns positive/negative records that are represented as (..., int label) where label is 0 or 1</p> +</li> +<li><p><code>categorical_features(array<string> featureNames, feature1, feature2, .. [, const string options])</code> - Returns a feature vector array<string></p> +</li> +<li><p><code>ffm_features(const array<string> featureNames, feature1, feature2, .. [, const string options])</code> - Takes categroical variables and returns a feature vector array<string> in a libffm format <field>:<index>:<value></p> +</li> +<li><p><code>indexed_features(double v1, double v2, ...)</code> - Returns a list of features as array<string>: [1:v1, 2:v2, ..]</p> +</li> +<li><p><code>onehot_encoding(PRIMITIVE feature, ...)</code> - Compute onehot encoded label for each feature</p> +</li> +<li><p><code>quantified_features(boolean output, col1, col2, ...)</code> - Returns an identified features in a dense array<double></p> +</li> +<li><p><code>quantitative_features(array<string> featureNames, feature1, feature2, .. [, const string options])</code> - Returns a feature vector array<string></p> +</li> +<li><p><code>vectorize_features(array<string> featureNames, feature1, feature2, .. [, const string options])</code> - Returns a feature vector array<string></p> +</li> +</ul> +<h1 id="geospatial-functions">Geospatial functions</h1> +<ul> +<li><p><code>haversine_distance(double lat1, double lon1, double lat2, double lon2, [const boolean mile=false])</code>::double - return distance between two locations in km [or miles] using <code>haversine</code> formula</p> +<pre><code>Usage: select latlon_distance(lat1, lon1, lat2, lon2) from ... +</code></pre></li> +<li><p><code>lat2tiley(double lat, int zoom)</code>::int - Returns the tile number of the given latitude and zoom level</p> +</li> +<li><p><code>lon2tilex(double lon, int zoom)</code>::int - Returns the tile number of the given longitude and zoom level</p> +</li> +<li><p><code>map_url(double lat, double lon, int zoom [, const string option])</code> - Returns a URL string</p> +<pre><code>OpenStreetMap: http://tile.openstreetmap.org/${zoom}/${xtile}/${ytile}.png +Google Maps: https://www.google.com/maps/@${lat},${lon},${zoom}z +</code></pre></li> +<li><p><code>tile(double lat, double lon, int zoom)</code>::bigint - Returns a tile number 2^2n where n is zoom level.</p> +<pre><code>_FUNC_(lat,lon,zoom) = xtile(lon,zoom) + ytile(lat,zoom) * 2^zoomrefer http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames for detail +</code></pre></li> +<li><p><code>tilex2lon(int x, int zoom)</code>::double - Returns longitude of the given tile x and zoom level</p> +</li> +<li><p><code>tiley2lat(int y, int zoom)</code>::double - Returns latitude of the given tile y and zoom level</p> +</li> +</ul> +<h1 id="distance-measures">Distance measures</h1> +<ul> +<li><p><code>angular_distance(ftvec1, ftvec2)</code> - Returns an angular distance of the given two vectors</p> +</li> +<li><p><code>cosine_distance(ftvec1, ftvec2)</code> - Returns a cosine distance of the given two vectors</p> +</li> +<li><p><code>euclid_distance(ftvec1, ftvec2)</code> - Returns the square root of the sum of the squared differences: sqrt(sum((x - y)^2))</p> +</li> +<li><p><code>hamming_distance(A, B [,int k])</code> - Returns Hamming distance between A and B</p> +</li> +<li><p><code>jaccard_distance(A, B [,int k])</code> - Returns Jaccard distance between A and B</p> +</li> +<li><p><code>kld(double m1, double sigma1, double mu2, double sigma 2)</code> - Returns KL divergence between two distributions</p> +</li> +<li><p><code>manhattan_distance(list x, list y)</code> - Returns sum(|x - y|)</p> +</li> +<li><p><code>minkowski_distance(list x, list y, double p)</code> - Returns sum(|x - y|^p)^(1/p)</p> +</li> +<li><p><code>popcnt(a [, b])</code> - Returns a popcount value</p> +</li> +</ul> +<h1 id="locality-sensitive-hashing">Locality-sensitive hashing</h1> +<ul> +<li><p><code>bbit_minhash(array<> features [, int numHashes])</code> - Returns a b-bits minhash value</p> +</li> +<li><p><code>minhash(ANY item, array<int|bigint|string> features [, constant string options])</code> - Returns n differnce k-depth signatures (i.e., clusteid) for each item <clusteid, item></p> +</li> +<li><p><code>minhashes(array<> features [, int numHashes, int keyGroup [, boolean noWeight]])</code> - Returns minhash values</p> +</li> +</ul> +<h1 id="similarity-measures">Similarity measures</h1> +<ul> +<li><p><code>angular_similarity(ftvec1, ftvec2)</code> - Returns an angular similarity of the given two vectors</p> +</li> +<li><p><code>cosine_similarity(ftvec1, ftvec2)</code> - Returns a cosine similarity of the given two vectors</p> +</li> +<li><p><code>dimsum_mapper(array<string> row, map<int col_id, double norm> colNorms [, const string options])</code> - Returns column-wise partial similarities</p> +</li> +<li><p><code>distance2similarity(float d)</code> - Returns 1.0 / (1.0 + d)</p> +</li> +<li><p><code>euclid_similarity(ftvec1, ftvec2)</code> - Returns a euclid distance based similarity, which is <code>1.0 / (1.0 + distance)</code>, of the given two vectors</p> +</li> +<li><p><code>jaccard_similarity(A, B [,int k])</code> - Returns Jaccard similarity coefficient of A and B</p> +</li> +</ul> +<h1 id="evaluation">Evaluation</h1> +<ul> +<li><p><code>auc(array rankItems | double score, array correctItems | int label [, const int recommendSize = rankItems.size ])</code> - Returns AUC</p> +</li> +<li><p><code>average_precision(array rankItems, array correctItems [, const int recommendSize = rankItems.size])</code> - Returns MAP</p> +</li> +<li><p><code>f1score(array[int], array[int])</code> - Return a F1 score</p> +</li> +<li><p><code>fmeasure(array|int|boolean actual, array|int| boolean predicted [, const string options])</code> - Return a F-measure (f1score is the special with beta=1.0)</p> +</li> +<li><p><code>hitrate(array rankItems, array correctItems [, const int recommendSize = rankItems.size])</code> - Returns HitRate</p> +</li> +<li><p><code>logloss(double predicted, double actual)</code> - Return a Logrithmic Loss</p> +</li> +<li><p><code>mae(double predicted, double actual)</code> - Return a Mean Absolute Error</p> +</li> +<li><p><code>mrr(array rankItems, array correctItems [, const int recommendSize = rankItems.size])</code> - Returns MRR</p> +</li> +<li><p><code>mse(double predicted, double actual)</code> - Return a Mean Squared Error</p> +</li> +<li><p><code>ndcg(array rankItems, array correctItems [, const int recommendSize = rankItems.size])</code> - Returns nDCG</p> +</li> +<li><p><code>precision_at(array rankItems, array correctItems [, const int recommendSize = rankItems.size])</code> - Returns Precision</p> +</li> +<li><p><code>r2(double predicted, double actual)</code> - Return R Squared (coefficient of determination)</p> +</li> +<li><p><code>recall_at(array rankItems, array correctItems [, const int recommendSize = rankItems.size])</code> - Returns Recall</p> +</li> +<li><p><code>rmse(double predicted, double actual)</code> - Return a Root Mean Squared Error</p> +</li> +</ul> +<h1 id="sketching">Sketching</h1> +<ul> +<li><code>approx_count_distinct(expr x [, const string options])</code> - Returns an approximation of count(DISTINCT x) using HyperLogLogPlus algorithm</li> +</ul> +<h1 id="ensemble-learning">Ensemble learning</h1> +<h2 id="utils">Utils</h2> +<ul> +<li><p><code>argmin_kld(float mean, float covar)</code> - Returns mean or covar that minimize a KL-distance among distributions</p> +<pre><code>The returned value is (1.0 / (sum(1.0 / covar))) * (sum(mean / covar) +</code></pre></li> +<li><p><code>max_label(double value, string label)</code> - Returns a label that has the maximum value</p> +</li> +<li><p><code>maxrow(ANY compare, ...)</code> - Returns a row that has maximum value in the 1st argument</p> +</li> +</ul> +<h2 id="bagging">Bagging</h2> +<ul> +<li><p><code>voted_avg(double value)</code> - Returns an averaged value by bagging for classification</p> +</li> +<li><p><code>weight_voted_avg(expr)</code> - Returns an averaged value by considering sum of positive/negative weights</p> +</li> +</ul> +<h1 id="dicision-trees-and-randomforest">Dicision trees and RandomForest</h1> +<ul> +<li><p><code>train_gradient_tree_boosting_classifier(array<double|string> features, int label [, string options])</code> - Returns a relation consists of <int iteration, int model_type, array<string> pred_models, double intercept, double shrinkage, array<double> var_importance, float oob_error_rate></p> +</li> +<li><p><code>train_randomforest_classifier(array<double|string> features, int label [, const array<double> classWeights, const string options])</code> - Returns a relation consists of <int model_id, int model_type, string pred_model, array<double> var_importance, int oob_errors, int oob_tests, double weight></p> +</li> +<li><p><code>train_randomforest_regression(array<double|string> features, double target [, string options])</code> - Returns a relation consists of <int model_id, int model_type, string pred_model, array<double> var_importance, int oob_errors, int oob_tests></p> +</li> +<li><p><code>guess_attribute_types(ANY, ...)</code> - Returns attribute types</p> +<pre><code>select guess_attribute_types(*) from train limit 1; +> Q,Q,C,C,C,C,Q,C,C,C,Q,C,Q,Q,Q,Q,C,Q +</code></pre></li> +<li><p><code>rf_ensemble(int yhat [, array<double> proba [, double model_weight=1.0]])</code> - Returns emsebled prediction results in <int label, double probability, array<double> probabilities></p> +</li> +<li><p><code>tree_export(string model, const string options, optional array<string> featureNames=null, optional array<string> classNames=null)</code> - exports a Decision Tree model as javascript/dot]</p> +</li> +<li><p><code>tree_predict(string modelId, string model, array<double|string> features [, const string options | const boolean classification=false])</code> - Returns a prediction result of a random forest in <int value, array<double> posteriori> for classification and <double> for regression</p> +</li> +</ul> +<h1 id="xgboost">XGBoost</h1> +<ul> +<li><p><code>train_multiclass_xgboost_classifier(string[] features, double target [, string options])</code> - Returns a relation consisting of <string model_id, array<byte> pred_model></p> +</li> +<li><p><code>train_xgboost_classifier(string[] features, double target [, string options])</code> - Returns a relation consisting of <string model_id, array<byte> pred_model></p> +</li> +<li><p><code>train_xgboost_regr(string[] features, double target [, string options])</code> - Returns a relation consisting of <string model_id, array<byte> pred_model></p> +</li> +<li><p><code>xgboost_multiclass_predict(string rowid, string[] features, string model_id, array<byte> pred_model [, string options])</code> - Returns a prediction result as (string rowid, string label, float probability)</p> +</li> +<li><p><code>xgboost_predict(string rowid, string[] features, string model_id, array<byte> pred_model [, string options])</code> - Returns a prediction result as (string rowid, float predicted)</p> +</li> +</ul> +<h1 id="others">Others</h1> +<ul> +<li><p><code>hivemall_version()</code> - Returns the version of Hivemall</p> +</li> +<li><p><code>lr_datagen(options string)</code> - Generates a logistic regression dataset</p> +<pre><code class="lang-sql">WITH dual AS (<span class="hljs-keyword">SELECT</span> <span class="hljs-number">1</span>) <span class="hljs-keyword">SELECT</span> lr_datagen(<span class="hljs-string">'-n_examples 1k -n_features 10'</span>) <span class="hljs-keyword">FROM</span> dual; +</code></pre> +</li> +<li><p><code>tf(string text)</code> - Return a term frequency in <string, float> +<div id="page-footer" class="localized-footer"><hr><!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> +<p><sub><font color="gray"> +Apache Hivemall is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. +</font></sub></p> +</div></p> +</li> +</ul> + + + </section> + + </div> + <div class="search-results"> + <div class="has-results"> + + <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1> + <ul class="search-results-list"></ul
<TRUNCATED>
