http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/d9012d92/userguide/binaryclass/news20_pa.html ---------------------------------------------------------------------- diff --git a/userguide/binaryclass/news20_pa.html b/userguide/binaryclass/news20_pa.html index b13456e..1dcd198 100644 --- a/userguide/binaryclass/news20_pa.html +++ b/userguide/binaryclass/news20_pa.html @@ -972,7 +972,7 @@ <b>6.2.1.</b> - Data preparation + Data Preparation </a> @@ -980,13 +980,28 @@ </li> - <li class="chapter " data-level="6.2.2" data-path="a9a_lr.html"> + <li class="chapter " data-level="6.2.2" data-path="a9a_generic.html"> - <a href="a9a_lr.html"> + <a href="a9a_generic.html"> <b>6.2.2.</b> + General Binary Classifier + + </a> + + + + </li> + + <li class="chapter " data-level="6.2.3" data-path="a9a_lr.html"> + + <a href="a9a_lr.html"> + + + <b>6.2.3.</b> + Logistic Regression </a> @@ -995,14 +1010,14 @@ </li> - <li class="chapter " data-level="6.2.3" data-path="a9a_minibatch.html"> + <li class="chapter " data-level="6.2.4" data-path="a9a_minibatch.html"> <a href="a9a_minibatch.html"> - <b>6.2.3.</b> + <b>6.2.4.</b> - Mini-batch gradient descent + Mini-batch Gradient Descent </a> @@ -1038,7 +1053,7 @@ <b>6.3.1.</b> - Data preparation + Data Preparation </a> @@ -1076,13 +1091,28 @@ </li> - <li class="chapter " data-level="6.3.4" data-path="news20_adagrad.html"> + <li class="chapter " data-level="6.3.4" data-path="news20_generic.html"> - <a href="news20_adagrad.html"> + <a href="news20_generic.html"> <b>6.3.4.</b> + General Binary Classifier + + </a> + + + + </li> + + <li class="chapter " data-level="6.3.5" data-path="news20_adagrad.html"> + + <a href="news20_adagrad.html"> + + + <b>6.3.5.</b> + AdaGradRDA, AdaGrad, AdaDelta </a> @@ -1091,12 +1121,12 @@ </li> - <li class="chapter " data-level="6.3.5" data-path="news20_rf.html"> + <li class="chapter " data-level="6.3.6" data-path="news20_rf.html"> <a href="news20_rf.html"> - <b>6.3.5.</b> + <b>6.3.6.</b> Random Forest @@ -1134,7 +1164,7 @@ <b>6.4.1.</b> - Data preparation + Data Preparation </a> @@ -1185,7 +1215,7 @@ <b>6.5.1.</b> - Data preparation + Data Preparation </a> @@ -1236,7 +1266,7 @@ <b>6.6.1.</b> - Data pareparation + Data Pareparation </a> @@ -1302,7 +1332,7 @@ <b>6.8.1.</b> - Data preparation + Data Preparation </a> @@ -1360,7 +1390,7 @@ <b>7.1.1.</b> - Data preparation + Data Preparation </a> @@ -1375,7 +1405,7 @@ <b>7.1.2.</b> - Data preparation for one-vs-the-rest classifiers + Data Preparation for one-vs-the-rest classifiers </a> @@ -1435,7 +1465,7 @@ <b>7.1.6.</b> - one-vs-the-rest classifier + one-vs-the-rest Classifier </a> @@ -1559,7 +1589,7 @@ <b>8.2.1.</b> - Data preparation + Data Preparation </a> @@ -1567,13 +1597,28 @@ </li> - <li class="chapter " data-level="8.2.2" data-path="../regression/e2006_arow.html"> + <li class="chapter " data-level="8.2.2" data-path="../regression/e2006_generic.html"> - <a href="../regression/e2006_arow.html"> + <a href="../regression/e2006_generic.html"> <b>8.2.2.</b> + General Regessor + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.3" data-path="../regression/e2006_arow.html"> + + <a href="../regression/e2006_arow.html"> + + + <b>8.2.3.</b> + Passive Aggressive, AROW </a> @@ -1610,7 +1655,7 @@ <b>8.3.1.</b> - Data preparation + Data Preparation </a> @@ -1698,7 +1743,7 @@ <b>9.1.1.</b> - Item-based collaborative filtering + Item-based Collaborative Filtering </a> @@ -1734,7 +1779,7 @@ <b>9.2.1.</b> - Data preparation + Data Preparation </a> @@ -1749,7 +1794,7 @@ <b>9.2.2.</b> - LSH/MinHash and Jaccard similarity + LSH/MinHash and Jaccard Similarity </a> @@ -1764,7 +1809,7 @@ <b>9.2.3.</b> - LSH/MinHash and brute-force search + LSH/MinHash and Brute-force Search </a> @@ -1815,7 +1860,7 @@ <b>9.3.1.</b> - Data preparation + Data Preparation </a> @@ -1830,7 +1875,7 @@ <b>9.3.2.</b> - Item-based collaborative filtering + Item-based Collaborative Filtering </a> @@ -1875,7 +1920,7 @@ <b>9.3.5.</b> - SLIM for fast top-k recommendation + SLIM for fast top-k Recommendation </a> @@ -1890,7 +1935,7 @@ <b>9.3.6.</b> - 10-fold cross validation (Matrix Factorization) + 10-fold Cross Validation (Matrix Factorization) </a> @@ -2080,7 +2125,7 @@ <b>13.2.1.</b> - a9a tutorial for DataFrame + a9a Tutorial for DataFrame </a> @@ -2095,7 +2140,7 @@ <b>13.2.2.</b> - a9a tutorial for SQL + a9a Tutorial for SQL </a> @@ -2131,7 +2176,7 @@ <b>13.3.1.</b> - E2006-tfidf regression tutorial for DataFrame + E2006-tfidf Regression Tutorial for DataFrame </a> @@ -2146,7 +2191,7 @@ <b>13.3.2.</b> - E2006-tfidf regression tutorial for SQL + E2006-tfidf Regression Tutorial for SQL </a> @@ -2166,7 +2211,7 @@ <b>13.4.</b> - Generic features + Generic Features </a> @@ -2182,7 +2227,7 @@ <b>13.4.1.</b> - Top-k join processing + Top-k Join Processing </a> @@ -2197,7 +2242,7 @@ <b>13.4.2.</b> - Other utility functions + Other Utility Functions </a> @@ -2317,12 +2362,6 @@ specific language governing permissions and limitations under the License. --> -<h2 id="udf-preparation">UDF preparation</h2> -<pre><code>delete jar /home/myui/tmp/hivemall.jar; -add jar /home/myui/tmp/hivemall.jar; - -source /home/myui/tmp/define-all.hive; -</code></pre><hr> <h1 id="perceptron">[Perceptron]</h1> <h2 id="model-building">model building</h2> <pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_perceptron_model1; @@ -2332,7 +2371,7 @@ source /home/myui/tmp/define-all.hive; voted_avg(weight) <span class="hljs-keyword">as</span> weight <span class="hljs-keyword">from</span> (<span class="hljs-keyword">select</span> - perceptron(add_bias(features),label) <span class="hljs-keyword">as</span> (feature,weight) + train_perceptron(add_bias(features),label) <span class="hljs-keyword">as</span> (feature,weight) <span class="hljs-keyword">from</span> news20b_train_x3 ) t @@ -2355,23 +2394,19 @@ source /home/myui/tmp/define-all.hive; <pre><code class="lang-sql"><span class="hljs-keyword">create</span> <span class="hljs-keyword">or</span> <span class="hljs-keyword">replace</span> <span class="hljs-keyword">view</span> news20b_perceptron_submit1 <span class="hljs-keyword">as</span> <span class="hljs-keyword">select</span> t.label <span class="hljs-keyword">as</span> actual, - pd.label <span class="hljs-keyword">as</span> predicted + p.label <span class="hljs-keyword">as</span> predicted <span class="hljs-keyword">from</span> - news20b_test t <span class="hljs-keyword">JOIN</span> news20b_perceptron_predict1 pd - <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = pd.<span class="hljs-keyword">rowid</span>); + news20b_test t <span class="hljs-keyword">JOIN</span> news20b_perceptron_predict1 p + <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = p.<span class="hljs-keyword">rowid</span>); </code></pre> -<pre><code class="lang-sql"><span class="hljs-keyword">select</span> <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>)/<span class="hljs-number">4996</span> <span class="hljs-keyword">from</span> news20b_perceptron_submit1 -<span class="hljs-keyword">where</span> actual == predicted; +<pre><code class="lang-sql"><span class="hljs-keyword">select</span> + <span class="hljs-keyword">sum</span>(<span class="hljs-keyword">if</span>(actual = predicted, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>)) / <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>) <span class="hljs-keyword">as</span> accuracy +<span class="hljs-keyword">from</span> + news20b_perceptron_submit1; </code></pre> <blockquote> <p>0.9459567654123299</p> </blockquote> -<h2 id="cleaning">Cleaning</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_perceptron_model1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_perceptron_predict1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_perceptron_submit1; -</code></pre> -<hr> <h1 id="passive-aggressive">[Passive Aggressive]</h1> <h2 id="model-building">model building</h2> <pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_pa_model1; @@ -2408,18 +2443,14 @@ select from news20b_test t JOIN news20b_pa_predict1 pd on (t.rowid = pd.rowid); -</code></pre><pre><code class="lang-sql"><span class="hljs-keyword">select</span> <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>)/<span class="hljs-number">4996</span> <span class="hljs-keyword">from</span> news20b_pa_submit1 -<span class="hljs-keyword">where</span> actual == predicted; +</code></pre><pre><code class="lang-sql"><span class="hljs-keyword">select</span> + <span class="hljs-keyword">sum</span>(<span class="hljs-keyword">if</span>(actual = predicted, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>)) / <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>) <span class="hljs-keyword">as</span> accuracy +<span class="hljs-keyword">from</span> + news20b_pa_submit1; </code></pre> <blockquote> <p>0.9603682946357086</p> </blockquote> -<h2 id="cleaning">Cleaning</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_pa_model1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_pa_predict1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_pa_submit1; -</code></pre> -<hr> <h1 id="passive-aggressive-pa1">[Passive Aggressive (PA1)]</h1> <h2 id="model-building">model building</h2> <pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_pa1_model1; @@ -2443,8 +2474,9 @@ from <span class="hljs-keyword">sum</span>(m.weight * t.<span class="hljs-keyword">value</span>) <span class="hljs-keyword">as</span> total_weight, <span class="hljs-keyword">case</span> <span class="hljs-keyword">when</span> <span class="hljs-keyword">sum</span>(m.weight * t.<span class="hljs-keyword">value</span>) > <span class="hljs-number">0.0</span> <span class="hljs-keyword">then</span> <span class="hljs-number">1</span> <span class="hljs-keyword">else</span> <span class="hljs-number">-1</span> <span class="hljs-keyword">end</span> <span class="hljs-keyword">as</span> label <span class="hljs-keyword">from</span> - news20b_test_exploded t <span class="hljs-keyword">LEFT</span> <span class="hljs-keyword">OUTER</span> <span class="hljs-keyword">JOIN</span> - news20b_pa1_model1 m <span class="hljs-keyword">ON</span> (t.feature = m.feature) + news20b_test_exploded t + <span class="hljs-keyword">LEFT</span> <span class="hljs-keyword">OUTER</span> <span class="hljs-keyword">JOIN</span> news20b_pa1_model1 m + <span class="hljs-keyword">ON</span> (t.feature = m.feature) <span class="hljs-keyword">group</span> <span class="hljs-keyword">by</span> t.<span class="hljs-keyword">rowid</span>; </code></pre> @@ -2452,23 +2484,20 @@ from <pre><code class="lang-sql"><span class="hljs-keyword">create</span> <span class="hljs-keyword">or</span> <span class="hljs-keyword">replace</span> <span class="hljs-keyword">view</span> news20b_pa1_submit1 <span class="hljs-keyword">as</span> <span class="hljs-keyword">select</span> t.label <span class="hljs-keyword">as</span> actual, - pd.label <span class="hljs-keyword">as</span> predicted + p.label <span class="hljs-keyword">as</span> predicted <span class="hljs-keyword">from</span> - news20b_test t <span class="hljs-keyword">JOIN</span> news20b_pa1_predict1 pd - <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = pd.<span class="hljs-keyword">rowid</span>); + news20b_test t + <span class="hljs-keyword">JOIN</span> news20b_pa1_predict1 p + <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = p.<span class="hljs-keyword">rowid</span>); </code></pre> -<pre><code class="lang-sql"><span class="hljs-keyword">select</span> <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>)/<span class="hljs-number">4996</span> <span class="hljs-keyword">from</span> news20b_pa1_submit1 -<span class="hljs-keyword">where</span> actual == predicted; +<pre><code class="lang-sql"><span class="hljs-keyword">select</span> + <span class="hljs-keyword">sum</span>(<span class="hljs-keyword">if</span>(actual = predicted, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>)) / <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>) <span class="hljs-keyword">as</span> accuracy +<span class="hljs-keyword">from</span> + news20b_pa1_submit1; </code></pre> <blockquote> <p>0.9601681345076061</p> </blockquote> -<h2 id="cleaning">Cleaning</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_pa1_model1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_pa1_predict1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_pa1_submit1; -</code></pre> -<hr> <h1 id="passive-aggressive-pa2">[Passive Aggressive (PA2)]</h1> <h2 id="model-building">model building</h2> <pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_pa2_model1; @@ -2506,17 +2535,14 @@ from news20b_test t <span class="hljs-keyword">JOIN</span> news20b_pa2_predict1 pd <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = pd.<span class="hljs-keyword">rowid</span>); </code></pre> -<pre><code class="lang-sql"><span class="hljs-keyword">select</span> <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>)/<span class="hljs-number">4996</span> <span class="hljs-keyword">from</span> news20b_pa2_submit1 -<span class="hljs-keyword">where</span> actual == predicted; +<pre><code class="lang-sql"><span class="hljs-keyword">select</span> + <span class="hljs-keyword">sum</span>(<span class="hljs-keyword">if</span>(actual = predicted, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>)) / <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>) <span class="hljs-keyword">as</span> accuracy +<span class="hljs-keyword">from</span> + news20b_pa2_submit1; </code></pre> <blockquote> <p>0.9597678142514011</p> </blockquote> -<h2 id="cleaning">Cleaning</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_pa2_model1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_pa2_predict1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_pa2_submit1; -</code></pre> <p><div id="page-footer" class="localized-footer"><hr><!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file @@ -2572,7 +2598,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda <script> var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"Perceptron, Passive Aggressive","level":"6.3.2","depth":2,"next":{"title":"CW, AROW, SCW","level":"6.3.3","depth":2,"path":"binaryclass/news20_scw.md","ref":"binaryclass/news20_scw.md","articles":[]},"previous":{"title":"Data preparation","level":"6.3.1","depth":2,"path":"binaryclass/news20_dataset.md","ref":"binaryclass/news20_dataset.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"h2lb":3,"header":1,"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator -hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md","hline":"true"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"https://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/tree/master/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"sh owLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"https://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"binaryclass/news20_pa.md","mtime":"2018-10-18T10:26:56.677Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2018-11-13T09:32:29.643Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"Perceptron, Passive Aggressive","level":"6.3.2","depth":2,"next":{"title":"CW, AROW, SCW","level":"6.3.3","depth":2,"path":"binaryclass/news20_scw.md","ref":"binaryclass/news20_scw.md","articles":[]},"previous":{"title":"Data Preparation","level":"6.3.1","depth":2,"path":"binaryclass/news20_dataset.md","ref":"binaryclass/news20_dataset.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"h2lb":3,"header":1,"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator -hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md","hline":"true"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"https://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/tree/master/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"sh owLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"https://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"binaryclass/news20_pa.md","mtime":"2018-12-26T10:16:03.080Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2018-12-26T10:20:07.153Z"},"basePath":"..","book":{"language":""}}); }); </script> </div> @@ -2602,7 +2628,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda - <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/4.1.1/anchor.min.js";></script> + <script src="../gitbook/gitbook-plugin-anchorjs/anchor.min.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/d9012d92/userguide/binaryclass/news20_rf.html ---------------------------------------------------------------------- diff --git a/userguide/binaryclass/news20_rf.html b/userguide/binaryclass/news20_rf.html index e859b4e..f65a2e2 100644 --- a/userguide/binaryclass/news20_rf.html +++ b/userguide/binaryclass/news20_rf.html @@ -972,7 +972,7 @@ <b>6.2.1.</b> - Data preparation + Data Preparation </a> @@ -980,13 +980,28 @@ </li> - <li class="chapter " data-level="6.2.2" data-path="a9a_lr.html"> + <li class="chapter " data-level="6.2.2" data-path="a9a_generic.html"> - <a href="a9a_lr.html"> + <a href="a9a_generic.html"> <b>6.2.2.</b> + General Binary Classifier + + </a> + + + + </li> + + <li class="chapter " data-level="6.2.3" data-path="a9a_lr.html"> + + <a href="a9a_lr.html"> + + + <b>6.2.3.</b> + Logistic Regression </a> @@ -995,14 +1010,14 @@ </li> - <li class="chapter " data-level="6.2.3" data-path="a9a_minibatch.html"> + <li class="chapter " data-level="6.2.4" data-path="a9a_minibatch.html"> <a href="a9a_minibatch.html"> - <b>6.2.3.</b> + <b>6.2.4.</b> - Mini-batch gradient descent + Mini-batch Gradient Descent </a> @@ -1038,7 +1053,7 @@ <b>6.3.1.</b> - Data preparation + Data Preparation </a> @@ -1076,13 +1091,28 @@ </li> - <li class="chapter " data-level="6.3.4" data-path="news20_adagrad.html"> + <li class="chapter " data-level="6.3.4" data-path="news20_generic.html"> - <a href="news20_adagrad.html"> + <a href="news20_generic.html"> <b>6.3.4.</b> + General Binary Classifier + + </a> + + + + </li> + + <li class="chapter " data-level="6.3.5" data-path="news20_adagrad.html"> + + <a href="news20_adagrad.html"> + + + <b>6.3.5.</b> + AdaGradRDA, AdaGrad, AdaDelta </a> @@ -1091,12 +1121,12 @@ </li> - <li class="chapter active" data-level="6.3.5" data-path="news20_rf.html"> + <li class="chapter active" data-level="6.3.6" data-path="news20_rf.html"> <a href="news20_rf.html"> - <b>6.3.5.</b> + <b>6.3.6.</b> Random Forest @@ -1134,7 +1164,7 @@ <b>6.4.1.</b> - Data preparation + Data Preparation </a> @@ -1185,7 +1215,7 @@ <b>6.5.1.</b> - Data preparation + Data Preparation </a> @@ -1236,7 +1266,7 @@ <b>6.6.1.</b> - Data pareparation + Data Pareparation </a> @@ -1302,7 +1332,7 @@ <b>6.8.1.</b> - Data preparation + Data Preparation </a> @@ -1360,7 +1390,7 @@ <b>7.1.1.</b> - Data preparation + Data Preparation </a> @@ -1375,7 +1405,7 @@ <b>7.1.2.</b> - Data preparation for one-vs-the-rest classifiers + Data Preparation for one-vs-the-rest classifiers </a> @@ -1435,7 +1465,7 @@ <b>7.1.6.</b> - one-vs-the-rest classifier + one-vs-the-rest Classifier </a> @@ -1559,7 +1589,7 @@ <b>8.2.1.</b> - Data preparation + Data Preparation </a> @@ -1567,13 +1597,28 @@ </li> - <li class="chapter " data-level="8.2.2" data-path="../regression/e2006_arow.html"> + <li class="chapter " data-level="8.2.2" data-path="../regression/e2006_generic.html"> - <a href="../regression/e2006_arow.html"> + <a href="../regression/e2006_generic.html"> <b>8.2.2.</b> + General Regessor + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.3" data-path="../regression/e2006_arow.html"> + + <a href="../regression/e2006_arow.html"> + + + <b>8.2.3.</b> + Passive Aggressive, AROW </a> @@ -1610,7 +1655,7 @@ <b>8.3.1.</b> - Data preparation + Data Preparation </a> @@ -1698,7 +1743,7 @@ <b>9.1.1.</b> - Item-based collaborative filtering + Item-based Collaborative Filtering </a> @@ -1734,7 +1779,7 @@ <b>9.2.1.</b> - Data preparation + Data Preparation </a> @@ -1749,7 +1794,7 @@ <b>9.2.2.</b> - LSH/MinHash and Jaccard similarity + LSH/MinHash and Jaccard Similarity </a> @@ -1764,7 +1809,7 @@ <b>9.2.3.</b> - LSH/MinHash and brute-force search + LSH/MinHash and Brute-force Search </a> @@ -1815,7 +1860,7 @@ <b>9.3.1.</b> - Data preparation + Data Preparation </a> @@ -1830,7 +1875,7 @@ <b>9.3.2.</b> - Item-based collaborative filtering + Item-based Collaborative Filtering </a> @@ -1875,7 +1920,7 @@ <b>9.3.5.</b> - SLIM for fast top-k recommendation + SLIM for fast top-k Recommendation </a> @@ -1890,7 +1935,7 @@ <b>9.3.6.</b> - 10-fold cross validation (Matrix Factorization) + 10-fold Cross Validation (Matrix Factorization) </a> @@ -2080,7 +2125,7 @@ <b>13.2.1.</b> - a9a tutorial for DataFrame + a9a Tutorial for DataFrame </a> @@ -2095,7 +2140,7 @@ <b>13.2.2.</b> - a9a tutorial for SQL + a9a Tutorial for SQL </a> @@ -2131,7 +2176,7 @@ <b>13.3.1.</b> - E2006-tfidf regression tutorial for DataFrame + E2006-tfidf Regression Tutorial for DataFrame </a> @@ -2146,7 +2191,7 @@ <b>13.3.2.</b> - E2006-tfidf regression tutorial for SQL + E2006-tfidf Regression Tutorial for SQL </a> @@ -2166,7 +2211,7 @@ <b>13.4.</b> - Generic features + Generic Features </a> @@ -2182,7 +2227,7 @@ <b>13.4.1.</b> - Top-k join processing + Top-k Join Processing </a> @@ -2197,7 +2242,7 @@ <b>13.4.2.</b> - Other utility functions + Other Utility Functions </a> @@ -2317,9 +2362,9 @@ specific language governing permissions and limitations under the License. --> -<p>Hivemall Random Forest supports libsvm-like sparse inputs. </p> -<div class="panel panel-primary"><div class="panel-heading"><h3 class="panel-title" id="note"><i class="fa fa-edit"></i> Note</h3></div><div class="panel-body"><p>This feature, i.e., Sparse input support in Random Forest, is supported since Hivemall v0.5.0 or later._ -<a href="https://hivemall.incubator.apache.org/userguide/ft_engineering/hashing.html#featurehashing-function"; target="_blank"><code>feature_hashing</code></a> function is useful to prepare feature vectors for Random Forest.</p></div></div> +<p>Hivemall Random Forest supports libsvm-like sparse inputs. This page shows a classification example on 20-newsgroup dataset.</p> +<div class="panel panel-primary"><div class="panel-heading"><h3 class="panel-title" id="note"><i class="fa fa-edit"></i> Note</h3></div><div class="panel-body"><p>This feature, i.e., Sparse input support in Random Forest, is supported since Hivemall v0.5.0 or later. +<a href="http://hivemall.incubator.apache.org/userguide/ft_engineering/hashing.html#featurehashing-function"; target="_blank"><code>feature_hashing</code></a> function is useful to prepare feature vectors for Random Forest.</p></div></div> <!-- toc --><div id="toc" class="toc"> <ul> @@ -2441,7 +2486,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda <script> var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"Random Forest","level":"6.3.5","depth":2,"next":{"title":"KDD2010a Tutorial","level":"6.4","depth":1,"path":"binaryclass/kdd2010a.md","ref":"binaryclass/kdd2010a.md","articles":[{"title":"Data preparation","level":"6.4.1","depth":2,"path":"binaryclass/kdd2010a_dataset.md","ref":"binaryclass/kdd2010a_dataset.md","articles":[]},{"title":"PA, CW, AROW, SCW","level":"6.4.2","depth":2,"path":"binaryclass/kdd2010a_scw.md","ref":"binaryclass/kdd2010a_scw.md","articles":[]}]},"previous":{"title":"AdaGradRDA, AdaGrad, AdaDelta","level":"6.3.4","depth":2,"path":"binaryclass/news20_adagrad.md","ref":"binaryclass/news20_adagrad.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"s tyles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"h2lb":3,"header":1,"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md","hline":"true"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"https://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link"; :{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/tree/master/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"https://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"binaryclass/news20_rf.md","mtime":"2018-11-02T10:33:52.942Z", "type":"markdown"},"gitbook":{"version":"3.2.3","time":"2018-11-13T09:32:29.643Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"Random Forest","level":"6.3.6","depth":2,"next":{"title":"KDD2010a Tutorial","level":"6.4","depth":1,"path":"binaryclass/kdd2010a.md","ref":"binaryclass/kdd2010a.md","articles":[{"title":"Data Preparation","level":"6.4.1","depth":2,"path":"binaryclass/kdd2010a_dataset.md","ref":"binaryclass/kdd2010a_dataset.md","articles":[]},{"title":"PA, CW, AROW, SCW","level":"6.4.2","depth":2,"path":"binaryclass/kdd2010a_scw.md","ref":"binaryclass/kdd2010a_scw.md","articles":[]}]},"previous":{"title":"AdaGradRDA, AdaGrad, AdaDelta","level":"6.3.5","depth":2,"path":"binaryclass/news20_adagrad.md","ref":"binaryclass/news20_adagrad.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"s tyles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"h2lb":3,"header":1,"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md","hline":"true"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"https://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link"; :{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/tree/master/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"https://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"binaryclass/news20_rf.md","mtime":"2018-12-26T10:16:03.081Z", "type":"markdown"},"gitbook":{"version":"3.2.3","time":"2018-12-26T10:20:07.153Z"},"basePath":"..","book":{"language":""}}); }); </script> </div> @@ -2471,7 +2516,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda - <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/4.1.1/anchor.min.js";></script> + <script src="../gitbook/gitbook-plugin-anchorjs/anchor.min.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-hivemall-site/blob/d9012d92/userguide/binaryclass/news20_scw.html ---------------------------------------------------------------------- diff --git a/userguide/binaryclass/news20_scw.html b/userguide/binaryclass/news20_scw.html index 2b9e56f..d0d82ea 100644 --- a/userguide/binaryclass/news20_scw.html +++ b/userguide/binaryclass/news20_scw.html @@ -97,7 +97,7 @@ <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon"> - <link rel="next" href="news20_adagrad.html" /> + <link rel="next" href="news20_generic.html" /> <link rel="prev" href="news20_pa.html" /> @@ -972,7 +972,7 @@ <b>6.2.1.</b> - Data preparation + Data Preparation </a> @@ -980,13 +980,28 @@ </li> - <li class="chapter " data-level="6.2.2" data-path="a9a_lr.html"> + <li class="chapter " data-level="6.2.2" data-path="a9a_generic.html"> - <a href="a9a_lr.html"> + <a href="a9a_generic.html"> <b>6.2.2.</b> + General Binary Classifier + + </a> + + + + </li> + + <li class="chapter " data-level="6.2.3" data-path="a9a_lr.html"> + + <a href="a9a_lr.html"> + + + <b>6.2.3.</b> + Logistic Regression </a> @@ -995,14 +1010,14 @@ </li> - <li class="chapter " data-level="6.2.3" data-path="a9a_minibatch.html"> + <li class="chapter " data-level="6.2.4" data-path="a9a_minibatch.html"> <a href="a9a_minibatch.html"> - <b>6.2.3.</b> + <b>6.2.4.</b> - Mini-batch gradient descent + Mini-batch Gradient Descent </a> @@ -1038,7 +1053,7 @@ <b>6.3.1.</b> - Data preparation + Data Preparation </a> @@ -1076,13 +1091,28 @@ </li> - <li class="chapter " data-level="6.3.4" data-path="news20_adagrad.html"> + <li class="chapter " data-level="6.3.4" data-path="news20_generic.html"> - <a href="news20_adagrad.html"> + <a href="news20_generic.html"> <b>6.3.4.</b> + General Binary Classifier + + </a> + + + + </li> + + <li class="chapter " data-level="6.3.5" data-path="news20_adagrad.html"> + + <a href="news20_adagrad.html"> + + + <b>6.3.5.</b> + AdaGradRDA, AdaGrad, AdaDelta </a> @@ -1091,12 +1121,12 @@ </li> - <li class="chapter " data-level="6.3.5" data-path="news20_rf.html"> + <li class="chapter " data-level="6.3.6" data-path="news20_rf.html"> <a href="news20_rf.html"> - <b>6.3.5.</b> + <b>6.3.6.</b> Random Forest @@ -1134,7 +1164,7 @@ <b>6.4.1.</b> - Data preparation + Data Preparation </a> @@ -1185,7 +1215,7 @@ <b>6.5.1.</b> - Data preparation + Data Preparation </a> @@ -1236,7 +1266,7 @@ <b>6.6.1.</b> - Data pareparation + Data Pareparation </a> @@ -1302,7 +1332,7 @@ <b>6.8.1.</b> - Data preparation + Data Preparation </a> @@ -1360,7 +1390,7 @@ <b>7.1.1.</b> - Data preparation + Data Preparation </a> @@ -1375,7 +1405,7 @@ <b>7.1.2.</b> - Data preparation for one-vs-the-rest classifiers + Data Preparation for one-vs-the-rest classifiers </a> @@ -1435,7 +1465,7 @@ <b>7.1.6.</b> - one-vs-the-rest classifier + one-vs-the-rest Classifier </a> @@ -1559,7 +1589,7 @@ <b>8.2.1.</b> - Data preparation + Data Preparation </a> @@ -1567,13 +1597,28 @@ </li> - <li class="chapter " data-level="8.2.2" data-path="../regression/e2006_arow.html"> + <li class="chapter " data-level="8.2.2" data-path="../regression/e2006_generic.html"> - <a href="../regression/e2006_arow.html"> + <a href="../regression/e2006_generic.html"> <b>8.2.2.</b> + General Regessor + + </a> + + + + </li> + + <li class="chapter " data-level="8.2.3" data-path="../regression/e2006_arow.html"> + + <a href="../regression/e2006_arow.html"> + + + <b>8.2.3.</b> + Passive Aggressive, AROW </a> @@ -1610,7 +1655,7 @@ <b>8.3.1.</b> - Data preparation + Data Preparation </a> @@ -1698,7 +1743,7 @@ <b>9.1.1.</b> - Item-based collaborative filtering + Item-based Collaborative Filtering </a> @@ -1734,7 +1779,7 @@ <b>9.2.1.</b> - Data preparation + Data Preparation </a> @@ -1749,7 +1794,7 @@ <b>9.2.2.</b> - LSH/MinHash and Jaccard similarity + LSH/MinHash and Jaccard Similarity </a> @@ -1764,7 +1809,7 @@ <b>9.2.3.</b> - LSH/MinHash and brute-force search + LSH/MinHash and Brute-force Search </a> @@ -1815,7 +1860,7 @@ <b>9.3.1.</b> - Data preparation + Data Preparation </a> @@ -1830,7 +1875,7 @@ <b>9.3.2.</b> - Item-based collaborative filtering + Item-based Collaborative Filtering </a> @@ -1875,7 +1920,7 @@ <b>9.3.5.</b> - SLIM for fast top-k recommendation + SLIM for fast top-k Recommendation </a> @@ -1890,7 +1935,7 @@ <b>9.3.6.</b> - 10-fold cross validation (Matrix Factorization) + 10-fold Cross Validation (Matrix Factorization) </a> @@ -2080,7 +2125,7 @@ <b>13.2.1.</b> - a9a tutorial for DataFrame + a9a Tutorial for DataFrame </a> @@ -2095,7 +2140,7 @@ <b>13.2.2.</b> - a9a tutorial for SQL + a9a Tutorial for SQL </a> @@ -2131,7 +2176,7 @@ <b>13.3.1.</b> - E2006-tfidf regression tutorial for DataFrame + E2006-tfidf Regression Tutorial for DataFrame </a> @@ -2146,7 +2191,7 @@ <b>13.3.2.</b> - E2006-tfidf regression tutorial for SQL + E2006-tfidf Regression Tutorial for SQL </a> @@ -2166,7 +2211,7 @@ <b>13.4.</b> - Generic features + Generic Features </a> @@ -2182,7 +2227,7 @@ <b>13.4.1.</b> - Top-k join processing + Top-k Join Processing </a> @@ -2197,7 +2242,7 @@ <b>13.4.2.</b> - Other utility functions + Other Utility Functions </a> @@ -2317,13 +2362,6 @@ specific language governing permissions and limitations under the License. --> -<h2 id="udf-preparation">UDF preparation</h2> -<pre><code>use news20; - -delete jar /home/myui/tmp/hivemall.jar; -add jar /home/myui/tmp/hivemall.jar; -source /home/myui/tmp/define-all.hive; -</code></pre><hr> <h1 id="confidece-weighted-cw">Confidece Weighted (CW)</h1> <h2 id="training">training</h2> <pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_cw_model1; @@ -2355,28 +2393,22 @@ source /home/myui/tmp/define-all.hive; t.<span class="hljs-keyword">rowid</span>; </code></pre> <h2 id="evaluation">evaluation</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">create</span> <span class="hljs-keyword">or</span> <span class="hljs-keyword">replace</span> <span class="hljs-keyword">view</span> news20b_cw_submit1 -<span class="hljs-keyword">as</span> +<pre><code class="lang-sql">WITH submit as ( <span class="hljs-keyword">select</span> t.<span class="hljs-keyword">rowid</span>, t.label <span class="hljs-keyword">as</span> actual, - pd.label <span class="hljs-keyword">as</span> predicted + p.label <span class="hljs-keyword">as</span> predicted <span class="hljs-keyword">from</span> - news20b_test t <span class="hljs-keyword">JOIN</span> news20b_cw_predict1 pd - <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = pd.<span class="hljs-keyword">rowid</span>); -</code></pre> -<pre><code class="lang-sql"><span class="hljs-keyword">select</span> <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>)/<span class="hljs-number">4996</span> <span class="hljs-keyword">from</span> news20b_cw_submit1 -<span class="hljs-keyword">where</span> actual = predicted; + news20b_test t + <span class="hljs-keyword">JOIN</span> news20b_cw_predict1 p + <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = p.<span class="hljs-keyword">rowid</span>) +) +<span class="hljs-keyword">select</span> <span class="hljs-keyword">sum</span>(<span class="hljs-keyword">if</span>(actual = predicted, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>)) / <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>) <span class="hljs-keyword">as</span> accuracy +<span class="hljs-keyword">from</span> submit; </code></pre> <blockquote> <p>0.9655724579663731</p> </blockquote> -<h2 id="cleaning">Cleaning</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_cw_model1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_cw_predict1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_cw_submit1; -</code></pre> -<hr> <h1 id="adaptive-regularization-of-weight-vectors-arow">Adaptive Regularization of Weight Vectors (AROW)</h1> <h2 id="training">training</h2> <pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_arow_model1; @@ -2408,27 +2440,22 @@ source /home/myui/tmp/define-all.hive; t.<span class="hljs-keyword">rowid</span>; </code></pre> <h2 id="evaluation">evaluation</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">create</span> <span class="hljs-keyword">or</span> <span class="hljs-keyword">replace</span> <span class="hljs-keyword">view</span> news20b_arow_submit1 <span class="hljs-keyword">as</span> -<span class="hljs-keyword">select</span> +<pre><code class="lang-sql">WITH submit as ( +<span class="hljs-keyword">select</span> t.<span class="hljs-keyword">rowid</span>, t.label <span class="hljs-keyword">as</span> actual, - pd.label <span class="hljs-keyword">as</span> predicted + p.label <span class="hljs-keyword">as</span> predicted <span class="hljs-keyword">from</span> - news20b_test t <span class="hljs-keyword">JOIN</span> news20b_arow_predict1 pd - <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = pd.<span class="hljs-keyword">rowid</span>); -</code></pre> -<pre><code class="lang-sql"><span class="hljs-keyword">select</span> <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>)/<span class="hljs-number">4996</span> <span class="hljs-keyword">from</span> news20b_arow_submit1 -<span class="hljs-keyword">where</span> actual = predicted; + news20b_test t + <span class="hljs-keyword">JOIN</span> news20b_arow_predict1 p + <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = p.<span class="hljs-keyword">rowid</span>) +) +<span class="hljs-keyword">select</span> <span class="hljs-keyword">sum</span>(<span class="hljs-keyword">if</span>(actual = predicted, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>)) / <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>) <span class="hljs-keyword">as</span> accuracy +<span class="hljs-keyword">from</span> submit; </code></pre> <blockquote> <p>0.9659727782225781</p> </blockquote> -<h2 id="cleaning">Cleaning</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_arow_model1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_arow_predict1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_arow_submit1; -</code></pre> -<hr> <h1 id="soft-confidence-weighted-scw1">Soft Confidence-Weighted (SCW1)</h1> <h2 id="training">training</h2> <pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_scw_model1; @@ -2460,27 +2487,21 @@ source /home/myui/tmp/define-all.hive; t.<span class="hljs-keyword">rowid</span>; </code></pre> <h2 id="evaluation">evaluation</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">create</span> <span class="hljs-keyword">or</span> <span class="hljs-keyword">replace</span> <span class="hljs-keyword">view</span> news20b_scw_submit1 <span class="hljs-keyword">as</span> -<span class="hljs-keyword">select</span> - t.<span class="hljs-keyword">rowid</span>, - t.label <span class="hljs-keyword">as</span> actual, - pd.label <span class="hljs-keyword">as</span> predicted -<span class="hljs-keyword">from</span> - news20b_test t <span class="hljs-keyword">JOIN</span> news20b_scw_predict1 pd - <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = pd.<span class="hljs-keyword">rowid</span>); -</code></pre> -<pre><code class="lang-sql"><span class="hljs-keyword">select</span> <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>)/<span class="hljs-number">4996</span> <span class="hljs-keyword">from</span> news20b_scw_submit1 -<span class="hljs-keyword">where</span> actual = predicted; +<pre><code class="lang-sql">WITH submit as ( + <span class="hljs-keyword">select</span> + t.<span class="hljs-keyword">rowid</span>, + t.label <span class="hljs-keyword">as</span> actual, + p.label <span class="hljs-keyword">as</span> predicted + <span class="hljs-keyword">from</span> + news20b_test t <span class="hljs-keyword">JOIN</span> news20b_scw_predict1 p + <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = p.<span class="hljs-keyword">rowid</span>) +) +<span class="hljs-keyword">select</span> <span class="hljs-keyword">sum</span>(<span class="hljs-keyword">if</span>(actual = predicted, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>)) / <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>) <span class="hljs-keyword">as</span> accuracy +<span class="hljs-keyword">from</span> submit </code></pre> <blockquote> <p>0.9661729383506805</p> </blockquote> -<h2 id="cleaning">Cleaning</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_scw_model1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_scw_predict1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_scw_submit1; -</code></pre> -<hr> <h1 id="soft-confidence-weighted-scw2">Soft Confidence-Weighted (SCW2)</h1> <h2 id="training">training</h2> <pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_scw2_model1; @@ -2512,26 +2533,22 @@ source /home/myui/tmp/define-all.hive; t.<span class="hljs-keyword">rowid</span>; </code></pre> <h2 id="evaluation">evaluation</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">create</span> <span class="hljs-keyword">or</span> <span class="hljs-keyword">replace</span> <span class="hljs-keyword">view</span> news20b_scw2_submit1 <span class="hljs-keyword">as</span> +<pre><code class="lang-sql">WITH submit as ( <span class="hljs-keyword">select</span> t.<span class="hljs-keyword">rowid</span>, t.label <span class="hljs-keyword">as</span> actual, pd.label <span class="hljs-keyword">as</span> predicted <span class="hljs-keyword">from</span> - news20b_test t <span class="hljs-keyword">JOIN</span> news20b_scw2_predict1 pd - <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = pd.<span class="hljs-keyword">rowid</span>); -</code></pre> -<pre><code class="lang-sql"><span class="hljs-keyword">select</span> <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>)/<span class="hljs-number">4996</span> <span class="hljs-keyword">from</span> news20b_scw2_submit1 -<span class="hljs-keyword">where</span> actual = predicted; + news20b_test t + <span class="hljs-keyword">JOIN</span> news20b_scw2_predict1 pd + <span class="hljs-keyword">on</span> (t.<span class="hljs-keyword">rowid</span> = pd.<span class="hljs-keyword">rowid</span>) +) +<span class="hljs-keyword">select</span> <span class="hljs-keyword">sum</span>(<span class="hljs-keyword">if</span>(actual = predicted, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>)) / <span class="hljs-keyword">count</span>(<span class="hljs-number">1</span>) <span class="hljs-keyword">as</span> accuracy +<span class="hljs-keyword">from</span> submit; </code></pre> <blockquote> <p>0.9579663730984788</p> </blockquote> -<h2 id="cleaning">Cleaning</h2> -<pre><code class="lang-sql"><span class="hljs-keyword">drop</span> <span class="hljs-keyword">table</span> news20b_scw2_model1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_scw2_predict1; -<span class="hljs-keyword">drop</span> <span class="hljs-keyword">view</span> news20b_scw2_submit1; -</code></pre> <p>--</p> <table> <thead> @@ -2631,7 +2648,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda <script> var gitbook = gitbook || []; gitbook.push(function() { - gitbook.page.hasChanged({"page":{"title":"CW, AROW, SCW","level":"6.3.3","depth":2,"next":{"title":"AdaGradRDA, AdaGrad, AdaDelta","level":"6.3.4","depth":2,"path":"binaryclass/news20_adagrad.md","ref":"binaryclass/news20_adagrad.md","articles":[]},"previous":{"title":"Perceptron, Passive Aggressive","level":"6.3.2","depth":2,"path":"binaryclass/news20_pa.md","ref":"binaryclass/news20_pa.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"h2lb":3,"header":1,"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apach e/incubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md","hline":"true"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"https://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/tree/master/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/prin t.css"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"https://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"binaryclass/news20_scw.md","mtime":"2018-10-18T10:26:56.679Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2018-11-13T09:32:29.643Z"},"basePath":"..","book":{"language":""}}); + gitbook.page.hasChanged({"page":{"title":"CW, AROW, SCW","level":"6.3.3","depth":2,"next":{"title":"General Binary Classifier","level":"6.3.4","depth":2,"path":"binaryclass/news20_generic.md","ref":"binaryclass/news20_generic.md","articles":[]},"previous":{"title":"Perceptron, Passive Aggressive","level":"6.3.2","depth":2,"path":"binaryclass/news20_pa.md","ref":"binaryclass/news20_pa.md","articles":[]},"dir":"ltr"},"config":{"plugins":["theme-api","edit-link","github","splitter","sitemap","etoc","callouts","toggle-chapters","anchorjs","codeblock-filename","expandable-chapters","multipart","codeblock-filename","katex","emphasize","localized-footer"],"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"pluginsConfig":{"emphasize":{},"callouts":{},"etoc":{"h2lb":3,"header":1,"maxdepth":3,"mindepth":1,"notoc":true},"github":{"url":"https://github.com/apache/in cubator-hivemall/"},"splitter":{},"search":{},"downloadpdf":{"base":"https://github.com/apache/incubator-hivemall/docs/gitbook","label":"PDF","multilingual":false},"multipart":{},"localized-footer":{"filename":"FOOTER.md","hline":"true"},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"katex":{},"fontsettings":{"theme":"white","family":"sans","size":2,"font":"sans"},"highlight":{},"codeblock-filename":{},"sitemap":{"hostname":"https://hivemall.incubator.apache.org/"},"theme-api":{"languages":[],"split":false,"theme":"dark"},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"edit-link":{"label":"Edit","base":"https://github.com/apache/incubator-hivemall/tree/master/docs/gitbook"},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.cs s"},"showLevel":true},"anchorjs":{"selector":"h1,h2,h3,*:not(.callout) > h4,h5"},"toggle-chapters":{},"expandable-chapters":{}},"theme":"default","pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"variables":{},"title":"Hivemall User Manual","links":{"sidebar":{"<i class=\"fa fa-home\"></i> Home":"https://hivemall.incubator.apache.org/"}},"gitbook":"3.x.x","description":"User Manual for Apache Hivemall"},"file":{"path":"binaryclass/news20_scw.md","mtime":"2018-12-26T10:16:03.081Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2018-12-26T10:20:07.153Z"},"basePath":"..","book":{"language":""}}); }); </script> </div> @@ -2661,7 +2678,7 @@ Apache Hivemall is an effort undergoing incubation at The Apache Software Founda - <script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/4.1.1/anchor.min.js";></script> + <script src="../gitbook/gitbook-plugin-anchorjs/anchor.min.js"></script>
