Added: avro/site/publish/docs/1.11.3/mapreduce-guide/index.html URL: http://svn.apache.org/viewvc/avro/site/publish/docs/1.11.3/mapreduce-guide/index.html?rev=1919887&view=auto ============================================================================== --- avro/site/publish/docs/1.11.3/mapreduce-guide/index.html (added) +++ avro/site/publish/docs/1.11.3/mapreduce-guide/index.html Wed Aug 14 13:30:06 2024 @@ -0,0 +1,341 @@ +<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.113.0"><link rel=alternate type=text/html href=/docs/1.11.3/mapreduce-guide/_print/><link rel=alternate type=application/rss+xml href=/docs/1.11.3/mapreduce-guide/index.xml><meta name=robots content="index, follow"><link rel=apple-touch-icon sizes=57x57 href=https://apache.org/favicons/apple-touch-icon-57x57.png><link rel=apple-touch-icon sizes=60x60 href=https://apache.org/favicons/apple-touch-icon-60x60.png><link rel=apple-touch-icon sizes=72x72 href=https://apache.org/favicons/apple-touch-icon-72x72.png><link rel=apple-touch-icon sizes=76x76 href=https://apache.org/favicons/apple-touch-icon-76x76.png><link rel=apple-touch-icon sizes=114x114 href=https://apache.org/favicons/apple-touch-icon-114x114.png><link rel=apple-touch-icon sizes=120x120 href=https://apache.org/favicons/apple-touch-icon-120x12 0.png><link rel=apple-touch-icon sizes=144x144 href=https://apache.org/favicons/apple-touch-icon-144x144.png><link rel=apple-touch-icon sizes=152x152 href=https://apache.org/favicons/apple-touch-icon-152x152.png><link rel=apple-touch-icon sizes=180x180 href=https://apache.org/favicons/apple-touch-icon-180x180.png><link rel=icon type=image/png href=https://apache.org/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=https://apache.org/favicons/favicon-194x194.png sizes=194x194><link rel=icon type=image/png href=https://apache.org/favicons/favicon-96x96.png sizes=96x96><link rel=icon type=image/png href=https://apache.org/favicons/android-chrome-192x192.png sizes=192x192><link rel=icon type=image/png href=https://apache.org/favicons/favicon-16x16.png sizes=16x16><link rel=manifest href=https://apache.org/favicons/manifest.json><link rel="shortcut icon" href=https://apache.org/favicons/favicon.ico><title>MapReduce guide | Apache Avro</title><meta name=descriptio n content><meta property="og:title" content="MapReduce guide"><meta property="og:description" content><meta property="og:type" content="website"><meta property="og:url" content="/docs/1.11.3/mapreduce-guide/"><meta itemprop=name content="MapReduce guide"><meta itemprop=description content><meta name=twitter:card content="summary"><meta name=twitter:title content="MapReduce guide"><meta name=twitter:description content><link rel=preload href=/scss/main.min.e6198ccfb5f488caccc5f70e1d5d01bec7ba61523a1be10b1acc6ce8c9358f56.css as=style><link href=/scss/main.min.e6198ccfb5f488caccc5f70e1d5d01bec7ba61523a1be10b1acc6ce8c9358f56.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.5.1.min.js integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin=anonymous></script> +<link rel=stylesheet href=/css/prism.css></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar"><a class=navbar-brand href=/><span class=navbar-logo><img src=/docs/1.11.3/logo.svg width=100 height=30 style="margin:0 10px"></span><span class="text-uppercase font-weight-bold">Apache Avro</span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/project/><span>Project</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/blog/><span>Blog</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/community/><span>Community</span></a></li><li class="nav-item dropdown mr-4 d-none d-lg-block"><a class="nav-link dropdown-toggle" href=# id=navbarDropdown role=button data-toggle=dropdown aria-haspopup=true aria-expanded=false>Documentation</a><div class=dropdown- menu aria-labelledby=navbarDropdownMenuLink><a class=dropdown-item href=/docs/1.11.3/>1.11.3 (Current)</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.11.0/>1.11.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.10.2/>1.10.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.10.1/>1.10.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.10.0/>1.10.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.9.2/>1.9.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.9.1/>1.9.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.9.0/>1.9.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.8.2/>1.8.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.8.1/>1.8.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.8.0/>1.8.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.7/>1.7.7</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.6/>1.7.6</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.5/>1.7.5</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.4/>1.7.4</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.3/>1.7.3</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.2/>1.7.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.1/>1.7.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.0/>1.7.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.6.3/>1.6.3</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.6.2/>1.6.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.6.1/>1.6.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.6.0/>1.6.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.4/>1.5.4</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.3/>1.5.3</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.2/>1.5.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.1/>1.5.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.0/>1.5.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.4.1/>1.4.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.4.0/>1.4.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.3.3/>1.3.3</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.3.2/>1.3.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.3.1/>1.3.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.3.0/>1.3.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.2.0/>1.2.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.1.0/>1.1.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.0.0/>1.0.0</a></div></li></ul></div><div class="navbar-nav d-none d-lg-block"></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav foldable-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sid ebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs1113-li><input type=checkbox id=m-docs1113-check checked> +<label for=m-docs1113-check><a href=/docs/1.11.3/ title="Apache Avro⢠1.11.3 Documentation" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113><span>1.11.3</span></a></label><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113getting-started-java-li><input type=checkbox id=m-docs1113getting-started-java-check> +<label for=m-docs1113getting-started-java-check><a href=/docs/1.11.3/getting-started-java/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113getting-started-java><span>Getting Started (Java)</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113getting-started-python-li><input type=checkbox id=m-docs1113getting-started-python-check> +<label for=m-docs1113getting-started-python-check><a href=/docs/1.11.3/getting-started-python/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113getting-started-python><span>Getting Started (Python)</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113specification-li><input type=checkbox id=m-docs1113specification-check> +<label for=m-docs1113specification-check><a href=/docs/1.11.3/specification/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113specification><span>Specification</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-java-li><input type=checkbox id=m-docs1113api-java-check> +<label for=m-docs1113api-java-check><a href=/docs/1.11.3/api/java/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-java><span>Java API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-c-li><input type=checkbox id=m-docs1113api-c-check> +<label for=m-docs1113api-c-check><a href=/docs/1.11.3/api/c/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-c><span>C API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-c-li><input type=checkbox id=m-docs1113api-c-check> +<label for=m-docs1113api-c-check><a href=/docs/1.11.3/api/cpp/html/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-c><span>C++ API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-csharp-li><input type=checkbox id=m-docs1113api-csharp-check> +<label for=m-docs1113api-csharp-check><a href=/docs/1.11.3/api/csharp/html/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-csharp><span>C# API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-py-li><input type=checkbox id=m-docs1113api-py-check> +<label for=m-docs1113api-py-check><a href=/docs/1.11.3/api/py/html/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-py><span>Python API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113mapreduce-guide-li><input type=checkbox id=m-docs1113mapreduce-guide-check> +<label for=m-docs1113mapreduce-guide-check><a href=/docs/1.11.3/mapreduce-guide/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docs1113mapreduce-guide><span class=td-sidebar-nav-active-item>MapReduce guide</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113idl-language-li><input type=checkbox id=m-docs1113idl-language-check> +<label for=m-docs1113idl-language-check><a href=/docs/1.11.3/idl-language/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113idl-language><span>IDL Language</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113sasl-profile-li><input type=checkbox id=m-docs1113sasl-profile-check> +<label for=m-docs1113sasl-profile-check><a href=/docs/1.11.3/sasl-profile/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113sasl-profile><span>SASL profile</span></a></label></li></ul></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/apache/avro/tree/master/doc/content/en/docs/1.11.3/MapReduce%20guide/_index.md class=td-page-meta--view target=_blank rel=noopener><i class="fa fa-file-alt fa-fw"></i> View page source</a> +<a href=https://github.com/apache/avro/edit/master/doc/content/en/docs/1.11.3/MapReduce%20guide/_index.md class=td-page-meta--edit target=_blank rel=noopener><i class="fa fa-edit fa-fw"></i> Edit this page</a> +<a href="https://github.com/apache/avro/new/master/doc/content/en/docs/1.11.3/MapReduce%20guide/_index.md?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+the+page+%28weight+controls+how+its+ordered+amongst+other+pages+in+the+same+directory%3B+lowest+number+first%29.%0A%2A+Add+a+good+commit+message+at+the+bottom+of+the+page+%28%3C80+characters%3B+use+the+extended+description+field+for+more+detail%29.%0A%2A+Create+a+new+branch+so+you+can+preview+your+new+file+and+request+a+review+via+Pull+Request.%0A"; class=td-page-meta--child target=_blank rel=noopener><i class="fa fa-edit fa-fw"></i> Create child page</a> +<a href="https://github.com/apache/avro/issues/new?title=MapReduce%20guide"; class=td-page-meta--issue target=_blank rel=noopener><i class="fab fa-github fa-fw"></i> Create documentation issue</a> +<a href=https://github.com/apache/avro/issues/new class=td-page-meta--project-issue target=_blank rel=noopener><i class="fas fa-tasks fa-fw"></i> Create project issue</a> +<a id=print href=/docs/1.11.3/mapreduce-guide/_print/><i class="fa fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#setup>Setup</a></li><li><a href=#example-colorcount>Example: ColorCount</a></li><li><a href=#running-colorcount>Running ColorCount</a></li><li><a href=#avromapper---orgapachehadoopmapred-api>AvroMapper - org.apache.hadoop.mapred API</a></li><li><a href=#mapper---orgapachehadoopmapreduce-api>Mapper - org.apache.hadoop.mapreduce API</a></li><li><a href=#avroreducer---orgapachehadoopmapred-api>AvroReducer - org.apache.hadoop.mapred API</a></li><li><a href=#reduce---orgapachehadoopmapreduce-api>Reduce - org.apache.hadoop.mapreduce API</a></li><li><a href=#learning-more>Learning more</a></li></ul></nav></div><div class="taxonomy taxonomy-terms-cloud taxo-tags"><h5 class=taxonomy-title>Tag Cloud</h5><ul class=taxonomy-terms><li><a class=taxonomy-term href=/tags/java/ data-taxonomy-term=java><span class=taxonomy-lab el>java</span><span class=taxonomy-count>1</span></a></li><li><a class=taxonomy-term href=/tags/python/ data-taxonomy-term=python><span class=taxonomy-label>python</span><span class=taxonomy-count>1</span></a></li></ul></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/1.11.3/>1.11.3</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/1.11.3/mapreduce-guide/>MapReduce guide</a></li></ol></nav><div class=td-content><h1>MapReduce guide</h1><header class=article-meta><p class=reading-time><i class="fa fa-clock" aria-hidden=true></i> 9 minute read </p></header><p>Avro provides a convenient way to represent complex data structures within a Hadoop MapReduce job. Avro data can be used as both input to and output from a MapReduce job, as well as the intermediate forma t. The example in this guide uses Avro data for all three, but it’s possible to mix and match; for instance, MapReduce can be used to aggregate a particular field in an Avro record.</p><p>This guide assumes basic familiarity with both Hadoop MapReduce and Avro. See the <a href=https://hadoop.apache.org/docs/current/>Hadoop documentation</a> and the <a href=./getting-started-java/>Avro getting started guide</a> for introductions to these projects. This guide uses the old MapReduce API (<code>org.apache.hadoop.mapred</code>) and the new MapReduce API (<code>org.apache.hadoop.mapreduce</code>).</p><h2 id=setup>Setup</h2><p>The code from this guide is included in the Avro docs under examples/mr-example. The example is set up as a Maven project that includes the necessary Avro and MapReduce dependencies and the Avro Maven plugin for code generation, so no external jars are needed to run the example. In particular, the POM includes the following dependencies:</p><div class=highlight ><pre tabindex=0 >style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code >class=language-xml data-lang=xml><span style=display:flex><span><span >style=color:#204a87;font-weight:700><dependency></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><groupId></span>org.apache.avro<span style=color:#204a87;font-weight:700></groupId></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><artifactId></span>avro<span style=color:#204a87;font-weight:700></artifactId></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><version></span>1.11.3<span style=color:#204a87;font-weight:700></version></span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700></dependency></span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700><dependency></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><groupId></span>org.apache.avro<span style=color:#204a87;font-weight:700></groupId></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><artifactId></span>avro-mapred<span style=color:#204a87;font-weight:700></artifactId></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><version></span>1.11.3<span style=color:#204a87;font-weight:700></version></span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700></dependency></span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700><dependency></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><groupId></span>org.apache.hadoop<span style=color:#204a87;font-weight:700></groupId></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><artifactId></span>hadoop-client<span style=color:#204a87;font-weight:700></artifactId></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><version></span>3.1.2<span style=color:#204a87;font-weight:700></version></span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700></dependency></span> +</span></span></code></pre></div><p>And the following plugin:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-xml data-lang=xml><span style=display:flex><span><span style=color:#204a87;font-weight:700><plugin></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><groupId></span>org.apache.avro<span style=color:#204a87;font-weight:700></groupId></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><artifactId></span>avro-maven-plugin<span style=color:#204a87;font-weight:700></artifactId></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><version></span>1.11.3<span style=color:#204a87;font-weight:700></version></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><executions></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><execution></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><phase></span>generate-sources<span style=color:#204a87;font-weight:700></phase></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><goals></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><goal></span>schema<span style=color:#204a87;font-weight:700></goal></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700></goals></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><configuration></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><sourceDirectory></span>${project.basedir}/../<span style=color:#204a87;font-weight:700></sourceDirectory></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700><outputDirectory></span>${project.basedir}/target/generated-sources/<span style=color:#204a87;font-weight:700></outputDirectory></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700></configuration></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700></execution></span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700></executions></span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700></plugin></span> +</span></span></code></pre></div><p>If you do not configure the <em>sourceDirectory</em> and <em>outputDirectory</em> properties, the defaults will be used. The <em>sourceDirectory</em> property defaults to <em>src/main/avro</em>. The <em>outputDirectory</em> property defaults to <em>target/generated-sources</em>. You can change the paths to match your project layout.</p><p>Alternatively, Avro jars can be downloaded directly from the Apache Avro⢠Releases <a href=https://avro.apache.org/releases.html>page</a>. The relevant Avro jars for this guide are <em>avro-1.11.3.jar</em> and <em>avro-mapred-1.11.3.jar</em>, as well as <em>avro-tools-1.11.3.jar</em> for code generation and viewing Avro data files as JSON. In addition, you will need to install Hadoop in order to use MapReduce.</p><h2 id=example-colorcount>Example: ColorCount</h2><p>Below is a simple example of a MapReduce that uses Avro. There is an example for both the old (org.apache.hadoop.mapred) and new (org.apache.hado op.mapreduce) APIs under <em>examples/mr-example/src/main/java/example/</em>. <em>MapredColorCount</em> is the example for the older mapred API while <em>MapReduceColorCount</em> is the example for the newer mapreduce API. Both examples are below, but we will detail the mapred API in our subsequent examples.</p><p>MapredColorCount.java:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-java data-lang=java><span style=display:flex><span><span style=color:#204a87;font-weight:700>package</span> <span style=color:#000>example</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>java.io.IOException</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.avro.*</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.avro.Schema.Type</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.avro.mapred.*</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.conf.*</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.fs.Path</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.mapred.*</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.util.*</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>example.avro.User</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>class</span> <span style=color:#000>MapredColorCount</span> <span style=color:#204a87;font-weight:700>extends</span> <span style=color:#000>Configured</span> <span style=color:#204a87;font-weight:700>implements</span> <span style=color:#000>Tool</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>static</span> <span style=color:#204a87;font-weight:700>class</span> <span style=color:#000>ColorCountMapper</span> <span style=color:#204a87;font-weight:700>extends</span> <span style=color:#000>AvroMapper</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>User</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>></span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#5c35cc;font-weight:700>@Override</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>void</span> <span style=color:#000>map</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>User</span> <span style=color:#000>user</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>AvroCollector</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>></span> <span style=color:#000>collector</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Reporter</span> <span style=color:#000>reporter</span><span style=color:#ce5c00;font-weight:700>)</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>throws</span> <span style=color:#000>IOException</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#000>CharSequence</span> <span style=color:#000>color</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000>user</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>getFavoriteColor</span><span style=color:#ce5c00;font-weight:700>();</span> +</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>// We need this check because the User.favorite_color field has type ["string", "null"] +</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>color</span> <span style=color:#ce5c00;font-weight:700>==</span> <span style=color:#204a87;font-weight:700>null</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#000>color</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#4e9a06>"none"</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#000>collector</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>collect</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>(</span><span style=color:#000>color</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#0000cf;font-weight:700>1</span><span style=color:#ce5c00;font-weight:700>));</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>static</span> <span style=color:#204a87;font-weight:700>class</span> <span style=color:#000>ColorCountReducer</span> <span style=color:#204a87;font-weight:700>extends</span> <span style=color:#000>AvroReducer</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>></span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#5c35cc;font-weight:700>@Override</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>void</span> <span style=color:#000>reduce</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>CharSequence</span> <span style=color:#000>key</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Iterable</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>></span> <span style=color:#000>values</span><span style=color:#ce5c00;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#000>AvroCollector</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>></span> <span style=color:#000>collector</span><span style=color:#ce5c00;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#000>Reporter</span> <span style=color:#000>reporter</span><span style=color:#ce5c00;font-weight:700>)</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>throws</span> <span style=color:#000>IOException</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>int</span> <span style=color:#000>sum</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#0000cf;font-weight:700>0</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>Integer</span> <span style=color:#000>value</span> <span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>values</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#000>sum</span> <span style=color:#ce5c00;font-weight:700>+=</span> <span style=color:#000>value</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#000>collector</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>collect</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>(</span><span style=color:#000>key</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>sum</span><span style=color:#ce5c00;font-weight:700>));</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>int</span> <span style=color:#000>run</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>String</span><span style=color:#ce5c00;font-weight:700>[]</span> <span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>throws</span> <span style=color:#000>Exception</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>length</span> <span style=color:#ce5c00;font-weight:700>!=</span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#000>System</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>err</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>println</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#4e9a06>"Usage: MapredColorCount <input path> <output path>"</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#ce5c00;font-weight:700>-</span><span style=color:#0000cf;font-weight:700>1</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#000>JobConf</span> <span style=color:#000>conf</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>JobConf</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>getConf</span><span style=color:#ce5c00;font-weight:700>(),</span> <span style=color:#000>MapredColorCount</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>conf</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setJobName</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#4e9a06>"colorcount"</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#000>FileInputFormat</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setInputPaths</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>conf</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Path</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>[</span><span style=color:#0000cf;font-weight:700>0</span><span style=color:#ce5c00;font-weight:700>]));</span> +</span></span><span style=display:flex><span> <span style=color:#000>FileOutputFormat</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setOutputPath</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>conf</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Path</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>[</span><span style=color:#0000cf;font-weight:700>1</span><span style=color:#ce5c00;font-weight:700>]));</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#000>AvroJob</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setMapperClass</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>conf</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>ColorCountMapper</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>AvroJob</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setReducerClass</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>conf</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>ColorCountReducer</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>// Note that AvroJob.setInputSchema and AvroJob.setOutputSchema set +</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#8f5902;font-style:italic>// relevant config options such as input/output format, map output +</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#8f5902;font-style:italic>// classes, and output key class. +</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>AvroJob</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setInputSchema</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>conf</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>User</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>getClassSchema</span><span style=color:#ce5c00;font-weight:700>());</span> +</span></span><span style=display:flex><span> <span style=color:#000>AvroJob</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setOutputSchema</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>conf</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>getPairSchema</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>Schema</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>create</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>Type</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>STRING</span><span style=color:#ce5c00;font-weight:700>),</span> +</span></span><span style=display:flex><span> <span style=color:#000>Schema</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>create</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>Type</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>INT</span><span style=color:#ce5c00;font-weight:700>)));</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#000>JobClient</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>runJob</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>conf</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#0000cf;font-weight:700>0</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>static</span> <span style=color:#204a87;font-weight:700>void</span> <span style=color:#000>main</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>String</span><span style=color:#ce5c00;font-weight:700>[]</span> <span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>throws</span> <span style=color:#000>Exception</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>int</span> <span style=color:#000>res</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000>ToolRunner</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>run</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Configuration</span><span style=color:#ce5c00;font-weight:700>(),</span> <span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>MapredColorCount</span><span style=color:#ce5c00;font-weight:700>(),</span> <span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>System</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>exit</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>res</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>}</span> +</span></span></code></pre></div><p>MapReduceColorCount.java:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-java data-lang=java><span style=display:flex><span><span style=color:#204a87;font-weight:700>package</span> <span style=color:#000>example</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>java.io.IOException</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.avro.Schema</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.avro.mapred.AvroKey</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.avro.mapred.AvroValue</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.avro.mapreduce.AvroJob</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.avro.mapreduce.AvroKeyInputFormat</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.avro.mapreduce.AvroKeyValueOutputFormat</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.conf.Configured</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.fs.Path</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.io.IntWritable</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.io.NullWritable</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.io.Text</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.mapreduce.Job</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.mapreduce.Mapper</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.mapreduce.Reducer</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.mapreduce.lib.input.FileInputFormat</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.mapreduce.lib.output.FileOutputFormat</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.util.Tool</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>org.apache.hadoop.util.ToolRunner</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>import</span> <span style=color:#000>example.avro.User</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>class</span> <span style=color:#000>MapReduceColorCount</span> <span style=color:#204a87;font-weight:700>extends</span> <span style=color:#000>Configured</span> <span style=color:#204a87;font-weight:700>implements</span> <span style=color:#000>Tool</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>static</span> <span style=color:#204a87;font-weight:700>class</span> <span style=color:#000>ColorCountMapper</span> <span style=color:#204a87;font-weight:700>extends</span> +</span></span><span style=display:flex><span> <span style=color:#000>Mapper</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>AvroKey</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>User</span><span style=color:#ce5c00;font-weight:700>>,</span> <span style=color:#000>NullWritable</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Text</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>IntWritable</span><span style=color:#ce5c00;font-weight:700>></span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#5c35cc;font-weight:700>@Override</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>void</span> <span style=color:#000>map</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>AvroKey</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>User</span><span style=color:#ce5c00;font-weight:700>></span> <span style=color:#000>key</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>NullWritable</span> <span style=color:#000>value</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Context</span> <span style=color:#000>context</span><span style=color:#ce5c00;font-weight:700>)</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>throws</span> <span style=color:#000>IOException</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>InterruptedException</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#000>CharSequence</span> <span style=color:#000>color</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000>key</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>datum</span><span style=color:#ce5c00;font-weight:700>().</span><span style=color:#c4a000>getFavoriteColor</span><span style=color:#ce5c00;font-weight:700>();</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>color</span> <span style=color:#ce5c00;font-weight:700>==</span> <span style=color:#204a87;font-weight:700>null</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#000>color</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#4e9a06>"none"</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#000>context</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>write</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Text</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>color</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>toString</span><span style=color:#ce5c00;font-weight:700>()),</span> <span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>IntWritable</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#0000cf;font-weight:700>1</span><span style=color:#ce5c00;font-weight:700>));</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>static</span> <span style=color:#204a87;font-weight:700>class</span> <span style=color:#000>ColorCountReducer</span> <span style=color:#204a87;font-weight:700>extends</span> +</span></span><span style=display:flex><span> <span style=color:#000>Reducer</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>Text</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>IntWritable</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>AvroKey</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>>,</span> <span style=color:#000>AvroValue</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>></span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#5c35cc;font-weight:700>@Override</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>void</span> <span style=color:#000>reduce</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>Text</span> <span style=color:#000>key</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Iterable</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>IntWritable</span><span style=color:#ce5c00;font-weight:700>></span> <span style=color:#000>values</span><span style=color:#ce5c00;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#000>Context</span> <span style=color:#000>context</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>throws</span> <span style=color:#000>IOException</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>InterruptedException</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>int</span> <span style=color:#000>sum</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#0000cf;font-weight:700>0</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>IntWritable</span> <span style=color:#000>value</span> <span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>values</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#000>sum</span> <span style=color:#ce5c00;font-weight:700>+=</span> <span style=color:#000>value</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>get</span><span style=color:#ce5c00;font-weight:700>();</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#000>context</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>write</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>AvroKey</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>>(</span><span style=color:#000>key</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>toString</span><span style=color:#ce5c00;font-weight:700>()),</span> <span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>AvroValue</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>(</span><span style=color:#000>sum</span><span style=color:#ce5c00;font-weight:700>));</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>int</span> <span style=color:#000>run</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>String</span><span style=color:#ce5c00;font-weight:700>[]</span> <span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>throws</span> <span style=color:#000>Exception</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>length</span> <span style=color:#ce5c00;font-weight:700>!=</span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#000>System</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>err</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>println</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#4e9a06>"Usage: MapReduceColorCount <input path> <output path>"</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#ce5c00;font-weight:700>-</span><span style=color:#0000cf;font-weight:700>1</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#000>Job</span> <span style=color:#000>job</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Job</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>getConf</span><span style=color:#ce5c00;font-weight:700>());</span> +</span></span><span style=display:flex><span> <span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setJarByClass</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>MapReduceColorCount</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setJobName</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#4e9a06>"Color Count"</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#000>FileInputFormat</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setInputPaths</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Path</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>[</span><span style=color:#0000cf;font-weight:700>0</span><span style=color:#ce5c00;font-weight:700>]));</span> +</span></span><span style=display:flex><span> <span style=color:#000>FileOutputFormat</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setOutputPath</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Path</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>[</span><span style=color:#0000cf;font-weight:700>1</span><span style=color:#ce5c00;font-weight:700>]));</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setInputFormatClass</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>AvroKeyInputFormat</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setMapperClass</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>ColorCountMapper</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>AvroJob</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setInputKeySchema</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>User</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>getClassSchema</span><span style=color:#ce5c00;font-weight:700>());</span> +</span></span><span style=display:flex><span> <span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setMapOutputKeyClass</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>Text</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setMapOutputValueClass</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>IntWritable</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setOutputFormatClass</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>AvroKeyValueOutputFormat</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setReducerClass</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>ColorCountReducer</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>class</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>AvroJob</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setOutputKeySchema</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Schema</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>create</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>Schema</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>Type</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>STRING</span><span style=color:#ce5c00;font-weight:700>));</span> +</span></span><span style=display:flex><span> <span style=color:#000>AvroJob</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>setOutputValueSchema</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Schema</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>create</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>Schema</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>Type</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>INT</span><span style=color:#ce5c00;font-weight:700>));</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>job</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>waitForCompletion</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#204a87;font-weight:700>true</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#ce5c00;font-weight:700>?</span> <span style=color:#0000cf;font-weight:700>0</span> <span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#0000cf;font-weight:700>1</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>static</span> <span style=color:#204a87;font-weight:700>void</span> <span style=color:#000>main</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>String</span><span style=color:#ce5c00;font-weight:700>[]</span> <span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>throws</span> <span style=color:#000>Exception</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>int</span> <span style=color:#000>res</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000>ToolRunner</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>run</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>MapReduceColorCount</span><span style=color:#ce5c00;font-weight:700>(),</span> <span style=color:#000>args</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#000>System</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>exit</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>res</span><span style=color:#ce5c00;font-weight:700>);</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>}</span> +</span></span></code></pre></div><p>ColorCount reads in data files containing <em>User</em> records, defined in <em>examples/user.avsc</em>, and counts the number of instances of each favorite color. (This example draws inspiration from the canonical <em>WordCount</em> MapReduce application.) This example uses the old MapReduce API. See MapReduceAvroWordCount, found under <em>doc/examples/mr-example/src/main/java/example/</em> to see the new MapReduce API example. The User schema is defined as follows:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"namespace"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"example.avro"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"record"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"User"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"fields"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"name"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"string"</span><span style=color:#000;font-weight:700>},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"favorite_number"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4e9a06>"int"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"null"</span><span style=color:#000;font-weight:700>]},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"favorite_color"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4e9a06>"string"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"null"</span><span style=color:#000;font-weight:700>]}</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>]</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><p>This schema is compiled into the <em>User</em> class used by <em>ColorCount</em> via the Avro Maven plugin (see <em>examples/mr-example/pom.xml</em> for how this is set up).</p><p><em>ColorCountMapper</em> essentially takes a <em>User</em> as input and extracts the User’s favorite color, emitting the key-value pair <code><favoriteColor, 1></code>. <em>ColorCountReducer</em> then adds up how many occurrences of a particular favorite color were emitted, and outputs the result as a Pair record. These Pairs are serialized to an Avro data file.</p><h2 id=running-colorcount>Running ColorCount</h2><p>The <em>ColorCount</em> application is provided as a Maven project in the Avro docs under <em>examples/mr-example</em>. To build the project, including the code generation of the User schema, run:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-shell data-lang=shell ><span style=display:flex><span>mvn compile +</span></span></code></pre></div><p>Next, run <em>GenerateData</em> from <code>examples/mr-examples</code> to create an Avro data file, <code>input/users.avro</code>, containing 20 Users with favorite colors chosen randomly from a list:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-shell data-lang=shell><span style=display:flex><span>mvn exec:java -q -Dexec.mainClass<span style=color:#ce5c00;font-weight:700>=</span>example.GenerateData +</span></span></code></pre></div><p>Besides creating the data file, GenerateData prints the JSON representations of the Users generated to stdout, for example:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"user"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"favorite_number"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#204a87;font-weight:700>null</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"favorite_color"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"red"</span><span styl e=color:#000;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"user"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"favorite_number"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#204a87;font-weight:700>null</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"favorite_color"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"green"</span><span style=color:#000;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"user"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"favorite_number"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#204a87;font-weight:700>null</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"favorite_color"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"purple"</span><span style=color:#000;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"user"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"favorite_number"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#204a87;font-weight:700>null</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"favorite_color"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#204a87;font-weight:700>null</span><span style=color:#000;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#a40000>...</span> +</span></span></code></pre></div><p>Now we’re ready to run ColorCount. We specify our freshly-generated input folder as the input path and output as our output folder (note that MapReduce will not start a job if the output folder already exists):</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-shell data-lang=shell><span style=display:flex><span>mvn exec:java -q -Dexec.mainClass<span style=color:#ce5c00;font-weight:700>=</span>example.MapredColorCount -Dexec.args<span style=color:#ce5c00;font-weight:700>=</span><span style=color:#4e9a06>"input output"</span> +</span></span></code></pre></div><p>Once ColorCount completes, checking the contents of the new output directory should yield the following:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-shell data-lang=shell><span style=display:flex><span>$ ls output/ +</span></span><span style=display:flex><span>part-00000.avro _SUCCESS +</span></span></code></pre></div><p>You can check the contents of the generated Avro file using the avro-tools jar:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-shell data-lang=shell><span style=display:flex><span>$ java -jar /path/to/avro-tools-1.11.3.jar tojson output/part-00000.avro +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>{</span><span style=color:#4e9a06>"value"</span>: 3, <span style=color:#4e9a06>"key"</span>: <span style=color:#4e9a06>"blue"</span><span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>{</span><span style=color:#4e9a06>"value"</span>: 7, <span style=color:#4e9a06>"key"</span>: <span style=color:#4e9a06>"green"</span><span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>{</span><span style=color:#4e9a06>"value"</span>: 1, <span style=color:#4e9a06>"key"</span>: <span style=color:#4e9a06>"none"</span><span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>{</span><span style=color:#4e9a06>"value"</span>: 2, <span style=color:#4e9a06>"key"</span>: <span style=color:#4e9a06>"orange"</span><span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>{</span><span style=color:#4e9a06>"value"</span>: 3, <span style=color:#4e9a06>"key"</span>: <span style=color:#4e9a06>"purple"</span><span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>{</span><span style=color:#4e9a06>"value"</span>: 2, <span style=color:#4e9a06>"key"</span>: <span style=color:#4e9a06>"red"</span><span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>{</span><span style=color:#4e9a06>"value"</span>: 2, <span style=color:#4e9a06>"key"</span>: <span style=color:#4e9a06>"yellow"</span><span style=color:#ce5c00;font-weight:700>}</span> +</span></span></code></pre></div><p>Now let’s go over the ColorCount example in detail.</p><h2 id=avromapper---orgapachehadoopmapred-api>AvroMapper - org.apache.hadoop.mapred API</h2><p>The easiest way to use Avro data files as input to a MapReduce job is to subclass <code>AvroMapper</code>. An <code>AvroMapper</code> defines a <code>map</code> function that takes an Avro datum as input and outputs a key/value pair represented as a Pair record. In the ColorCount example, ColorCountMapper is an AvroMapper that takes a User as input and outputs a <code>Pair<CharSequence, Integer>></code>, where the CharSequence key is the user’s favorite color and the Integer value is 1.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-java data-lang=java><span style=display:flex><span><span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>static</span> <span st yle=color:#204a87;font-weight:700>class</span> <span style=color:#000>ColorCountMapper</span> <span style=color:#204a87;font-weight:700>extends</span> <span style=color:#000>AvroMapper</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>User</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>></span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#5c35cc;font-weight:700>@Override</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>public</span> <span style=color:#204a87;font-weight:700>void</span> <span style=color:#000>map</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>User</span> <span style=color:#000>user</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>AvroCollector</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>></span> <span style=color:#000>collector</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Reporter</span> <span style=color:#000>reporter</span><span style=color:#ce5c00;font-weight:700>)</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>throws</span> <span style=color:#000>IOException</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#000>CharSequence</span> <span style=color:#000>color</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000>user</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>getFavoriteColor</span><span style=color:#ce5c00;font-weight:700>();</span> +</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>// We need this check because the User.favorite_color field has type ["string", "null"] +</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#ce5c00;font-weight:700>(</span><span style=color:#000>color</span> <span style=color:#ce5c00;font-weight:700>==</span> <span style=color:#204a87;font-weight:700>null</span><span style=color:#ce5c00;font-weight:700>)</span> <span style=color:#ce5c00;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#000>color</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#4e9a06>"none"</span><span style=color:#ce5c00;font-weight:700>;</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#000>collector</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#c4a000>collect</span><span style=color:#ce5c00;font-weight:700>(</span><span style=color:#204a87;font-weight:700>new</span> <span style=color:#000>Pair</span><span style=color:#ce5c00;font-weight:700><</span><span style=color:#000>CharSequence</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#000>Integer</span><span style=color:#ce5c00;font-weight:700>>(</span><span style=color:#000>color</span><span style=color:#ce5c00;font-weight:700>,</span> <span style=color:#0000cf;font-weight:700>1</span><span style=color:#ce5c00;font-weight:700>));</span> +</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#ce5c00;font-weight:700>}</span>
[... 54 lines stripped ...]
