Added: avro/site/publish/docs/1.11.3/specification/index.html URL: http://svn.apache.org/viewvc/avro/site/publish/docs/1.11.3/specification/index.html?rev=1919887&view=auto ============================================================================== --- avro/site/publish/docs/1.11.3/specification/index.html (added) +++ avro/site/publish/docs/1.11.3/specification/index.html Wed Aug 14 13:30:06 2024 @@ -0,0 +1,252 @@ +<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.113.0"><link rel=alternate type=text/html href=/docs/1.11.3/specification/_print/><link rel=alternate type=application/rss+xml href=/docs/1.11.3/specification/index.xml><meta name=robots content="index, follow"><link rel=apple-touch-icon sizes=57x57 href=https://apache.org/favicons/apple-touch-icon-57x57.png><link rel=apple-touch-icon sizes=60x60 href=https://apache.org/favicons/apple-touch-icon-60x60.png><link rel=apple-touch-icon sizes=72x72 href=https://apache.org/favicons/apple-touch-icon-72x72.png><link rel=apple-touch-icon sizes=76x76 href=https://apache.org/favicons/apple-touch-icon-76x76.png><link rel=apple-touch-icon sizes=114x114 href=https://apache.org/favicons/apple-touch-icon-114x114.png><link rel=apple-touch-icon sizes=120x120 href=https://apache.org/favicons/apple-touch-icon-120x120.pn g><link rel=apple-touch-icon sizes=144x144 href=https://apache.org/favicons/apple-touch-icon-144x144.png><link rel=apple-touch-icon sizes=152x152 href=https://apache.org/favicons/apple-touch-icon-152x152.png><link rel=apple-touch-icon sizes=180x180 href=https://apache.org/favicons/apple-touch-icon-180x180.png><link rel=icon type=image/png href=https://apache.org/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=https://apache.org/favicons/favicon-194x194.png sizes=194x194><link rel=icon type=image/png href=https://apache.org/favicons/favicon-96x96.png sizes=96x96><link rel=icon type=image/png href=https://apache.org/favicons/android-chrome-192x192.png sizes=192x192><link rel=icon type=image/png href=https://apache.org/favicons/favicon-16x16.png sizes=16x16><link rel=manifest href=https://apache.org/favicons/manifest.json><link rel="shortcut icon" href=https://apache.org/favicons/favicon.ico><title>Specification | Apache Avro</title><meta name=description cont ent><meta property="og:title" content="Specification"><meta property="og:description" content><meta property="og:type" content="website"><meta property="og:url" content="/docs/1.11.3/specification/"><meta itemprop=name content="Specification"><meta itemprop=description content><meta name=twitter:card content="summary"><meta name=twitter:title content="Specification"><meta name=twitter:description content><link rel=preload href=/scss/main.min.e6198ccfb5f488caccc5f70e1d5d01bec7ba61523a1be10b1acc6ce8c9358f56.css as=style><link href=/scss/main.min.e6198ccfb5f488caccc5f70e1d5d01bec7ba61523a1be10b1acc6ce8c9358f56.css rel=stylesheet integrity><script src=https://code.jquery.com/jquery-3.5.1.min.js integrity="sha256-9/aliU8dGd2tb6OSsuzixeV4y/faTqgFtohetphbbj0=" crossorigin=anonymous></script> +<link rel=stylesheet href=/css/prism.css></head><body class=td-section><header><nav class="js-navbar-scroll navbar navbar-expand navbar-dark flex-column flex-md-row td-navbar"><a class=navbar-brand href=/><span class=navbar-logo><img src=/docs/1.11.3/logo.svg width=100 height=30 style="margin:0 10px"></span><span class="text-uppercase font-weight-bold">Apache Avro</span></a><div class="td-navbar-nav-scroll ml-md-auto" id=main_navbar><ul class="navbar-nav mt-2 mt-lg-0"><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/project/><span>Project</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/blog/><span>Blog</span></a></li><li class="nav-item mr-4 mb-2 mb-lg-0"><a class=nav-link href=/community/><span>Community</span></a></li><li class="nav-item dropdown mr-4 d-none d-lg-block"><a class="nav-link dropdown-toggle" href=# id=navbarDropdown role=button data-toggle=dropdown aria-haspopup=true aria-expanded=false>Documentation</a><div class=dropdown- menu aria-labelledby=navbarDropdownMenuLink><a class=dropdown-item href=/docs/1.11.3/>1.11.3 (Current)</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.11.0/>1.11.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.10.2/>1.10.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.10.1/>1.10.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.10.0/>1.10.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.9.2/>1.9.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.9.1/>1.9.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.9.0/>1.9.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.8.2/>1.8.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.8.1/>1.8.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.8.0/>1.8.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.7/>1.7.7</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.6/>1.7.6</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.5/>1.7.5</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.4/>1.7.4</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.3/>1.7.3</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.2/>1.7.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.1/>1.7.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.7.0/>1.7.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.6.3/>1.6.3</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.6.2/>1.6.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.6.1/>1.6.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.6.0/>1.6.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.4/>1.5.4</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.3/>1.5.3</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.2/>1.5.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.1/>1.5.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.5.0/>1.5.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.4.1/>1.4.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.4.0/>1.4.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.3.3/>1.3.3</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.3.2/>1.3.2</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.3.1/>1.3.1</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.3.0/>1.3.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.2.0/>1.2.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.1.0/>1.1.0</a> +<a class=dropdown-item href=https://avro.apache.org/docs/1.0.0/>1.0.0</a></div></li></ul></div><div class="navbar-nav d-none d-lg-block"></div></nav></header><div class="container-fluid td-outer"><div class=td-main><div class="row flex-xl-nowrap"><aside class="col-12 col-md-3 col-xl-2 td-sidebar d-print-none"><div id=td-sidebar-menu class=td-sidebar__inner><div id=content-mobile><form class="td-sidebar__search d-flex align-items-center"><button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form></div><div id=content-desktop></div><nav class="collapse td-sidebar-nav foldable-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-sidebar-link td-sid ebar-link__section tree-root" id=m-docs><span>Documentation</span></a><ul class=ul-1><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs1113-li><input type=checkbox id=m-docs1113-check checked> +<label for=m-docs1113-check><a href=/docs/1.11.3/ title="Apache Avro⢠1.11.3 Documentation" class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113><span>1.11.3</span></a></label><ul class="ul-2 foldable"><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113getting-started-java-li><input type=checkbox id=m-docs1113getting-started-java-check> +<label for=m-docs1113getting-started-java-check><a href=/docs/1.11.3/getting-started-java/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113getting-started-java><span>Getting Started (Java)</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113getting-started-python-li><input type=checkbox id=m-docs1113getting-started-python-check> +<label for=m-docs1113getting-started-python-check><a href=/docs/1.11.3/getting-started-python/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113getting-started-python><span>Getting Started (Python)</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113specification-li><input type=checkbox id=m-docs1113specification-check> +<label for=m-docs1113specification-check><a href=/docs/1.11.3/specification/ class="align-left pl-0 active td-sidebar-link td-sidebar-link__section" id=m-docs1113specification><span class=td-sidebar-nav-active-item>Specification</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-java-li><input type=checkbox id=m-docs1113api-java-check> +<label for=m-docs1113api-java-check><a href=/docs/1.11.3/api/java/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-java><span>Java API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-c-li><input type=checkbox id=m-docs1113api-c-check> +<label for=m-docs1113api-c-check><a href=/docs/1.11.3/api/c/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-c><span>C API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-c-li><input type=checkbox id=m-docs1113api-c-check> +<label for=m-docs1113api-c-check><a href=/docs/1.11.3/api/cpp/html/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-c><span>C++ API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-csharp-li><input type=checkbox id=m-docs1113api-csharp-check> +<label for=m-docs1113api-csharp-check><a href=/docs/1.11.3/api/csharp/html/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-csharp><span>C# API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113api-py-li><input type=checkbox id=m-docs1113api-py-check> +<label for=m-docs1113api-py-check><a href=/docs/1.11.3/api/py/html/ class="align-left pl-0 td-sidebar-link td-sidebar-link__page" id=m-docs1113api-py><span>Python API</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113mapreduce-guide-li><input type=checkbox id=m-docs1113mapreduce-guide-check> +<label for=m-docs1113mapreduce-guide-check><a href=/docs/1.11.3/mapreduce-guide/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113mapreduce-guide><span>MapReduce guide</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113idl-language-li><input type=checkbox id=m-docs1113idl-language-check> +<label for=m-docs1113idl-language-check><a href=/docs/1.11.3/idl-language/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113idl-language><span>IDL Language</span></a></label></li><li class="td-sidebar-nav__section-title td-sidebar-nav__section without-child" id=m-docs1113sasl-profile-li><input type=checkbox id=m-docs1113sasl-profile-check> +<label for=m-docs1113sasl-profile-check><a href=/docs/1.11.3/sasl-profile/ class="align-left pl-0 td-sidebar-link td-sidebar-link__section" id=m-docs1113sasl-profile><span>SASL profile</span></a></label></li></ul></li></ul></li></ul></nav></div></aside><aside class="d-none d-xl-block col-xl-2 td-sidebar-toc d-print-none"><div class="td-page-meta ml-2 pb-1 pt-2 mb-0"><a href=https://github.com/apache/avro/tree/master/doc/content/en/docs/1.11.3/Specification/_index.md class=td-page-meta--view target=_blank rel=noopener><i class="fa fa-file-alt fa-fw"></i> View page source</a> +<a href=https://github.com/apache/avro/edit/master/doc/content/en/docs/1.11.3/Specification/_index.md class=td-page-meta--edit target=_blank rel=noopener><i class="fa fa-edit fa-fw"></i> Edit this page</a> +<a href="https://github.com/apache/avro/new/master/doc/content/en/docs/1.11.3/Specification/_index.md?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+the+page+%28weight+controls+how+its+ordered+amongst+other+pages+in+the+same+directory%3B+lowest+number+first%29.%0A%2A+Add+a+good+commit+message+at+the+bottom+of+the+page+%28%3C80+characters%3B+use+the+extended+description+field+for+more+detail%29.%0A%2A+Create+a+new+branch+so+you+can+preview+your+new+file+and+request+a+review+via+Pull+Request.%0A"; class=td-page-meta--child target=_blank rel=noopener><i class="fa fa-edit fa-fw"></i> Create child page</a> +<a href="https://github.com/apache/avro/issues/new?title=Specification"; class=td-page-meta--issue target=_blank rel=noopener><i class="fab fa-github fa-fw"></i> Create documentation issue</a> +<a href=https://github.com/apache/avro/issues/new class=td-page-meta--project-issue target=_blank rel=noopener><i class="fas fa-tasks fa-fw"></i> Create project issue</a> +<a id=print href=/docs/1.11.3/specification/_print/><i class="fa fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#introduction>Introduction</a></li><li><a href=#schema-declaration>Schema Declaration</a></li><li><a href=#primitive-types>Primitive Types</a></li><li><a href=#complex-types>Complex Types</a><ul><li><a href=#schema-record>Records</a></li><li><a href=#enums>Enums</a></li><li><a href=#arrays>Arrays</a></li><li><a href=#maps>Maps</a></li><li><a href=#unions>Unions</a></li><li><a href=#fixed>Fixed</a></li><li><a href=#names>Names</a></li><li><a href=#aliases>Aliases</a></li></ul></li><li><a href=#data-serialization-and-deserialization>Data Serialization and Deserialization</a><ul><li><a href=#encodings>Encodings</a></li><li><a href=#binary-encoding>Binary Encoding</a></li><li><a href=#complex-types-1>Complex Types</a></li><li><a href=#json-encoding>JSON Encoding</a></li><li><a href=#single-object-encoding>Single-obj ect encoding</a></li></ul></li><li><a href=#sort-order>Sort Order</a></li><li><a href=#object-container-files>Object Container Files</a><ul><li><a href=#required-codecs>Required Codecs</a></li><li><a href=#optional-codecs>Optional Codecs</a></li><li><a href=#protocol-declaration>Protocol Declaration</a></li><li><a href=#messages>Messages</a></li><li><a href=#sample-protocol>Sample Protocol</a></li></ul></li><li><a href=#protocol-wire-format>Protocol Wire Format</a><ul><li><a href=#message-transport>Message Transport</a></li><li><a href=#message-framing>Message Framing</a></li><li><a href=#handshake>Handshake</a></li><li><a href=#call-format>Call Format</a></li><li><a href=#schema-resolution>Schema Resolution</a></li><li><a href=#parsing-canonical-form-for-schemas>Parsing Canonical Form for Schemas</a></li></ul></li><li><a href=#logical-types>Logical Types</a><ul><li><a href=#decimal>Decimal</a></li><li><a href=#uuid>UUID</a></li><li><a href=#date>Date</a></li><li><a href=#time-milli second-precision>Time (millisecond precision)</a></li><li><a href=#time-microsecond-precision>Time (microsecond precision)</a></li><li><a href=#timestamp-millisecond-precision>Timestamp (millisecond precision)</a></li><li><a href=#timestamp-microsecond-precision>Timestamp (microsecond precision)</a></li><li><a href=#local-timestamp-millisecond-precision>Local timestamp (millisecond precision)</a></li><li><a href=#local-timestamp-microsecond-precision>Local timestamp (microsecond precision)</a></li><li><a href=#duration>Duration</a></li></ul></li></ul></nav></div><div class="taxonomy taxonomy-terms-cloud taxo-tags"><h5 class=taxonomy-title>Tag Cloud</h5><ul class=taxonomy-terms><li><a class=taxonomy-term href=/tags/java/ data-taxonomy-term=java><span class=taxonomy-label>java</span><span class=taxonomy-count>1</span></a></li><li><a class=taxonomy-term href=/tags/python/ data-taxonomy-term=python><span class=taxonomy-label>python</span><span class=taxonomy-count>1</span></a></li></ul> </div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/1.11.3/>1.11.3</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/1.11.3/specification/>Specification</a></li></ol></nav><div class=td-content><h1>Specification</h1><header class=article-meta><p class=reading-time><i class="fa fa-clock" aria-hidden=true></i> 39 minute read </p></header><h2 id=introduction>Introduction</h2><p>This document defines Apache Avro. It is intended to be the authoritative specification. Implementations of Avro must adhere to this document.</p><h2 id=schema-declaration>Schema Declaration</h2><p>A Schema is represented in <a href=https://www.json.org/>JSON</a> by one of:</p><ul><li>A JSON string, naming a defined type.</li><li>A JSON object, of the form:</li></ul><div class=highlight>< pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-js data-lang=js><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#4e9a06>"type"</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#4e9a06>"typeName"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000;font-weight:700>...</span><span style=color:#000>attributes</span><span style=color:#000;font-weight:700>...}</span> +</span></span></code></pre></div><p>where <em>typeName</em> is either a primitive or derived type name, as defined below. Attributes not defined in this document are permitted as metadata, but must not affect the format of serialized data.</p><ul><li>A JSON array, representing a union of embedded types.</li></ul><h2 id=primitive-types>Primitive Types</h2><p>The set of primitive type names is:</p><ul><li><em>null</em>: no value</li><li><em>boolean</em>: a binary value</li><li><em>int</em>: 32-bit signed integer</li><li><em>long</em>: 64-bit signed integer</li><li><em>float</em>: single precision (32-bit) IEEE 754 floating-point number</li><li><em>double</em>: double precision (64-bit) IEEE 754 floating-point number</li><li><em>bytes</em>: sequence of 8-bit unsigned bytes</li><li><em>string</em>: unicode character sequence</li></ul><p>Primitive types have no specified attributes.</p><p>Primitive type names are also defined type names. Thus, for example, the schema “string” is equivalent to:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"string"</span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><h2 id=complex-types>Complex Types</h2><p>Avro supports six kinds of complex types: <em>records</em>, <em>enums</em>, <em>arrays</em>, <em>maps</em>, <em>unions</em> and <em>fixed</em>.</p><h3 id=schema-record>Records</h3><p>Records use the type name “record” and support the following attributes:</p><ul><li><em>name</em>: a JSON string providing the name of the record (required).</li><li><em>namespace</em>, a JSON string that qualifies the name (optional);</li><li><em>doc</em>: a JSON string providing documentation to the user of this schema (optional).</li><li><em>aliases</em>: a JSON array of strings, providing alternate names for this record (optional).</li><li><em>fields</em>: a JSON array, listing fields (required). Each field is a JSON object with the following attributes:<ul><li><em>name</em>: a JSON string providing the name of the field (required), and</li><li><em>doc</em>: a JSON string describing this field for users (optional) .</li><li><em>type</em>: a <a href=#schema-declaration title="Schema declaration">schema</a>, as defined above</li><li><em>order</em>: specifies how this field impacts sort ordering of this record (optional). Valid values are “ascending” (the default), “descending”, or “ignore”. For more details on how this is used, see the sort order section below.</li><li><em>aliases</em>: a JSON array of strings, providing alternate names for this field (optional).</li><li><em>default</em>: A default value for this field, only used when reading instances that lack the field for schema evolution purposes. The presence of a default value does not make the field optional at encoding time. Permitted values depend on the field’s schema type, according to the table below. Default values for union fields correspond to the first schema in the union. Default values for bytes and fixed fields are JSON strings, where Unicode code points 0-255 are mapped to unsigned 8-bit byte values 0-255. Avro encodes a field even if its value is equal to its default.</li></ul></li></ul><p><em>field default values</em></p><table><thead><tr><th><strong>avro type</strong></th><th><strong>json type</strong></th><th><strong>example</strong></th></tr></thead><tbody><tr><td>null</td><td>null</td><td><code>null</code></td></tr><tr><td>boolean</td><td>boolean</td><td><code>true</code></td></tr><tr><td>int,long</td><td>integer</td><td><code>1</code></td></tr><tr><td>float,double</td><td>number</td><td><code>1.1</code></td></tr><tr><td>bytes</td><td>string</td><td><code>"\u00FF"</code></td></tr><tr><td>string</td><td>string</td><td><code>"foo"</code></td></tr><tr><td>record</td><td>object</td><td><code>{"a": 1}</code></td></tr><tr><td>enum</td><td>string</td><td><code>"FOO"</code></td></tr><tr><td>array</td><td>array</td><td><code>[1]</code></td></tr><tr><td>map</td><td>object</td><td><code>{"a": 1}</code></td></tr><tr><td>fixed</td><td>string</td><td><code>"\u00ff"</c ode></td></tr></tbody></table><p>For example, a linked-list of 64-bit values may be defined with:</p><pre tabindex=0><code class=language-jsonc data-lang=jsonc>{ + "type": "record", + "name": "LongList", + "aliases": ["LinkedLongs"], // old name for this + "fields" : [ + {"name": "value", "type": "long"}, // each element has a long + {"name": "next", "type": ["null", "LongList"]} // optional next element + ] +} +</code></pre><h3 id=enums>Enums</h3><p>Enums use the type name “enum” and support the following attributes:</p><ul><li><em>name</em>: a JSON string providing the name of the enum (required).</li><li><em>namespace</em>, a JSON string that qualifies the name (optional);</li><li><em>aliases</em>: a JSON array of strings, providing alternate names for this enum (optional).</li><li><em>doc</em>: a JSON string providing documentation to the user of this schema (optional).</li><li><em>symbols</em>: a JSON array, listing symbols, as JSON strings (required). All symbols in an enum must be unique; duplicates are prohibited. Every symbol must match the regular expression [A-Za-z_][A-Za-z0-9_]* (the same requirement as for <a href=#names title=Names>names</a>).</li><li><em>default</em>: A default value for this enumeration, used during resolution when the reader encounters a symbol from the writer that isn’t defined in the reader’s schema (optional). The value provided h ere must be a JSON string that’s a member of the symbols array. See documentation on schema resolution for how this gets used.</li></ul><p>For example, playing card suits might be defined with:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"enum"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Suit"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"symbols"</span> <span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4e9a06>"SPADES"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"HEARTS"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"DIAMONDS"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"CLUBS"</span><span style=color:#000;font-weight:700>]</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><h3 id=arrays>Arrays</h3><p>Arrays use the type name “array” and support a single attribute:</p><ul><li><em>items</em>: the schema of the array’s items.</li></ul><p>For example, an array of strings is declared with:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"array"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"items"</span> <span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"string"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"default"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[]</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><h3 id=maps>Maps</h3><p>Maps use the type name “map” and support one attribute:</p><ul><li><em>values</em>: the schema of the map’s values.</li></ul><p>Map keys are assumed to be strings.</p><p>For example, a map from string to long is declared with:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"map"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"values"</span> <span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"long"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"default"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>{}</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><h3 id=unions>Unions</h3><p>Unions, as mentioned above, are represented using JSON arrays. For example, <code>["null", "string"]</code> declares a schema which may be either a null or string.</p><p>(Note that when a <a href=#schema-record title="Schema record">default value</a> is specified for a record field whose type is a union, the type of the default value must match the first element of the union. Thus, for unions containing “null”, the “null” is usually listed first, since the default value of such unions is typically null.)</p><p>Unions may not contain more than one schema with the same type, except for the named types record, fixed and enum. For example, unions containing two array types or two map types are not permitted, but two types with different names are permitted. (Names permit efficient resolution when reading and writing unions.)</p><p>Unions may not immediately contain other unions.</p><h3 id=fixed>Fixed</h 3><p>Fixed uses the type name “fixed” and supports the following attributes:</p><ul><li><em>name</em>: a string naming this fixed (required).</li><li><em>namespace</em>, a string that qualifies the name (optional);</li><li><em>aliases</em>: a JSON array of strings, providing alternate names for this enum (optional).</li><li><em>size</em>: an integer, specifying the number of bytes per value (required).</li></ul><p>For example, 16-byte quantity may be declared with:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"fixed"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"size"</span><span style=co lor:#000;font-weight:700>:</span> <span style=color:#0000cf;font-weight:700>16</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"md5"</span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><h3 id=names>Names</h3><p>Record, enums and fixed are named types. Each has a fullname that is composed of two parts; a name and a namespace, separated by a dot. Equality of names is defined on the fullname.</p><p>Record fields and enum symbols have names as well (but no namespace). Equality of fields and enum symbols is defined on the name of the field/symbol within its scope (the record/enum that defines it). Fields and enum symbols across scopes are never equal.</p><p>The name portion of the fullname of named types, record field names, and enum symbols must:</p><ul><li>start with [A-Za-z_]</li><li>subsequently contain only [A-Za-z0-9_]</li></ul><p>A namespace is a dot-separated sequence of such names. The empty string may also be used as a namespace to indicate the null namespace. Equality of names (including field names and enum symbols) as well as fullnames is case-sensitive.</p><p>The null namespace may not be used in a dot-separated sequence o f names. So the grammar for a namespace is:</p><pre tabindex=0><code> <empty> | <name>[(<dot><name>)*] +</code></pre><p>In record, enum and fixed definitions, the fullname is determined according to the algorithm below the example:</p><pre tabindex=0><code>{ + "type": "record", + "name": "Example", + "doc": "A simple name (attribute) and no namespace attribute: use the null namespace (\"\"); the fullname is 'Example'.", + "fields": [ + { + "name": "inheritNull", + "type": { + "type": "enum", + "name": "Simple", + "doc": "A simple name (attribute) and no namespace attribute: inherit the null namespace of the enclosing type 'Example'. The fullname is 'Simple'.", + "symbols": ["a", "b"] + } + }, { + "name": "explicitNamespace", + "type": { + "type": "fixed", + "name": "Simple", + "namespace": "explicit", + "doc": "A simple name (attribute) and a namespace (attribute); the fullname is 'explicit.Simple' (this is a different type than of the 'inheritNull' field).", + "size": 12 + } + }, { + "name": "fullName", + "type": { + "type": "record", + "name": "a.full.Name", + "namespace": "ignored", + "doc": "A name attribute with a fullname, so the namespace attribute is ignored. The fullname is 'a.full.Name', and the namespace is 'a.full'.", + "fields": [ + { + "name": "inheritNamespace", + "type": { + "type": "enum", + "name": "Understanding", + "doc": "A simple name (attribute) and no namespace attribute: inherit the namespace of the enclosing type 'a.full.Name'. The fullname is 'a.full.Understanding'.", + "symbols": ["d", "e"] + } + } + ] + } + } + ] +} +</code></pre><p>The fullname of a record, enum or fixed definition is determined by the required <code>name</code> and optional <code>namespace</code> attributes like this:</p><ul><li>A fullname is specified. If the name specified contains a dot, then it is assumed to be a fullname, and any namespace also specified is ignored. For example, use “name”: “org.foo.X” to indicate the fullname org.foo.X.</li><li>A simple name (a name that contains no dots) and namespace are both specified. For example, one might use “name”: “X”, “namespace”: “org.foo” to indicate the fullname org.foo.X.</li><li>A simple name only is specified (a name that contains no dots). In this case the namespace is taken from the most tightly enclosing named schema or protocol, and the fullname is constructed from that namespace and the name. For example, if “name”: “X” is specified, and this occurs within a field of the r ecord definition of org.foo.Y, then the fullname is org.foo.X. This also happens if there is no enclosing namespace (i.e., the enclosing schema definition has the null namespace).</li></ul><p>References to previously defined names are as in the latter two cases above: if they contain a dot they are a fullname, if they do not contain a dot, the namespace is the namespace of the enclosing definition.</p><p>Primitive type names (<code>null</code>, <code>boolean</code>, <code>int</code>, <code>long</code>, <code>float</code>, <code>double</code>, <code>bytes</code>, <code>string</code>) have no namespace and their names may not be defined in any namespace.</p><p>Complex types (<code>record</code>, <code>enum</code>, <code>array</code>, <code>map</code>, <code>fixed</code>) have no namespace, but their names (as well as <code>union</code>) are permitted to be reused as type names. This can be confusing to the human reader, but is always unambiguous for binary serialization. Due to the li mitations of JSON encoding, it is a best practice to use a namespace when using these names.</p><p>A schema or protocol may not contain multiple definitions of a fullname. Further, a name must be defined before it is used (“before” in the depth-first, left-to-right traversal of the JSON parse tree, where the types attribute of a protocol is always deemed to come “before” the messages attribute.)</p><h3 id=aliases>Aliases</h3><p>Named types and fields may have aliases. An implementation may optionally use aliases to map a writer’s schema to the reader’s. This facilitates both schema evolution as well as processing disparate datasets.</p><p>Aliases function by re-writing the writer’s schema using aliases from the reader’s schema. For example, if the writer’s schema was named “Foo” and the reader’s schema is named “Bar” and has an alias of “Foo”, then the implementation would act as though & ldquo;Foo” were named “Bar” when reading. Similarly, if data was written as a record with a field named “x” and is read as a record with a field named “y” with alias “x”, then the implementation would act as though “x” were named “y” when reading.</p><p>A type alias may be specified either as a fully namespace-qualified, or relative to the namespace of the name it is an alias for. For example, if a type named “a.b” has aliases of “c” and “x.y”, then the fully qualified names of its aliases are “a.c” and “x.y”.</p><h2 id=data-serialization-and-deserialization>Data Serialization and Deserialization</h2><p>Binary encoded Avro data does not include type information or field names. The benefit is that the serialized data is small, but as a result a schema must always be used in order to read Avro data correctly. The best way to ensure that the schema i s structurally identical to the one used to write the data is to use the exact same schema.</p><p>Therefore, files or systems that store Avro data should always include the writer’s schema for that data. Avro-based remote procedure call (RPC) systems must also guarantee that remote recipients of data have a copy of the schema used to write that data. In general, it is advisable that any reader of Avro data should use a schema that is the same (as defined more fully in <a href=#parsing-canonical-form-for-schemas title="Parsing Canonical Form for Schemas">Parsing Canonical Form for Schemas</a>) as the schema that was used to write the data in order to deserialize it correctly. Deserializing data into a newer schema is accomplished by specifying an additional schema, the results of which are described in <a href=#schema-resolution>Schema Resolution</a>.</p><p>In general, both serialization and deserialization proceed as a depth-first, left-to-right traversal of the schema, serial izing or deserializing primitive types as they are encountered. Therefore, it is possible, though not advisable, to read Avro data with a schema that does not have the same Parsing Canonical Form as the schema with which the data was written. In order for this to work, the serialized primitive values must be compatible, in order value by value, with the items in the deserialization schema. For example, int and long are always serialized the same way, so an int could be deserialized as a long. Since the compatibility of two schemas depends on both the data and the serialization format (eg. binary is more permissive than JSON because JSON includes field names, eg. a long that is too large will overflow an int), it is simpler and more reliable to use schemas with identical Parsing Canonical Form.</p><h3 id=encodings>Encodings</h3><p>Avro specifies two serialization encodings: binary and JSON. Most applications will use the binary encoding, as it is smaller and faster. But, for debuggin g and web-based applications, the JSON encoding may sometimes be appropriate.</p><h3 id=binary-encoding>Binary Encoding</h3><p>Binary encoding does not include field names, self-contained information about the types of individual bytes, nor field or record separators. Therefore readers are wholly reliant on the schema used when the data was encoded.</p><h4 id=primitive-types-1>Primitive Types</h4><p>Primitive types are encoded in binary as follows:</p><ul><li><em>null</em> is written as zero bytes.</li><li>a <em>boolean</em> is written as a single byte whose value is either 0 (false) or 1 (true).</li><li><em>int</em> and <em>long</em> values are written using <a href=https://lucene.apache.org/java/3_5_0/fileformats.html#VInt>variable-length</a> <a href=https://code.google.com/apis/protocolbuffers/docs/encoding.html#types>zig-zag</a> coding. Some examples:</li></ul><table><thead><tr><th><em>value</em></th><th><em>hex</em></th></tr></thead><tbody><tr><td>0</td><td>00</td></tr><tr><td> -1</td><td>01</td></tr><tr><td>1</td><td>02</td></tr><tr><td>-2</td><td>03</td></tr><tr><td>2</td><td>04</td></tr><tr><td>…</td><td>…</td></tr><tr><td>-64</td><td>7f</td></tr><tr><td>64</td><td>80 01</td></tr><tr><td>…</td><td>…</td></tr></tbody></table><ul><li>a <em>float</em> is written as 4 bytes. The float is converted into a 32-bit integer using a method equivalent to Java’s <a href=https://docs.oracle.com/javase/8/docs/api/java/lang/Float.html#floatToIntBits-float->floatToIntBits</a> and then encoded in little-endian format.</li><li>a <em>double</em> is written as 8 bytes. The double is converted into a 64-bit integer using a method equivalent to Java’s <a href=https://docs.oracle.com/javase/8/docs/api/java/lang/Double.html#doubleToLongBits-double->doubleToLongBits</a> and then encoded in little-endian format.</li><li><em>bytes</em> are encoded as a long followed by that many bytes of data.</li><li>a <em>string</em> is encoded as a long followed by that many bytes of UTF-8 encoded character data. +For example, the three-character string “foo” would be encoded as the long value 3 (encoded as hex 06) followed by the UTF-8 encoding of ‘f’, ‘o’, and ‘o’ (the hex bytes 66 6f 6f):</li></ul><pre tabindex=0><code>06 66 6f 6f +</code></pre><h3 id=complex-types-1>Complex Types</h3><p>Complex types are encoded in binary as follows:</p><h4 id=records>Records</h4><p>A record is encoded by encoding the values of its fields in the order that they are declared. In other words, a record is encoded as just the concatenation of the encodings of its fields. Field values are encoded per their schema.</p><p>For example, the record schema</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"record"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"test"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"fields"</span> <span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"a"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"long"</span><span style=color:#000;font-weight:700>},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"b"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"string"</span><span style=color:#000;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>]</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><p>An instance of this record whose a field has value 27 (encoded as hex 36) and whose b field has value “foo” (encoded as hex bytes 06 66 6f 6f), would be encoded simply as the concatenation of these, namely the hex byte sequence:</p><pre tabindex=0><code>36 06 66 6f 6f +</code></pre><h4 id=enums-1>Enums</h4><p>An enum is encoded by a int, representing the zero-based position of the symbol in the schema.</p><p>For example, consider the enum:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"enum"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Foo"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"symbols"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4 e9a06>"A"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"B"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"C"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"D"</span><span style=color:#000;font-weight:700>]</span> <span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><p>This would be encoded by an int between zero and three, with zero indicating “A”, and 3 indicating “D”.</p><h4 id=arrays-1>Arrays</h4><p>Arrays are encoded as a series of blocks. Each block consists of a long count value, followed by that many array items. A block with count zero indicates the end of the array. Each item is encoded per the array’s item schema.</p><p>If a block’s count is negative, its absolute value is used, and the count is followed immediately by a long block size indicating the number of bytes in the block. This block size permits fast skipping through data, e.g., when projecting a record to a subset of its fields.</p><p>For example, the array schema</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=col or:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"array"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"items"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"long"</span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><p>an array containing the items 3 and 27 could be encoded as the long value 2 (encoded as hex 04) followed by long values 3 and 27 (encoded as hex 06 36) terminated by zero:</p><pre tabindex=0><code>04 06 36 00 +</code></pre><p>The blocked representation permits one to read and write arrays larger than can be buffered in memory, since one can start writing items without knowing the full length of the array.</p><h4 id=schema-maps>Maps</h4><p>Maps are encoded as a series of <em>blocks</em>. Each block consists of a <code>long</code> <em>count</em> value, followed by that many key/value pairs. A block with count zero indicates the end of the map. Each item is encoded per the map’s value schema.</p><p>If a block’s count is negative, its absolute value is used, and the count is followed immediately by a <code>long</code> block size indicating the number of bytes in the block. This block size permits fast skipping through data, e.g., when projecting a record to a subset of its fields.</p><p>The blocked representation permits one to read and write maps larger than can be buffered in memory, since one can start writing items without knowing the full length of the map.</p><h4 id=unions-1 >Unions</h4><p>A union is encoded by first writing an <code>int</code> value >indicating the zero-based position within the union of the schema of its >value. The value is then encoded per the indicated schema within the >union.</p><p>For example, the union schema <code>["null","string"]</code> >would encode:</p><ul><li><em>null</em> as zero (the index of >“null” in the union): +<code>00</code></li><li>the string “a” as one (the index of “string” in the union, 1, encoded as hex 02), followed by the serialized string: +<code>02 02 61</code> +NOTE: Currently for C/C++ implementations, the positions are practically an int, but theoretically a long. In reality, we don’t expect unions with 215M members</li></ul><h4 id=fixed-1>Fixed</h4><p>Fixed instances are encoded using the number of bytes declared in the schema.</p><h3 id=json-encoding>JSON Encoding</h3><p>Except for unions, the JSON encoding is the same as is used to encode <a href=#schema-record>field default values</a>.</p><p>The value of a union is encoded in JSON as follows:</p><ul><li>if its type is <em>null</em>, then it is encoded as a JSON <em>null</em>;</li><li>otherwise it is encoded as a JSON object with one name/value pair whose name is the type’s name and whose value is the recursively encoded value. For Avro’s named types (record, fixed or enum) the user-specified name is used, for other types the type name is used.</li></ul><p>For example, the union schema <code>["null","string","Foo"]</code>, where Foo is a record name, would encode:</p ><ul><li><em>null</em> as <em>null</em>;</li><li>the string “a” as ><code>{"string": "a"}</code> and</li><li>a Foo instance as <code>{"Foo": >{...}}</code>, where <code>{...}</code> indicates the JSON encoding of a Foo >instance.</li></ul><p>Note that the original schema is still required to >correctly process JSON-encoded data. For example, the JSON encoding does not >distinguish between <em>int</em> and <em>long</em>, <em>float</em> and ><em>double</em>, records and maps, enums and strings, etc.</p><h3 >id=single-object-encoding>Single-object encoding</h3><p>In some situations a >single Avro serialized object is to be stored for a longer period of time. >One very common example is storing Avro records for several weeks in an <a >href=https://kafka.apache.org/>Apache Kafka</a> topic.</p><p>In the period >after a schema change this persistence system will contain records that have >been written with different schemas. So the need arises to know which schema >was used to write a recor d to support schema evolution correctly. In most cases the schema itself is too large to include in the message, so this binary wrapper format supports the use case more effectively.</p><h4 id=single-object-encoding-specification>Single object encoding specification</h4><p>Single Avro objects are encoded as follows:</p><ol><li>A two-byte marker, <code>C3 01</code>, to show that the message is Avro and uses this single-record format (version 1).</li><li>The 8-byte little-endian CRC-64-AVRO <a href=#schema-fingerprints title="Schema fingerprints">fingerprint</a> of the object’s schema.</li><li>The Avro object encoded using <a href=#binary-encoding>Avro’s binary encoding</a>.</li></ol><p>Implementations use the 2-byte marker to determine whether a payload is Avro. This check helps avoid expensive lookups that resolve the schema from a fingerprint, when the message is not an encoded Avro payload.</p><h2 id=sort-order>Sort Order</h2><p>Avro defines a standard sort order for d ata. This permits data written by one system to be efficiently sorted by another system. This can be an important optimization, as sort order comparisons are sometimes the most frequent per-object operation. Note also that Avro binary-encoded data can be efficiently ordered without deserializing it to objects.</p><p>Data items may only be compared if they have identical schemas. Pairwise comparisons are implemented recursively with a depth-first, left-to-right traversal of the schema. The first mismatch encountered determines the order of the items.</p><p>Two items with the same schema are compared according to the following rules.</p><ul><li><em>null</em> data is always equal.</li><li><em>boolean</em> data is ordered with false before true.</li><li><em>int</em>, <em>long</em>, <em>float</em> and <em>double</em> data is ordered by ascending numeric value.</li><li><em>bytes</em> and fixed data are compared lexicographically by unsigned 8-bit values.</li><li><em>string</em> data is co mpared lexicographically by Unicode code point. Note that since UTF-8 is used as the binary encoding for strings, sorting of bytes and string binary data is identical.</li><li><em>array</em> data is compared lexicographically by element.</li><li><em>enum</em> data is ordered by the symbol’s position in the enum schema. For example, an enum whose symbols are <code>["z", "a"]</code> would sort “z” values before “a” values.</li><li><em>union</em> data is first ordered by the branch within the union, and, within that, by the type of the branch. For example, an <code>["int", "string"]</code> union would order all int values before all string values, with the ints and strings themselves ordered as defined above.</li><li><em>record</em> data is ordered lexicographically by field. If a field specifies that its order is:<ul><li>“ascending”, then the order of its values is unaltered.</li><li>“descending”, then the order of its values is re versed.</li><li>“ignore”, then its values are ignored when sorting.</li></ul></li><li><em>map</em> data may not be compared. It is an error to attempt to compare data containing maps unless those maps are in an <code>"order":"ignore"</code> record field.</li></ul><h2 id=object-container-files>Object Container Files</h2><p>Avro includes a simple object container file format. A file has a schema, and all objects stored in the file must be written according to that schema, using binary encoding. Objects are stored in blocks that may be compressed. Syncronization markers are used between blocks to permit efficient splitting of files for MapReduce processing.</p><p>Files may include arbitrary user-specified metadata.</p><p>A file consists of:</p><ul><li>A file header, followed by</li><li>one or more file data blocks.</li></ul><p>A file header consists of:</p><ul><li>Four bytes, ASCII ‘O’, ‘b’, ‘j’, followed by 1.</li><li>file metadata, incl uding the schema.</li><li>The 16-byte, randomly-generated sync marker for this file.</li></ul><p>File metadata is written as if defined by the following <a href=#schema-maps>map</a> schema:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"map"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"values"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"bytes"</span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><p>All metadata properties that start with “avro.” are reserved. The following file metadata properties are currently used:</p><ul><li><strong>avro.schema</strong> contains the schema of objects stored in the file, as JSON data (required).</li><li><strong>avro.codec</strong> the name of the compression codec used to compress blocks, as a string. Implementations are required to support the following codecs: “null” and “deflate”. If codec is absent, it is assumed to be “null”. The codecs are described with more detail below.</li></ul><p>A file header is thus described by the following schema:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;f ont-weight:700>:</span> <span style=color:#4e9a06>"record"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"org.apache.avro.file.Header"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"fields"</span> <span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"magic"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"fixed"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Magic"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"size"</span><span style=color:#000;font-weight:700>:</span> <span style= color:#0000cf;font-weight:700>4</span><span style=color:#000;font-weight:700>}},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"meta"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"map"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"values"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"bytes"</span><span style=color:#000;font-weight:700>}},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"sync"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"fixed"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Sync"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"size"</span><span style=color:#000;font-weight:700>:</span> <span style=co lor:#0000cf;font-weight:700>16</span><span style=color:#000;font-weight:700>}}</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>]</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><p>A file data block consists of:</p><ul><li>A long indicating the count of objects in this block.</li><li>A long indicating the size in bytes of the serialized objects in the current block, after any codec is applied</li><li>The serialized objects. If a codec is specified, this is compressed by that codec.</li><li>The file’s 16-byte sync marker.</li></ul><p>A file data block is thus described by the following schema:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"record"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight :700>:</span> <span style=color:#4e9a06>"org.apache.avro.file.DataBlock"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"fields"</span> <span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"count"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"long"</span><span style=color:#000;font-weight:700>},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"data"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"bytes"</span><span style=color:#000;font-weight:700>},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"sync"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"fixed"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Sync"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"size"</span><span style=color:#000;font-weight:700>:</span> <span style=co lor:#0000cf;font-weight:700>16</span><span style=color:#000;font-weight:700>}}</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>]</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><p>Each block’s binary data can be efficiently extracted or skipped without deserializing the contents. The combination of block size, object counts, and sync markers enable detection of corrupt blocks and help ensure data integrity.</p><h3 id=required-codecs>Required Codecs</h3><p><em>null</em></p><p>The “null” codec simply passes through data uncompressed.</p><p><em>deflate</em></p><p>The “deflate” codec writes the data block using the deflate algorithm as specified in <a href=https://www.isi.edu/in-notes/rfc1951.txt>RFC 1951</a>, and typically implemented using the zlib library. Note that this format (unlike the “zlib format” in RFC 1950) does not have a checksum.</p><h3 id=optional-codecs>Optional Codecs</h3><p><em>bzip2</em></p><p>The “bzip2” codec uses the <a href=https://sourceware.org/bzip2/>bzip2</a> compression library.</p><p><em>snappy</em></p><p>The “snappy” codec uses Goog le’s <a href=https://code.google.com/p/snappy/>Snappy</a> compression library. Each compressed block is followed by the 4-byte, big-endian CRC32 checksum of the uncompressed data in the block.</p><p><em>xz</em></p><p>The “xz” codec uses the <a href=https://tukaani.org/xz/>XZ</a> compression library.</p><p><em>zstandard</em></p><p>The “zstandard” codec uses Facebook’s <a href=https://facebook.github.io/zstd/>Zstandard</a> compression library.</p><h3 id=protocol-declaration>Protocol Declaration</h3><p>Avro protocols describe RPC interfaces. Like schemas, they are defined with JSON text.</p><p>A protocol is a JSON object with the following attributes:</p><ul><li><em>protocol</em>, a string, the name of the protocol (required);</li><li><em>namespace</em>, an optional string that qualifies the name (optional);</li><li><em>doc</em>, an optional string describing this protocol;</li><li><em>types</em>, an optional list of definitions of named types (recor ds, enums, fixed and errors). An error definition is just like a record definition except it uses “error” instead of “record”. Note that forward references to named types are not permitted.</li><li><em>messages</em>, an optional JSON object whose keys are message names and whose values are objects whose attributes are described below. No two messages may have the same name.</li></ul><p>The name and namespace qualification rules defined for schema objects apply to protocols as well.</p><h3 id=messages>Messages</h3><p>A message has attributes:</p><ul><li>a <em>doc</em>, an optional description of the message,</li><li>a <em>request</em>, a list of named, typed parameter schemas (this has the same form as the fields of a record declaration);</li><li>a <em>response</em> schema;</li><li>an optional union of declared error schemas. The effective union has “string” prepended to the declared union, to permit transmission of undeclared “system” errors. For example, if the declared error union is <code>["AccessError"]</code>, then the effective union is <code>["string", "AccessError"]</code>. When no errors are declared, the effective error union is <code>["string"]</code>. Errors are serialized using the effective union; however, a protocol’s JSON declaration contains only the declared union.</li><li>an optional one-way boolean parameter.</li></ul><p>A request parameter list is processed equivalently to an anonymous record. Since record field lists may vary between reader and writer, request parameters may also differ between the caller and responder, and such differences are resolved in the same manner as record field differences.</p><p>The one-way parameter may only be true when the response type is <code>"null"</code> and no errors are listed.</p><h3 id=sample-protocol>Sample Protocol</h3><p>For example, one may define a simple HelloWorld protocol with:</p><div class=highlight><pre tabindex=0 style=background-colo r:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"namespace"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"com.acme"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"protocol"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"HelloWorld"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"doc"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Protocol Greetings"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"types"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Greeting"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"record"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"fields"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"message"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"string"</span><span style=color:#000;font-weight:700>}]},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Curse"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"error"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"fields"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"message"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"string"</span><span style=color:#000;font-weight:700>}]}</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>],</span> +</span></span><span style=display:flex><span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"messages"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"hello"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"doc"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Say hello."</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"request"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"greeting"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Greeting"</span> <span style=color:#000;font-weight:700>}],</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"response"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"Greeting"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"errors"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4e9a06>"Curse"</span><span style=color:#000;font-weight:700>]</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>}</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span> +</span></span></code></pre></div><h2 id=protocol-wire-format>Protocol Wire Format</h2><h3 id=message-transport>Message Transport</h3><p>Messages may be transmitted via different transport mechanisms.</p><p>To the transport, a <em>message</em> is an opaque byte sequence.</p><p>A transport is a system that supports:</p><ul><li><strong>transmission of request messages</strong></li><li><strong>receipt of corresponding response messages</strong> +Servers may send a response message back to the client corresponding to a request message. The mechanism of correspondence is transport-specific. For example, in HTTP it is implicit, since HTTP directly supports requests and responses. But a transport that multiplexes many client threads over a single socket would need to tag messages with unique identifiers.</li></ul><p>Transports may be either stateless or stateful. In a stateless transport, messaging assumes no established connection state, while stateful transports establish connections that may be used for multiple messages. This distinction is discussed further in the <a href=#handshake>handshake</a> section below.</p><h4 id=http-as-transport>HTTP as Transport</h4><p>When <a href=https://www.w3.org/Protocols/rfc2616/rfc2616.html>HTTP</a> is used as a transport, each Avro message exchange is an HTTP request/response pair. All messages of an Avro protocol should share a single URL at an HTTP server. Other protocols may also use that URL. Both normal and error Avro response messages should use the 200 (OK) response code. The chunked encoding may be used for requests and responses, but, regardless the Avro request and response are the entire content of an HTTP request and response. The HTTP Content-Type of requests and responses should be specified as “avro/binary”. Requests should be made using the POST method.</p><p>HTTP is used by Avro as a stateless transport.</p><h3 id=message-framing>Message Framing</h3><p>Avro messages are <em>framed</em> as a list of buffers.</p><p>Framing is a layer between messages and the transport. It exists to optimize certain operations.</p><p>The format of framed message data is:</p><ul><li>a series of buffers, where each buffer consists of:<ul><li>a four-byte, big-endian <em>buffer length</em>, followed by</li><li>that many bytes of <em>buffer</em> data.</li></ul></li><li>a message is always terminated by a zero-length buffer.</li></ul><p>Framing is transparent to request and response message formats (described below). Any message may be presented as a single or multiple buffers.</p><p>Framing can permit readers to more efficiently get different buffers from different sources and for writers to more efficiently store different buffers to different destinations. In particular, it can reduce the number of times large binary objects are copied. For example, if an RPC parameter consists of a megabyte of file data, that data can be copied directly to a socket from a file descriptor, and, on the other end, it could be written directly to a file descriptor, never entering user space.</p><p>A simple, recommended, framing policy is for writers to create a new segment whenever a single binary object is written that is larger than a normal output buffer. Small objects are then appended in buffers, while larger objects are written as their own buffers. When a reader then tries to read a large object the runtime can hand it an entire buffer directly, wit hout having to copy it.</p><h3 id=handshake>Handshake</h3><p>The purpose of the handshake is to ensure that the client and the server have each other’s protocol definition, so that the client can correctly deserialize responses, and the server can correctly deserialize requests. Both clients and servers should maintain a cache of recently seen protocols, so that, in most cases, a handshake will be completed without extra round-trip network exchanges or the transmission of full protocol text.</p><p>RPC requests and responses may not be processed until a handshake has been completed. With a stateless transport, all requests and responses are prefixed by handshakes. With a stateful transport, handshakes are only attached to requests and responses until a successful handshake response has been returned over a connection. After this, request and response payloads are sent without handshakes for the lifetime of that connection.</p><p>The handshake process uses the following record s chemas:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-json data-lang=json><span style=display:flex><span><span style=color:#000;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"record"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"HandshakeRequest"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"namespace"</span><span style=color:#000;font-weight:700>:</span><span style=color:#4e9a06>"org.apache.avro.ipc"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"fields"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"clientHash"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"fixed"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"MD5"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"size"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#0000cf;font-weight:700>16</span><span style=color:#000;font-weight:700>}},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"clientProtocol"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4e9a06>"null"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"string"</span><span style=color:#000;font-weight:700>]},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"serverHash"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"MD5"</span><span style=color:#000;font-weight:700>},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"meta"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4e9a06>"null"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"map"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"values"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"bytes"</span><span style=color:#000;font-w eight:700>}]}</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>]</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span> +</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>{</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"record"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"HandshakeResponse"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"namespace"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"org.apache.avro.ipc"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"fields"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"match"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"enum"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"HandshakeMatch"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"symbols"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4e9a06>"BOTH"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"CLIENT"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"NONE"</span><span style=color:#000;font-weight:700>]}},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"serverProtocol"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4e9a06>"null"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"string"</span><span style=color:#000;font-weight:700>]},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"serverHash"</span><span style=color:#000;font-weight:700>,</span> +</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000;font-weight:700>[</span><span style=color:#4e9a06>"null"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"type"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"fixed"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"MD5"</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;font-weight:700>"size"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#0000cf;font-weight:700>16</span><span style=color:#000;font-weight:700>}]},</span> +</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>{</span><span style=color:#204a87;font-weight:700>"name"</span><span style=color:#000;font-weight:700>:</span> <span style=color:#4e9a06>"meta"</span><span style=color:#000;font-weight:700>,</span>
[... 51 lines stripped ...]
