This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/arrow-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new f0d19f0 Updating built site (build
2605501fae8d9d0d985d5af8917db1b96fdf4d70)
f0d19f0 is described below
commit f0d19f0ab5a82913768f9e372939ac74cde96e7d
Author: Andy Grove <[email protected]>
AuthorDate: Tue Oct 27 15:46:42 2020 +0000
Updating built site (build 2605501fae8d9d0d985d5af8917db1b96fdf4d70)
---
...manifest-aae08af9e12444e7eef76ff60a14f030.json} | 2 +-
blog/2020/10/27/rust-2.0.0-release/index.html | 449 ++++++++++++++++++++
blog/index.html | 15 +
feed.xml | 465 +++++++++------------
4 files changed, 658 insertions(+), 273 deletions(-)
diff --git a/assets/.sprockets-manifest-59b8ad9b3f186aacb3e1ce1050f0c6bf.json
b/assets/.sprockets-manifest-aae08af9e12444e7eef76ff60a14f030.json
similarity index 79%
rename from assets/.sprockets-manifest-59b8ad9b3f186aacb3e1ce1050f0c6bf.json
rename to assets/.sprockets-manifest-aae08af9e12444e7eef76ff60a14f030.json
index b2136bb..7ad3810 100644
--- a/assets/.sprockets-manifest-59b8ad9b3f186aacb3e1ce1050f0c6bf.json
+++ b/assets/.sprockets-manifest-aae08af9e12444e7eef76ff60a14f030.json
@@ -1 +1 @@
-{"files":{"main-18cd3029557f73c1ee82e41113127b04f6fcd84c56d9db0cb9c40ebe26ef6e33.js":{"logical_path":"main.js","mtime":"2020-10-23T11:24:18-04:00","size":124531,"digest":"18cd3029557f73c1ee82e41113127b04f6fcd84c56d9db0cb9c40ebe26ef6e33","integrity":"sha256-GM0wKVV/c8HuguQRExJ7BPb82ExW2dsMucQOvibvbjM="}},"assets":{"main.js":"main-18cd3029557f73c1ee82e41113127b04f6fcd84c56d9db0cb9c40ebe26ef6e33.js"}}
\ No newline at end of file
+{"files":{"main-18cd3029557f73c1ee82e41113127b04f6fcd84c56d9db0cb9c40ebe26ef6e33.js":{"logical_path":"main.js","mtime":"2020-10-27T11:46:30-04:00","size":124531,"digest":"18cd3029557f73c1ee82e41113127b04f6fcd84c56d9db0cb9c40ebe26ef6e33","integrity":"sha256-GM0wKVV/c8HuguQRExJ7BPb82ExW2dsMucQOvibvbjM="}},"assets":{"main.js":"main-18cd3029557f73c1ee82e41113127b04f6fcd84c56d9db0cb9c40ebe26ef6e33.js"}}
\ No newline at end of file
diff --git a/blog/2020/10/27/rust-2.0.0-release/index.html
b/blog/2020/10/27/rust-2.0.0-release/index.html
new file mode 100644
index 0000000..406517f
--- /dev/null
+++ b/blog/2020/10/27/rust-2.0.0-release/index.html
@@ -0,0 +1,449 @@
+<!DOCTYPE html>
+<html lang="en-US">
+ <head>
+ <meta charset="UTF-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above meta tags *must* come first in the head; any other head
content must come *after* these tags -->
+
+ <title>Apache Arrow 2.0.0 Rust Highlights | Apache Arrow</title>
+
+
+ <!-- Begin Jekyll SEO tag v2.7.1 -->
+<meta name="generator" content="Jekyll v3.8.4" />
+<meta property="og:title" content="Apache Arrow 2.0.0 Rust Highlights" />
+<meta name="author" content="pmc" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="Apache Arrow 2.0.0 is a significant release
for the Apache Arrow project in general (release notes), and the Rust
subproject in particular, with almost 200 issues resolved by 15 contributors.
In this blog post, we will go through the main changes affecting core Arrow,
Parquet support, and DataFusion query engine. The full list of resolved issues
can be found here. While the Java and C/C++ (used by Python and R) Arrow
implementations likely remain the mos [...]
+<meta property="og:description" content="Apache Arrow 2.0.0 is a significant
release for the Apache Arrow project in general (release notes), and the Rust
subproject in particular, with almost 200 issues resolved by 15 contributors.
In this blog post, we will go through the main changes affecting core Arrow,
Parquet support, and DataFusion query engine. The full list of resolved issues
can be found here. While the Java and C/C++ (used by Python and R) Arrow
implementations likely remain [...]
+<link rel="canonical"
href="https://arrow.apache.org/blog/2020/10/27/rust-2.0.0-release/"; />
+<meta property="og:url"
content="https://arrow.apache.org/blog/2020/10/27/rust-2.0.0-release/"; />
+<meta property="og:site_name" content="Apache Arrow" />
+<meta property="og:image" content="https://arrow.apache.org/img/arrow.png"; />
+<meta property="og:type" content="article" />
+<meta property="article:published_time" content="2020-10-27T02:00:00-04:00" />
+<meta name="twitter:card" content="summary_large_image" />
+<meta property="twitter:image"
content="https://arrow.apache.org/img/arrow.png"; />
+<meta property="twitter:title" content="Apache Arrow 2.0.0 Rust Highlights" />
+<meta name="twitter:site" content="@ApacheArrow" />
+<meta name="twitter:creator" content="@pmc" />
+<script type="application/ld+json">
+{"image":"https://arrow.apache.org/img/arrow.png","@type":"BlogPosting","url":"https://arrow.apache.org/blog/2020/10/27/rust-2.0.0-release/","headline":"Apache
Arrow 2.0.0 Rust
Highlights","dateModified":"2020-10-27T02:00:00-04:00","datePublished":"2020-10-27T02:00:00-04:00","author":{"@type":"Person","name":"pmc"},"mainEntityOfPage":{"@type":"WebPage","@id":"https://arrow.apache.org/blog/2020/10/27/rust-2.0.0-release/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject",";
[...]
+<!-- End Jekyll SEO tag -->
+
+
+ <!-- favicons -->
+ <link rel="icon" type="image/png" sizes="16x16"
href="/img/favicon-16x16.png" id="light1">
+ <link rel="icon" type="image/png" sizes="32x32"
href="/img/favicon-32x32.png" id="light2">
+ <link rel="apple-touch-icon" type="image/png" sizes="180x180"
href="/img/apple-touch-icon.png" id="light3">
+ <link rel="apple-touch-icon" type="image/png" sizes="120x120"
href="/img/apple-touch-icon-120x120.png" id="light4">
+ <link rel="apple-touch-icon" type="image/png" sizes="76x76"
href="/img/apple-touch-icon-76x76.png" id="light5">
+ <link rel="apple-touch-icon" type="image/png" sizes="60x60"
href="/img/apple-touch-icon-60x60.png" id="light6">
+ <!-- dark mode favicons -->
+ <link rel="icon" type="image/png" sizes="16x16"
href="/img/favicon-16x16-dark.png" id="dark1">
+ <link rel="icon" type="image/png" sizes="32x32"
href="/img/favicon-32x32-dark.png" id="dark2">
+ <link rel="apple-touch-icon" type="image/png" sizes="180x180"
href="/img/apple-touch-icon-dark.png" id="dark3">
+ <link rel="apple-touch-icon" type="image/png" sizes="120x120"
href="/img/apple-touch-icon-120x120-dark.png" id="dark4">
+ <link rel="apple-touch-icon" type="image/png" sizes="76x76"
href="/img/apple-touch-icon-76x76-dark.png" id="dark5">
+ <link rel="apple-touch-icon" type="image/png" sizes="60x60"
href="/img/apple-touch-icon-60x60-dark.png" id="dark6">
+
+ <script>
+ // Switch to the dark-mode favicons if prefers-color-scheme: dark
+ function onUpdate() {
+ light1 = document.querySelector('link#light1');
+ light2 = document.querySelector('link#light2');
+ light3 = document.querySelector('link#light3');
+ light4 = document.querySelector('link#light4');
+ light5 = document.querySelector('link#light5');
+ light6 = document.querySelector('link#light6');
+
+ dark1 = document.querySelector('link#dark1');
+ dark2 = document.querySelector('link#dark2');
+ dark3 = document.querySelector('link#dark3');
+ dark4 = document.querySelector('link#dark4');
+ dark5 = document.querySelector('link#dark5');
+ dark6 = document.querySelector('link#dark6');
+
+ if (matcher.matches) {
+ light1.remove();
+ light2.remove();
+ light3.remove();
+ light4.remove();
+ light5.remove();
+ light6.remove();
+ document.head.append(dark1);
+ document.head.append(dark2);
+ document.head.append(dark3);
+ document.head.append(dark4);
+ document.head.append(dark5);
+ document.head.append(dark6);
+ } else {
+ dark1.remove();
+ dark2.remove();
+ dark3.remove();
+ dark4.remove();
+ dark5.remove();
+ dark6.remove();
+ document.head.append(light1);
+ document.head.append(light2);
+ document.head.append(light3);
+ document.head.append(light4);
+ document.head.append(light5);
+ document.head.append(light6);
+ }
+ }
+ matcher = window.matchMedia('(prefers-color-scheme: dark)');
+ matcher.addListener(onUpdate);
+ onUpdate();
+ </script>
+
+ <link rel="stylesheet"
href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+ <link href="/css/main.css" rel="stylesheet">
+ <link href="/css/syntax.css" rel="stylesheet">
+ <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js";
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
crossorigin="anonymous"></script>
+ <script
src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js";
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
crossorigin="anonymous"></script>
+
+ <!-- Global Site Tag (gtag.js) - Google Analytics -->
+<script async
src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1";></script>
+<script>
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments)};
+ gtag('js', new Date());
+
+ gtag('config', 'UA-107500873-1');
+</script>
+
+
+ </head>
+
+
+<body class="wrap">
+ <header>
+ <nav class="navbar navbar-expand-md navbar-dark bg-dark">
+
+ <a class="navbar-brand no-padding" href="/"><img
src="/img/arrow-inverse-300px.png" height="40px"/></a>
+
+ <button class="navbar-toggler ml-auto" type="button" data-toggle="collapse"
data-target="#arrow-navbar" aria-controls="arrow-navbar" aria-expanded="false"
aria-label="Toggle navigation">
+ <span class="navbar-toggler-icon"></span>
+ </button>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse justify-content-end"
id="arrow-navbar">
+ <ul class="nav navbar-nav">
+ <li class="nav-item"><a class="nav-link" href="/overview/"
role="button" aria-haspopup="true" aria-expanded="false">Overview</a></li>
+ <li class="nav-item"><a class="nav-link" href="/faq/" role="button"
aria-haspopup="true" aria-expanded="false">FAQ</a></li>
+ <li class="nav-item"><a class="nav-link" href="/blog" role="button"
aria-haspopup="true" aria-expanded="false">Blog</a></li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownGetArrow" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Get Arrow
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownGetArrow">
+ <a class="dropdown-item" href="/install/">Install</a>
+ <a class="dropdown-item" href="/release/">Releases</a>
+ <a class="dropdown-item"
href="https://github.com/apache/arrow";>Source Code</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownDocumentation" role="button"
data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Documentation
+ </a>
+ <div class="dropdown-menu"
aria-labelledby="navbarDropdownDocumentation">
+ <a class="dropdown-item" href="/docs">Project Docs</a>
+ <a class="dropdown-item"
href="/docs/format/Columnar.html">Format</a>
+ <hr/>
+ <a class="dropdown-item" href="/docs/c_glib">C GLib</a>
+ <a class="dropdown-item" href="/docs/cpp">C++</a>
+ <a class="dropdown-item"
href="https://github.com/apache/arrow/blob/master/csharp/README.md";>C#</a>
+ <a class="dropdown-item"
href="https://godoc.org/github.com/apache/arrow/go/arrow";>Go</a>
+ <a class="dropdown-item" href="/docs/java">Java</a>
+ <a class="dropdown-item" href="/docs/js">JavaScript</a>
+ <a class="dropdown-item"
href="https://github.com/apache/arrow/blob/master/matlab/README.md";>MATLAB</a>
+ <a class="dropdown-item" href="/docs/python">Python</a>
+ <a class="dropdown-item" href="/docs/r">R</a>
+ <a class="dropdown-item"
href="https://github.com/apache/arrow/blob/master/ruby/README.md";>Ruby</a>
+ <a class="dropdown-item"
href="https://docs.rs/crate/arrow/";>Rust</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownCommunity" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ Community
+ </a>
+ <div class="dropdown-menu" aria-labelledby="navbarDropdownCommunity">
+ <a class="dropdown-item" href="/community/">Communication</a>
+ <a class="dropdown-item"
href="/docs/developers/contributing.html">Contributing</a>
+ <a class="dropdown-item"
href="https://issues.apache.org/jira/browse/ARROW";>Issue Tracker</a>
+ <a class="dropdown-item" href="/committers/">Governance</a>
+ <a class="dropdown-item" href="/use_cases/">Use Cases</a>
+ <a class="dropdown-item" href="/powered_by/">Powered By</a>
+ <a class="dropdown-item" href="/security/">Security</a>
+ <a class="dropdown-item"
href="https://www.apache.org/foundation/policies/conduct.html";>Code of
Conduct</a>
+ </div>
+ </li>
+ <li class="nav-item dropdown">
+ <a class="nav-link dropdown-toggle" href="#"
+ id="navbarDropdownASF" role="button" data-toggle="dropdown"
+ aria-haspopup="true" aria-expanded="false">
+ ASF Links
+ </a>
+ <div class="dropdown-menu dropdown-menu-right"
aria-labelledby="navbarDropdownASF">
+ <a class="dropdown-item" href="http://www.apache.org/";>ASF
Website</a>
+ <a class="dropdown-item"
href="http://www.apache.org/licenses/";>License</a>
+ <a class="dropdown-item"
href="http://www.apache.org/foundation/sponsorship.html";>Donate</a>
+ <a class="dropdown-item"
href="http://www.apache.org/foundation/thanks.html";>Thanks</a>
+ <a class="dropdown-item"
href="http://www.apache.org/security/";>Security</a>
+ </div>
+ </li>
+ </ul>
+ </div><!-- /.navbar-collapse -->
+ </nav>
+
+ </header>
+
+ <div class="container p-4 pt-5">
+ <div class="col-md-8 mx-auto">
+ <main role="main" class="pb-5">
+
+<h1>
+ Apache Arrow 2.0.0 Rust Highlights
+</h1>
+<hr class="mt-4 mb-3">
+
+
+
+<p class="mb-4 pb-1">
+ <span class="badge badge-secondary">Published</span>
+ <span class="published mr-3">
+ 27 Oct 2020
+ </span>
+ <br />
+ <span class="badge badge-secondary">By</span>
+
+ <a class="mr-3" href="https://arrow.apache.org";>The Apache Arrow PMC (pmc)
</a>
+
+
+
+</p>
+
+
+ <!--
+
+-->
+
+<p>Apache Arrow 2.0.0 is a significant release for the Apache Arrow project in
general (<a
href="https://arrow.apache.org/blog/2020/10/22/2.0.0-release/";>release
notes</a>), and the Rust subproject
+in particular, with almost 200 issues resolved by 15 contributors. In this
blog post, we will go through the main changes
+affecting core Arrow, Parquet support, and DataFusion query engine. The full
list of resolved issues can be found
+<a
href="https://issues.apache.org/jira/browse/ARROW-10295?jql=project%20%3D%20ARROW%20AND%20status%20not%20in%20%20(Open%2C%20%22In%20Progress%22%2C%20Reopened)%20and%20fixVersion%20%3D%202.0.0%20AND%20text%20~%20rust%20ORDER%20BY%20created%20DESC">here</a>.</p>
+
+<p>While the Java and C/C++ (used by Python and R) Arrow implementations
likely remain the most feature-rich, with the
+2.0.0 release, the Rust implementation is closing the feature gap quickly.
Here are some of the highlights for this
+release.</p>
+
+<h1 id="core-arrow-crate">Core Arrow Crate</h1>
+
+<h2 id="iterator-trait">Iterator Trait</h2>
+
+<ul>
+ <li>Primitive arrays (e.g., array of integers) can now be converted to and
initialized from an iterator. This exposes a
+very popular API - iterators - to arrow arrays. Work for other types will
continue throughout 3.0.0.</li>
+</ul>
+
+<h2 id="improved-variable-sized-arrays">Improved Variable-sized Arrays</h2>
+
+<ul>
+ <li>Variable sized arrays (e.g., array of strings) have been internally
refactored to more easily support their larger (64-bit
+size offset) version. This allowed us to generalize some of the kernels to
both (32 and 64) versions and
+perform type checks when building them.</li>
+</ul>
+
+<h2 id="kernels">Kernels</h2>
+
+<p>There have been numerous improvements in the Arrow compute kernels,
including:</p>
+
+<ul>
+ <li>New kernels have been added for string operations, including substring,
min, max, concat, and length.</li>
+ <li>Aggregate sum is now implemented for SIMD with a 5x improvement over the
non-SIMD operation</li>
+ <li>Many kernels have been improved to support dictionary-encoded arrays</li>
+ <li>Some kernels were optimized for arrays without nulls, making them
significantly faster in that case.</li>
+ <li>Many kernels were optimized in the number of memory copies that are
needed to apply them and also on their
+implementation.</li>
+</ul>
+
+<h2 id="other-improvements">Other Improvements</h2>
+
+<p>The Array trait now has <code
class="highlighter-rouge">get_buffer_memory_size</code> and <code
class="highlighter-rouge">get_array_memory_size</code> methods for determining
the amount of
+memory allocated for the array.</p>
+
+<h1 id="parquet">Parquet</h1>
+
+<p>A significant effort is underway to create a Parquet writer for Arrow data.
This work has not been released as part of
+2.0.0, and is planned for the 3.0.0 release. The development of this writer is
being carried out on the
+<a
href="https://github.com/apache/arrow/tree/rust-parquet-arrow-writer";>rust-parquet-arrow-writer</a>
branch, and the branch is regularly synchronized with the main branch.
+As part of the writer, the necessary improvements and features are being added
to the reader.</p>
+
+<p>The main focus areas are:</p>
+<ul>
+ <li>Supporting nested Arrow types, such as <code
class="highlighter-rouge">List<Struct<[Dictionary,
String]>></code></li>
+ <li>Ensuring correct round-trip between the reader and writer by encoding
Arrow schemas in the Parquet metadata</li>
+ <li>Improve null value writing for Parquet</li>
+</ul>
+
+<p>A new <code class="highlighter-rouge">parquet_derive</code> crate has been
created, which allows users to derive Parquet records for simple structs. Refer
to
+the <a
href="https://github.com/apache/arrow/tree/master/rust/parquet_derive";>parquet_derive
crate</a> for usage examples.</p>
+
+<h1 id="datafusion">DataFusion</h1>
+
+<p>DataFusion is an in-memory query engine with DataFrame and SQL APIs, built
on top of base Arrow support.</p>
+
+<h2 id="dataframe-api">DataFrame API</h2>
+
+<p>DataFusion now has a richer <a
href="https://docs.rs/datafusion/2.0.0/datafusion/dataframe/trait.DataFrame.html";>DataFrame
API</a> with improved documentation showing example usage,
+supporting the following operations:</p>
+
+<ul>
+ <li>select_columns</li>
+ <li>select</li>
+ <li>filter</li>
+ <li>aggregate</li>
+ <li>limit</li>
+ <li>sort</li>
+ <li>collect</li>
+ <li>explain</li>
+</ul>
+
+<h2 id="performance--scalability">Performance & Scalability</h2>
+
+<p>DataFusion query execution now uses <code
class="highlighter-rouge">async</code>/<code
class="highlighter-rouge">await</code> with the tokio threaded runtime rather
than launching dedicated
+threads, making queries scale much better across available cores.</p>
+
+<p>The hash aggregate physical operator has been largely re-written, resulting
in significant performance improvements.</p>
+
+<h2 id="expressions-and-compute">Expressions and Compute</h2>
+
+<h3 id="improved-scalar-functions">Improved Scalar Functions</h3>
+
+<p>DataFusion has many new functions, both in the SQL and the DataFrame
API:</p>
+<ul>
+ <li>Length of an string</li>
+ <li>COUNT(DISTINCT column)</li>
+ <li>to_timestamp</li>
+ <li>IsNull and IsNotNull</li>
+ <li>Min/Max for strings (lexicographic order)</li>
+ <li>Array of columns</li>
+ <li>Concatenation of strings</li>
+ <li>Aliases of aggregate expressions</li>
+</ul>
+
+<p>Many existing expressions were also significantly optimized (2-3x speedups)
by avoiding memory copies and leveraging
+Arrow format’s invariants.</p>
+
+<p>Unary mathematical functions (such as sqrt) now support both 32 and 64-bit
floats and return the corresponding type,
+thereby allowing faster operations when higher precision is not needed.</p>
+
+<h3 id="improved-user-defined-functions-udfs">Improved User-defined Functions
(UDFs)</h3>
+<p>The API to use and register UDFs has been significantly improved, allowing
users to register UDFs and call them both
+via SQL and the DataFrame API. UDFs now also have the same generality as
DataFusion’s functions, including variadic
+and dynamically typed arguments.</p>
+
+<h3 id="user-defined-aggregate-functions-udafs">User-defined Aggregate
Functions (UDAFs)</h3>
+<p>DataFusion now supports user-defined aggregate functions that can be used
to perform operations than span multiple
+rows, batches, and partitions. UDAFs have the same generality as DataFusion’s
functions and support both row updates
+and batch updates. You can check out <a
href="https://github.com/apache/arrow/blob/master/rust/datafusion/examples/simple_udaf.rs";>this
example</a> to learn how to declare and use a UDAF.</p>
+
+<h3 id="user-defined-constants">User-defined Constants</h3>
+<p>DataFusion now supports registering constants (e.g. “@version”) that live
for the duration of the execution context
+and can be accessed from SQL.</p>
+
+<h2 id="query-planning--optimization">Query Planning & Optimization</h2>
+
+<h3 id="user-defined-logical-plans">User-defined logical plans</h3>
+
+<p>The Logical Plan enum is now extensible through an Extension variant which
accepts a UserDefinedLogicalPlan trait using
+dynamic dispatch. Consequently, DataFusion now supports user-defined logical
nodes, thereby allowing complex nodes to
+be planned and executed. You can check this example to learn how to declare a
new node.</p>
+
+<h3 id="predicate-push-down">Predicate push-down</h3>
+
+<p>DataFusion now has a Predicate push-down optimizer rule that pushes filter
operations as close as possible to scans,
+thereby speeding up the physical execution of suboptimal queries created via
the DataFrame API.</p>
+
+<h3 id="sql">SQL</h3>
+
+<p>DataFusion now uses a more recent release of the sqlparser crate, which has
much more comprehensive support for SQL
+syntax and also supports multiple dialects (Postgres, MS SQL, and MySQL).</p>
+
+<p>It is now possible to see the query plan for a SQL statement using EXPLAIN
syntax.</p>
+
+<h1 id="benchmarks">Benchmarks</h1>
+
+<p>The benchmark crate now contains a new benchmark based on TPC-H that can
execute TPC-H query 1 against CSV, Parquet,
+and memory data sources. This is useful for running benchmarks against larger
data sets.</p>
+
+<h1 id="integration-testing--ipc">Integration Testing / IPC</h1>
+
+<p>Arrow IPC is the format for serialization and interprocess communication.
It is described in <a href="https://arrow.apache.org/";>arrow.apache.org</a> and
is
+the format used for file and stream I/O between applications wishing to
interchange Arrow data.</p>
+
+<p>The Arrow project released IPC version 5 of the Arrow IPC format in version
1.0.0. Before that, a message padding
+change was made in version 0.15.0 to change the default padding to 8 bytes,
while remaining in IPC version 4. Arrow
+release 0.14.1 and earlier were the last releases to use the legacy 4 byte
alignment.
+As part of 2.0.0, the Rust implementation was updated to comply with the
changes up to release 0.15.0 of Arrow.
+Work on supporting IPC version 5 is underway, and is expected to be completed
in time for 3.0.0.</p>
+
+<p>As part of the conformance work, Rust is being added to the Arrow
integration suite, which tests that supported
+language implementations <a
href="https://issues.apache.org/jira/browse/ARROW-3690";>(ARROW-3690)</a>:</p>
+
+<ul>
+ <li>Comply with the Arrow IPC format</li>
+ <li>Can read and write each other’s generated data</li>
+ <li>IPC version 4 is being verified through the above work.</li>
+</ul>
+
+<h1 id="roadmap-for-300-and-beyond">Roadmap for 3.0.0 and Beyond</h1>
+
+<p>Here are some of the initiatives that contributors are currently working on
for future releases:</p>
+
+<ul>
+ <li>Support stable Rust</li>
+ <li>Improved DictionaryArray support and performance</li>
+ <li>Implement inner equijoins</li>
+ <li>Support for various platforms like ARMv8</li>
+ <li>Supporting the C Data Interface from Rust to better support
interoperability with other Arrow implementations</li>
+</ul>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to the Rust subproject in Apache
Arrow, you can find a list of open issues
+suitable for beginners <a
href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20in%20(Rust%2C%20%22Rust%20-%20DataFusion%22)%20AND%20labels%20%3D%20beginner">here</a>
and the full list <a
href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20in%20(Rust%2C%20%22Rust%20-%20DataFusion%22)%20ORDER%20BY%20updated%20DESC%2C%20created%20DESC%2C%20
[...]
+
+<p>Other ways to get involved include trying out Arrow on some of your data
and filing bug reports, and helping to
+improve the documentation.</p>
+
+
+ </main>
+ </div>
+
+ <hr/>
+<footer class="footer">
+ <div class="row">
+ <div class="col-md-9">
+ <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache
Arrow project logo are either registered trademarks or trademarks of The Apache
Software Foundation in the United States and other countries.</p>
+ <p>© 2016-2020 The Apache Software Foundation</p>
+ </div>
+ <div class="col-md-3">
+ <a class="d-sm-none d-md-inline pr-2"
href="https://www.apache.org/events/current-event.html";>
+ <img src="https://www.apache.org/events/current-event-234x60.png"/>
+ </a>
+ </div>
+ <script type="text/javascript"
src="/assets/main-18cd3029557f73c1ee82e41113127b04f6fcd84c56d9db0cb9c40ebe26ef6e33.js"
integrity="sha256-GM0wKVV/c8HuguQRExJ7BPb82ExW2dsMucQOvibvbjM="
crossorigin="anonymous"></script>
+</footer>
+
+ </div>
+</body>
+</html>
diff --git a/blog/index.html b/blog/index.html
index eb7d72a..eec0c8e 100644
--- a/blog/index.html
+++ b/blog/index.html
@@ -215,6 +215,21 @@
<p>
<h3>
+ <a href="/blog/2020/10/27/rust-2.0.0-release/">Apache Arrow 2.0.0 Rust
Highlights</a>
+ </h3>
+
+ <p>
+ <span class="blog-list-date">
+ 27 October 2020
+ </span>
+ </p>
+ Apache Arrow 2.0.0 is a significant release for the Apache Arrow project
in general (release notes), and the Rust subproject in particular, with almost
200 issues resolved by 15 contributors. In this blog post, we will go through
the main changes affecting core Arrow, Parquet support, and DataFusion query
engine....
+ </p>
+
+
+
+ <p>
+ <h3>
<a href="/blog/2020/10/22/2.0.0-release/">Apache Arrow 2.0.0 Release</a>
</h3>
diff --git a/feed.xml b/feed.xml
index 06f0caf..54a1db3 100644
--- a/feed.xml
+++ b/feed.xml
@@ -1,4 +1,195 @@
-<?xml version="1.0" encoding="utf-8"?><feed
xmlns="http://www.w3.org/2005/Atom"; ><generator uri="https://jekyllrb.com/";
version="3.8.4">Jekyll</generator><link
href="https://arrow.apache.org/feed.xml"; rel="self" type="application/atom+xml"
/><link href="https://arrow.apache.org/"; rel="alternate" type="text/html"
/><updated>2020-10-23T11:24:09-04:00</updated><id>https://arrow.apache.org/feed.xml</id><title
type="html">Apache Arrow</title><subtitle>Apache Arrow is a cross-language
developm [...]
+<?xml version="1.0" encoding="utf-8"?><feed
xmlns="http://www.w3.org/2005/Atom"; ><generator uri="https://jekyllrb.com/";
version="3.8.4">Jekyll</generator><link
href="https://arrow.apache.org/feed.xml"; rel="self" type="application/atom+xml"
/><link href="https://arrow.apache.org/"; rel="alternate" type="text/html"
/><updated>2020-10-27T11:46:20-04:00</updated><id>https://arrow.apache.org/feed.xml</id><title
type="html">Apache Arrow</title><subtitle>Apache Arrow is a cross-language
developm [...]
+
+-->
+
+<p>Apache Arrow 2.0.0 is a significant release for the Apache Arrow
project in general (<a
href="https://arrow.apache.org/blog/2020/10/22/2.0.0-release/">release
notes</a>), and the Rust subproject
+in particular, with almost 200 issues resolved by 15 contributors. In this
blog post, we will go through the main changes
+affecting core Arrow, Parquet support, and DataFusion query engine. The full
list of resolved issues can be found
+<a
href="https://issues.apache.org/jira/browse/ARROW-10295?jql=project%20%3D%20ARROW%20AND%20status%20not%20in%20%20(Open%2C%20%22In%20Progress%22%2C%20Reopened)%20and%20fixVersion%20%3D%202.0.0%20AND%20text%20~%20rust%20ORDER%20BY%20created%20DESC">here</a>.</p>
+
+<p>While the Java and C/C++ (used by Python and R) Arrow implementations
likely remain the most feature-rich, with the
+2.0.0 release, the Rust implementation is closing the feature gap quickly.
Here are some of the highlights for this
+release.</p>
+
+<h1 id="core-arrow-crate">Core Arrow Crate</h1>
+
+<h2 id="iterator-trait">Iterator Trait</h2>
+
+<ul>
+ <li>Primitive arrays (e.g., array of integers) can now be converted to
and initialized from an iterator. This exposes a
+very popular API - iterators - to arrow arrays. Work for other types will
continue throughout 3.0.0.</li>
+</ul>
+
+<h2 id="improved-variable-sized-arrays">Improved
Variable-sized Arrays</h2>
+
+<ul>
+ <li>Variable sized arrays (e.g., array of strings) have been
internally refactored to more easily support their larger (64-bit
+size offset) version. This allowed us to generalize some of the kernels to
both (32 and 64) versions and
+perform type checks when building them.</li>
+</ul>
+
+<h2 id="kernels">Kernels</h2>
+
+<p>There have been numerous improvements in the Arrow compute kernels,
including:</p>
+
+<ul>
+ <li>New kernels have been added for string operations, including
substring, min, max, concat, and length.</li>
+ <li>Aggregate sum is now implemented for SIMD with a 5x improvement
over the non-SIMD operation</li>
+ <li>Many kernels have been improved to support dictionary-encoded
arrays</li>
+ <li>Some kernels were optimized for arrays without nulls, making them
significantly faster in that case.</li>
+ <li>Many kernels were optimized in the number of memory copies that
are needed to apply them and also on their
+implementation.</li>
+</ul>
+
+<h2 id="other-improvements">Other Improvements</h2>
+
+<p>The Array trait now has <code
class="highlighter-rouge">get_buffer_memory_size</code> and
<code
class="highlighter-rouge">get_array_memory_size</code>
methods for determining the amount of
+memory allocated for the array.</p>
+
+<h1 id="parquet">Parquet</h1>
+
+<p>A significant effort is underway to create a Parquet writer for Arrow
data. This work has not been released as part of
+2.0.0, and is planned for the 3.0.0 release. The development of this writer is
being carried out on the
+<a
href="https://github.com/apache/arrow/tree/rust-parquet-arrow-writer">rust-parquet-arrow-writer</a>
branch, and the branch is regularly synchronized with the main branch.
+As part of the writer, the necessary improvements and features are being added
to the reader.</p>
+
+<p>The main focus areas are:</p>
+<ul>
+ <li>Supporting nested Arrow types, such as <code
class="highlighter-rouge">List&lt;Struct&lt;[Dictionary,
String]&gt;&gt;</code></li>
+ <li>Ensuring correct round-trip between the reader and writer by
encoding Arrow schemas in the Parquet metadata</li>
+ <li>Improve null value writing for Parquet</li>
+</ul>
+
+<p>A new <code
class="highlighter-rouge">parquet_derive</code> crate has
been created, which allows users to derive Parquet records for simple structs.
Refer to
+the <a
href="https://github.com/apache/arrow/tree/master/rust/parquet_derive">parquet_derive
crate</a> for usage examples.</p>
+
+<h1 id="datafusion">DataFusion</h1>
+
+<p>DataFusion is an in-memory query engine with DataFrame and SQL APIs,
built on top of base Arrow support.</p>
+
+<h2 id="dataframe-api">DataFrame API</h2>
+
+<p>DataFusion now has a richer <a
href="https://docs.rs/datafusion/2.0.0/datafusion/dataframe/trait.DataFrame.html">DataFrame
API</a> with improved documentation showing example usage,
+supporting the following operations:</p>
+
+<ul>
+ <li>select_columns</li>
+ <li>select</li>
+ <li>filter</li>
+ <li>aggregate</li>
+ <li>limit</li>
+ <li>sort</li>
+ <li>collect</li>
+ <li>explain</li>
+</ul>
+
+<h2 id="performance--scalability">Performance &amp;
Scalability</h2>
+
+<p>DataFusion query execution now uses <code
class="highlighter-rouge">async</code>/<code
class="highlighter-rouge">await</code> with the tokio
threaded runtime rather than launching dedicated
+threads, making queries scale much better across available cores.</p>
+
+<p>The hash aggregate physical operator has been largely re-written,
resulting in significant performance improvements.</p>
+
+<h2 id="expressions-and-compute">Expressions and
Compute</h2>
+
+<h3 id="improved-scalar-functions">Improved Scalar
Functions</h3>
+
+<p>DataFusion has many new functions, both in the SQL and the DataFrame
API:</p>
+<ul>
+ <li>Length of an string</li>
+ <li>COUNT(DISTINCT column)</li>
+ <li>to_timestamp</li>
+ <li>IsNull and IsNotNull</li>
+ <li>Min/Max for strings (lexicographic order)</li>
+ <li>Array of columns</li>
+ <li>Concatenation of strings</li>
+ <li>Aliases of aggregate expressions</li>
+</ul>
+
+<p>Many existing expressions were also significantly optimized (2-3x
speedups) by avoiding memory copies and leveraging
+Arrow format’s invariants.</p>
+
+<p>Unary mathematical functions (such as sqrt) now support both 32 and
64-bit floats and return the corresponding type,
+thereby allowing faster operations when higher precision is not
needed.</p>
+
+<h3 id="improved-user-defined-functions-udfs">Improved
User-defined Functions (UDFs)</h3>
+<p>The API to use and register UDFs has been significantly improved,
allowing users to register UDFs and call them both
+via SQL and the DataFrame API. UDFs now also have the same generality as
DataFusion’s functions, including variadic
+and dynamically typed arguments.</p>
+
+<h3 id="user-defined-aggregate-functions-udafs">User-defined
Aggregate Functions (UDAFs)</h3>
+<p>DataFusion now supports user-defined aggregate functions that can be
used to perform operations than span multiple
+rows, batches, and partitions. UDAFs have the same generality as DataFusion’s
functions and support both row updates
+and batch updates. You can check out <a
href="https://github.com/apache/arrow/blob/master/rust/datafusion/examples/simple_udaf.rs">this
example</a> to learn how to declare and use a UDAF.</p>
+
+<h3 id="user-defined-constants">User-defined
Constants</h3>
+<p>DataFusion now supports registering constants (e.g. “@version”) that
live for the duration of the execution context
+and can be accessed from SQL.</p>
+
+<h2 id="query-planning--optimization">Query Planning &amp;
Optimization</h2>
+
+<h3 id="user-defined-logical-plans">User-defined logical
plans</h3>
+
+<p>The Logical Plan enum is now extensible through an Extension variant
which accepts a UserDefinedLogicalPlan trait using
+dynamic dispatch. Consequently, DataFusion now supports user-defined logical
nodes, thereby allowing complex nodes to
+be planned and executed. You can check this example to learn how to declare a
new node.</p>
+
+<h3 id="predicate-push-down">Predicate push-down</h3>
+
+<p>DataFusion now has a Predicate push-down optimizer rule that pushes
filter operations as close as possible to scans,
+thereby speeding up the physical execution of suboptimal queries created via
the DataFrame API.</p>
+
+<h3 id="sql">SQL</h3>
+
+<p>DataFusion now uses a more recent release of the sqlparser crate,
which has much more comprehensive support for SQL
+syntax and also supports multiple dialects (Postgres, MS SQL, and
MySQL).</p>
+
+<p>It is now possible to see the query plan for a SQL statement using
EXPLAIN syntax.</p>
+
+<h1 id="benchmarks">Benchmarks</h1>
+
+<p>The benchmark crate now contains a new benchmark based on TPC-H that
can execute TPC-H query 1 against CSV, Parquet,
+and memory data sources. This is useful for running benchmarks against larger
data sets.</p>
+
+<h1 id="integration-testing--ipc">Integration Testing /
IPC</h1>
+
+<p>Arrow IPC is the format for serialization and interprocess
communication. It is described in <a
href="https://arrow.apache.org/">arrow.apache.org</a> and is
+the format used for file and stream I/O between applications wishing to
interchange Arrow data.</p>
+
+<p>The Arrow project released IPC version 5 of the Arrow IPC format in
version 1.0.0. Before that, a message padding
+change was made in version 0.15.0 to change the default padding to 8 bytes,
while remaining in IPC version 4. Arrow
+release 0.14.1 and earlier were the last releases to use the legacy 4 byte
alignment.
+As part of 2.0.0, the Rust implementation was updated to comply with the
changes up to release 0.15.0 of Arrow.
+Work on supporting IPC version 5 is underway, and is expected to be completed
in time for 3.0.0.</p>
+
+<p>As part of the conformance work, Rust is being added to the Arrow
integration suite, which tests that supported
+language implementations <a
href="https://issues.apache.org/jira/browse/ARROW-3690">(ARROW-3690)</a>:</p>
+
+<ul>
+ <li>Comply with the Arrow IPC format</li>
+ <li>Can read and write each other’s generated data</li>
+ <li>IPC version 4 is being verified through the above work.</li>
+</ul>
+
+<h1 id="roadmap-for-300-and-beyond">Roadmap for 3.0.0 and
Beyond</h1>
+
+<p>Here are some of the initiatives that contributors are currently
working on for future releases:</p>
+
+<ul>
+ <li>Support stable Rust</li>
+ <li>Improved DictionaryArray support and performance</li>
+ <li>Implement inner equijoins</li>
+ <li>Support for various platforms like ARMv8</li>
+ <li>Supporting the C Data Interface from Rust to better support
interoperability with other Arrow implementations</li>
+</ul>
+
+<h1 id="how-to-get-involved">How to Get Involved</h1>
+
+<p>If you are interested in contributing to the Rust subproject in
Apache Arrow, you can find a list of open issues
+suitable for beginners <a
href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20in%20(Rust%2C%20%22Rust%20-%20DataFusion%22)%20AND%20labels%20%3D%20beginner">here</a>
and the full list <a
href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20resolution%20%3D%20Unresolved%20AND%20component%20in%20(Rust%2C%20%22Rust%20-%20DataFusion%22)%20ORDER%20BY%20updated%20
[...]
+
+<p>Other ways to get involved include trying out Arrow on some of your
data and filing bug reports, and helping to
+improve the
documentation.</p></content><author><name>pmc</name></author><category
term="release" /><summary type="html">Apache Arrow 2.0.0 is a significant
release for the Apache Arrow project in general (release notes), and the Rust
subproject in particular, with almost 200 issues resolved by 15 contributors.
In this blog post, we will go through the main changes affecting core Arrow,
Parquet support, and DataFusion query engine. The full list of resolved issues
can be found here [...]
-->
@@ -1812,274 +2003,4 @@ lot of the Flight work from here will be creating
user-facing Flight-enabled
services. Since Flight is a development framework, we expect that user-facing
APIs will utilize a layer of API veneer that hides many general Flight details
and details related to a particular application of Flight in a custom data
-service.</p></content><author><name>wesm</name></author><category
term="application" /><summary type="html">Over the last 18 months, the Apache
Arrow community has been busy designing and implementing Flight, a new
general-purpose client-server framework to simplify high performance transport
of large datasets over network interfaces. Flight initially is focused on
optimized transport of the Arrow columnar format (i.e. “Arrow record batches”)
over gRPC, Google’s popular HTTP/2-base [...]
-
--->
-
-<p>The Apache Arrow team is pleased to announce the 0.15.0 release. This
covers
-about 3 months of development work and includes <a
href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20%3D%20Resolved%20AND%20fixVersion%20%3D%200.15.0"><strong>687
resolved issues</strong></a>
-from <a
href="https://arrow.apache.org/release/0.15.0.html#contributors"><strong>80
distinct contributors</strong></a>. See the Install Page to learn
how to
-get the libraries for your platform. The <a
href="https://arrow.apache.org/release/0.15.0.html">complete
changelog</a> is also
-available.</p>
-
-<p>About a third of issues closed (240) were classified as bug fixes, so
this
-release brings many stability, memory use, and performance improvements over
-0.14.x. We will discuss some of the language-specific improvements and new
-features below.</p>
-
-<h2 id="new-committers">New committers</h2>
-
-<p>Since the 0.14.0 release, we’ve added four new committers:</p>
-
-<ul>
- <li><a href="https://github.com/bkietz">Ben
Kietzman</a></li>
- <li><a href="https://github.com/lidavidm">David
Li</a></li>
- <li><a href="https://github.com/mrkn">Kenta
Murata</a></li>
- <li><a href="https://github.com/nealrichardson">Neal
Richardson</a></li>
-</ul>
-
-<p>In addition, <a
href="https://github.com/sbinet">Sebastien Binet</a> and
<a href="https://github.com/emkornfield">Micah
Kornfield</a> have joined the PMC.</p>
-
-<p>Thank you for all your contributions!</p>
-
-<h2 id="columnar-format-notes">Columnar Format Notes</h2>
-
-<p>The format gets new datatypes : LargeList(ARROW-4810), LargeBinary and
-LargeString (ARROW-750). LargeList is similar to List but with 64-bit
-offsets instead of 32-bit. The same relationship holds for LargeBinary
-and LargeString with respect to Binary and String.</p>
-
-<p>Since the last major release, we have also made a significant
overhaul of the
-<a
href="https://github.com/apache/arrow/blob/master/docs/source/format/Columnar.rst">columnar
format documentation</a> to be clearer and easier to follow for
-implementation creators.</p>
-
-<h2
id="upcoming-columnar-format-stability-and-library--format-version-split">Upcoming
Columnar Format Stability and Library / Format Version Split</h2>
-
-<p>The Arrow community has decided to make a 1.0.0 release of the
project marking
-formal stability of the columnar format and binary protocol, including explicit
-forward and backward compatibility guarantees. You can read about these
-guarantees in the <a
href="https://github.com/apache/arrow/blob/master/docs/source/format/Versioning.rst">new
documentation page</a> about versioning.</p>
-
-<p>Starting with 1.0.0, we will give the columnar format and libraries
separate
-version numbers. This will allow the library versions to evolve without
-creating confusion or uncertainty about whether the Arrow columnar format
-remains stable or not.</p>
-
-<h2
id="columnar-streaming-protocol-change-since-0140">Columnar
“Streaming Protocol” Change since 0.14.0</h2>
-
-<p>Since 0.14.0 we have modified the IPC “encapsulated message” format
to insert 4
-bytes of additional data in the message preamble to ensure that the Flatbuffers
-metadata starts on an aligned offset. By default, IPC streams generated by
-0.15.0 and later will not be readable by library versions 0.14.1 and
-prior. Implementations have offered options to write messages using the now
-“legacy” message format.</p>
-
-<p>For users who cannot upgrade to version 0.15.0 in all parts of their
system,
-such as Apache Spark users, we recommend one of the two routes:</p>
-
-<ul>
- <li>If using pyarrow, set the environment variable <code
class="highlighter-rouge">ARROW_PRE_0_15_IPC_FORMAT=1</code>
-when using 0.15.0 and sending data to an old library</li>
- <li>Wait to upgrade all components simultaneously</li>
-</ul>
-
-<p>We do not anticipate making this kind of change again in the near
future and
-would not have made such a non-forward-compatible change unless we deemed it
-very important.</p>
-
-<h2 id="arrow-flight-notes">Arrow Flight notes</h2>
-
-<p>A GetFlightSchema method is added to the Flight RPC protocol
(ARROW-6094).
-As the name suggests, it returns the schema for a given Flight descriptor
-on the server. This is useful for cases where the Flight locations are
-not immediately available, depending on the server implementation.</p>
-
-<p>Flight implementations for C++ and Java now implement half-closed
-semantics for DoPut (ARROW-6063). The client can close the writing
-end of the stream to signal that it has finished sending the Flight
-data, but still receive the batch-specific response and its associated
-metadata.</p>
-
-<h2 id="c-notes">C++ notes</h2>
-
-<p>C++ now supports the LargeList, LargeBinary and LargeString
datatypes.</p>
-
-<p>The Status class gains the ability to carry additional
subsystem-specific
-data with it, under the form of an opaque StatusDetail interface (ARROW-4036).
-This allows, for example, to store not only an exception message coming from
-Python but the actual Python exception object, such as to raise it again if
-the Status is propagated back to Python. It can also enable the consumer
-of Status to inspect the subsystem-specific error, such as a finer-grained
-Flight error code.</p>
-
-<p>DataType and Schema equality are significantly faster
(ARROW-6038).</p>
-
-<p>The Column class is completely removed, as it did not have a strong
enough
-motivation for existing between ChunkedArray and RecordBatch / Table
-(ARROW-5893).</p>
-
-<h3 id="c-parquet">C++: Parquet</h3>
-
-<p>The 0.15 release includes many improvements to the Apache Parquet C++
internals,
-resulting in greatly improved read and write performance. We described the work
-and published some benchmarks in a <a
href="https://arrow.apache.org/blog/2019/09/05/faster-strings-cpp-parquet/">recent
blog post</a>.</p>
-
-<h3 id="c-csv-reader">C++: CSV reader</h3>
-
-<p>The CSV reader is now more flexible in terms of how column names are
chosen
-(ARROW-6231) and column selection (ARROW-5977).</p>
-
-<h3 id="c-memory-allocation-layer">C++: Memory Allocation
Layer</h3>
-
-<p>Arrow now has the option to allocate memory using the mimalloc memory
-allocator. jemalloc is still preferred for best performance, but mimalloc
-is a reasonable alternative to the system allocator on Windows where jemalloc
-is not currently supported.</p>
-
-<p>Also, we now expose explicit global functions to get a MemoryPool for
each
-of the jemalloc allocator, mimalloc allocator and system allocator
(ARROW-6292).</p>
-
-<p>The vendored jemalloc version is bumped from 4.5.x to 5.2.x
(ARROW-6549).
-Performance characteristics may differ on memory allocation-heavy workloads,
-though we did not notice any significant regression on our suite of
-micro-benchmarks (and a multi-threaded benchmark of reading a CSV file
-showed a 25% speedup).</p>
-
-<h3 id="c-filesystem-layer">C++: Filesystem layer</h3>
-
-<p>A FileSystem implementation to access Amazon S3-compatible
filesystems is now
-available. It depends on the AWS SDK for C++.</p>
-
-<h3 id="c-io-layer">C++: I/O layer</h3>
-
-<p>Significant improvements were made to the Arrow I/O stack.</p>
-
-<ul>
- <li>ARROW-6180: Add RandomAccessFile::GetStream that returns an
InputStream over
-a fixed subset of the file.</li>
- <li>ARROW-6381: Improve performance of small writes with
BufferOutputStream</li>
- <li>ARROW-2490: Streamline concurrency semantics of InputStream
implementations,
-and add debug checks for race conditions between non-thread-safe InputStream
-operations.</li>
- <li>ARROW-6527: Add an OutputStream::Write overload that takes an
owned Buffer
-rather than a raw memory area. This allows OutputStream implementations
-to safely implement delayed writing without having to copy the data.</li>
-</ul>
-
-<h3 id="c-tensors">C++: Tensors</h3>
-
-<p>There are three improvements of Tensor and SparseTensor in this
release.</p>
-
-<ul>
- <li>Add Tensor::Value template function for element access</li>
- <li>Add EqualOptions support in Tensor::Equals function, that allows
us to control the way to compare two float tensors</li>
- <li>Add smaller bit-width index supports in SparseTensor</li>
-</ul>
-
-<h2 id="c-notes-1">C# Notes</h2>
-
-<p>We have fixed some bugs causing incompatibilities between C# and
other Arrow
-implementations.</p>
-
-<h2 id="java-notes">Java notes</h2>
-
-<ul>
- <li>Added an initial version of an Avro adapter</li>
- <li>To improve the JDBC adapter performance, refactored consume data
logic and
-implemented an iterator API to prevent loading all data into one
vector</li>
- <li>Implemented subField encoding for complex type, now List and
Struct vectors
-subField encoding is available</li>
- <li>Implemented visitor API for vector/range/type/approx equals
compare</li>
- <li>Performed a lot of optimization and refactoring for
DictionaryEncoder,
-supporting all data types and avoiding memory copy via hash table and visitor
-API</li>
- <li>Introduced <a
href="https://github.com/google/error-prone">Error Prone</a>
into code base to catch more potential errors
-earlier</li>
- <li>Fixed the bug where dictionary entries were required in IPC
streams even when
-empty; readers can now also read interleaved messages</li>
-</ul>
-
-<h2 id="python-notes">Python notes</h2>
-
-<p>The FileSystem API, implemented in C++, is now available in Python
(ARROW-5494).</p>
-
-<p>The API for extension types has been straightened and definition of
-custom extension types in Python is now more powerful (ARROW-5610).</p>
-
-<p>Sparse tensors are now available in Python (ARROW-4453).</p>
-
-<p>A potential crash when handling Python dates and datetimes was fixed
-(ARROW-6597).</p>
-
-<p>Based on a mailing list discussion, we are looking for help with
maintaining
-our Python wheels. Community members have found that the wheels take up a great
-deal of maintenance time, so if you or your organization depend on <code
class="highlighter-rouge">pip install
-pyarrow</code> working, we would appreciate your assistance.</p>
-
-<h2 id="ruby-and-c-glib-notes">Ruby and C GLib notes</h2>
-
-<p>Ruby and C GLib continues to follow the features in the C++ project.
-Ruby includes the following backward incompatible changes.</p>
-
-<ul>
- <li>Remove Arrow::Struct and use Hash instead.</li>
- <li>Add Arrow::Time for Arrow::Time{32,64}DataType value.</li>
- <li>Arrow::Decimal128Array#get_value returns BigDecimal.</li>
-</ul>
-
-<p>Ruby improves the performance of Arrow#values.</p>
-
-<h2 id="rust-notes">Rust notes</h2>
-
-<p>A number of core Arrow improvements were made to the Rust
library.</p>
-
-<ul>
- <li>Add explicit SIMD vectorization for the divide kernel</li>
- <li>Add a feature to disable SIMD</li>
- <li>Use “if cfg!” pattern</li>
- <li>Optimizations to BooleanBufferBuilder::append_slice</li>
- <li>Implemented Debug trait for List/Struct/BinaryArray</li>
-</ul>
-
-<p>Improvements related to Rust Parquet and DataFusion are detailed
next.</p>
-
-<h3 id="rust-parquet">Rust Parquet</h3>
-
-<ul>
- <li>Implement Arrow record reader</li>
- <li>Add converter that is used to convert record reader’s content to
arrow primitive array.</li>
-</ul>
-
-<h3 id="rust-datafusion">Rust DataFusion</h3>
-
-<ul>
- <li>Preview of new query execution engine using an extensible
trait-based
-physical execution plan that supports parallel execution using
threads</li>
- <li>ExecutionContext now has a register_parquet convenience method for
-registering Parquet data sources</li>
- <li>Fixed bug in type coercion optimizer rule</li>
- <li>TableProvider.scan() now returns a thread-safe
BatchIterator</li>
- <li>Remove use of bare trait objects (switched to using dyn
syntax)</li>
- <li>Adds casting from unsigned to signed integer data types</li>
-</ul>
-
-<h2 id="r-notes">R notes</h2>
-
-<p>A major development since the 0.14 release was the arrival of the
<code class="highlighter-rouge">arrow</code> R
-package on <a
href="https://cran.r-project.org/package=arrow">CRAN</a>. We
wrote about this in August on the <a
href="https://arrow.apache.org/blog/2019/08/08/r-package-on-cran/">Arrow
blog</a>.
-In addition to the package availability on CRAN, we also published
-<a href="https://arrow.apache.org/docs/r">package
documentation</a> on the Arrow website.</p>
-
-<p>The 0.15 R package includes many of the enhancements in the C++
library
-release, such as the Parquet performance improvements and the FileSystem API.
-In addition, there are a number of upgrades that make it easier to read and
-write data, specify types and schema, and interact with Arrow tables and record
-batches in R.</p>
-
-<p>For more on what’s in the 0.15 R package, see the <a
href="http://arrow.apache.org/docs/r/news/">changelog</a>.</p>
-
-<h2 id="community-discussions-ongoing">Community Discussions
Ongoing</h2>
-
-<p>There are a number of active discussions ongoing on the developer
[email protected] mailing list. We look forward to hearing from the
-community there.</p></content><author><name>pmc</name></author><category
term="release" /><summary type="html">The Apache Arrow team is pleased to
announce the 0.15.0 release. This covers about 3 months of development work and
includes 687 resolved issues from 80 distinct contributors. See the Install
Page to learn how to get the libraries for your platform. The complete
changelog is also available. About a third of issues closed (240) were
classified as bug fixes, so this release [...]
\ No newline at end of file
+service.</p></content><author><name>wesm</name></author><category
term="application" /><summary type="html">Over the last 18 months, the Apache
Arrow community has been busy designing and implementing Flight, a new
general-purpose client-server framework to simplify high performance transport
of large datasets over network interfaces. Flight initially is focused on
optimized transport of the Arrow columnar format (i.e. “Arrow record batches”)
over gRPC, Google’s popular HTTP/2-base [...]
\ No newline at end of file
![https://arrow.apache.org/img/arrow.png"](https://arrow.apache.org/img/arrow.png"){kind=link}
![https://arrow.apache.org/img/arrow.png","@type":"BlogPosting","url":"https://arrow.apache.org/blog/2020/10/27/rust-2.0.0-release/","headline":"Apache](https://arrow.apache.org/img/arrow.png","@type":"BlogPosting","url":"https://arrow.apache.org/blog/2020/10/27/rust-2.0.0-release/","headline":"Apache){kind=link}
![https://www.apache.org/events/current-event-234x60.png"/](https://www.apache.org/events/current-event-234x60.png"/){kind=link}