http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/c_glib/writer-classes.html ---------------------------------------------------------------------- diff --git a/docs/c_glib/writer-classes.html b/docs/c_glib/writer-classes.html new file mode 100644 index 0000000..6b048f7 --- /dev/null +++ b/docs/c_glib/writer-classes.html @@ -0,0 +1,585 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> +<title> Reference Manual: Writer classes</title> +<meta name="generator" content="DocBook XSL Stylesheets V1.78.1"> +<link rel="home" href="index.html" title="Reference Manual"> +<link rel="up" href="writer.html" title="Writer"> +<link rel="prev" href="writer.html" title="Writer"> +<link rel="next" href="object-tree.html" title="Object Hierarchy"> +<meta name="generator" content="GTK-Doc V1.20 (XML mode)"> +<link rel="stylesheet" href="style.css" type="text/css"> +</head> +<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"> +<table class="navigation" id="top" width="100%" summary="Navigation header" cellpadding="2" cellspacing="10"><tr valign="middle"> +<td width="100%" align="left" class="shortcuts"> +<a href="#" class="shortcut">Top</a><span id="nav_description"> <span class="dim">|</span> + <a href="#writer-classes.description" class="shortcut">Description</a></span><span id="nav_hierarchy"> <span class="dim">|</span> + <a href="#writer-classes.object-hierarchy" class="shortcut">Object Hierarchy</a></span><span id="nav_properties"> <span class="dim">|</span> + <a href="#writer-classes.properties" class="shortcut">Properties</a></span> +</td> +<td><a accesskey="h" href="index.html"><img src="home.png" width="16" height="16" border="0" alt="Home"></a></td> +<td><a accesskey="u" href="writer.html"><img src="up.png" width="16" height="16" border="0" alt="Up"></a></td> +<td><a accesskey="p" href="writer.html"><img src="left.png" width="16" height="16" border="0" alt="Prev"></a></td> +<td><a accesskey="n" href="object-tree.html"><img src="right.png" width="16" height="16" border="0" alt="Next"></a></td> +</tr></table> +<div class="refentry"> +<a name="writer-classes"></a><div class="titlepage"></div> +<div class="refnamediv"><table width="100%"><tr> +<td valign="top"> +<h2><span class="refentrytitle"><a name="writer-classes.top_of_page"></a>Writer classes</span></h2> +<p>Writer classes</p> +</td> +<td class="gallery_image" valign="top" align="right"></td> +</tr></table></div> +<div class="refsect1"> +<a name="writer-classes.functions"></a><h2>Functions</h2> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="functions_return"> +<col class="functions_name"> +</colgroup> +<tbody> +<tr> +<td class="function_type"> +<GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK> +</td> +<td class="function_name"> +<a class="link" href="writer-classes.html#garrow-record-batch-writer-write-record-batch" title="garrow_record_batch_writer_write_record_batch ()">garrow_record_batch_writer_write_record_batch</a> <span class="c_punctuation">()</span> +</td> +</tr> +<tr> +<td class="function_type"> +<GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK> +</td> +<td class="function_name"> +<a class="link" href="writer-classes.html#garrow-record-batch-writer-close" title="garrow_record_batch_writer_close ()">garrow_record_batch_writer_close</a> <span class="c_punctuation">()</span> +</td> +</tr> +<tr> +<td class="function_type"> +<a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter"><span class="returnvalue">GArrowRecordBatchStreamWriter</span></a> * +</td> +<td class="function_name"> +<a class="link" href="writer-classes.html#garrow-record-batch-stream-writer-new" title="garrow_record_batch_stream_writer_new ()">garrow_record_batch_stream_writer_new</a> <span class="c_punctuation">()</span> +</td> +</tr> +<tr> +<td class="function_type"> +<a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter"><span class="returnvalue">GArrowRecordBatchFileWriter</span></a> * +</td> +<td class="function_name"> +<a class="link" href="writer-classes.html#garrow-record-batch-file-writer-new" title="garrow_record_batch_file_writer_new ()">garrow_record_batch_file_writer_new</a> <span class="c_punctuation">()</span> +</td> +</tr> +<tr> +<td class="function_type"> +<a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="returnvalue">GArrowFeatherFileWriter</span></a> * +</td> +<td class="function_name"> +<a class="link" href="writer-classes.html#garrow-feather-file-writer-new" title="garrow_feather_file_writer_new ()">garrow_feather_file_writer_new</a> <span class="c_punctuation">()</span> +</td> +</tr> +<tr> +<td class="function_type"> +<GTKDOCLINK HREF="void"><span class="returnvalue">void</span></GTKDOCLINK> +</td> +<td class="function_name"> +<a class="link" href="writer-classes.html#garrow-feather-file-writer-set-description" title="garrow_feather_file_writer_set_description ()">garrow_feather_file_writer_set_description</a> <span class="c_punctuation">()</span> +</td> +</tr> +<tr> +<td class="function_type"> +<GTKDOCLINK HREF="void"><span class="returnvalue">void</span></GTKDOCLINK> +</td> +<td class="function_name"> +<a class="link" href="writer-classes.html#garrow-feather-file-writer-set-n-rows" title="garrow_feather_file_writer_set_n_rows ()">garrow_feather_file_writer_set_n_rows</a> <span class="c_punctuation">()</span> +</td> +</tr> +<tr> +<td class="function_type"> +<GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK> +</td> +<td class="function_name"> +<a class="link" href="writer-classes.html#garrow-feather-file-writer-append" title="garrow_feather_file_writer_append ()">garrow_feather_file_writer_append</a> <span class="c_punctuation">()</span> +</td> +</tr> +<tr> +<td class="function_type"> +<GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK> +</td> +<td class="function_name"> +<a class="link" href="writer-classes.html#garrow-feather-file-writer-close" title="garrow_feather_file_writer_close ()">garrow_feather_file_writer_close</a> <span class="c_punctuation">()</span> +</td> +</tr> +</tbody> +</table></div> +</div> +<div class="refsect1"> +<a name="writer-classes.properties"></a><h2>Properties</h2> +<div class="informaltable"><table border="0"> +<colgroup> +<col width="150px" class="properties_type"> +<col width="300px" class="properties_name"> +<col width="200px" class="properties_flags"> +</colgroup> +<tbody> +<tr> +<td class="property_type"><GTKDOCLINK HREF="gpointer"><span class="type">gpointer</span></GTKDOCLINK></td> +<td class="property_name"><a class="link" href="writer-classes.html#GArrowRecordBatchWriter--record-batch-writer" title="The ârecord-batch-writerâ property">record-batch-writer</a></td> +<td class="property_flags">Write / Construct Only</td> +</tr> +<tr> +<td class="property_type"><GTKDOCLINK HREF="gpointer"><span class="type">gpointer</span></GTKDOCLINK></td> +<td class="property_name"><a class="link" href="writer-classes.html#GArrowFeatherFileWriter--feather-table-writer" title="The âfeather-table-writerâ property">feather-table-writer</a></td> +<td class="property_flags">Write / Construct Only</td> +</tr> +</tbody> +</table></div> +</div> +<a name="GArrowRecordBatchWriter"></a><a name="GArrowRecordBatchStreamWriter"></a><a name="GArrowRecordBatchFileWriter"></a><a name="GArrowFeatherFileWriter"></a><div class="refsect1"> +<a name="writer-classes.other"></a><h2>Types and Values</h2> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="name"> +<col class="description"> +</colgroup> +<tbody> +<tr> +<td class="datatype_keyword">struct</td> +<td class="function_name"><a class="link" href="writer-classes.html#GArrowRecordBatchWriter-struct" title="struct GArrowRecordBatchWriter">GArrowRecordBatchWriter</a></td> +</tr> +<tr> +<td class="datatype_keyword">struct</td> +<td class="function_name"><a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter-struct" title="struct GArrowRecordBatchStreamWriter">GArrowRecordBatchStreamWriter</a></td> +</tr> +<tr> +<td class="datatype_keyword">struct</td> +<td class="function_name"><a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter-struct" title="struct GArrowRecordBatchFileWriter">GArrowRecordBatchFileWriter</a></td> +</tr> +<tr> +<td class="datatype_keyword">struct</td> +<td class="function_name"><a class="link" href="writer-classes.html#GArrowFeatherFileWriter-struct" title="struct GArrowFeatherFileWriter">GArrowFeatherFileWriter</a></td> +</tr> +</tbody> +</table></div> +</div> +<div class="refsect1"> +<a name="writer-classes.object-hierarchy"></a><h2>Object Hierarchy</h2> +<pre class="screen"> <GTKDOCLINK HREF="GObject">GObject</GTKDOCLINK> + <span class="lineart">âââ</span> GArrowFeatherFileWriter + <span class="lineart">âââ</span> <a class="link" href="writer-classes.html#GArrowRecordBatchWriter">GArrowRecordBatchWriter</a> + <span class="lineart">â</span> <span class="lineart">â°ââ</span> GArrowRecordBatchStreamWriter + <span class="lineart">â</span> <span class="lineart">â°ââ</span> <a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter">GArrowRecordBatchFileWriter</a> + <span class="lineart">â°ââ</span> GArrowRecordBatchWriter + <span class="lineart">â°ââ</span> <a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter">GArrowRecordBatchStreamWriter</a> + <span class="lineart">â°ââ</span> GArrowRecordBatchFileWriter +</pre> +</div> +<div class="refsect1"> +<a name="writer-classes.includes"></a><h2>Includes</h2> +<pre class="synopsis">#include <arrow-glib/arrow-glib.h> +</pre> +</div> +<div class="refsect1"> +<a name="writer-classes.description"></a><h2>Description</h2> +<p><a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a> is a base class for writing record batches +in stream format into output.</p> +<p><a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter"><span class="type">GArrowRecordBatchStreamWriter</span></a> is a base class for writing record +batches in stream format into output synchronously.</p> +<p><a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter"><span class="type">GArrowRecordBatchFileWriter</span></a> is a class for writing record +batches in file format into output.</p> +<p><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> is a class for writing arrays +in Feather file format into output.</p> +</div> +<div class="refsect1"> +<a name="writer-classes.functions_details"></a><h2>Functions</h2> +<div class="refsect2"> +<a name="garrow-record-batch-writer-write-record-batch"></a><h3>garrow_record_batch_writer_write_record_batch ()</h3> +<pre class="programlisting"><GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK> +garrow_record_batch_writer_write_record_batch + (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a> *writer</code></em>, + <em class="parameter"><code><a class="link" href="GArrowRecordBatch.html" title="GArrowRecordBatch"><span class="type">GArrowRecordBatch</span></a> *record_batch</code></em>, + <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre> +<div class="refsect3"> +<a name="id-1.4.4.2.13.2.4"></a><h4>Parameters</h4> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="parameters_name"> +<col class="parameters_description"> +<col width="200px" class="parameters_annotations"> +</colgroup> +<tbody> +<tr> +<td class="parameter_name"><p>writer</p></td> +<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a>.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>record_batch</p></td> +<td class="parameter_description"><p>The record batch to be written.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>error</p></td> +<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td> +<td class="parameter_annotations"><span class="annotation">[nullable]</span></td> +</tr> +</tbody> +</table></div> +</div> +<div class="refsect3"> +<a name="id-1.4.4.2.13.2.5"></a><h4>Returns</h4> +<p> <GTKDOCLINK HREF="TRUE:CAPS"><code class="literal">TRUE</code></GTKDOCLINK> on success, <GTKDOCLINK HREF="FALSE:CAPS"><code class="literal">FALSE</code></GTKDOCLINK> if there was an error.</p> +<p></p> +</div> +<p class="since">Since 0.4.0</p> +</div> +<hr> +<div class="refsect2"> +<a name="garrow-record-batch-writer-close"></a><h3>garrow_record_batch_writer_close ()</h3> +<pre class="programlisting"><GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK> +garrow_record_batch_writer_close (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a> *writer</code></em>, + <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre> +<div class="refsect3"> +<a name="id-1.4.4.2.13.3.4"></a><h4>Parameters</h4> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="parameters_name"> +<col class="parameters_description"> +<col width="200px" class="parameters_annotations"> +</colgroup> +<tbody> +<tr> +<td class="parameter_name"><p>writer</p></td> +<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a>.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>error</p></td> +<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td> +<td class="parameter_annotations"><span class="annotation">[nullable]</span></td> +</tr> +</tbody> +</table></div> +</div> +<div class="refsect3"> +<a name="id-1.4.4.2.13.3.5"></a><h4>Returns</h4> +<p> <GTKDOCLINK HREF="TRUE:CAPS"><code class="literal">TRUE</code></GTKDOCLINK> on success, <GTKDOCLINK HREF="FALSE:CAPS"><code class="literal">FALSE</code></GTKDOCLINK> if there was an error.</p> +<p></p> +</div> +<p class="since">Since 0.4.0</p> +</div> +<hr> +<div class="refsect2"> +<a name="garrow-record-batch-stream-writer-new"></a><h3>garrow_record_batch_stream_writer_new ()</h3> +<pre class="programlisting"><a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter"><span class="returnvalue">GArrowRecordBatchStreamWriter</span></a> * +garrow_record_batch_stream_writer_new (<em class="parameter"><code><a class="link" href="output-stream-classes.html#GArrowOutputStream"><span class="type">GArrowOutputStream</span></a> *sink</code></em>, + <em class="parameter"><code><a class="link" href="GArrowSchema.html" title="GArrowSchema"><span class="type">GArrowSchema</span></a> *schema</code></em>, + <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre> +<div class="refsect3"> +<a name="id-1.4.4.2.13.4.4"></a><h4>Parameters</h4> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="parameters_name"> +<col class="parameters_description"> +<col width="200px" class="parameters_annotations"> +</colgroup> +<tbody> +<tr> +<td class="parameter_name"><p>sink</p></td> +<td class="parameter_description"><p>The output of the writer.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>schema</p></td> +<td class="parameter_description"><p>The schema of the writer.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>error</p></td> +<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td> +<td class="parameter_annotations"><span class="annotation">[nullable]</span></td> +</tr> +</tbody> +</table></div> +</div> +<div class="refsect3"> +<a name="id-1.4.4.2.13.4.5"></a><h4>Returns</h4> +<p> A newly created <a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter"><span class="type">GArrowRecordBatchStreamWriter</span></a> +or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK> on error. </p> +<p><span class="annotation">[nullable]</span></p> +</div> +<p class="since">Since 0.4.0</p> +</div> +<hr> +<div class="refsect2"> +<a name="garrow-record-batch-file-writer-new"></a><h3>garrow_record_batch_file_writer_new ()</h3> +<pre class="programlisting"><a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter"><span class="returnvalue">GArrowRecordBatchFileWriter</span></a> * +garrow_record_batch_file_writer_new (<em class="parameter"><code><a class="link" href="output-stream-classes.html#GArrowOutputStream"><span class="type">GArrowOutputStream</span></a> *sink</code></em>, + <em class="parameter"><code><a class="link" href="GArrowSchema.html" title="GArrowSchema"><span class="type">GArrowSchema</span></a> *schema</code></em>, + <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre> +<div class="refsect3"> +<a name="id-1.4.4.2.13.5.4"></a><h4>Parameters</h4> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="parameters_name"> +<col class="parameters_description"> +<col width="200px" class="parameters_annotations"> +</colgroup> +<tbody> +<tr> +<td class="parameter_name"><p>sink</p></td> +<td class="parameter_description"><p>The output of the writer.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>schema</p></td> +<td class="parameter_description"><p>The schema of the writer.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>error</p></td> +<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td> +<td class="parameter_annotations"><span class="annotation">[nullable]</span></td> +</tr> +</tbody> +</table></div> +</div> +<div class="refsect3"> +<a name="id-1.4.4.2.13.5.5"></a><h4>Returns</h4> +<p> A newly created <a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter"><span class="type">GArrowRecordBatchFileWriter</span></a> +or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK> on error. </p> +<p><span class="annotation">[nullable]</span></p> +</div> +<p class="since">Since 0.4.0</p> +</div> +<hr> +<div class="refsect2"> +<a name="garrow-feather-file-writer-new"></a><h3>garrow_feather_file_writer_new ()</h3> +<pre class="programlisting"><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="returnvalue">GArrowFeatherFileWriter</span></a> * +garrow_feather_file_writer_new (<em class="parameter"><code><a class="link" href="output-stream-classes.html#GArrowOutputStream"><span class="type">GArrowOutputStream</span></a> *sink</code></em>, + <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre> +<div class="refsect3"> +<a name="id-1.4.4.2.13.6.4"></a><h4>Parameters</h4> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="parameters_name"> +<col class="parameters_description"> +<col width="200px" class="parameters_annotations"> +</colgroup> +<tbody> +<tr> +<td class="parameter_name"><p>sink</p></td> +<td class="parameter_description"><p>The output of the writer.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>error</p></td> +<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td> +<td class="parameter_annotations"><span class="annotation">[nullable]</span></td> +</tr> +</tbody> +</table></div> +</div> +<div class="refsect3"> +<a name="id-1.4.4.2.13.6.5"></a><h4>Returns</h4> +<p> A newly created <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> +or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK> on error. </p> +<p><span class="annotation">[nullable]</span></p> +</div> +<p class="since">Since 0.4.0</p> +</div> +<hr> +<div class="refsect2"> +<a name="garrow-feather-file-writer-set-description"></a><h3>garrow_feather_file_writer_set_description ()</h3> +<pre class="programlisting"><GTKDOCLINK HREF="void"><span class="returnvalue">void</span></GTKDOCLINK> +garrow_feather_file_writer_set_description + (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> *writer</code></em>, + <em class="parameter"><code>const <GTKDOCLINK HREF="gchar"><span class="type">gchar</span></GTKDOCLINK> *description</code></em>);</pre> +<div class="refsect3"> +<a name="id-1.4.4.2.13.7.4"></a><h4>Parameters</h4> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="parameters_name"> +<col class="parameters_description"> +<col width="200px" class="parameters_annotations"> +</colgroup> +<tbody> +<tr> +<td class="parameter_name"><p>writer</p></td> +<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a>.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>description</p></td> +<td class="parameter_description"><p>The description of the file.</p></td> +<td class="parameter_annotations"> </td> +</tr> +</tbody> +</table></div> +</div> +<p class="since">Since 0.4.0</p> +</div> +<hr> +<div class="refsect2"> +<a name="garrow-feather-file-writer-set-n-rows"></a><h3>garrow_feather_file_writer_set_n_rows ()</h3> +<pre class="programlisting"><GTKDOCLINK HREF="void"><span class="returnvalue">void</span></GTKDOCLINK> +garrow_feather_file_writer_set_n_rows (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> *writer</code></em>, + <em class="parameter"><code><GTKDOCLINK HREF="gint64"><span class="type">gint64</span></GTKDOCLINK> n_rows</code></em>);</pre> +<div class="refsect3"> +<a name="id-1.4.4.2.13.8.4"></a><h4>Parameters</h4> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="parameters_name"> +<col class="parameters_description"> +<col width="200px" class="parameters_annotations"> +</colgroup> +<tbody> +<tr> +<td class="parameter_name"><p>writer</p></td> +<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a>.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>n_rows</p></td> +<td class="parameter_description"><p>The number of rows in the file.</p></td> +<td class="parameter_annotations"> </td> +</tr> +</tbody> +</table></div> +</div> +<p class="since">Since 0.4.0</p> +</div> +<hr> +<div class="refsect2"> +<a name="garrow-feather-file-writer-append"></a><h3>garrow_feather_file_writer_append ()</h3> +<pre class="programlisting"><GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK> +garrow_feather_file_writer_append (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> *writer</code></em>, + <em class="parameter"><code>const <GTKDOCLINK HREF="gchar"><span class="type">gchar</span></GTKDOCLINK> *name</code></em>, + <em class="parameter"><code><a class="link" href="array-classes.html#GArrowArray"><span class="type">GArrowArray</span></a> *array</code></em>, + <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre> +<div class="refsect3"> +<a name="id-1.4.4.2.13.9.4"></a><h4>Parameters</h4> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="parameters_name"> +<col class="parameters_description"> +<col width="200px" class="parameters_annotations"> +</colgroup> +<tbody> +<tr> +<td class="parameter_name"><p>writer</p></td> +<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a>.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>name</p></td> +<td class="parameter_description"><p>The name of the array to be appended.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>array</p></td> +<td class="parameter_description"><p>The array to be appended.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>error</p></td> +<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td> +<td class="parameter_annotations"><span class="annotation">[nullable]</span></td> +</tr> +</tbody> +</table></div> +</div> +<div class="refsect3"> +<a name="id-1.4.4.2.13.9.5"></a><h4>Returns</h4> +<p> <GTKDOCLINK HREF="TRUE:CAPS"><code class="literal">TRUE</code></GTKDOCLINK> on success, <GTKDOCLINK HREF="FALSE:CAPS"><code class="literal">FALSE</code></GTKDOCLINK> if there was an error.</p> +<p></p> +</div> +<p class="since">Since 0.4.0</p> +</div> +<hr> +<div class="refsect2"> +<a name="garrow-feather-file-writer-close"></a><h3>garrow_feather_file_writer_close ()</h3> +<pre class="programlisting"><GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK> +garrow_feather_file_writer_close (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> *writer</code></em>, + <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre> +<div class="refsect3"> +<a name="id-1.4.4.2.13.10.4"></a><h4>Parameters</h4> +<div class="informaltable"><table width="100%" border="0"> +<colgroup> +<col width="150px" class="parameters_name"> +<col class="parameters_description"> +<col width="200px" class="parameters_annotations"> +</colgroup> +<tbody> +<tr> +<td class="parameter_name"><p>writer</p></td> +<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a>.</p></td> +<td class="parameter_annotations"> </td> +</tr> +<tr> +<td class="parameter_name"><p>error</p></td> +<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td> +<td class="parameter_annotations"><span class="annotation">[nullable]</span></td> +</tr> +</tbody> +</table></div> +</div> +<div class="refsect3"> +<a name="id-1.4.4.2.13.10.5"></a><h4>Returns</h4> +<p> <GTKDOCLINK HREF="TRUE:CAPS"><code class="literal">TRUE</code></GTKDOCLINK> on success, <GTKDOCLINK HREF="FALSE:CAPS"><code class="literal">FALSE</code></GTKDOCLINK> if there was an error.</p> +<p></p> +</div> +<p class="since">Since 0.4.0</p> +</div> +</div> +<div class="refsect1"> +<a name="writer-classes.other_details"></a><h2>Types and Values</h2> +<div class="refsect2"> +<a name="GArrowRecordBatchWriter-struct"></a><h3>struct GArrowRecordBatchWriter</h3> +<pre class="programlisting">struct GArrowRecordBatchWriter;</pre> +<p>It wraps <code class="literal">arrow::ipc::RecordBatchWriter</code>.</p> +</div> +<hr> +<div class="refsect2"> +<a name="GArrowRecordBatchStreamWriter-struct"></a><h3>struct GArrowRecordBatchStreamWriter</h3> +<pre class="programlisting">struct GArrowRecordBatchStreamWriter;</pre> +<p>It wraps <code class="literal">arrow::ipc::RecordBatchStreamWriter</code>.</p> +</div> +<hr> +<div class="refsect2"> +<a name="GArrowRecordBatchFileWriter-struct"></a><h3>struct GArrowRecordBatchFileWriter</h3> +<pre class="programlisting">struct GArrowRecordBatchFileWriter;</pre> +<p>It wraps <code class="literal">arrow::ipc::RecordBatchFileWriter</code>.</p> +</div> +<hr> +<div class="refsect2"> +<a name="GArrowFeatherFileWriter-struct"></a><h3>struct GArrowFeatherFileWriter</h3> +<pre class="programlisting">struct GArrowFeatherFileWriter;</pre> +<p>It wraps <code class="literal">arrow::ipc::feather::TableWriter</code>.</p> +</div> +</div> +<div class="refsect1"> +<a name="writer-classes.property-details"></a><h2>Property Details</h2> +<div class="refsect2"> +<a name="GArrowRecordBatchWriter--record-batch-writer"></a><h3>The <code class="literal">ârecord-batch-writerâ</code> property</h3> +<pre class="programlisting"> ârecord-batch-writerâ <GTKDOCLINK HREF="gpointer"><span class="type">gpointer</span></GTKDOCLINK></pre> +<p>The raw std::shared<arrow::ipc::RecordBatchWriter> *.</p> +<p>Flags: Write / Construct Only</p> +</div> +<hr> +<div class="refsect2"> +<a name="GArrowFeatherFileWriter--feather-table-writer"></a><h3>The <code class="literal">âfeather-table-writerâ</code> property</h3> +<pre class="programlisting"> âfeather-table-writerâ <GTKDOCLINK HREF="gpointer"><span class="type">gpointer</span></GTKDOCLINK></pre> +<p>The raw std::shared<arrow::ipc::feather::TableWriter> *.</p> +<p>Flags: Write / Construct Only</p> +</div> +</div> +</div> +<div class="footer"> +<hr> + Generated by GTK-Doc V1.20</div> +</body> +</html>
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/c_glib/writer.html ---------------------------------------------------------------------- diff --git a/docs/c_glib/writer.html b/docs/c_glib/writer.html index ccb09c1..ea5d730 100644 --- a/docs/c_glib/writer.html +++ b/docs/c_glib/writer.html @@ -6,8 +6,8 @@ <meta name="generator" content="DocBook XSL Stylesheets V1.78.1"> <link rel="home" href="index.html" title="Reference Manual"> <link rel="up" href="ipc.html" title="Part III. IPC"> -<link rel="prev" href="GArrowStreamReader.html" title="GArrowStreamReader"> -<link rel="next" href="GArrowFileWriter.html" title="GArrowFileWriter"> +<link rel="prev" href="reader-classes.html" title="Reader classes"> +<link rel="next" href="writer-classes.html" title="Writer classes"> <meta name="generator" content="GTK-Doc V1.20 (XML mode)"> <link rel="stylesheet" href="style.css" type="text/css"> </head> @@ -16,20 +16,15 @@ <td width="100%" align="left" class="shortcuts"></td> <td><a accesskey="h" href="index.html"><img src="home.png" width="16" height="16" border="0" alt="Home"></a></td> <td><a accesskey="u" href="ipc.html"><img src="up.png" width="16" height="16" border="0" alt="Up"></a></td> -<td><a accesskey="p" href="GArrowStreamReader.html"><img src="left.png" width="16" height="16" border="0" alt="Prev"></a></td> -<td><a accesskey="n" href="GArrowFileWriter.html"><img src="right.png" width="16" height="16" border="0" alt="Next"></a></td> +<td><a accesskey="p" href="reader-classes.html"><img src="left.png" width="16" height="16" border="0" alt="Prev"></a></td> +<td><a accesskey="n" href="writer-classes.html"><img src="right.png" width="16" height="16" border="0" alt="Next"></a></td> </tr></table> <div class="chapter"> <div class="titlepage"><div><div><h2 class="title"> <a name="writer"></a>Writer</h2></div></div></div> -<div class="toc"><dl class="toc"> -<dt> -<span class="refentrytitle"><a href="GArrowFileWriter.html">GArrowFileWriter</a></span><span class="refpurpose"> â File writer class</span> -</dt> -<dt> -<span class="refentrytitle"><a href="GArrowStreamWriter.html">GArrowStreamWriter</a></span><span class="refpurpose"> â Stream writer class</span> -</dt> -</dl></div> +<div class="toc"><dl class="toc"><dt> +<span class="refentrytitle"><a href="writer-classes.html">Writer classes</a></span><span class="refpurpose"></span> +</dt></dl></div> </div> <div class="footer"> <hr> http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/_modules/index.html ---------------------------------------------------------------------- diff --git a/docs/python/_modules/index.html b/docs/python/_modules/index.html index 48c9094..6070788 100644 --- a/docs/python/_modules/index.html +++ b/docs/python/_modules/index.html @@ -71,7 +71,8 @@ <li class="toctree-l1"><a class="reference internal" href="../memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1"><a class="reference internal" href="../data.html">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="../ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">Filesystem Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="../pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="../parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1"><a class="reference internal" href="../api.html">API Reference</a></li> @@ -123,9 +124,12 @@ <div class="col-md-12 content"> <h1>All modules for which code is available</h1> -<ul><li><a href="pyarrow/ipc.html">pyarrow.ipc</a></li> +<ul><li><a href="pyarrow/filesystem.html">pyarrow.filesystem</a></li> +<li><a href="pyarrow/hdfs.html">pyarrow.hdfs</a></li> +<li><a href="pyarrow/ipc.html">pyarrow.ipc</a></li> <li><a href="pyarrow/lib.html">pyarrow.lib</a></li> <li><a href="pyarrow/parquet.html">pyarrow.parquet</a></li> +<li><a href="pyarrow/plasma.html">pyarrow.plasma</a></li> </ul> </div> http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/_modules/pyarrow/filesystem.html ---------------------------------------------------------------------- diff --git a/docs/python/_modules/pyarrow/filesystem.html b/docs/python/_modules/pyarrow/filesystem.html new file mode 100644 index 0000000..b194357 --- /dev/null +++ b/docs/python/_modules/pyarrow/filesystem.html @@ -0,0 +1,472 @@ +<!DOCTYPE html> + + +<html xmlns="http://www.w3.org/1999/xhtml";> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + + <title>pyarrow.filesystem — pyarrow documentation</title> + + <link rel="stylesheet" href="../../_static/bootstrap-sphinx.css" type="text/css" /> + <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" /> + + <script type="text/javascript"> + var DOCUMENTATION_OPTIONS = { + URL_ROOT: '../../', + VERSION: '', + COLLAPSE_INDEX: false, + FILE_SUFFIX: '.html', + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' + }; + </script> + <script type="text/javascript" src="../../_static/jquery.js"></script> + <script type="text/javascript" src="../../_static/underscore.js"></script> + <script type="text/javascript" src="../../_static/doctools.js"></script> + <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML";></script> + <script type="text/javascript" src="../../_static/js/jquery-1.11.0.min.js"></script> + <script type="text/javascript" src="../../_static/js/jquery-fix.js"></script> + <script type="text/javascript" src="../../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script> + <script type="text/javascript" src="../../_static/bootstrap-sphinx.js"></script> + <link rel="index" title="Index" href="../../genindex.html" /> + <link rel="search" title="Search" href="../../search.html" /> +<meta charset='utf-8'> +<meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'> +<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'> +<meta name="apple-mobile-web-app-capable" content="yes"> + + </head> + <body role="document"> + + <div id="navbar" class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <!-- .btn-navbar is used as the toggle for collapsed navbar content --> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="../../index.html"> + pyarrow</a> + <span class="navbar-text navbar-version pull-left"><b></b></span> + </div> + + <div class="collapse navbar-collapse nav-collapse"> + <ul class="nav navbar-nav"> + + + <li class="dropdown globaltoc-container"> + <a role="button" + id="dLabelGlobalToc" + data-toggle="dropdown" + data-target="#" + href="../../index.html">Site <b class="caret"></b></a> + <ul class="dropdown-menu globaltoc" + role="menu" + aria-labelledby="dLabelGlobalToc"><p class="caption"><span class="caption-text">Getting Started</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../../install.html">Install PyArrow</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../development.html">Development</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../memory.html">Memory and IO Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../data.html">In-Memory Data Model</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../ipc.html">IPC: Fast Streaming and Serialization</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../plasma.html">The Plasma In-Memory Object Store</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../pandas.html">Using PyArrow with pandas</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../parquet.html">Reading and Writing the Apache Parquet Format</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../api.html">API Reference</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../getting_involved.html">Getting Involved</a></li> +</ul> +</ul> +</li> + + <li class="dropdown"> + <a role="button" + id="dLabelLocalToc" + data-toggle="dropdown" + data-target="#" + href="#">Page <b class="caret"></b></a> + <ul class="dropdown-menu localtoc" + role="menu" + aria-labelledby="dLabelLocalToc"></ul> +</li> + + + + + + + + + + + <li class="hidden-sm"></li> + + </ul> + + + +<form class="navbar-form navbar-right" action="../../search.html" method="get"> + <div class="form-group"> + <input type="text" name="q" class="form-control" placeholder="Search" /> + </div> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> +</form> + + </div> + </div> + </div> + +<div class="container"> + <div class="row"> + <div class="col-md-12 content"> + + <h1>Source code for pyarrow.filesystem</h1><div class="highlight"><pre> +<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span> +<span class="c1"># or more contributor license agreements. See the NOTICE file</span> +<span class="c1"># distributed with this work for additional information</span> +<span class="c1"># regarding copyright ownership. The ASF licenses this file</span> +<span class="c1"># to you under the Apache License, Version 2.0 (the</span> +<span class="c1"># "License"); you may not use this file except in compliance</span> +<span class="c1"># with the License. You may obtain a copy of the License at</span> +<span class="c1">#</span> +<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">#</span> +<span class="c1"># Unless required by applicable law or agreed to in writing,</span> +<span class="c1"># software distributed under the License is distributed on an</span> +<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span> +<span class="c1"># KIND, either express or implied. See the License for the</span> +<span class="c1"># specific language governing permissions and limitations</span> +<span class="c1"># under the License.</span> + +<span class="kn">from</span> <span class="nn">os.path</span> <span class="k">import</span> <span class="n">join</span> <span class="k">as</span> <span class="n">pjoin</span> +<span class="kn">import</span> <span class="nn">os</span> +<span class="kn">import</span> <span class="nn">posixpath</span> + +<span class="kn">from</span> <span class="nn">pyarrow.util</span> <span class="k">import</span> <span class="n">implements</span> + + +<span class="k">class</span> <span class="nc">FileSystem</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Abstract filesystem interface</span> +<span class="sd"> """</span> + <span class="k">def</span> <span class="nf">cat</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Return contents of file as a bytes object</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> contents : bytes</span> +<span class="sd"> """</span> + <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> + <span class="k">return</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> + + <span class="k">def</span> <span class="nf">ls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Return list of file paths</span> +<span class="sd"> """</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span> + + <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Delete the indicated file or directory</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> path : string</span> +<span class="sd"> recursive : boolean, default False</span> +<span class="sd"> If True, also delete child paths for directories</span> +<span class="sd"> """</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span> + + <span class="k">def</span> <span class="nf">disk_usage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Compute bytes used by all contents under indicated path in file tree</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> path : string</span> +<span class="sd"> Can be a file path or directory</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> usage : int</span> +<span class="sd"> """</span> + <span class="n">path_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> + <span class="k">if</span> <span class="n">path_info</span><span class="p">[</span><span class="s1">'kind'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'file'</span><span class="p">:</span> + <span class="k">return</span> <span class="n">path_info</span><span class="p">[</span><span class="s1">'size'</span><span class="p">]</span> + + <span class="n">total</span> <span class="o">=</span> <span class="mi">0</span> + <span class="k">for</span> <span class="n">root</span><span class="p">,</span> <span class="n">directories</span><span class="p">,</span> <span class="n">files</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">path</span><span class="p">):</span> + <span class="k">for</span> <span class="n">child_path</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span> + <span class="n">abspath</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_path_join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">child_path</span><span class="p">)</span> + <span class="n">total</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">abspath</span><span class="p">)[</span><span class="s1">'size'</span><span class="p">]</span> + + <span class="k">return</span> <span class="n">total</span> + + <span class="k">def</span> <span class="nf">_path_join</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">pathsep</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">stat</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="sd">"""</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> stat : dict</span> +<span class="sd"> """</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">'FileSystem.stat'</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">rm</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Alias for FileSystem.delete</span> +<span class="sd"> """</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="n">recursive</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">mv</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">new_path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Alias for FileSystem.rename</span> +<span class="sd"> """</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">new_path</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">new_path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Rename file, like UNIX mv command</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> path : string</span> +<span class="sd"> Path to alter</span> +<span class="sd"> new_path : string</span> +<span class="sd"> Path to move to</span> +<span class="sd"> """</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">'FileSystem.rename'</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">create_parents</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span> + + <span class="k">def</span> <span class="nf">exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span> + + <span class="k">def</span> <span class="nf">isdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Return True if path is a directory</span> +<span class="sd"> """</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span> + + <span class="k">def</span> <span class="nf">isfile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Return True if path is a file</span> +<span class="sd"> """</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span> + + <span class="k">def</span> <span class="nf">read_parquet</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">nthreads</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">use_pandas_metadata</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Read Parquet data from path in file system. Can read from a single file</span> +<span class="sd"> or a directory of files</span> + +<span class="sd"> Parameters</span> +<span class="sd"> ----------</span> +<span class="sd"> path : str</span> +<span class="sd"> Single file path or directory</span> +<span class="sd"> columns : List[str], optional</span> +<span class="sd"> Subset of columns to read</span> +<span class="sd"> metadata : pyarrow.parquet.FileMetaData</span> +<span class="sd"> Known metadata to validate files against</span> +<span class="sd"> schema : pyarrow.parquet.Schema</span> +<span class="sd"> Known schema to validate files against. Alternative to metadata</span> +<span class="sd"> argument</span> +<span class="sd"> nthreads : int, default 1</span> +<span class="sd"> Number of columns to read in parallel. If > 1, requires that the</span> +<span class="sd"> underlying file source is threadsafe</span> +<span class="sd"> use_pandas_metadata : boolean, default False</span> +<span class="sd"> If True and file has custom pandas schema metadata, ensure that</span> +<span class="sd"> index columns are also loaded</span> + +<span class="sd"> Returns</span> +<span class="sd"> -------</span> +<span class="sd"> table : pyarrow.Table</span> +<span class="sd"> """</span> + <span class="kn">from</span> <span class="nn">pyarrow.parquet</span> <span class="k">import</span> <span class="n">ParquetDataset</span> + <span class="n">dataset</span> <span class="o">=</span> <span class="n">ParquetDataset</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="n">metadata</span><span class="p">,</span> + <span class="n">filesystem</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span> + <span class="k">return</span> <span class="n">dataset</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span> + <span class="n">use_pandas_metadata</span><span class="o">=</span><span class="n">use_pandas_metadata</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">'rb'</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Open file for reading or writing</span> +<span class="sd"> """</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span> + + <span class="nd">@property</span> + <span class="k">def</span> <span class="nf">pathsep</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> + <span class="k">return</span> <span class="s1">'/'</span> + + +<div class="viewcode-block" id="LocalFileSystem"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem">[docs]</a><span class="k">class</span> <span class="nc">LocalFileSystem</span><span class="p">(</span><span class="n">FileSystem</span><span class="p">):</span> + + <span class="n">_instance</span> <span class="o">=</span> <span class="kc">None</span> + + <span class="nd">@classmethod</span> +<div class="viewcode-block" id="LocalFileSystem.get_instance"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.get_instance">[docs]</a> <span class="k">def</span> <span class="nf">get_instance</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span> + <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instance</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="bp">cls</span><span class="o">.</span><span class="n">_instance</span> <span class="o">=</span> <span class="n">LocalFileSystem</span><span class="p">()</span> + <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instance</span></div> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">ls</span><span class="p">)</span> +<div class="viewcode-block" id="LocalFileSystem.ls"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.ls">[docs]</a> <span class="k">def</span> <span class="nf">ls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">return</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">pjoin</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">path</span><span class="p">))</span></div> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">mkdir</span><span class="p">)</span> +<div class="viewcode-block" id="LocalFileSystem.mkdir"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.mkdir">[docs]</a> <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">create_parents</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span> + <span class="k">if</span> <span class="n">create_parents</span><span class="p">:</span> + <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">os</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isdir</span><span class="p">)</span> +<div class="viewcode-block" id="LocalFileSystem.isdir"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.isdir">[docs]</a> <span class="k">def</span> <span class="nf">isdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isfile</span><span class="p">)</span> +<div class="viewcode-block" id="LocalFileSystem.isfile"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.isfile">[docs]</a> <span class="k">def</span> <span class="nf">isfile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">exists</span><span class="p">)</span> +<div class="viewcode-block" id="LocalFileSystem.exists"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.exists">[docs]</a> <span class="k">def</span> <span class="nf">exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">open</span><span class="p">)</span> +<div class="viewcode-block" id="LocalFileSystem.open"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.open">[docs]</a> <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">'rb'</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Open file for reading or writing</span> +<span class="sd"> """</span> + <span class="k">return</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="n">mode</span><span class="p">)</span></div> + + <span class="nd">@property</span> + <span class="k">def</span> <span class="nf">pathsep</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> + <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">sep</span> + +<div class="viewcode-block" id="LocalFileSystem.walk"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.walk">[docs]</a> <span class="k">def</span> <span class="nf">walk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">top_dir</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Directory tree generator, see os.walk</span> +<span class="sd"> """</span> + <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">top_dir</span><span class="p">)</span></div></div> + + +<span class="k">class</span> <span class="nc">DaskFileSystem</span><span class="p">(</span><span class="n">FileSystem</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.</span> +<span class="sd"> """</span> + + <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fs</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">fs</span> <span class="o">=</span> <span class="n">fs</span> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isdir</span><span class="p">)</span> + <span class="k">def</span> <span class="nf">isdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"Unsupported file system API"</span><span class="p">)</span> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isfile</span><span class="p">)</span> + <span class="k">def</span> <span class="nf">isfile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">"Unsupported file system API"</span><span class="p">)</span> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">delete</span><span class="p">)</span> + <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">rm</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="n">recursive</span><span class="p">)</span> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">mkdir</span><span class="p">)</span> + <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">open</span><span class="p">)</span> + <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">'rb'</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Open file for reading or writing</span> +<span class="sd"> """</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="n">mode</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">ls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">detail</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">ls</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">detail</span><span class="o">=</span><span class="n">detail</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">walk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">top_path</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Directory tree generator, like os.walk</span> +<span class="sd"> """</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">top_path</span><span class="p">)</span> + + +<span class="k">class</span> <span class="nc">S3FSWrapper</span><span class="p">(</span><span class="n">DaskFileSystem</span><span class="p">):</span> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isdir</span><span class="p">)</span> + <span class="k">def</span> <span class="nf">isdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">try</span><span class="p">:</span> + <span class="n">contents</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">ls</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> + <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">contents</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">contents</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="n">path</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">False</span> + <span class="k">else</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">True</span> + <span class="k">except</span> <span class="ne">OSError</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">False</span> + + <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isfile</span><span class="p">)</span> + <span class="k">def</span> <span class="nf">isfile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span> + <span class="k">try</span><span class="p">:</span> + <span class="n">contents</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">ls</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> + <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">contents</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">contents</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="n">path</span> + <span class="k">except</span> <span class="ne">OSError</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">False</span> + + <span class="k">def</span> <span class="nf">walk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">refresh</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Directory tree generator, like os.walk</span> + +<span class="sd"> Generator version of what is in s3fs, which yields a flattened list of</span> +<span class="sd"> files</span> +<span class="sd"> """</span> + <span class="n">path</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'s3://'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> + <span class="n">directories</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span> + <span class="n">files</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span> + + <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">_ls</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">refresh</span><span class="o">=</span><span class="n">refresh</span><span class="p">)):</span> + <span class="n">path</span> <span class="o">=</span> <span class="n">key</span><span class="p">[</span><span class="s1">'Key'</span><span class="p">]</span> + <span class="k">if</span> <span class="n">key</span><span class="p">[</span><span class="s1">'StorageClass'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'DIRECTORY'</span><span class="p">:</span> + <span class="n">directories</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> + <span class="k">elif</span> <span class="n">key</span><span class="p">[</span><span class="s1">'StorageClass'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'BUCKET'</span><span class="p">:</span> + <span class="k">pass</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">files</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> + + <span class="c1"># s3fs creates duplicate 'DIRECTORY' entries</span> + <span class="n">files</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">([</span><span class="n">posixpath</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">f</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">files</span> + <span class="k">if</span> <span class="n">f</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">directories</span><span class="p">])</span> + <span class="n">directories</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">([</span><span class="n">posixpath</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">x</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span> + <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">directories</span><span class="p">])</span> + + <span class="k">yield</span> <span class="n">path</span><span class="p">,</span> <span class="n">directories</span><span class="p">,</span> <span class="n">files</span> + + <span class="k">for</span> <span class="n">directory</span> <span class="ow">in</span> <span class="n">directories</span><span class="p">:</span> + <span class="k">for</span> <span class="n">tup</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">directory</span><span class="p">,</span> <span class="n">refresh</span><span class="o">=</span><span class="n">refresh</span><span class="p">):</span> + <span class="k">yield</span> <span class="n">tup</span> +</pre></div> + + </div> + + </div> +</div> +<footer class="footer"> + <div class="container"> + <p class="pull-right"> + <a href="#">Back to top</a> + + </p> + <p> + © Copyright 2016-2017 Apache Software Foundation.<br/> + Created using <a href="http://sphinx-doc.org/";>Sphinx</a> 1.5.5.<br/> + </p> + </div> +</footer> + </body> +</html> \ No newline at end of file
