http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/mysql_hook.html ---------------------------------------------------------------------- diff --git a/_modules/mysql_hook.html b/_modules/mysql_hook.html index a8f6aea..1045e28 100644 --- a/_modules/mysql_hook.html +++ b/_modules/mysql_hook.html @@ -13,6 +13,8 @@ + + @@ -80,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -183,18 +188,22 @@ <div class="viewcode-block" id="MySqlHook"><a class="viewcode-back" href="../code.html#airflow.hooks.MySqlHook">[docs]</a><span class="k">class</span> <span class="nc">MySqlHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span> - <span class="sd">'''</span> + <span class="sd">"""</span> <span class="sd"> Interact with MySQL.</span> <span class="sd"> You can specify charset in the extra field of your connection</span> <span class="sd"> as ``{"charset": "utf8"}``. Also you can choose cursor as</span> <span class="sd"> ``{"cursor": "SSCursor"}``. Refer to the MySQLdb.cursors for more details.</span> -<span class="sd"> '''</span> +<span class="sd"> """</span> <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'mysql_conn_id'</span> <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'mysql_default'</span> <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">True</span> + <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> + <span class="nb">super</span><span class="p">(</span><span class="n">MySqlHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"schema"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> + <div class="viewcode-block" id="MySqlHook.get_conn"><a class="viewcode-back" href="../code.html#airflow.hooks.MySqlHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Returns a mysql connection object</span> @@ -202,17 +211,16 @@ <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span> <span class="n">conn_config</span> <span class="o">=</span> <span class="p">{</span> <span class="s2">"user"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> - <span class="s2">"passwd"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="s1">''</span> + <span class="s2">"passwd"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span> + <span class="s2">"host"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> <span class="ow">or</span> <span class="s1">'localhost'</span><span class="p">,</span> + <span class="s2">"db"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="s1">''</span> <span class="p">}</span> - <span class="n">conn_config</span><span class="p">[</span><span class="s2">"host"</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> <span class="ow">or</span> <span class="s1">'localhost'</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span> <span class="n">conn_config</span><span class="p">[</span><span class="s2">"port"</span><span class="p">]</span> <span class="o">=</span> <span class="mi">3306</span> <span class="k">else</span><span class="p">:</span> <span class="n">conn_config</span><span class="p">[</span><span class="s2">"port"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span> - <span class="n">conn_config</span><span class="p">[</span><span class="s2">"db"</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="s1">''</span> - <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'charset'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span> <span class="n">conn_config</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">]</span> <span class="k">if</span> <span class="p">(</span><span class="n">conn_config</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'utf8'</span> <span class="ow">or</span>\ @@ -248,17 +256,18 @@ <span class="nd">@staticmethod</span> <span class="k">def</span> <span class="nf">_serialize_cell</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">conn</span><span class="p">):</span> <span class="sd">"""</span> -<span class="sd"> Returns the MySQL literal of the cell as a string.</span> +<span class="sd"> MySQLdb converts an argument to a literal when passing those seperately to execute.</span> +<span class="sd"> Hence, this method does nothing.</span> <span class="sd"> :param cell: The cell to insert into the table</span> <span class="sd"> :type cell: object</span> <span class="sd"> :param conn: The database connection</span> <span class="sd"> :type conn: connection object</span> -<span class="sd"> :return: The serialized cell</span> -<span class="sd"> :rtype: str</span> +<span class="sd"> :return: The same cell</span> +<span class="sd"> :rtype: object</span> <span class="sd"> """</span> - <span class="k">return</span> <span class="n">conn</span><span class="o">.</span><span class="n">literal</span><span class="p">(</span><span class="n">cell</span><span class="p">)</span></div> + <span class="k">return</span> <span class="n">cell</span></div> </pre></div> </div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/mysql_operator.html ---------------------------------------------------------------------- diff --git a/_modules/mysql_operator.html b/_modules/mysql_operator.html index a60fab2..6acf146 100644 --- a/_modules/mysql_operator.html +++ b/_modules/mysql_operator.html @@ -13,6 +13,8 @@ + + @@ -80,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -175,9 +180,6 @@ <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> - -<span class="kn">import</span> <span class="nn">logging</span> - <span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="k">import</span> <span class="n">MySqlHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span> <span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span> @@ -193,6 +195,8 @@ <span class="sd"> :type sql: Can receive a str representing a sql statement,</span> <span class="sd"> a list of str (sql statements), or reference to a template file.</span> <span class="sd"> Template reference are recognized by str ending in '.sql'</span> +<span class="sd"> :param database: name of database which overwrite defined one in connection</span> +<span class="sd"> :type database: string</span> <span class="sd"> """</span> <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span> @@ -202,16 +206,18 @@ <span class="nd">@apply_defaults</span> <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span> <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">mysql_conn_id</span><span class="o">=</span><span class="s1">'mysql_default'</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> - <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="nb">super</span><span class="p">(</span><span class="n">MySqlOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">database</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> + <span class="nb">super</span><span class="p">(</span><span class="n">MySqlOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span> <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span> + <span class="bp">self</span><span class="o">.</span><span class="n">database</span> <span class="o">=</span> <span class="n">database</span> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: '</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">))</span> - <span class="n">hook</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span> + <span class="n">hook</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">,</span> + <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">)</span> <span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span><span class="p">,</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/mysql_to_hive.html ---------------------------------------------------------------------- diff --git a/_modules/mysql_to_hive.html b/_modules/mysql_to_hive.html index cc8496b..007b81c 100644 --- a/_modules/mysql_to_hive.html +++ b/_modules/mysql_to_hive.html @@ -13,6 +13,8 @@ + + @@ -30,6 +32,9 @@ + <link rel="index" title="Index" + href="../genindex.html"/> + <link rel="search" title="Search" href="../search.html"/> <link rel="top" title="Airflow Documentation" href="../index.html"/> <link rel="up" title="Module code" href="index.html"/> @@ -40,6 +45,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -76,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -90,6 +99,8 @@ <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> </ul> @@ -104,8 +115,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Airflow</a> + </nav> @@ -118,19 +131,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../index.html">Docs</a> »</li> - + + <li><a href="../index.html">Docs</a> »</li> + <li><a href="index.html">Module code</a> »</li> - - <li>mysql_to_hive</li> + + <li>mysql_to_hive</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> @@ -151,17 +181,16 @@ <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> -<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span> -<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span> -<span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span> -<span class="kn">import</span> <span class="nn">logging</span> -<span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span> +<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">chr</span> +<span class="kn">from</span> <span class="nn">collections</span> <span class="k">import</span> <span class="n">OrderedDict</span> +<span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="k">as</span> <span class="nn">csv</span> +<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span> <span class="kn">import</span> <span class="nn">MySQLdb</span> -<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span> -<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span> -<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> -<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveCliHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="k">import</span> <span class="n">MySqlHook</span> +<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span> +<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span> <div class="viewcode-block" id="MySqlToHiveTransfer"><a class="viewcode-back" href="../code.html#airflow.operators.MySqlToHiveTransfer">[docs]</a><span class="k">class</span> <span class="nc">MySqlToHiveTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -197,6 +226,8 @@ <span class="sd"> :type mysql_conn_id: str</span> <span class="sd"> :param hive_conn_id: destination hive connection</span> <span class="sd"> :type hive_conn_id: str</span> +<span class="sd"> :param tblproperties: TBLPROPERTIES of the hive table being created</span> +<span class="sd"> :type tblproperties: dict</span> <span class="sd"> """</span> <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'partition'</span><span class="p">,</span> <span class="s1">'hive_table'</span><span class="p">)</span> @@ -208,14 +239,15 @@ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">hive_table</span><span class="p">,</span> - <span class="n">create</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> - <span class="n">recreate</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> - <span class="n">partition</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> + <span class="n">create</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> + <span class="n">recreate</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> + <span class="n">partition</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="nb">chr</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">mysql_conn_id</span><span class="o">=</span><span class="s1">'mysql_default'</span><span class="p">,</span> <span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="s1">'hive_cli_default'</span><span class="p">,</span> + <span class="n">tblproperties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="nb">super</span><span class="p">(</span><span class="n">MySqlToHiveTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="nb">super</span><span class="p">(</span><span class="n">MySqlToHiveTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_table</span> <span class="o">=</span> <span class="n">hive_table</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span> <span class="o">=</span> <span class="n">partition</span> @@ -225,9 +257,10 @@ <span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span> <span class="o">=</span> <span class="n">hive_cli_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">partition</span> <span class="o">=</span> <span class="n">partition</span> <span class="ow">or</span> <span class="p">{}</span> + <span class="bp">self</span><span class="o">.</span><span class="n">tblproperties</span> <span class="o">=</span> <span class="n">tblproperties</span> <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">type_map</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">mysql_type</span><span class="p">):</span> + <span class="k">def</span> <span class="nf">type_map</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">mysql_type</span><span class="p">):</span> <span class="n">t</span> <span class="o">=</span> <span class="n">MySQLdb</span><span class="o">.</span><span class="n">constants</span><span class="o">.</span><span class="n">FIELD_TYPE</span> <span class="n">d</span> <span class="o">=</span> <span class="p">{</span> <span class="n">t</span><span class="o">.</span><span class="n">BIT</span><span class="p">:</span> <span class="s1">'INT'</span><span class="p">,</span> @@ -246,7 +279,7 @@ <span class="n">hive</span> <span class="o">=</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">)</span> <span class="n">mysql</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Dumping MySQL query results to local file"</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Dumping MySQL query results to local file"</span><span class="p">)</span> <span class="n">conn</span> <span class="o">=</span> <span class="n">mysql</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span> <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span> @@ -259,7 +292,7 @@ <span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span> <span class="n">cursor</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> <span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Loading file into Hive"</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Loading file into Hive"</span><span class="p">)</span> <span class="n">hive</span><span class="o">.</span><span class="n">load_file</span><span class="p">(</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_table</span><span class="p">,</span> @@ -267,10 +300,14 @@ <span class="n">create</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">create</span><span class="p">,</span> <span class="n">partition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">partition</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">,</span> - <span class="n">recreate</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">recreate</span><span class="p">)</span></div> + <span class="n">recreate</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">recreate</span><span class="p">,</span> + <span class="n">tblproperties</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tblproperties</span><span class="p">)</span></div> </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -303,7 +340,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/postgres_hook.html ---------------------------------------------------------------------- diff --git a/_modules/postgres_hook.html b/_modules/postgres_hook.html index 3ecfa69..b1e8237 100644 --- a/_modules/postgres_hook.html +++ b/_modules/postgres_hook.html @@ -13,6 +13,8 @@ + + @@ -30,6 +32,9 @@ + <link rel="index" title="Index" + href="../genindex.html"/> + <link rel="search" title="Search" href="../search.html"/> <link rel="top" title="Airflow Documentation" href="../index.html"/> <link rel="up" title="Module code" href="index.html"/> @@ -40,6 +45,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -76,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -90,6 +99,8 @@ <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> </ul> @@ -104,8 +115,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Airflow</a> + </nav> @@ -118,19 +131,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../index.html">Docs</a> »</li> - + + <li><a href="../index.html">Docs</a> »</li> + <li><a href="index.html">Module code</a> »</li> - - <li>postgres_hook</li> + + <li>postgres_hook</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> @@ -153,19 +183,27 @@ <span class="kn">import</span> <span class="nn">psycopg2</span> <span class="kn">import</span> <span class="nn">psycopg2.extensions</span> +<span class="kn">from</span> <span class="nn">contextlib</span> <span class="k">import</span> <span class="n">closing</span> -<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span> <div class="viewcode-block" id="PostgresHook"><a class="viewcode-back" href="../code.html#airflow.hooks.PostgresHook">[docs]</a><span class="k">class</span> <span class="nc">PostgresHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span> - <span class="sd">'''</span> + <span class="sd">"""</span> <span class="sd"> Interact with Postgres.</span> <span class="sd"> You can specify ssl parameters in the extra field of your connection</span> <span class="sd"> as ``{"sslmode": "require", "sslcert": "/path/to/cert.pem", etc}``.</span> -<span class="sd"> '''</span> + +<span class="sd"> Note: For Redshift, use keepalives_idle in the extra connection parameters</span> +<span class="sd"> and set it to less than 300 seconds.</span> +<span class="sd"> """</span> <span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'postgres_conn_id'</span> <span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'postgres_default'</span> - <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="bp">False</span> + <span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">True</span> + + <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> + <span class="nb">super</span><span class="p">(</span><span class="n">PostgresHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"schema"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">postgres_conn_id</span><span class="p">)</span> @@ -173,23 +211,51 @@ <span class="n">host</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> - <span class="n">dbname</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> + <span class="n">dbname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span> <span class="c1"># check for ssl parameters in conn.extra</span> <span class="k">for</span> <span class="n">arg_name</span><span class="p">,</span> <span class="n">arg_val</span> <span class="ow">in</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">items</span><span class="p">():</span> - <span class="k">if</span> <span class="n">arg_name</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'sslmode'</span><span class="p">,</span> <span class="s1">'sslcert'</span><span class="p">,</span> <span class="s1">'sslkey'</span><span class="p">,</span> <span class="s1">'sslrootcert'</span><span class="p">,</span> <span class="s1">'sslcrl'</span><span class="p">]:</span> + <span class="k">if</span> <span class="n">arg_name</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'sslmode'</span><span class="p">,</span> <span class="s1">'sslcert'</span><span class="p">,</span> <span class="s1">'sslkey'</span><span class="p">,</span> + <span class="s1">'sslrootcert'</span><span class="p">,</span> <span class="s1">'sslcrl'</span><span class="p">,</span> <span class="s1">'application_name'</span><span class="p">,</span> + <span class="s1">'keepalives_idle'</span><span class="p">]:</span> <span class="n">conn_args</span><span class="p">[</span><span class="n">arg_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">arg_val</span> + <span class="n">psycopg2_conn</span> <span class="o">=</span> <span class="n">psycopg2</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="o">**</span><span class="n">conn_args</span><span class="p">)</span> - <span class="k">if</span> <span class="n">psycopg2_conn</span><span class="o">.</span><span class="n">server_version</span> <span class="o"><</span> <span class="mi">70400</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">supports_autocommit</span> <span class="o">=</span> <span class="bp">True</span> <span class="k">return</span> <span class="n">psycopg2_conn</span> +<div class="viewcode-block" id="PostgresHook.copy_expert"><a class="viewcode-back" href="../code.html#airflow.hooks.PostgresHook.copy_expert">[docs]</a> <span class="k">def</span> <span class="nf">copy_expert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="nb">open</span><span class="o">=</span><span class="nb">open</span><span class="p">):</span> + <span class="sd">'''</span> +<span class="sd"> Executes SQL using psycopg2 copy_expert method</span> +<span class="sd"> Necessary to execute COPY command without access to a superuser</span> +<span class="sd"> '''</span> + <span class="n">f</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">)</span> + <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">())</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span> + <span class="k">with</span> <span class="n">closing</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">())</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span> + <span class="n">cur</span><span class="o">.</span><span class="n">copy_expert</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="n">f</span><span class="p">)</span></div> + <span class="nd">@staticmethod</span> - <span class="k">def</span> <span class="nf">_serialize_cell</span><span class="p">(</span><span class="n">cell</span><span class="p">):</span> - <span class="k">return</span> <span class="n">psycopg2</span><span class="o">.</span><span class="n">extensions</span><span class="o">.</span><span class="n">adapt</span><span class="p">(</span><span class="n">cell</span><span class="p">)</span><span class="o">.</span><span class="n">getquoted</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span></div> + <span class="k">def</span> <span class="nf">_serialize_cell</span><span class="p">(</span><span class="n">cell</span><span class="p">,</span> <span class="n">conn</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Postgresql will adapt all arguments to the execute() method internally,</span> +<span class="sd"> hence we return cell without any conversion.</span> + +<span class="sd"> See http://initd.org/psycopg/docs/advanced.html#adapting-new-types for</span> +<span class="sd"> more information.</span> + +<span class="sd"> :param cell: The cell to insert into the table</span> +<span class="sd"> :type cell: object</span> +<span class="sd"> :param conn: The database connection</span> +<span class="sd"> :type conn: connection object</span> +<span class="sd"> :return: The cell</span> +<span class="sd"> :rtype: object</span> +<span class="sd"> """</span> + <span class="k">return</span> <span class="n">cell</span></div> </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -222,7 +288,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/postgres_operator.html ---------------------------------------------------------------------- diff --git a/_modules/postgres_operator.html b/_modules/postgres_operator.html index 5ade37d..61cf4a2 100644 --- a/_modules/postgres_operator.html +++ b/_modules/postgres_operator.html @@ -13,6 +13,8 @@ + + @@ -30,6 +32,9 @@ + <link rel="index" title="Index" + href="../genindex.html"/> + <link rel="search" title="Search" href="../search.html"/> <link rel="top" title="Airflow Documentation" href="../index.html"/> <link rel="up" title="Module code" href="index.html"/> @@ -40,6 +45,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -76,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -90,6 +99,8 @@ <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> </ul> @@ -104,8 +115,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Airflow</a> + </nav> @@ -118,19 +131,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../index.html">Docs</a> »</li> - + + <li><a href="../index.html">Docs</a> »</li> + <li><a href="index.html">Module code</a> »</li> - - <li>postgres_operator</li> + + <li>postgres_operator</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> @@ -150,12 +180,9 @@ <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> - -<span class="kn">import</span> <span class="nn">logging</span> - -<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span> -<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> -<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="k">import</span> <span class="n">PostgresHook</span> +<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span> +<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span> <div class="viewcode-block" id="PostgresOperator"><a class="viewcode-back" href="../code.html#airflow.operators.PostgresOperator">[docs]</a><span class="k">class</span> <span class="nc">PostgresOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -168,6 +195,8 @@ <span class="sd"> :type sql: Can receive a str representing a sql statement,</span> <span class="sd"> a list of str (sql statements), or reference to a template file.</span> <span class="sd"> Template reference are recognized by str ending in '.sql'</span> +<span class="sd"> :param database: name of database which overwrite defined one in connection</span> +<span class="sd"> :type database: string</span> <span class="sd"> """</span> <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span> @@ -177,22 +206,28 @@ <span class="nd">@apply_defaults</span> <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span> <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> - <span class="n">postgres_conn_id</span><span class="o">=</span><span class="s1">'postgres_default'</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> - <span class="n">parameters</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> + <span class="n">postgres_conn_id</span><span class="o">=</span><span class="s1">'postgres_default'</span><span class="p">,</span> <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> + <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">database</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="nb">super</span><span class="p">(</span><span class="n">PostgresOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="nb">super</span><span class="p">(</span><span class="n">PostgresOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span> <span class="bp">self</span><span class="o">.</span><span class="n">postgres_conn_id</span> <span class="o">=</span> <span class="n">postgres_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span> + <span class="bp">self</span><span class="o">.</span><span class="n">database</span> <span class="o">=</span> <span class="n">database</span> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: '</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">))</span> - <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">postgres_conn_id</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">postgres_conn_id</span><span class="p">,</span> + <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">)</span></div> </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -225,7 +260,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/presto_check_operator.html ---------------------------------------------------------------------- diff --git a/_modules/presto_check_operator.html b/_modules/presto_check_operator.html index 325c884..58be6bb 100644 --- a/_modules/presto_check_operator.html +++ b/_modules/presto_check_operator.html @@ -13,6 +13,8 @@ + + @@ -80,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -220,7 +225,7 @@ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">'presto_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="nb">super</span><span class="p">(</span><span class="n">PrestoCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="nb">super</span><span class="p">(</span><span class="n">PrestoCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span> <span class="o">=</span> <span class="n">presto_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span> @@ -244,7 +249,7 @@ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">pass_value</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">'presto_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="nb">super</span><span class="p">(</span><span class="n">PrestoValueCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span> + <span class="nb">super</span><span class="p">(</span><span class="n">PrestoValueCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span> <span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="n">pass_value</span><span class="o">=</span><span class="n">pass_value</span><span class="p">,</span> <span class="n">tolerance</span><span class="o">=</span><span class="n">tolerance</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">presto_conn_id</span> <span class="o">=</span> <span class="n">presto_conn_id</span> @@ -275,7 +280,7 @@ <span class="n">date_filter_column</span><span class="o">=</span><span class="s1">'ds'</span><span class="p">,</span> <span class="n">days_back</span><span class="o">=-</span><span class="mi">7</span><span class="p">,</span> <span class="n">presto_conn_id</span><span class="o">=</span><span class="s1">'presto_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="nb">super</span><span class="p">(</span><span class="n">PrestoIntervalCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span> + <span class="nb">super</span><span class="p">(</span><span class="n">PrestoIntervalCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span> <span class="n">table</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">metrics_thresholds</span><span class="o">=</span><span class="n">metrics_thresholds</span><span class="p">,</span> <span class="n">date_filter_column</span><span class="o">=</span><span class="n">date_filter_column</span><span class="p">,</span> <span class="n">days_back</span><span class="o">=</span><span class="n">days_back</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/presto_hook.html ---------------------------------------------------------------------- diff --git a/_modules/presto_hook.html b/_modules/presto_hook.html index 573ddc8..717b871 100644 --- a/_modules/presto_hook.html +++ b/_modules/presto_hook.html @@ -13,6 +13,8 @@ + + @@ -80,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -177,16 +182,12 @@ <span class="c1"># limitations under the License.</span> <span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span> -<span class="kn">import</span> <span class="nn">logging</span> <span class="kn">from</span> <span class="nn">pyhive</span> <span class="k">import</span> <span class="n">presto</span> <span class="kn">from</span> <span class="nn">pyhive.exc</span> <span class="k">import</span> <span class="n">DatabaseError</span> <span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="k">import</span> <span class="n">DbApiHook</span> -<span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"pyhive"</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span> - - <span class="k">class</span> <span class="nc">PrestoException</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span> <span class="k">pass</span> @@ -218,6 +219,19 @@ <span class="k">def</span> <span class="nf">_strip_sql</span><span class="p">(</span><span class="n">sql</span><span class="p">):</span> <span class="k">return</span> <span class="n">sql</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s1">';'</span><span class="p">)</span> + <span class="k">def</span> <span class="nf">_get_pretty_exception_message</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">e</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Parses some DatabaseError to provide a better error message</span> +<span class="sd"> """</span> + <span class="k">if</span> <span class="p">(</span><span class="nb">hasattr</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="s1">'message'</span><span class="p">)</span> + <span class="ow">and</span> <span class="s1">'errorName'</span> <span class="ow">in</span> <span class="n">e</span><span class="o">.</span><span class="n">message</span> + <span class="ow">and</span> <span class="s1">'message'</span> <span class="ow">in</span> <span class="n">e</span><span class="o">.</span><span class="n">message</span><span class="p">):</span> + <span class="k">return</span> <span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">: </span><span class="si">{message}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> + <span class="n">name</span><span class="o">=</span><span class="n">e</span><span class="o">.</span><span class="n">message</span><span class="p">[</span><span class="s1">'errorName'</span><span class="p">],</span> + <span class="n">message</span><span class="o">=</span><span class="n">e</span><span class="o">.</span><span class="n">message</span><span class="p">[</span><span class="s1">'message'</span><span class="p">]))</span> + <span class="k">else</span><span class="p">:</span> + <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> + <div class="viewcode-block" id="PrestoHook.get_records"><a class="viewcode-back" href="../code.html#airflow.hooks.PrestoHook.get_records">[docs]</a> <span class="k">def</span> <span class="nf">get_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Get a set of records from Presto</span> @@ -226,14 +240,7 @@ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">get_records</span><span class="p">(</span> <span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span> <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> - <span class="k">if</span> <span class="p">(</span><span class="nb">hasattr</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="s1">'message'</span><span class="p">)</span> <span class="ow">and</span> - <span class="s1">'errorName'</span> <span class="ow">in</span> <span class="n">e</span><span class="o">.</span><span class="n">message</span> <span class="ow">and</span> - <span class="s1">'message'</span> <span class="ow">in</span> <span class="n">e</span><span class="o">.</span><span class="n">message</span><span class="p">):</span> - <span class="c1"># Use the structured error data in the raised exception</span> - <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">: </span><span class="si">{message}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> - <span class="n">name</span><span class="o">=</span><span class="n">e</span><span class="o">.</span><span class="n">message</span><span class="p">[</span><span class="s1">'errorName'</span><span class="p">],</span> <span class="n">message</span><span class="o">=</span><span class="n">e</span><span class="o">.</span><span class="n">message</span><span class="p">[</span><span class="s1">'message'</span><span class="p">]))</span> - <span class="k">else</span><span class="p">:</span> - <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div> + <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_exception_message</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div> <div class="viewcode-block" id="PrestoHook.get_first"><a class="viewcode-back" href="../code.html#airflow.hooks.PrestoHook.get_first">[docs]</a> <span class="k">def</span> <span class="nf">get_first</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="sd">"""</span> @@ -244,7 +251,7 @@ <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">PrestoHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span> <span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span> <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> - <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'message'</span><span class="p">])</span></div> + <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_exception_message</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div> <div class="viewcode-block" id="PrestoHook.get_pandas_df"><a class="viewcode-back" href="../code.html#airflow.hooks.PrestoHook.get_pandas_df">[docs]</a> <span class="k">def</span> <span class="nf">get_pandas_df</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="sd">"""</span> @@ -256,7 +263,7 @@ <span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_strip_sql</span><span class="p">(</span><span class="n">hql</span><span class="p">),</span> <span class="n">parameters</span><span class="p">)</span> <span class="n">data</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span> <span class="k">except</span> <span class="n">DatabaseError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> - <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'message'</span><span class="p">])</span> + <span class="k">raise</span> <span class="n">PrestoException</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_parse_exception_message</span><span class="p">(</span><span class="n">e</span><span class="p">))</span> <span class="n">column_descriptions</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">description</span> <span class="k">if</span> <span class="n">data</span><span class="p">:</span> <span class="n">df</span> <span class="o">=</span> <span class="n">pandas</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
