http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/hive_operator.html ---------------------------------------------------------------------- diff --git a/_modules/hive_operator.html b/_modules/hive_operator.html index 4d8c62c..b0e3265 100644 --- a/_modules/hive_operator.html +++ b/_modules/hive_operator.html @@ -13,6 +13,8 @@ + + @@ -30,6 +32,9 @@ + <link rel="index" title="Index" + href="../genindex.html"/> + <link rel="search" title="Search" href="../search.html"/> <link rel="top" title="Airflow Documentation" href="../index.html"/> <link rel="up" title="Module code" href="index.html"/> @@ -40,6 +45,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -76,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -90,6 +99,8 @@ <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> </ul> @@ -104,8 +115,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Airflow</a> + </nav> @@ -118,19 +131,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../index.html">Docs</a> »</li> - + + <li><a href="../index.html">Docs</a> »</li> + <li><a href="index.html">Module code</a> »</li> - - <li>hive_operator</li> + + <li>hive_operator</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> @@ -150,14 +180,12 @@ <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> - -<span class="kn">import</span> <span class="nn">logging</span> <span class="kn">import</span> <span class="nn">re</span> -<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span> -<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> -<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> -<span class="kn">from</span> <span class="nn">airflow.utils.operator_helpers</span> <span class="kn">import</span> <span class="n">context_to_airflow_vars</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveCliHook</span> +<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span> +<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span> +<span class="kn">from</span> <span class="nn">airflow.utils.operator_helpers</span> <span class="k">import</span> <span class="n">context_to_airflow_vars</span> <div class="viewcode-block" id="HiveOperator"><a class="viewcode-back" href="../code.html#airflow.operators.HiveOperator">[docs]</a><span class="k">class</span> <span class="nc">HiveOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -196,21 +224,21 @@ <span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span> <span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="s1">'hive_cli_default'</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s1">'default'</span><span class="p">,</span> - <span class="n">hiveconf_jinja_translate</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> - <span class="n">script_begin_tag</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> - <span class="n">run_as_owner</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> - <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> - <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> - <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> + <span class="n">hiveconf_jinja_translate</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> + <span class="n">script_begin_tag</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">run_as_owner</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> + <span class="n">mapred_queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">mapred_job_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="nb">super</span><span class="p">(</span><span class="n">HiveOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="nb">super</span><span class="p">(</span><span class="n">HiveOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">hiveconf_jinja_translate</span> <span class="o">=</span> <span class="n">hiveconf_jinja_translate</span> <span class="bp">self</span><span class="o">.</span><span class="n">hql</span> <span class="o">=</span> <span class="n">hql</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span> <span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span> <span class="o">=</span> <span class="n">hive_cli_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">script_begin_tag</span> <span class="o">=</span> <span class="n">script_begin_tag</span> - <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="bp">None</span> + <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="kc">None</span> <span class="k">if</span> <span class="n">run_as_owner</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">owner</span> @@ -234,7 +262,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">hql</span> <span class="o">=</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">script_begin_tag</span><span class="p">)[</span><span class="mi">1</span><span class="p">:])</span> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: '</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="n">hive_conf</span><span class="o">=</span><span class="n">context_to_airflow_vars</span><span class="p">(</span><span class="n">context</span><span class="p">))</span> @@ -248,6 +276,9 @@ </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -280,7 +311,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/hive_to_druid.html ---------------------------------------------------------------------- diff --git a/_modules/hive_to_druid.html b/_modules/hive_to_druid.html index d85bfb3..35dc4ff 100644 --- a/_modules/hive_to_druid.html +++ b/_modules/hive_to_druid.html @@ -13,6 +13,8 @@ + + @@ -30,6 +32,9 @@ + <link rel="index" title="Index" + href="../genindex.html"/> + <link rel="search" title="Search" href="../search.html"/> <link rel="top" title="Airflow Documentation" href="../index.html"/> <link rel="up" title="Module code" href="index.html"/> @@ -40,6 +45,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -76,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -90,6 +99,8 @@ <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> </ul> @@ -104,8 +115,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Airflow</a> + </nav> @@ -118,19 +131,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../index.html">Docs</a> »</li> - + + <li><a href="../index.html">Docs</a> »</li> + <li><a href="index.html">Module code</a> »</li> - - <li>hive_to_druid</li> + + <li>hive_to_druid</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> @@ -150,13 +180,13 @@ <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.druid_hook</span> <span class="k">import</span> <span class="n">DruidHook</span> +<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span> +<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span> -<span class="kn">import</span> <span class="nn">logging</span> - -<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span> -<span class="kn">from</span> <span class="nn">airflow.hooks.druid_hook</span> <span class="kn">import</span> <span class="n">DruidHook</span> -<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> -<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> +<span class="n">LOAD_CHECK_INTERVAL</span> <span class="o">=</span> <span class="mi">5</span> +<span class="n">DEFAULT_TARGET_PARTITION_SIZE</span> <span class="o">=</span> <span class="mi">5000000</span> <div class="viewcode-block" id="HiveToDruidTransfer"><a class="viewcode-back" href="../code.html#airflow.operators.HiveToDruidTransfer">[docs]</a><span class="k">class</span> <span class="nc">HiveToDruidTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -189,7 +219,6 @@ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'intervals'</span><span class="p">)</span> <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span> - <span class="c1">#ui_color = '#a0e08c'</span> <span class="nd">@apply_defaults</span> <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span> @@ -197,22 +226,26 @@ <span class="n">sql</span><span class="p">,</span> <span class="n">druid_datasource</span><span class="p">,</span> <span class="n">ts_dim</span><span class="p">,</span> - <span class="n">metric_spec</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> + <span class="n">metric_spec</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="s1">'hive_cli_default'</span><span class="p">,</span> <span class="n">druid_ingest_conn_id</span><span class="o">=</span><span class="s1">'druid_ingest_default'</span><span class="p">,</span> <span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">,</span> - <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> - <span class="n">intervals</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> + <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">intervals</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span> + <span class="n">query_granularity</span><span class="o">=</span><span class="s2">"NONE"</span><span class="p">,</span> + <span class="n">segment_granularity</span><span class="o">=</span><span class="s2">"DAY"</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="nb">super</span><span class="p">(</span><span class="n">HiveToDruidTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="nb">super</span><span class="p">(</span><span class="n">HiveToDruidTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span> <span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span> <span class="o">=</span> <span class="n">druid_datasource</span> <span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span> <span class="o">=</span> <span class="n">ts_dim</span> <span class="bp">self</span><span class="o">.</span><span class="n">intervals</span> <span class="o">=</span> <span class="n">intervals</span> <span class="ow">or</span> <span class="p">[</span><span class="s1">'{{ ds }}/{{ tomorrow_ds }}'</span><span class="p">]</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span> <span class="o">=</span> <span class="n">num_shards</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span> <span class="o">=</span> <span class="n">target_partition_size</span> + <span class="bp">self</span><span class="o">.</span><span class="n">query_granularity</span> <span class="o">=</span> <span class="n">query_granularity</span> + <span class="bp">self</span><span class="o">.</span><span class="n">segment_granularity</span> <span class="o">=</span> <span class="n">segment_granularity</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span> <span class="o">=</span> <span class="n">metric_spec</span> <span class="ow">or</span> <span class="p">[{</span> <span class="s2">"name"</span><span class="p">:</span> <span class="s2">"count"</span><span class="p">,</span> <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"count"</span><span class="p">}]</span> @@ -223,28 +256,30 @@ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hive</span> <span class="o">=</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from Hive"</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from Hive"</span><span class="p">)</span> <span class="n">hive_table</span> <span class="o">=</span> <span class="s1">'druid.'</span> <span class="o">+</span> <span class="n">context</span><span class="p">[</span><span class="s1">'task_instance_key_str'</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">)</span> <span class="n">sql</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">';'</span><span class="p">)</span> <span class="n">hql</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span> -<span class="s2"> set mapred.output.compress=false;</span> -<span class="s2"> set hive.exec.compress.output=false;</span> -<span class="s2"> DROP TABLE IF EXISTS {hive_table};</span> -<span class="s2"> CREATE TABLE {hive_table}</span> +<span class="s2"> SET mapred.output.compress=false;</span> +<span class="s2"> SET hive.exec.compress.output=false;</span> +<span class="s2"> DROP TABLE IF EXISTS </span><span class="si">{hive_table}</span><span class="s2">;</span> +<span class="s2"> CREATE TABLE </span><span class="si">{hive_table}</span><span class="s2"></span> <span class="s2"> ROW FORMAT DELIMITED FIELDS TERMINATED BY '</span><span class="se">\t</span><span class="s2">'</span> <span class="s2"> STORED AS TEXTFILE</span> <span class="s2"> TBLPROPERTIES ('serialization.null.format' = '')</span> <span class="s2"> AS</span> -<span class="s2"> {sql}</span> +<span class="s2"> </span><span class="si">{sql}</span><span class="s2"></span> <span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command:</span><span class="se">\n</span><span class="s2"> {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hql</span><span class="p">))</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command:</span><span class="se">\n</span><span class="s2"> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">hql</span><span class="p">)</span> <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span> <span class="n">m</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span> - <span class="n">t</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">get_table</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span> + <span class="c1"># Get the Hive table and extract the columns</span> + <span class="n">t</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">get_table</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span> <span class="n">columns</span> <span class="o">=</span> <span class="p">[</span><span class="n">col</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">t</span><span class="o">.</span><span class="n">sd</span><span class="o">.</span><span class="n">cols</span><span class="p">]</span> + <span class="c1"># Get the path on hdfs</span> <span class="n">hdfs_uri</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">get_table</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span><span class="o">.</span><span class="n">sd</span><span class="o">.</span><span class="n">location</span> <span class="n">pos</span> <span class="o">=</span> <span class="n">hdfs_uri</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'/user'</span><span class="p">)</span> <span class="n">static_path</span> <span class="o">=</span> <span class="n">hdfs_uri</span><span class="p">[</span><span class="n">pos</span><span class="p">:]</span> @@ -252,26 +287,113 @@ <span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">hive_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span> <span class="n">druid</span> <span class="o">=</span> <span class="n">DruidHook</span><span class="p">(</span><span class="n">druid_ingest_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_ingest_conn_id</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into Druid"</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"HDFS path: "</span> <span class="o">+</span> <span class="n">static_path</span><span class="p">)</span> <span class="k">try</span><span class="p">:</span> - <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span> - <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span> - <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span> - <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span> - <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span> - <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Load seems to have succeeded!"</span><span class="p">)</span> + <span class="n">index_spec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">construct_ingest_query</span><span class="p">(</span> + <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> + <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> + <span class="p">)</span> + + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into Druid, hdfs path: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">static_path</span><span class="p">)</span> + + <span class="n">druid</span><span class="o">.</span><span class="n">submit_indexing_job</span><span class="p">(</span><span class="n">index_spec</span><span class="p">)</span> + + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Load seems to have succeeded!"</span><span class="p">)</span> <span class="k">finally</span><span class="p">:</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> - <span class="s2">"Cleaning up by dropping the temp "</span> - <span class="s2">"Hive table {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span> - <span class="n">hql</span> <span class="o">=</span> <span class="s2">"DROP TABLE IF EXISTS {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span> - <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> + <span class="s2">"Cleaning up by dropping the temp Hive table </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> + <span class="n">hive_table</span> + <span class="p">)</span> + <span class="n">hql</span> <span class="o">=</span> <span class="s2">"DROP TABLE IF EXISTS </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span> + <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span> + +<div class="viewcode-block" id="HiveToDruidTransfer.construct_ingest_query"><a class="viewcode-back" href="../code.html#airflow.operators.HiveToDruidTransfer.construct_ingest_query">[docs]</a> <span class="k">def</span> <span class="nf">construct_ingest_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">static_path</span><span class="p">,</span> <span class="n">columns</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Builds an ingest query for an HDFS TSV load.</span> + +<span class="sd"> :param static_path: The path on hdfs where the data is</span> +<span class="sd"> :type static_path: str</span> +<span class="sd"> :param columns: List of all the columns that are available</span> +<span class="sd"> :type columns: list</span> +<span class="sd"> """</span> + + <span class="c1"># backward compatibilty for num_shards, but target_partition_size is the default setting</span> + <span class="c1"># and overwrites the num_shards</span> + <span class="n">num_shards</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span> + <span class="n">target_partition_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span> + <span class="n">target_partition_size</span> <span class="o">=</span> <span class="n">DEFAULT_TARGET_PARTITION_SIZE</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">num_shards</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span> + + <span class="n">metric_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">m</span><span class="p">[</span><span class="s1">'fieldName'</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span> <span class="k">if</span> <span class="n">m</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'count'</span><span class="p">]</span> + + <span class="c1"># Take all the columns, which are not the time dimension or a metric, as the dimension columns</span> + <span class="n">dimensions</span> <span class="o">=</span> <span class="p">[</span><span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">columns</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">metric_names</span> <span class="ow">and</span> <span class="n">c</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">]</span> + + <span class="n">ingest_query_dict</span> <span class="o">=</span> <span class="p">{</span> + <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"index_hadoop"</span><span class="p">,</span> + <span class="s2">"spec"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"dataSchema"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"metricsSpec"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> + <span class="s2">"granularitySpec"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"queryGranularity"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">query_granularity</span><span class="p">,</span> + <span class="s2">"intervals"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span> + <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"uniform"</span><span class="p">,</span> + <span class="s2">"segmentGranularity"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">segment_granularity</span><span class="p">,</span> + <span class="p">},</span> + <span class="s2">"parser"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"string"</span><span class="p">,</span> + <span class="s2">"parseSpec"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"columns"</span><span class="p">:</span> <span class="n">columns</span><span class="p">,</span> + <span class="s2">"dimensionsSpec"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"dimensionExclusions"</span><span class="p">:</span> <span class="p">[],</span> + <span class="s2">"dimensions"</span><span class="p">:</span> <span class="n">dimensions</span><span class="p">,</span> <span class="c1"># list of names</span> + <span class="s2">"spatialDimensions"</span><span class="p">:</span> <span class="p">[]</span> + <span class="p">},</span> + <span class="s2">"timestampSpec"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"column"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span> + <span class="s2">"format"</span><span class="p">:</span> <span class="s2">"auto"</span> + <span class="p">},</span> + <span class="s2">"format"</span><span class="p">:</span> <span class="s2">"tsv"</span> + <span class="p">}</span> + <span class="p">},</span> + <span class="s2">"dataSource"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span> + <span class="p">},</span> + <span class="s2">"tuningConfig"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"hadoop"</span><span class="p">,</span> + <span class="s2">"jobProperties"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"mapreduce.job.user.classpath.first"</span><span class="p">:</span> <span class="s2">"false"</span><span class="p">,</span> + <span class="s2">"mapreduce.map.output.compress"</span><span class="p">:</span> <span class="s2">"false"</span><span class="p">,</span> + <span class="s2">"mapreduce.output.fileoutputformat.compress"</span><span class="p">:</span> <span class="s2">"false"</span><span class="p">,</span> + <span class="p">},</span> + <span class="s2">"partitionsSpec"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"hashed"</span><span class="p">,</span> + <span class="s2">"targetPartitionSize"</span><span class="p">:</span> <span class="n">target_partition_size</span><span class="p">,</span> + <span class="s2">"numShards"</span><span class="p">:</span> <span class="n">num_shards</span><span class="p">,</span> + <span class="p">},</span> + <span class="p">},</span> + <span class="s2">"ioConfig"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"inputSpec"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"paths"</span><span class="p">:</span> <span class="n">static_path</span><span class="p">,</span> + <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"static"</span> + <span class="p">},</span> + <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"hadoop"</span> + <span class="p">}</span> + <span class="p">}</span> + <span class="p">}</span> + + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">:</span> + <span class="n">ingest_query_dict</span><span class="p">[</span><span class="s1">'hadoopDependencyCoordinates'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span> + + <span class="k">return</span> <span class="n">ingest_query_dict</span></div></div> </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -304,7 +426,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/hive_to_mysql.html ---------------------------------------------------------------------- diff --git a/_modules/hive_to_mysql.html b/_modules/hive_to_mysql.html index fd8b342..6262205 100644 --- a/_modules/hive_to_mysql.html +++ b/_modules/hive_to_mysql.html @@ -13,6 +13,8 @@ + + @@ -30,6 +32,9 @@ + <link rel="index" title="Index" + href="../genindex.html"/> + <link rel="search" title="Search" href="../search.html"/> <link rel="top" title="Airflow Documentation" href="../index.html"/> <link rel="up" title="Module code" href="index.html"/> @@ -40,6 +45,7 @@ <body class="wy-body-for-nav" role="document"> + <div class="wy-grid-for-nav"> @@ -76,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -90,6 +99,8 @@ <li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> <li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> <li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li> +<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li> <li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> <li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> </ul> @@ -104,8 +115,10 @@ <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> - <i data-toggle="wy-nav-top" class="fa fa-bars"></i> - <a href="../index.html">Airflow</a> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Airflow</a> + </nav> @@ -118,19 +131,36 @@ + + + + + + + + + + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> - <li><a href="../index.html">Docs</a> »</li> - + + <li><a href="../index.html">Docs</a> »</li> + <li><a href="index.html">Module code</a> »</li> - - <li>hive_to_mysql</li> + + <li>hive_to_mysql</li> + + <li class="wy-breadcrumbs-aside"> - + </li> + </ul> + + <hr/> </div> <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> @@ -150,15 +180,12 @@ <span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span> <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveServer2Hook</span> +<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="k">import</span> <span class="n">MySqlHook</span> +<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span> +<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span> -<span class="kn">import</span> <span class="nn">logging</span> - -<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span> -<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span> -<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> -<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> - -<span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span> +<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span> <div class="viewcode-block" id="HiveToMySqlTransfer"><a class="viewcode-back" href="../code.html#airflow.operators.HiveToMySqlTransfer">[docs]</a><span class="k">class</span> <span class="nc">HiveToMySqlTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -204,11 +231,11 @@ <span class="n">mysql_table</span><span class="p">,</span> <span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="s1">'hiveserver2_default'</span><span class="p">,</span> <span class="n">mysql_conn_id</span><span class="o">=</span><span class="s1">'mysql_default'</span><span class="p">,</span> - <span class="n">mysql_preoperator</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> - <span class="n">mysql_postoperator</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> - <span class="n">bulk_load</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> + <span class="n">mysql_preoperator</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">mysql_postoperator</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">bulk_load</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="nb">super</span><span class="p">(</span><span class="n">HiveToMySqlTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="nb">super</span><span class="p">(</span><span class="n">HiveToMySqlTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span> <span class="bp">self</span><span class="o">.</span><span class="n">mysql_table</span> <span class="o">=</span> <span class="n">mysql_table</span> <span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span> @@ -219,22 +246,21 @@ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hive</span> <span class="o">=</span> <span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hiveserver2_conn_id</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from Hive"</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from Hive: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bulk_load</span><span class="p">:</span> <span class="n">tmpfile</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">()</span> <span class="n">hive</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span> <span class="n">tmpfile</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">'</span><span class="se">\t</span><span class="s1">'</span><span class="p">,</span> - <span class="n">lineterminator</span><span class="o">=</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span> <span class="n">output_header</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span> + <span class="n">lineterminator</span><span class="o">=</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span> <span class="n">output_header</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> <span class="n">results</span> <span class="o">=</span> <span class="n">hive</span><span class="o">.</span><span class="n">get_records</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span> <span class="n">mysql</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mysql_preoperator</span><span class="p">:</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running MySQL preoperator"</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running MySQL preoperator"</span><span class="p">)</span> <span class="n">mysql</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_preoperator</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into MySQL"</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into MySQL"</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bulk_load</span><span class="p">:</span> <span class="n">mysql</span><span class="o">.</span><span class="n">bulk_load</span><span class="p">(</span><span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_table</span><span class="p">,</span> <span class="n">tmp_file</span><span class="o">=</span><span class="n">tmpfile</span><span class="o">.</span><span class="n">name</span><span class="p">)</span> @@ -243,13 +269,16 @@ <span class="n">mysql</span><span class="o">.</span><span class="n">insert_rows</span><span class="p">(</span><span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_table</span><span class="p">,</span> <span class="n">rows</span><span class="o">=</span><span class="n">results</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mysql_postoperator</span><span class="p">:</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running MySQL postoperator"</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running MySQL postoperator"</span><span class="p">)</span> <span class="n">mysql</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_postoperator</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done."</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done."</span><span class="p">)</span></div> </pre></div> </div> + <div class="articleComments"> + + </div> </div> <footer> @@ -282,7 +311,8 @@ VERSION:'', COLLAPSE_INDEX:false, FILE_SUFFIX:'.html', - HAS_SOURCE: true + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' }; </script> <script type="text/javascript" src="../_static/jquery.js"></script> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/http_hook.html ---------------------------------------------------------------------- diff --git a/_modules/http_hook.html b/_modules/http_hook.html index 5458ec1..ec71e54 100644 --- a/_modules/http_hook.html +++ b/_modules/http_hook.html @@ -13,6 +13,8 @@ + + @@ -80,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -177,7 +182,6 @@ <span class="c1"># limitations under the License.</span> <span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span> -<span class="kn">import</span> <span class="nn">logging</span> <span class="kn">import</span> <span class="nn">requests</span> @@ -201,9 +205,13 @@ <span class="sd"> """</span> <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span><span class="p">)</span> <span class="n">session</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span> - <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> - <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'http'</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="s1">'http://'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> + + <span class="k">if</span> <span class="s2">"://"</span> <span class="ow">in</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> + <span class="k">else</span><span class="p">:</span> + <span class="c1"># schema defaults to HTTP</span> + <span class="n">schema</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="k">else</span> <span class="s2">"http"</span> + <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">schema</span> <span class="o">+</span> <span class="s2">"://"</span> <span class="o">+</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"/"</span> @@ -230,6 +238,11 @@ <span class="n">url</span><span class="p">,</span> <span class="n">params</span><span class="o">=</span><span class="n">data</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span> + <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s1">'HEAD'</span><span class="p">:</span> + <span class="c1"># HEAD doesn't use params</span> + <span class="n">req</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Request</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> + <span class="n">url</span><span class="p">,</span> + <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span> <span class="k">else</span><span class="p">:</span> <span class="c1"># Others use data</span> <span class="n">req</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Request</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> @@ -238,7 +251,7 @@ <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span> <span class="n">prepped_request</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">prepare_request</span><span class="p">(</span><span class="n">req</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Sending '"</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">+</span> <span class="s2">"' to url: "</span> <span class="o">+</span> <span class="n">url</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Sending '</span><span class="si">%s</span><span class="s2">' to url: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_and_check</span><span class="p">(</span><span class="n">session</span><span class="p">,</span> <span class="n">prepped_request</span><span class="p">,</span> <span class="n">extra_options</span><span class="p">)</span></div> <div class="viewcode-block" id="HttpHook.run_and_check"><a class="viewcode-back" href="../code.html#airflow.hooks.HttpHook.run_and_check">[docs]</a> <span class="k">def</span> <span class="nf">run_and_check</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">prepped_request</span><span class="p">,</span> <span class="n">extra_options</span><span class="p">):</span> @@ -263,13 +276,13 @@ <span class="c1"># Tried rewrapping, but not supported. This way, it's possible</span> <span class="c1"># to get reason and code for failure by checking first 3 chars</span> <span class="c1"># for the code, or do a split on ':'</span> - <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"HTTP error: "</span> <span class="o">+</span> <span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span> - <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">!=</span> <span class="s1">'GET'</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"HTTP error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'GET'</span><span class="p">,</span> <span class="s1">'HEAD'</span><span class="p">):</span> <span class="c1"># The sensor uses GET, so this prevents filling up the log</span> <span class="c1"># with the body every time the GET 'misses'.</span> <span class="c1"># That's ok to do, because GETs should be repeatable and</span> <span class="c1"># all data should be visible in the log (no post data)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">)</span><span class="o">+</span><span class="s2">":"</span><span class="o">+</span><span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span> <span class="k">return</span> <span class="n">response</span></div></div> </pre></div> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/http_operator.html ---------------------------------------------------------------------- diff --git a/_modules/http_operator.html b/_modules/http_operator.html index a22b710..8a42e96 100644 --- a/_modules/http_operator.html +++ b/_modules/http_operator.html @@ -13,6 +13,8 @@ + + @@ -80,7 +82,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -176,8 +181,6 @@ <span class="c1"># See the License for the specific language governing permissions and</span> <span class="c1"># limitations under the License.</span> -<span class="kn">import</span> <span class="nn">logging</span> - <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span> <span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="k">import</span> <span class="n">HttpHook</span> <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span> @@ -209,7 +212,7 @@ <span class="sd"> depends on the option that's being modified.</span> <span class="sd"> """</span> - <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span><span class="s1">'data'</span><span class="p">,)</span> + <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span> <span class="s1">'data'</span><span class="p">,)</span> <span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span> <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f4a460'</span> @@ -227,7 +230,7 @@ <span class="sd"> If xcom_push is True, response of an HTTP request will also</span> <span class="sd"> be pushed to an XCom.</span> <span class="sd"> """</span> - <span class="nb">super</span><span class="p">(</span><span class="n">SimpleHttpOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="nb">super</span><span class="p">(</span><span class="n">SimpleHttpOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="n">method</span> <span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span> <span class="o">=</span> <span class="n">endpoint</span> @@ -239,7 +242,9 @@ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">http</span> <span class="o">=</span> <span class="n">HttpHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span><span class="p">)</span> - <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Calling HTTP method"</span><span class="p">)</span> + + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Calling HTTP method"</span><span class="p">)</span> + <span class="n">response</span> <span class="o">=</span> <span class="n">http</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">,</span> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/index.html ---------------------------------------------------------------------- diff --git a/_modules/index.html b/_modules/index.html index b1faeb5..6bce09a 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -13,6 +13,8 @@ + + @@ -79,7 +81,10 @@ - <ul> + + + + <ul> <li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> <li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> <li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> @@ -159,32 +164,72 @@ <div itemprop="articleBody"> <h1>All modules for which code is available</h1> -<ul><li><a href="airflow/contrib/executors/mesos_executor.html">airflow.contrib.executors.mesos_executor</a></li> +<ul><li><a href="S3_hook.html">S3_hook</a></li> +<li><a href="airflow/contrib/executors/mesos_executor.html">airflow.contrib.executors.mesos_executor</a></li> +<li><a href="airflow/contrib/hooks/bigquery_hook.html">airflow.contrib.hooks.bigquery_hook</a></li> +<li><a href="airflow/contrib/hooks/datastore_hook.html">airflow.contrib.hooks.datastore_hook</a></li> +<li><a href="airflow/contrib/hooks/emr_hook.html">airflow.contrib.hooks.emr_hook</a></li> +<li><a href="airflow/contrib/hooks/gcp_dataflow_hook.html">airflow.contrib.hooks.gcp_dataflow_hook</a></li> +<li><a href="airflow/contrib/hooks/gcp_mlengine_hook.html">airflow.contrib.hooks.gcp_mlengine_hook</a></li> +<li><a href="airflow/contrib/hooks/gcs_hook.html">airflow.contrib.hooks.gcs_hook</a></li> +<li><a href="airflow/contrib/hooks/wasb_hook.html">airflow.contrib.hooks.wasb_hook</a></li> +<li><a href="airflow/contrib/operators/bigquery_check_operator.html">airflow.contrib.operators.bigquery_check_operator</a></li> +<li><a href="airflow/contrib/operators/bigquery_operator.html">airflow.contrib.operators.bigquery_operator</a></li> +<li><a href="airflow/contrib/operators/bigquery_to_bigquery.html">airflow.contrib.operators.bigquery_to_bigquery</a></li> +<li><a href="airflow/contrib/operators/bigquery_to_gcs.html">airflow.contrib.operators.bigquery_to_gcs</a></li> +<li><a href="airflow/contrib/operators/databricks_operator.html">airflow.contrib.operators.databricks_operator</a></li> +<li><a href="airflow/contrib/operators/dataflow_operator.html">airflow.contrib.operators.dataflow_operator</a></li> +<li><a href="airflow/contrib/operators/dataproc_operator.html">airflow.contrib.operators.dataproc_operator</a></li> +<li><a href="airflow/contrib/operators/ecs_operator.html">airflow.contrib.operators.ecs_operator</a></li> +<li><a href="airflow/contrib/operators/emr_add_steps_operator.html">airflow.contrib.operators.emr_add_steps_operator</a></li> +<li><a href="airflow/contrib/operators/emr_create_job_flow_operator.html">airflow.contrib.operators.emr_create_job_flow_operator</a></li> +<li><a href="airflow/contrib/operators/emr_terminate_job_flow_operator.html">airflow.contrib.operators.emr_terminate_job_flow_operator</a></li> +<li><a href="airflow/contrib/operators/file_to_wasb.html">airflow.contrib.operators.file_to_wasb</a></li> +<li><a href="airflow/contrib/operators/gcs_download_operator.html">airflow.contrib.operators.gcs_download_operator</a></li> +<li><a href="airflow/contrib/operators/gcs_to_bq.html">airflow.contrib.operators.gcs_to_bq</a></li> <li><a href="airflow/contrib/operators/hipchat_operator.html">airflow.contrib.operators.hipchat_operator</a></li> +<li><a href="airflow/contrib/operators/mlengine_operator.html">airflow.contrib.operators.mlengine_operator</a></li> +<li><a href="airflow/contrib/sensors/wasb_sensor.html">airflow.contrib.sensors.wasb_sensor</a></li> <li><a href="airflow/executors/local_executor.html">airflow.executors.local_executor</a></li> <li><a href="airflow/executors/sequential_executor.html">airflow.executors.sequential_executor</a></li> +<li><a href="airflow/hooks/S3_hook.html">airflow.hooks.S3_hook</a></li> <li><a href="airflow/macros.html">airflow.macros</a></li> <ul><li><a href="airflow/macros/hive.html">airflow.macros.hive</a></li> </ul><li><a href="airflow/models.html">airflow.models</a></li> +<li><a href="airflow/operators/docker_operator.html">airflow.operators.docker_operator</a></li> +<li><a href="airflow/operators/redshift_to_s3_operator.html">airflow.operators.redshift_to_s3_operator</a></li> +<li><a href="airflow/operators/s3_file_transform_operator.html">airflow.operators.s3_file_transform_operator</a></li> +<li><a href="airflow/operators/s3_to_hive_operator.html">airflow.operators.s3_to_hive_operator</a></li> <li><a href="airflow/operators/sensors.html">airflow.operators.sensors</a></li> <li><a href="bash_operator.html">bash_operator</a></li> +<li><a href="bigquery_hook.html">bigquery_hook</a></li> <li><a href="dagrun_operator.html">dagrun_operator</a></li> <li><a href="dbapi_hook.html">dbapi_hook</a></li> +<li><a href="druid_hook.html">druid_hook</a></li> <li><a href="dummy_operator.html">dummy_operator</a></li> <li><a href="email_operator.html">email_operator</a></li> <li><a href="ftp_hook.html">ftp_hook</a></li> +<li><a href="gcs_hook.html">gcs_hook</a></li> <li><a href="generic_transfer.html">generic_transfer</a></li> +<li><a href="hive_hooks.html">hive_hooks</a></li> +<li><a href="hive_operator.html">hive_operator</a></li> +<li><a href="hive_to_druid.html">hive_to_druid</a></li> +<li><a href="hive_to_mysql.html">hive_to_mysql</a></li> <li><a href="http_hook.html">http_hook</a></li> <li><a href="http_operator.html">http_operator</a></li> <li><a href="mysql_hook.html">mysql_hook</a></li> <li><a href="mysql_operator.html">mysql_operator</a></li> +<li><a href="mysql_to_hive.html">mysql_to_hive</a></li> +<li><a href="postgres_hook.html">postgres_hook</a></li> +<li><a href="postgres_operator.html">postgres_operator</a></li> <li><a href="presto_check_operator.html">presto_check_operator</a></li> <li><a href="presto_hook.html">presto_hook</a></li> <li><a href="python_operator.html">python_operator</a></li> +<li><a href="qubole_operator.html">qubole_operator</a></li> +<li><a href="s3_to_hive_operator.html">s3_to_hive_operator</a></li> <li><a href="sensors.html">sensors</a></li> +<li><a href="slack_operator.html">slack_operator</a></li> <li><a href="sqlite_hook.html">sqlite_hook</a></li> -<li><a href="ssh_execute_operator.html">ssh_execute_operator</a></li> -<li><a href="ssh_hook.html">ssh_hook</a></li> </ul> </div>
