http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/segment_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/segment_hook.html b/_modules/airflow/contrib/hooks/segment_hook.html index ccb8fd0..fb4396f 100644 --- a/_modules/airflow/contrib/hooks/segment_hook.html +++ b/_modules/airflow/contrib/hooks/segment_hook.html @@ -281,20 +281,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> +
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/sftp_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/sftp_hook.html b/_modules/airflow/contrib/hooks/sftp_hook.html index efc5485..84a9fe8 100644 --- a/_modules/airflow/contrib/hooks/sftp_hook.html +++ b/_modules/airflow/contrib/hooks/sftp_hook.html @@ -182,11 +182,14 @@ <span class="kn">import</span> <span class="nn">pysftp</span> <span class="kn">import</span> <span class="nn">logging</span> <span class="kn">import</span> <span class="nn">datetime</span> -<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span> +<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.ssh_hook</span> <span class="k">import</span> <span class="n">SSHHook</span> -<div class="viewcode-block" id="SFTPHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook">[docs]</a><span class="k">class</span> <span class="nc">SFTPHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span> +<div class="viewcode-block" id="SFTPHook"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook">[docs]</a><span class="k">class</span> <span class="nc">SFTPHook</span><span class="p">(</span><span class="n">SSHHook</span><span class="p">):</span> <span class="sd">"""</span> +<span class="sd"> This hook is inherited from SSH hook. Please refer to SSH hook for the input</span> +<span class="sd"> arguments.</span> + <span class="sd"> Interact with SFTP. Aims to be interchangeable with FTPHook.</span> <span class="sd"> Pitfalls: - In contrast with FTPHook describe_directory only returns size, type and</span> @@ -200,32 +203,74 @@ <span class="sd"> Errors that may occur throughout but should be handled downstream.</span> <span class="sd"> """</span> - <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ftp_conn_id</span><span class="o">=</span><span class="s1">'sftp_default'</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span> <span class="o">=</span> <span class="n">ftp_conn_id</span> + <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ftp_conn_id</span><span class="o">=</span><span class="s1">'sftp_default'</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> + <span class="n">kwargs</span><span class="p">[</span><span class="s1">'ssh_conn_id'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ftp_conn_id</span> + <span class="nb">super</span><span class="p">(</span><span class="n">SFTPHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="kc">None</span> + <span class="bp">self</span><span class="o">.</span><span class="n">private_key_pass</span> <span class="o">=</span> <span class="kc">None</span> + + <span class="c1"># Fail for unverified hosts, unless this is explicitly allowed</span> + <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">False</span> + + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span> + <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">extra_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span> + <span class="k">if</span> <span class="s1">'private_key_pass'</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">private_key_pass</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'private_key_pass'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> + + <span class="c1"># For backward compatibility</span> + <span class="c1"># TODO: remove in Airflow 2.1</span> + <span class="kn">import</span> <span class="nn">warnings</span> + <span class="k">if</span> <span class="s1">'ignore_hostkey_verification'</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span> + <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> + <span class="s1">'Extra option `ignore_hostkey_verification` is deprecated.'</span> + <span class="s1">'Please use `no_host_key_check` instead.'</span> + <span class="s1">'This option will be removed in Airflow 2.1'</span><span class="p">,</span> + <span class="ne">DeprecationWarning</span><span class="p">,</span> + <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> + <span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span> + <span class="n">extra_options</span><span class="p">[</span><span class="s1">'ignore_hostkey_verification'</span><span class="p">]</span> + <span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'true'</span> + + <span class="k">if</span> <span class="s1">'no_host_key_check'</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span> + <span class="n">extra_options</span><span class="p">[</span><span class="s1">'no_host_key_check'</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'true'</span> + + <span class="k">if</span> <span class="s1">'private_key'</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span> + <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span> + <span class="s1">'Extra option `private_key` is deprecated.'</span> + <span class="s1">'Please use `key_file` instead.'</span> + <span class="s1">'This option will be removed in Airflow 2.1'</span><span class="p">,</span> + <span class="ne">DeprecationWarning</span><span class="p">,</span> + <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> + <span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'private_key'</span><span class="p">)</span> <div class="viewcode-block" id="SFTPHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.sftp_hook.SFTPHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="sd">"""</span> <span class="sd"> Returns an SFTP connection object</span> <span class="sd"> """</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> - <span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ftp_conn_id</span><span class="p">)</span> <span class="n">cnopts</span> <span class="o">=</span> <span class="n">pysftp</span><span class="o">.</span><span class="n">CnOpts</span><span class="p">()</span> - <span class="k">if</span> <span class="p">(</span><span class="s1">'ignore_hostkey_verification'</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span> <span class="ow">and</span> - <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'ignore_hostkey_verification'</span><span class="p">]):</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span><span class="p">:</span> <span class="n">cnopts</span><span class="o">.</span><span class="n">hostkeys</span> <span class="o">=</span> <span class="kc">None</span> + <span class="n">cnopts</span><span class="o">.</span><span class="n">compression</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span> - <span class="s1">'host'</span><span class="p">:</span> <span class="n">params</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> - <span class="s1">'port'</span><span class="p">:</span> <span class="n">params</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> - <span class="s1">'username'</span><span class="p">:</span> <span class="n">params</span><span class="o">.</span><span class="n">login</span><span class="p">,</span> + <span class="s1">'host'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> + <span class="s1">'port'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> + <span class="s1">'username'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> <span class="s1">'cnopts'</span><span class="p">:</span> <span class="n">cnopts</span> <span class="p">}</span> - <span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">password</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> - <span class="n">conn_params</span><span class="p">[</span><span class="s1">'password'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">password</span> - <span class="k">if</span> <span class="s1">'private_key'</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span> - <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span> - <span class="k">if</span> <span class="s1">'private_key_pass'</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">:</span> - <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key_pass'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s1">'private_key_pass'</span><span class="p">]</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span> + <span class="n">conn_params</span><span class="p">[</span><span class="s1">'password'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">:</span> + <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">private_key_pass</span><span class="p">:</span> + <span class="n">conn_params</span><span class="p">[</span><span class="s1">'private_key_pass'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">private_key_pass</span> + <span class="bp">self</span><span class="o">.</span><span class="n">conn</span> <span class="o">=</span> <span class="n">pysftp</span><span class="o">.</span><span class="n">Connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn</span></div> @@ -358,20 +403,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/slack_webhook_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/slack_webhook_hook.html b/_modules/airflow/contrib/hooks/slack_webhook_hook.html index 8112c68..4ac5d9b 100644 --- a/_modules/airflow/contrib/hooks/slack_webhook_hook.html +++ b/_modules/airflow/contrib/hooks/slack_webhook_hook.html @@ -318,20 +318,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/snowflake_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/snowflake_hook.html b/_modules/airflow/contrib/hooks/snowflake_hook.html index aa15463..757efeb 100644 --- a/_modules/airflow/contrib/hooks/snowflake_hook.html +++ b/_modules/airflow/contrib/hooks/snowflake_hook.html @@ -287,20 +287,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/spark_jdbc_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/spark_jdbc_hook.html b/_modules/airflow/contrib/hooks/spark_jdbc_hook.html index f6fe261..1af0775 100644 --- a/_modules/airflow/contrib/hooks/spark_jdbc_hook.html +++ b/_modules/airflow/contrib/hooks/spark_jdbc_hook.html @@ -435,20 +435,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/spark_sql_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/spark_sql_hook.html b/_modules/airflow/contrib/hooks/spark_sql_hook.html index ba78c7b..95e9e4b 100644 --- a/_modules/airflow/contrib/hooks/spark_sql_hook.html +++ b/_modules/airflow/contrib/hooks/spark_sql_hook.html @@ -351,20 +351,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/spark_submit_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/spark_submit_hook.html b/_modules/airflow/contrib/hooks/spark_submit_hook.html index 9e47eef..db35e88 100644 --- a/_modules/airflow/contrib/hooks/spark_submit_hook.html +++ b/_modules/airflow/contrib/hooks/spark_submit_hook.html @@ -756,20 +756,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/sqoop_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/sqoop_hook.html b/_modules/airflow/contrib/hooks/sqoop_hook.html index 9c13d5f..b6acb04 100644 --- a/_modules/airflow/contrib/hooks/sqoop_hook.html +++ b/_modules/airflow/contrib/hooks/sqoop_hook.html @@ -541,20 +541,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/ssh_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/ssh_hook.html b/_modules/airflow/contrib/hooks/ssh_hook.html index 453721f..3123f1d 100644 --- a/_modules/airflow/contrib/hooks/ssh_hook.html +++ b/_modules/airflow/contrib/hooks/ssh_hook.html @@ -180,11 +180,12 @@ <span class="kn">import</span> <span class="nn">getpass</span> <span class="kn">import</span> <span class="nn">os</span> +<span class="kn">import</span> <span class="nn">warnings</span> <span class="kn">import</span> <span class="nn">paramiko</span> <span class="kn">from</span> <span class="nn">paramiko.config</span> <span class="k">import</span> <span class="n">SSH_PORT</span> +<span class="kn">from</span> <span class="nn">sshtunnel</span> <span class="k">import</span> <span class="n">SSHTunnelForwarder</span> -<span class="kn">from</span> <span class="nn">contextlib</span> <span class="k">import</span> <span class="n">contextmanager</span> <span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span> <span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="k">import</span> <span class="n">BaseHook</span> <span class="kn">from</span> <span class="nn">airflow.utils.log.logging_mixin</span> <span class="k">import</span> <span class="n">LoggingMixin</span> @@ -223,7 +224,7 @@ <span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">key_file</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> - <span class="n">port</span><span class="o">=</span><span class="n">SSH_PORT</span><span class="p">,</span> + <span class="n">port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">keepalive_interval</span><span class="o">=</span><span class="mi">30</span> <span class="p">):</span> @@ -233,165 +234,170 @@ <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">username</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">password</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">key_file</span> + <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">port</span> <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span> <span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span> <span class="o">=</span> <span class="n">keepalive_interval</span> + <span class="c1"># Default values, overridable from Connection</span> <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">True</span> <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">True</span> + <span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span> <span class="o">=</span> <span class="kc">None</span> + + <span class="c1"># Placeholder for deprecated __enter__</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span> - <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">port</span> - <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Creating SSH client for conn_id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span> - <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> - <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span> - <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span> - <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span> - <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> - <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span> - <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> - <span class="n">extra_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span> - <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"key_file"</span><span class="p">)</span> - - <span class="k">if</span> <span class="s2">"timeout"</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"timeout"</span><span class="p">],</span> <span class="mi">10</span><span class="p">)</span> - - <span class="k">if</span> <span class="s2">"compress"</span> <span class="ow">in</span> <span class="n">extra_options</span> \ - <span class="ow">and</span> <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"compress"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">False</span> - <span class="k">if</span> <span class="s2">"no_host_key_check"</span> <span class="ow">in</span> <span class="n">extra_options</span> \ - <span class="ow">and</span> \ - <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"no_host_key_check"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">False</span> - - <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">:</span> - <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Missing required param: remote_host"</span><span class="p">)</span> - - <span class="c1"># Auto detecting username values from system</span> - <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span> - <span class="s2">"username to ssh to host: </span><span class="si">%s</span><span class="s2"> is not specified for connection id"</span> - <span class="s2">" </span><span class="si">%s</span><span class="s2">. Using system's default provided by getpass.getuser()"</span><span class="p">,</span> - <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> - <span class="p">)</span> - <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">getpass</span><span class="o">.</span><span class="n">getuser</span><span class="p">()</span> - - <span class="n">host_proxy</span> <span class="o">=</span> <span class="kc">None</span> - <span class="n">user_ssh_config_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="s1">'~/.ssh/config'</span><span class="p">)</span> - <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">):</span> - <span class="n">ssh_conf</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHConfig</span><span class="p">()</span> - <span class="n">ssh_conf</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">))</span> - <span class="n">host_info</span> <span class="o">=</span> <span class="n">ssh_conf</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">)</span> - <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">):</span> - <span class="n">host_proxy</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">ProxyCommand</span><span class="p">(</span><span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">))</span> - - <span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">):</span> - <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> - - <span class="k">try</span><span class="p">:</span> - <span class="n">client</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHClient</span><span class="p">()</span> - <span class="n">client</span><span class="o">.</span><span class="n">load_system_host_keys</span><span class="p">()</span> - <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span><span class="p">:</span> - <span class="c1"># Default is RejectPolicy</span> - <span class="n">client</span><span class="o">.</span><span class="n">set_missing_host_key_policy</span><span class="p">(</span><span class="n">paramiko</span><span class="o">.</span><span class="n">AutoAddPolicy</span><span class="p">())</span> - - <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span> - <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> - <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> - <span class="n">password</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> - <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span> - <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span> - <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> - <span class="n">sock</span><span class="o">=</span><span class="n">host_proxy</span><span class="p">)</span> - <span class="k">else</span><span class="p">:</span> - <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> - <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> - <span class="n">key_filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span> - <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span> - <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span> - <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> - <span class="n">sock</span><span class="o">=</span><span class="n">host_proxy</span><span class="p">)</span> - - <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">:</span> - <span class="n">client</span><span class="o">.</span><span class="n">get_transport</span><span class="p">()</span><span class="o">.</span><span class="n">set_keepalive</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">)</span> - - <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">client</span> - <span class="k">except</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">AuthenticationException</span> <span class="k">as</span> <span class="n">auth_error</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span> - <span class="s2">"Auth failed while connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> - <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">auth_error</span> - <span class="p">)</span> - <span class="k">except</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHException</span> <span class="k">as</span> <span class="n">ssh_error</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span> - <span class="s2">"Failed connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> - <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">ssh_error</span> - <span class="p">)</span> - <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span> - <span class="s2">"Error connecting to host: </span><span class="si">%s</span><span class="s2">, error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> - <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">error</span> - <span class="p">)</span> - <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span> - -<div class="viewcode-block" id="SSHHook.create_tunnel"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ssh_hook.SSHHook.create_tunnel">[docs]</a> <span class="nd">@contextmanager</span> - <span class="k">def</span> <span class="nf">create_tunnel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_port</span><span class="p">,</span> <span class="n">remote_port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">remote_host</span><span class="o">=</span><span class="s2">"localhost"</span><span class="p">):</span> + <span class="c1"># Use connection to override defaults</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span> + <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">extra_options</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span> + <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">extra_options</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"key_file"</span><span class="p">)</span> + + <span class="k">if</span> <span class="s2">"timeout"</span> <span class="ow">in</span> <span class="n">extra_options</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"timeout"</span><span class="p">],</span> <span class="mi">10</span><span class="p">)</span> + + <span class="k">if</span> <span class="s2">"compress"</span> <span class="ow">in</span> <span class="n">extra_options</span>\ + <span class="ow">and</span> <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"compress"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">compress</span> <span class="o">=</span> <span class="kc">False</span> + <span class="k">if</span> <span class="s2">"no_host_key_check"</span> <span class="ow">in</span> <span class="n">extra_options</span>\ + <span class="ow">and</span>\ + <span class="nb">str</span><span class="p">(</span><span class="n">extra_options</span><span class="p">[</span><span class="s2">"no_host_key_check"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'false'</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span> <span class="o">=</span> <span class="kc">False</span> + + <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">:</span> + <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Missing required param: remote_host"</span><span class="p">)</span> + + <span class="c1"># Auto detecting username values from system</span> + <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span> + <span class="s2">"username to ssh to host: </span><span class="si">%s</span><span class="s2"> is not specified for connection id"</span> + <span class="s2">" </span><span class="si">%s</span><span class="s2">. Using system's default provided by getpass.getuser()"</span><span class="p">,</span> + <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span> + <span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">username</span> <span class="o">=</span> <span class="n">getpass</span><span class="o">.</span><span class="n">getuser</span><span class="p">()</span> + + <span class="n">user_ssh_config_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">expanduser</span><span class="p">(</span><span class="s1">'~/.ssh/config'</span><span class="p">)</span> + <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">):</span> + <span class="n">ssh_conf</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHConfig</span><span class="p">()</span> + <span class="n">ssh_conf</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="nb">open</span><span class="p">(</span><span class="n">user_ssh_config_filename</span><span class="p">))</span> + <span class="n">host_info</span> <span class="o">=</span> <span class="n">ssh_conf</span><span class="o">.</span><span class="n">lookup</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">)</span> + <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">ProxyCommand</span><span class="p">(</span><span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'proxycommand'</span><span class="p">))</span> + + <span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">):</span> + <span class="k">if</span> <span class="n">host_info</span> <span class="ow">and</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">key_file</span> <span class="o">=</span> <span class="n">host_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'identityfile'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> + + <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="ow">or</span> <span class="n">SSH_PORT</span> + +<div class="viewcode-block" id="SSHHook.get_conn"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ssh_hook.SSHHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="sd">"""</span> -<span class="sd"> Creates a tunnel between two hosts. Like ssh -L <LOCAL_PORT>:host:<REMOTE_PORT>.</span> -<span class="sd"> Remember to close() the returned "tunnel" object in order to clean up</span> -<span class="sd"> after yourself when you are done with the tunnel.</span> +<span class="sd"> Opens a ssh connection to the remote host.</span> -<span class="sd"> :param local_port:</span> -<span class="sd"> :type local_port: int</span> -<span class="sd"> :param remote_port:</span> -<span class="sd"> :type remote_port: int</span> -<span class="sd"> :param remote_host:</span> -<span class="sd"> :type remote_host: str</span> -<span class="sd"> :return:</span> +<span class="sd"> :return paramiko.SSHClient object</span> <span class="sd"> """</span> - <span class="kn">import</span> <span class="nn">subprocess</span> - <span class="c1"># this will ensure the connection to the ssh.remote_host from where the tunnel</span> - <span class="c1"># is getting created</span> - <span class="bp">self</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> - - <span class="n">tunnel_host</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">:</span><span class="si">{1}</span><span class="s2">:</span><span class="si">{2}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">local_port</span><span class="p">,</span> <span class="n">remote_host</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">)</span> - - <span class="n">ssh_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"ssh"</span><span class="p">,</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">@</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">),</span> - <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"ControlMaster=no"</span><span class="p">,</span> - <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"UserKnownHostsFile=/dev/null"</span><span class="p">,</span> - <span class="s2">"-o"</span><span class="p">,</span> <span class="s2">"StrictHostKeyChecking=no"</span><span class="p">]</span> - - <span class="n">ssh_tunnel_cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"-L"</span><span class="p">,</span> <span class="n">tunnel_host</span><span class="p">,</span> - <span class="s2">"echo -n ready && cat"</span> - <span class="p">]</span> - - <span class="n">ssh_cmd</span> <span class="o">+=</span> <span class="n">ssh_tunnel_cmd</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating tunnel with cmd: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">ssh_cmd</span><span class="p">)</span> - - <span class="n">proc</span> <span class="o">=</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">Popen</span><span class="p">(</span><span class="n">ssh_cmd</span><span class="p">,</span> - <span class="n">stdin</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> - <span class="n">stdout</span><span class="o">=</span><span class="n">subprocess</span><span class="o">.</span><span class="n">PIPE</span><span class="p">,</span> - <span class="n">close_fds</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> - <span class="n">ready</span> <span class="o">=</span> <span class="n">proc</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span> - <span class="k">assert</span> <span class="n">ready</span> <span class="o">==</span> <span class="sa">b</span><span class="s2">"ready"</span><span class="p">,</span> \ - <span class="s2">"Did not get 'ready' from remote, got '</span><span class="si">{0}</span><span class="s2">' instead"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">ready</span><span class="p">)</span> - <span class="k">yield</span> - <span class="n">proc</span><span class="o">.</span><span class="n">communicate</span><span class="p">()</span> - <span class="k">assert</span> <span class="n">proc</span><span class="o">.</span><span class="n">returncode</span> <span class="o">==</span> <span class="mi">0</span><span class="p">,</span> \ - <span class="s2">"Tunnel process did unclean exit (returncode </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">proc</span><span class="o">.</span><span class="n">returncode</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'Creating SSH client for conn_id: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ssh_conn_id</span><span class="p">)</span> + <span class="n">client</span> <span class="o">=</span> <span class="n">paramiko</span><span class="o">.</span><span class="n">SSHClient</span><span class="p">()</span> + <span class="n">client</span><span class="o">.</span><span class="n">load_system_host_keys</span><span class="p">()</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">no_host_key_check</span><span class="p">:</span> + <span class="c1"># Default is RejectPolicy</span> + <span class="n">client</span><span class="o">.</span><span class="n">set_missing_host_key_policy</span><span class="p">(</span><span class="n">paramiko</span><span class="o">.</span><span class="n">AutoAddPolicy</span><span class="p">())</span> + + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span> + <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> + <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> + <span class="n">password</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> + <span class="n">key_filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span> + <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span> + <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span> + <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> + <span class="n">sock</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">client</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> + <span class="n">username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> + <span class="n">key_filename</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span> + <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">,</span> + <span class="n">compress</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">compress</span><span class="p">,</span> + <span class="n">port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> + <span class="n">sock</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span><span class="p">)</span> + + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">:</span> + <span class="n">client</span><span class="o">.</span><span class="n">get_transport</span><span class="p">()</span><span class="o">.</span><span class="n">set_keepalive</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keepalive_interval</span><span class="p">)</span> + + <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">client</span> + <span class="k">return</span> <span class="n">client</span></div> <span class="k">def</span> <span class="nf">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> + <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'The contextmanager of SSHHook is deprecated.'</span> + <span class="s1">'Please use get_conn() as a contextmanager instead.'</span> + <span class="s1">'This method will be removed in Airflow 2.0'</span><span class="p">,</span> + <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span> <span class="k">return</span> <span class="bp">self</span> <span class="k">def</span> <span class="nf">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_val</span><span class="p">,</span> <span class="n">exc_tb</span><span class="p">):</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> - <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> + <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="kc">None</span> + +<div class="viewcode-block" id="SSHHook.get_tunnel"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.hooks.ssh_hook.SSHHook.get_tunnel">[docs]</a> <span class="k">def</span> <span class="nf">get_tunnel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">,</span> <span class="n">remote_host</span><span class="o">=</span><span class="s2">"localhost"</span><span class="p">,</span> <span class="n">local_port</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Creates a tunnel between two hosts. Like ssh -L <LOCAL_PORT>:host:<REMOTE_PORT>.</span> + +<span class="sd"> :param remote_port: The remote port to create a tunnel to</span> +<span class="sd"> :type remote_port: int</span> +<span class="sd"> :param remote_host: The remote host to create a tunnel to (default localhost)</span> +<span class="sd"> :type remote_host: str</span> +<span class="sd"> :param local_port: The local port to attach the tunnel to</span> +<span class="sd"> :type local_port: int</span> + +<span class="sd"> :return: sshtunnel.SSHTunnelForwarder object</span> +<span class="sd"> """</span> + + <span class="k">if</span> <span class="n">local_port</span><span class="p">:</span> + <span class="n">local_bind_address</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'localhost'</span><span class="p">,</span> <span class="n">local_port</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">local_bind_address</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'localhost'</span><span class="p">,)</span> + + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span> + <span class="n">client</span> <span class="o">=</span> <span class="n">SSHTunnelForwarder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> + <span class="n">ssh_port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> + <span class="n">ssh_username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> + <span class="n">ssh_password</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> + <span class="n">ssh_pkey</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span> + <span class="n">ssh_proxy</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span><span class="p">,</span> + <span class="n">local_bind_address</span><span class="o">=</span><span class="n">local_bind_address</span><span class="p">,</span> + <span class="n">remote_bind_address</span><span class="o">=</span><span class="p">(</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">),</span> + <span class="n">logger</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">client</span> <span class="o">=</span> <span class="n">SSHTunnelForwarder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">remote_host</span><span class="p">,</span> + <span class="n">ssh_port</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">port</span><span class="p">,</span> + <span class="n">ssh_username</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">username</span><span class="p">,</span> + <span class="n">ssh_pkey</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">key_file</span><span class="p">,</span> + <span class="n">ssh_proxy</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host_proxy</span><span class="p">,</span> + <span class="n">local_bind_address</span><span class="o">=</span><span class="n">local_bind_address</span><span class="p">,</span> + <span class="n">remote_bind_address</span><span class="o">=</span><span class="p">(</span><span class="n">remote_host</span><span class="p">,</span> <span class="n">remote_port</span><span class="p">),</span> + <span class="n">host_pkey_directories</span><span class="o">=</span><span class="p">[],</span> + <span class="n">logger</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="p">)</span> + + <span class="k">return</span> <span class="n">client</span></div> + + <span class="k">def</span> <span class="nf">create_tunnel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_port</span><span class="p">,</span> <span class="n">remote_port</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">remote_host</span><span class="o">=</span><span class="s2">"localhost"</span><span class="p">):</span> + <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s1">'SSHHook.create_tunnel is deprecated, Please'</span> + <span class="s1">'use get_tunnel() instead. But please note that the'</span> + <span class="s1">'order of the parameters have changed'</span> + <span class="s1">'This method will be removed in Airflow 2.0'</span><span class="p">,</span> + <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span> + + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_tunnel</span><span class="p">(</span><span class="n">remote_port</span><span class="p">,</span> <span class="n">remote_host</span><span class="p">,</span> <span class="n">local_port</span><span class="p">)</span></div> </pre></div> </div> @@ -422,20 +428,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/hooks/vertica_hook.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/hooks/vertica_hook.html b/_modules/airflow/contrib/hooks/vertica_hook.html index f60fa92..8193e44 100644 --- a/_modules/airflow/contrib/hooks/vertica_hook.html +++ b/_modules/airflow/contrib/hooks/vertica_hook.html @@ -242,20 +242,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> +
