http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/executor/use-celery.html ---------------------------------------------------------------------- diff --git a/howto/executor/use-celery.html b/howto/executor/use-celery.html index d29401f..a28a857 100644 --- a/howto/executor/use-celery.html +++ b/howto/executor/use-celery.html @@ -216,7 +216,7 @@ to start a Flower web server.</p> <ul class="simple"> <li>Make sure to use a database backed result backend</li> <li>Make sure to set a visibility timeout in [celery_broker_transport_options] that exceeds the ETA of your longest running task</li> -<li>Tasks can and consume resources, make sure your worker as enough resources to run <cite>worker_concurrency</cite> tasks</li> +<li>Tasks can consume resources. Make sure your worker has enough resources to run <cite>worker_concurrency</cite> tasks</li> </ul> </div> @@ -258,20 +258,13 @@ to start a Flower web server.</p> - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../_static/jquery.js"></script> - <script type="text/javascript" src="../../_static/underscore.js"></script> - <script type="text/javascript" src="../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../_static/jquery.js"></script> + <script type="text/javascript" src="../../_static/underscore.js"></script> + <script type="text/javascript" src="../../_static/doctools.js"></script> +
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/executor/use-dask.html ---------------------------------------------------------------------- diff --git a/howto/executor/use-dask.html b/howto/executor/use-dask.html index bc18ef5..bb7433d 100644 --- a/howto/executor/use-dask.html +++ b/howto/executor/use-dask.html @@ -244,20 +244,13 @@ warning will be raised but the task will be submitted to the cluster.</li> - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../_static/jquery.js"></script> - <script type="text/javascript" src="../../_static/underscore.js"></script> - <script type="text/javascript" src="../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../_static/jquery.js"></script> + <script type="text/javascript" src="../../_static/underscore.js"></script> + <script type="text/javascript" src="../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/executor/use-mesos.html ---------------------------------------------------------------------- diff --git a/howto/executor/use-mesos.html b/howto/executor/use-mesos.html index ba905ea..f958feb 100644 --- a/howto/executor/use-mesos.html +++ b/howto/executor/use-mesos.html @@ -283,20 +283,13 @@ For any queries/bugs on <cite>MesosExecutor</cite>, please contact <a class="ref - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../_static/jquery.js"></script> - <script type="text/javascript" src="../../_static/underscore.js"></script> - <script type="text/javascript" src="../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../_static/jquery.js"></script> + <script type="text/javascript" src="../../_static/underscore.js"></script> + <script type="text/javascript" src="../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/index.html ---------------------------------------------------------------------- diff --git a/howto/index.html b/howto/index.html index 4189bc4..6147610 100644 --- a/howto/index.html +++ b/howto/index.html @@ -259,20 +259,13 @@ configuring an Airflow environment.</p> - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/initialize-database.html ---------------------------------------------------------------------- diff --git a/howto/initialize-database.html b/howto/initialize-database.html index b03d757..21bd6de 100644 --- a/howto/initialize-database.html +++ b/howto/initialize-database.html @@ -249,20 +249,13 @@ airflow initdb - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/manage-connections.html ---------------------------------------------------------------------- diff --git a/howto/manage-connections.html b/howto/manage-connections.html index 0cedef4..0859080 100644 --- a/howto/manage-connections.html +++ b/howto/manage-connections.html @@ -94,6 +94,7 @@ <li class="toctree-l3"><a class="reference internal" href="#creating-a-connection-with-environment-variables">Creating a Connection with Environment Variables</a></li> <li class="toctree-l3"><a class="reference internal" href="#connection-types">Connection Types</a><ul> <li class="toctree-l4"><a class="reference internal" href="#google-cloud-platform">Google Cloud Platform</a></li> +<li class="toctree-l4"><a class="reference internal" href="#mysql">MySQL</a></li> </ul> </li> </ul> @@ -191,7 +192,7 @@ <h1>Managing Connections<a class="headerlink" href="#managing-connections" title="Permalink to this headline">¶</a></h1> <p>Airflow needs to know how to connect to your environment. Information such as hostname, port, login and passwords to other systems and services is -handled in the <code class="docutils literal notranslate"><span class="pre">Admin->Connection</span></code> section of the UI. The pipeline code you +handled in the <code class="docutils literal notranslate"><span class="pre">Admin->Connections</span></code> section of the UI. The pipeline code you will author will reference the âconn_idâ of the Connection objects.</p> <img alt="../_images/connections.png" src="../_images/connections.png" /> <p>Connections can be created and managed using either the UI or environment @@ -200,7 +201,7 @@ variables.</p> more information.</p> <div class="section" id="creating-a-connection-with-the-ui"> <h2>Creating a Connection with the UI<a class="headerlink" href="#creating-a-connection-with-the-ui" title="Permalink to this headline">¶</a></h2> -<p>Open the <code class="docutils literal notranslate"><span class="pre">Admin->Connection</span></code> section of the UI. Click the <code class="docutils literal notranslate"><span class="pre">Create</span></code> link +<p>Open the <code class="docutils literal notranslate"><span class="pre">Admin->Connections</span></code> section of the UI. Click the <code class="docutils literal notranslate"><span class="pre">Create</span></code> link to create a new connection.</p> <img alt="../_images/connection_create.png" src="../_images/connection_create.png" /> <ol class="arabic simple"> @@ -216,7 +217,7 @@ belonging to the different connection types.</li> </div> <div class="section" id="editing-a-connection-with-the-ui"> <h2>Editing a Connection with the UI<a class="headerlink" href="#editing-a-connection-with-the-ui" title="Permalink to this headline">¶</a></h2> -<p>Open the <code class="docutils literal notranslate"><span class="pre">Admin->Connection</span></code> section of the UI. Click the pencil icon next +<p>Open the <code class="docutils literal notranslate"><span class="pre">Admin->Connections</span></code> section of the UI. Click the pencil icon next to the connection you wish to edit in the connection list.</p> <img alt="../_images/connection_edit.png" src="../_images/connection_edit.png" /> <p>Modify the connection properties and click the <code class="docutils literal notranslate"><span class="pre">Save</span></code> button to save your @@ -297,6 +298,78 @@ issue <a class="reference external" href="https://issues.apache.org/jira/browse/ </dl> </div> </div> +<div class="section" id="mysql"> +<h3>MySQL<a class="headerlink" href="#mysql" title="Permalink to this headline">¶</a></h3> +<p>The MySQL connect type allows to connect with MySQL database.</p> +<div class="section" id="id3"> +<h4>Configuring the Connection<a class="headerlink" href="#id3" title="Permalink to this headline">¶</a></h4> +<dl class="docutils"> +<dt>Host (required)</dt> +<dd>The host to connect to.</dd> +<dt>Schema (optional)</dt> +<dd>Specify the schema name to be used in the database.</dd> +<dt>Login (required)</dt> +<dd>Specify the user name to connect.</dd> +<dt>Password (required)</dt> +<dd>Specify the password to connect.</dd> +<dt>Extra (optional)</dt> +<dd><p class="first">Specify the extra parameters (as json dictionary) that can be used in mysql +connection. The following parameters are supported:</p> +<ul class="simple"> +<li><strong>charset</strong>: specify charset of the connection</li> +<li><strong>cursor</strong>: one of âsscursorâ, âdictcursor, âssdictcursorâ - specifies cursor class to be +used</li> +<li><strong>local_infile</strong>: controls MySQLâs LOCAL capability (permitting local data loading by +clients). See <a class="reference external" href="https://mysqlclient.readthedocs.io/user_guide.html";>MySQLdb docs</a> +for details.</li> +<li><strong>unix_socket</strong>: UNIX socket used instead of the default socket</li> +<li><strong>ssl</strong>: Dictionary of SSL parameters that control connecting using SSL (those +parameters are server specific and should contain âcaâ, âcertâ, âkeyâ, âcapathâ, +âcipherâ parameters. See +<a class="reference external" href="https://mysqlclient.readthedocs.io/user_guide.html";>MySQLdb docs</a> for details. +Note that in order to be useful in URL notation, this parameter might also be +a string where the SSL dictionary is a string-encoded JSON dictionary.</li> +</ul> +<p>Example âextrasâ field:</p> +<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span> + <span class="nt">"charset"</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">,</span> + <span class="nt">"cursorclass"</span><span class="p">:</span> <span class="s2">"sscursor"</span><span class="p">,</span> + <span class="nt">"local_infile"</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span> + <span class="nt">"unix_socket"</span><span class="p">:</span> <span class="s2">"/var/socket"</span><span class="p">,</span> + <span class="nt">"ssl"</span><span class="p">:</span> <span class="p">{</span> + <span class="nt">"cert"</span><span class="p">:</span> <span class="s2">"/tmp/client-cert.pem"</span><span class="p">,</span> + <span class="nt">"ca"</span><span class="p">:</span> <span class="s2">"/tmp/server-ca.pem'"</span><span class="p">,</span> + <span class="nt">"key"</span><span class="p">:</span> <span class="s2">"/tmp/client-key.pem"</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> +</div> +<p>or</p> +<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span> + <span class="nt">"charset"</span><span class="p">:</span> <span class="s2">"utf8"</span><span class="p">,</span> + <span class="nt">"cursorclass"</span><span class="p">:</span> <span class="s2">"sscursor"</span><span class="p">,</span> + <span class="nt">"local_infile"</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span> + <span class="nt">"unix_socket"</span><span class="p">:</span> <span class="s2">"/var/socket"</span><span class="p">,</span> + <span class="nt">"ssl"</span><span class="p">:</span> <span class="s2">"{\"cert\": \"/tmp/client-cert.pem\", \"ca\": \"/tmp/server-ca.pem\", \"key\": \"/tmp/client-key.pem\"}"</span> +<span class="p">}</span> +</pre></div> +</div> +<p>When specifying the connection as URI (in AIRFLOW_CONN_* variable) you should specify it +following the standard syntax of DB connections, where extras as passed as parameters +of the URI (note that all components of the URI should be URL-encoded).</p> +<p>For example:</p> +<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>mysql://mysql_user:XXXXXXXXXXXX@1.1.1.1:3306/mysqldb?ssl<span class="o">=</span>%7B%22cert%22%3A+%22%2Ftmp%2Fclient-cert.pem%22%2C+%22ca%22%3A+%22%2Ftmp%2Fserver-ca.pem%22%2C+%22key%22%3A+%22%2Ftmp%2Fclient-key.pem%22%7D +</pre></div> +</div> +<div class="last admonition note"> +<p class="first admonition-title">Note</p> +<p class="last">If encounter UnicodeDecodeError while working with MySQL connection check +the charset defined is matched to the database charset.</p> +</div> +</dd> +</dl> +</div> +</div> </div> </div> @@ -338,20 +411,13 @@ issue <a class="reference external" href="https://issues.apache.org/jira/browse/ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> +
