http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/operator.html ---------------------------------------------------------------------- diff --git a/howto/operator.html b/howto/operator.html index e2605df..4aae06f 100644 --- a/howto/operator.html +++ b/howto/operator.html @@ -100,6 +100,17 @@ </li> <li class="toctree-l3"><a class="reference internal" href="#google-cloud-platform-operators">Google Cloud Platform Operators</a><ul> <li class="toctree-l4"><a class="reference internal" href="#googlecloudstoragetobigqueryoperator">GoogleCloudStorageToBigQueryOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#gceinstancestartoperator">GceInstanceStartOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#gceinstancestopoperator">GceInstanceStopOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#gcesetmachinetypeoperator">GceSetMachineTypeOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#gcffunctiondeleteoperator">GcfFunctionDeleteOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#gcffunctiondeployoperator">GcfFunctionDeployOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedatabasecreateoperator">CloudSqlInstanceDatabaseCreateOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedatabasedeleteoperator">CloudSqlInstanceDatabaseDeleteOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedatabasepatchoperator">CloudSqlInstanceDatabasePatchOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancedeleteoperator">CloudSqlInstanceDeleteOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancecreateoperator">CloudSqlInstanceCreateOperator</a></li> +<li class="toctree-l4"><a class="reference internal" href="#cloudsqlinstancepatchoperator">CloudSqlInstancePatchOperator</a></li> </ul> </li> </ul> @@ -203,48 +214,105 @@ determine what actually executes when your DAG runs.</p> information.</p> <div class="contents local topic" id="contents"> <ul class="simple"> -<li><a class="reference internal" href="#bashoperator" id="id2">BashOperator</a><ul> -<li><a class="reference internal" href="#templating" id="id3">Templating</a></li> -<li><a class="reference internal" href="#troubleshooting" id="id4">Troubleshooting</a><ul> -<li><a class="reference internal" href="#jinja-template-not-found" id="id5">Jinja template not found</a></li> +<li><a class="reference internal" href="#bashoperator" id="id28">BashOperator</a><ul> +<li><a class="reference internal" href="#templating" id="id29">Templating</a></li> +<li><a class="reference internal" href="#troubleshooting" id="id30">Troubleshooting</a><ul> +<li><a class="reference internal" href="#jinja-template-not-found" id="id31">Jinja template not found</a></li> </ul> </li> </ul> </li> -<li><a class="reference internal" href="#pythonoperator" id="id6">PythonOperator</a><ul> -<li><a class="reference internal" href="#passing-in-arguments" id="id7">Passing in arguments</a></li> -<li><a class="reference internal" href="#id1" id="id8">Templating</a></li> +<li><a class="reference internal" href="#pythonoperator" id="id32">PythonOperator</a><ul> +<li><a class="reference internal" href="#passing-in-arguments" id="id33">Passing in arguments</a></li> +<li><a class="reference internal" href="#id1" id="id34">Templating</a></li> +</ul> +</li> +<li><a class="reference internal" href="#google-cloud-platform-operators" id="id35">Google Cloud Platform Operators</a><ul> +<li><a class="reference internal" href="#googlecloudstoragetobigqueryoperator" id="id36">GoogleCloudStorageToBigQueryOperator</a></li> +<li><a class="reference internal" href="#gceinstancestartoperator" id="id37">GceInstanceStartOperator</a></li> +<li><a class="reference internal" href="#gceinstancestopoperator" id="id38">GceInstanceStopOperator</a></li> +<li><a class="reference internal" href="#gcesetmachinetypeoperator" id="id39">GceSetMachineTypeOperator</a></li> +<li><a class="reference internal" href="#gcffunctiondeleteoperator" id="id40">GcfFunctionDeleteOperator</a><ul> +<li><a class="reference internal" href="#id2" id="id41">Troubleshooting</a></li> +</ul> +</li> +<li><a class="reference internal" href="#gcffunctiondeployoperator" id="id42">GcfFunctionDeployOperator</a><ul> +<li><a class="reference internal" href="#id3" id="id43">Troubleshooting</a></li> +</ul> +</li> +<li><a class="reference internal" href="#cloudsqlinstancedatabasecreateoperator" id="id44">CloudSqlInstanceDatabaseCreateOperator</a><ul> +<li><a class="reference internal" href="#arguments" id="id45">Arguments</a></li> +<li><a class="reference internal" href="#using-the-operator" id="id46">Using the operator</a></li> +<li><a class="reference internal" href="#id5" id="id47">Templating</a></li> +<li><a class="reference internal" href="#more-information" id="id48">More information</a></li> +</ul> +</li> +<li><a class="reference internal" href="#cloudsqlinstancedatabasedeleteoperator" id="id49">CloudSqlInstanceDatabaseDeleteOperator</a><ul> +<li><a class="reference internal" href="#id6" id="id50">Arguments</a></li> +<li><a class="reference internal" href="#id7" id="id51">Using the operator</a></li> +<li><a class="reference internal" href="#id8" id="id52">Templating</a></li> +<li><a class="reference internal" href="#id9" id="id53">More information</a></li> +</ul> +</li> +<li><a class="reference internal" href="#cloudsqlinstancedatabasepatchoperator" id="id54">CloudSqlInstanceDatabasePatchOperator</a><ul> +<li><a class="reference internal" href="#id10" id="id55">Arguments</a></li> +<li><a class="reference internal" href="#id11" id="id56">Using the operator</a></li> +<li><a class="reference internal" href="#id12" id="id57">Templating</a></li> +<li><a class="reference internal" href="#id13" id="id58">More information</a></li> +</ul> +</li> +<li><a class="reference internal" href="#cloudsqlinstancedeleteoperator" id="id59">CloudSqlInstanceDeleteOperator</a><ul> +<li><a class="reference internal" href="#id14" id="id60">Arguments</a></li> +<li><a class="reference internal" href="#id15" id="id61">Using the operator</a></li> +<li><a class="reference internal" href="#id16" id="id62">Templating</a></li> +<li><a class="reference internal" href="#id17" id="id63">More information</a></li> +</ul> +</li> +<li><a class="reference internal" href="#cloudsqlinstancecreateoperator" id="id64">CloudSqlInstanceCreateOperator</a><ul> +<li><a class="reference internal" href="#id19" id="id65">Arguments</a></li> +<li><a class="reference internal" href="#id20" id="id66">Using the operator</a></li> +<li><a class="reference internal" href="#id21" id="id67">Templating</a></li> +<li><a class="reference internal" href="#id22" id="id68">More information</a></li> +</ul> +</li> +<li><a class="reference internal" href="#cloudsqlinstancepatchoperator" id="id69">CloudSqlInstancePatchOperator</a><ul> +<li><a class="reference internal" href="#id24" id="id70">Arguments</a></li> +<li><a class="reference internal" href="#id25" id="id71">Using the operator</a></li> +<li><a class="reference internal" href="#id26" id="id72">Templating</a></li> +<li><a class="reference internal" href="#id27" id="id73">More information</a></li> </ul> </li> -<li><a class="reference internal" href="#google-cloud-platform-operators" id="id9">Google Cloud Platform Operators</a><ul> -<li><a class="reference internal" href="#googlecloudstoragetobigqueryoperator" id="id10">GoogleCloudStorageToBigQueryOperator</a></li> </ul> </li> </ul> </div> <div class="section" id="bashoperator"> -<h2><a class="toc-backref" href="#id2">BashOperator</a><a class="headerlink" href="#bashoperator" title="Permalink to this headline">¶</a></h2> +<h2><a class="toc-backref" href="#id28">BashOperator</a><a class="headerlink" href="#bashoperator" title="Permalink to this headline">¶</a></h2> <p>Use the <a class="reference internal" href="../code.html#airflow.operators.bash_operator.BashOperator" title="airflow.operators.bash_operator.BashOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">BashOperator</span></code></a> to execute commands in a <a class="reference external" href="https://www.gnu.org/software/bash/";>Bash</a> shell.</p> <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">run_this</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span> - <span class="n">task_id</span><span class="o">=</span><span class="s1">'run_after_loop'</span><span class="p">,</span> <span class="n">bash_command</span><span class="o">=</span><span class="s1">'echo 1'</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'run_after_loop'</span><span class="p">,</span> + <span class="n">bash_command</span><span class="o">=</span><span class="s1">'echo 1'</span><span class="p">,</span> + <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> +<span class="p">)</span> </pre></div> </div> <div class="section" id="templating"> -<h3><a class="toc-backref" href="#id3">Templating</a><a class="headerlink" href="#templating" title="Permalink to this headline">¶</a></h3> +<h3><a class="toc-backref" href="#id29">Templating</a><a class="headerlink" href="#templating" title="Permalink to this headline">¶</a></h3> <p>You can use <a class="reference internal" href="../concepts.html#jinja-templating"><span class="std std-ref">Jinja templates</span></a> to parameterize the <code class="docutils literal notranslate"><span class="pre">bash_command</span></code> argument.</p> -<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">task</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">also_run_this</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span> <span class="n">task_id</span><span class="o">=</span><span class="s1">'also_run_this'</span><span class="p">,</span> <span class="n">bash_command</span><span class="o">=</span><span class="s1">'echo "run_id={{ run_id }} | dag_run={{ dag_run }}"'</span><span class="p">,</span> - <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span> + <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> +<span class="p">)</span> </pre></div> </div> </div> <div class="section" id="troubleshooting"> -<h3><a class="toc-backref" href="#id4">Troubleshooting</a><a class="headerlink" href="#troubleshooting" title="Permalink to this headline">¶</a></h3> +<h3><a class="toc-backref" href="#id30">Troubleshooting</a><a class="headerlink" href="#troubleshooting" title="Permalink to this headline">¶</a></h3> <div class="section" id="jinja-template-not-found"> -<h4><a class="toc-backref" href="#id5">Jinja template not found</a><a class="headerlink" href="#jinja-template-not-found" title="Permalink to this headline">¶</a></h4> +<h4><a class="toc-backref" href="#id31">Jinja template not found</a><a class="headerlink" href="#jinja-template-not-found" title="Permalink to this headline">¶</a></h4> <p>Add a space after the script name when directly calling a Bash script with the <code class="docutils literal notranslate"><span class="pre">bash_command</span></code> argument. This is because Airflow tries to apply a Jinja template to it, which will fail.</p> @@ -263,7 +331,7 @@ template to it, which will fail.</p> </div> </div> <div class="section" id="pythonoperator"> -<h2><a class="toc-backref" href="#id6">PythonOperator</a><a class="headerlink" href="#pythonoperator" title="Permalink to this headline">¶</a></h2> +<h2><a class="toc-backref" href="#id32">PythonOperator</a><a class="headerlink" href="#pythonoperator" title="Permalink to this headline">¶</a></h2> <p>Use the <a class="reference internal" href="../code.html#airflow.operators.python_operator.PythonOperator" title="airflow.operators.python_operator.PythonOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">PythonOperator</span></code></a> to execute Python callables.</p> <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">print_context</span><span class="p">(</span><span class="n">ds</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> @@ -276,11 +344,12 @@ Python callables.</p> <span class="n">task_id</span><span class="o">=</span><span class="s1">'print_the_context'</span><span class="p">,</span> <span class="n">provide_context</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">python_callable</span><span class="o">=</span><span class="n">print_context</span><span class="p">,</span> - <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span> + <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> +<span class="p">)</span> </pre></div> </div> <div class="section" id="passing-in-arguments"> -<h3><a class="toc-backref" href="#id7">Passing in arguments</a><a class="headerlink" href="#passing-in-arguments" title="Permalink to this headline">¶</a></h3> +<h3><a class="toc-backref" href="#id33">Passing in arguments</a><a class="headerlink" href="#passing-in-arguments" title="Permalink to this headline">¶</a></h3> <p>Use the <code class="docutils literal notranslate"><span class="pre">op_args</span></code> and <code class="docutils literal notranslate"><span class="pre">op_kwargs</span></code> arguments to pass additional arguments to the Python callable.</p> <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">my_sleeping_function</span><span class="p">(</span><span class="n">random_base</span><span class="p">):</span> @@ -294,14 +363,15 @@ to the Python callable.</p> <span class="n">task_id</span><span class="o">=</span><span class="s1">'sleep_for_'</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span> <span class="n">python_callable</span><span class="o">=</span><span class="n">my_sleeping_function</span><span class="p">,</span> <span class="n">op_kwargs</span><span class="o">=</span><span class="p">{</span><span class="s1">'random_base'</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="o">/</span> <span class="mi">10</span><span class="p">},</span> - <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span> + <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">,</span> + <span class="p">)</span> - <span class="n">task</span><span class="o">.</span><span class="n">set_upstream</span><span class="p">(</span><span class="n">run_this</span><span class="p">)</span> + <span class="n">run_this</span> <span class="o">>></span> <span class="n">task</span> </pre></div> </div> </div> <div class="section" id="id1"> -<h3><a class="toc-backref" href="#id8">Templating</a><a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3> +<h3><a class="toc-backref" href="#id34">Templating</a><a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3> <p>When you set the <code class="docutils literal notranslate"><span class="pre">provide_context</span></code> argument to <code class="docutils literal notranslate"><span class="pre">True</span></code>, Airflow passes in an additional set of keyword arguments: one for each of the <a class="reference internal" href="../code.html#macros"><span class="std std-ref">Jinja template variables</span></a> and a <code class="docutils literal notranslate"><span class="pre">templates_dict</span></code> argument.</p> @@ -310,26 +380,508 @@ is evaluated as a <a class="reference internal" href="../concepts.html#jinja-tem </div> </div> <div class="section" id="google-cloud-platform-operators"> -<h2><a class="toc-backref" href="#id9">Google Cloud Platform Operators</a><a class="headerlink" href="#google-cloud-platform-operators" title="Permalink to this headline">¶</a></h2> +<h2><a class="toc-backref" href="#id35">Google Cloud Platform Operators</a><a class="headerlink" href="#google-cloud-platform-operators" title="Permalink to this headline">¶</a></h2> <div class="section" id="googlecloudstoragetobigqueryoperator"> -<h3><a class="toc-backref" href="#id10">GoogleCloudStorageToBigQueryOperator</a><a class="headerlink" href="#googlecloudstoragetobigqueryoperator" title="Permalink to this headline">¶</a></h3> +<h3><a class="toc-backref" href="#id36">GoogleCloudStorageToBigQueryOperator</a><a class="headerlink" href="#googlecloudstoragetobigqueryoperator" title="Permalink to this headline">¶</a></h3> <p>Use the <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator" title="airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GoogleCloudStorageToBigQueryOperator</span></code></a> to execute a BigQuery load job.</p> -<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">load_csv</span> <span class="o">=</span> <span class="n">gcs_to_bq</span><span class="o">.</span><span class="n">GoogleCloudStorageToBigQueryOperator</span><span class="p">(</span> - <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcs_to_bq_example'</span><span class="p">,</span> - <span class="n">bucket</span><span class="o">=</span><span class="s1">'cloud-samples-data'</span><span class="p">,</span> - <span class="n">source_objects</span><span class="o">=</span><span class="p">[</span><span class="s1">'bigquery/us-states/us-states.csv'</span><span class="p">],</span> - <span class="n">destination_project_dataset_table</span><span class="o">=</span><span class="s1">'airflow_test.gcs_to_bq_table'</span><span class="p">,</span> - <span class="n">schema_fields</span><span class="o">=</span><span class="p">[</span> - <span class="p">{</span><span class="s1">'name'</span><span class="p">:</span> <span class="s1">'name'</span><span class="p">,</span> <span class="s1">'type'</span><span class="p">:</span> <span class="s1">'STRING'</span><span class="p">,</span> <span class="s1">'mode'</span><span class="p">:</span> <span class="s1">'NULLABLE'</span><span class="p">},</span> - <span class="p">{</span><span class="s1">'name'</span><span class="p">:</span> <span class="s1">'post_abbr'</span><span class="p">,</span> <span class="s1">'type'</span><span class="p">:</span> <span class="s1">'STRING'</span><span class="p">,</span> <span class="s1">'mode'</span><span class="p">:</span> <span class="s1">'NULLABLE'</span><span class="p">},</span> - <span class="p">],</span> - <span class="n">write_disposition</span><span class="o">=</span><span class="s1">'WRITE_TRUNCATE'</span><span class="p">,</span> - <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span> +</div> +<div class="section" id="gceinstancestartoperator"> +<h3><a class="toc-backref" href="#id37">GceInstanceStartOperator</a><a class="headerlink" href="#gceinstancestartoperator" title="Permalink to this headline">¶</a></h3> +<p>Allows to start an existing Google Compute Engine instance.</p> +<p>In this example parameter values are extracted from Airflow variables. +Moreover, the <code class="docutils literal notranslate"><span class="pre">default_args</span></code> dict is used to pass common arguments to all operators in a single DAG.</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">LOCATION</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'LOCATION'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">INSTANCE</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">SHORT_MACHINE_TYPE_NAME</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SHORT_MACHINE_TYPE_NAME'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">SET_MACHINE_TYPE_BODY</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">'machineType'</span><span class="p">:</span> <span class="s1">'zones/{}/machineTypes/{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">LOCATION</span><span class="p">,</span> <span class="n">SHORT_MACHINE_TYPE_NAME</span><span class="p">)</span> +<span class="p">}</span> + +<span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">'start_date'</span><span class="p">:</span> <span class="n">airflow</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">dates</span><span class="o">.</span><span class="n">days_ago</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> +<span class="p">}</span> +</pre></div> +</div> +<p>Define the <code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStartOperator</span></code> by passing the required arguments to the constructor.</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_start</span> <span class="o">=</span> <span class="n">GceInstanceStartOperator</span><span class="p">(</span> + <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="n">zone</span><span class="o">=</span><span class="n">LOCATION</span><span class="p">,</span> + <span class="n">resource_id</span><span class="o">=</span><span class="n">INSTANCE</span><span class="p">,</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_start_task'</span> +<span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="gceinstancestopoperator"> +<h3><a class="toc-backref" href="#id38">GceInstanceStopOperator</a><a class="headerlink" href="#gceinstancestopoperator" title="Permalink to this headline">¶</a></h3> +<p>Allows to stop an existing Google Compute Engine instance.</p> +<p>For parameter definition take a look at <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator" title="airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStartOperator</span></code></a> above.</p> +<p>Define the <code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStopOperator</span></code> by passing the required arguments to the constructor.</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_instance_stop</span> <span class="o">=</span> <span class="n">GceInstanceStopOperator</span><span class="p">(</span> + <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="n">zone</span><span class="o">=</span><span class="n">LOCATION</span><span class="p">,</span> + <span class="n">resource_id</span><span class="o">=</span><span class="n">INSTANCE</span><span class="p">,</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_stop_task'</span> +<span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="gcesetmachinetypeoperator"> +<h3><a class="toc-backref" href="#id39">GceSetMachineTypeOperator</a><a class="headerlink" href="#gcesetmachinetypeoperator" title="Permalink to this headline">¶</a></h3> +<p>Allows to change the machine type for a stopped instance to the specified machine type.</p> +<p>For parameter definition take a look at <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator" title="airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GceInstanceStartOperator</span></code></a> above.</p> +<p>Define the <code class="xref py py-class docutils literal notranslate"><span class="pre">GceSetMachineTypeOperator</span></code> by passing the required arguments to the constructor.</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gce_set_machine_type</span> <span class="o">=</span> <span class="n">GceSetMachineTypeOperator</span><span class="p">(</span> + <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="n">zone</span><span class="o">=</span><span class="n">LOCATION</span><span class="p">,</span> + <span class="n">resource_id</span><span class="o">=</span><span class="n">INSTANCE</span><span class="p">,</span> + <span class="n">body</span><span class="o">=</span><span class="n">SET_MACHINE_TYPE_BODY</span><span class="p">,</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'gcp_compute_set_machine_type'</span> +<span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="gcffunctiondeleteoperator"> +<h3><a class="toc-backref" href="#id40">GcfFunctionDeleteOperator</a><a class="headerlink" href="#gcffunctiondeleteoperator" title="Permalink to this headline">¶</a></h3> +<p>Use the <code class="docutils literal notranslate"><span class="pre">default_args</span></code> dict to pass arguments to the operator.</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">LOCATION</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'LOCATION'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">ENTRYPOINT</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ENTRYPOINT'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="c1"># A fully-qualified name of the function to delete</span> + +<span class="n">FUNCTION_NAME</span> <span class="o">=</span> <span class="s1">'projects/{}/locations/{}/functions/{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">PROJECT_ID</span><span class="p">,</span> <span class="n">LOCATION</span><span class="p">,</span> + <span class="n">ENTRYPOINT</span><span class="p">)</span> +<span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">'start_date'</span><span class="p">:</span> <span class="n">airflow</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">dates</span><span class="o">.</span><span class="n">days_ago</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> +<span class="p">}</span> +</pre></div> +</div> +<p>Use the <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator" title="airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GcfFunctionDeleteOperator</span></code></a> +to delete a function from Google Cloud Functions.</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span> <span class="n">t1</span> <span class="o">=</span> <span class="n">GcfFunctionDeleteOperator</span><span class="p">(</span> + <span class="n">task_id</span><span class="o">=</span><span class="s2">"gcf_delete_task"</span><span class="p">,</span> + <span class="n">name</span><span class="o">=</span><span class="n">FUNCTION_NAME</span> + <span class="p">)</span> +</pre></div> +</div> +<div class="section" id="id2"> +<h4><a class="toc-backref" href="#id41">Troubleshooting</a><a class="headerlink" href="#id2" title="Permalink to this headline">¶</a></h4> +<p>If you want to run or deploy an operator using a service account and get âforbidden 403â +errors, it means that your service account does not have the correct +Cloud IAM permissions.</p> +<ol class="arabic simple"> +<li>Assign your Service Account the Cloud Functions Developer role.</li> +<li>Grant the user the Cloud IAM Service Account User role on the Cloud Functions runtime +service account.</li> +</ol> +<p>The typical way of assigning Cloud IAM permissions with <cite>gcloud</cite> is +shown below. Just replace PROJECT_ID with ID of your Google Cloud Platform project +and SERVICE_ACCOUNT_EMAIL with the email ID of your service account.</p> +<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>gcloud iam service-accounts add-iam-policy-binding <span class="se">\</span> + [email protected] <span class="se">\</span> + --member<span class="o">=</span><span class="s2">"serviceAccount:[SERVICE_ACCOUNT_EMAIL]"</span> <span class="se">\</span> + --role<span class="o">=</span><span class="s2">"roles/iam.serviceAccountUser"</span> +</pre></div> +</div> +<p>See <a class="reference external" href="https://cloud.google.com/functions/docs/reference/iam/roles#adding_the_iam_service_agent_user_role_to_the_runtime_service_account";>Adding the IAM service agent user role to the runtime service</a> for details</p> +</div> +</div> +<div class="section" id="gcffunctiondeployoperator"> +<h3><a class="toc-backref" href="#id42">GcfFunctionDeployOperator</a><a class="headerlink" href="#gcffunctiondeployoperator" title="Permalink to this headline">¶</a></h3> +<p>Use the <a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator" title="airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">GcfFunctionDeployOperator</span></code></a> +to deploy a function from Google Cloud Functions.</p> +<p>The following examples of Airflow variables show various variants and combinations +of default_args that you can use. The variables are defined as follows:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">LOCATION</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'LOCATION'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">SOURCE_ARCHIVE_URL</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SOURCE_ARCHIVE_URL'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">SOURCE_UPLOAD_URL</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SOURCE_UPLOAD_URL'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">SOURCE_REPOSITORY</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'SOURCE_REPOSITORY'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">ZIP_PATH</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ZIP_PATH'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">ENTRYPOINT</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'ENTRYPOINT'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> +<span class="n">FUNCTION_NAME</span> <span class="o">=</span> <span class="s1">'projects/{}/locations/{}/functions/{}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">PROJECT_ID</span><span class="p">,</span> <span class="n">LOCATION</span><span class="p">,</span> + <span class="n">ENTRYPOINT</span><span class="p">)</span> +<span class="n">RUNTIME</span> <span class="o">=</span> <span class="s1">'nodejs6'</span> +<span class="n">VALIDATE_BODY</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'VALIDATE_BODY'</span><span class="p">,</span> <span class="bp">True</span><span class="p">)</span> + +</pre></div> +</div> +<p>With those variables you can define the body of the request:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">body</span> <span class="o">=</span> <span class="p">{</span> + <span class="s2">"name"</span><span class="p">:</span> <span class="n">FUNCTION_NAME</span><span class="p">,</span> + <span class="s2">"entryPoint"</span><span class="p">:</span> <span class="n">ENTRYPOINT</span><span class="p">,</span> + <span class="s2">"runtime"</span><span class="p">:</span> <span class="n">RUNTIME</span><span class="p">,</span> + <span class="s2">"httpsTrigger"</span><span class="p">:</span> <span class="p">{}</span> +<span class="p">}</span> +</pre></div> +</div> +<p>When you create a DAG, the default_args dictionary can be used to pass the body and +other arguments:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">default_args</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">'start_date'</span><span class="p">:</span> <span class="n">dates</span><span class="o">.</span><span class="n">days_ago</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> + <span class="s1">'project_id'</span><span class="p">:</span> <span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="s1">'location'</span><span class="p">:</span> <span class="n">LOCATION</span><span class="p">,</span> + <span class="s1">'body'</span><span class="p">:</span> <span class="n">body</span><span class="p">,</span> + <span class="s1">'validate_body'</span><span class="p">:</span> <span class="n">VALIDATE_BODY</span> +<span class="p">}</span> +</pre></div> +</div> +<p>Note that the neither the body nor the default args are complete in the above examples. +Depending on the set variables, there might be different variants on how to pass source +code related fields. Currently, you can pass either sourceArchiveUrl, sourceRepository +or sourceUploadUrl as described in the +<a class="reference external" href="https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions#CloudFunction";>CloudFunction API specification</a>. +Additionally, default_args might contain zip_path parameter to run the extra step of +uploading the source code before deploying it. In the last case, you also need to +provide an empty <cite>sourceUploadUrl</cite> parameter in the body.</p> +<p>Based on the variables defined above, example logic of setting the source code +related fields is shown here:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">if</span> <span class="n">SOURCE_ARCHIVE_URL</span><span class="p">:</span> + <span class="n">body</span><span class="p">[</span><span class="s1">'sourceArchiveUrl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SOURCE_ARCHIVE_URL</span> +<span class="k">elif</span> <span class="n">SOURCE_REPOSITORY</span><span class="p">:</span> + <span class="n">body</span><span class="p">[</span><span class="s1">'sourceRepository'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span> + <span class="s1">'url'</span><span class="p">:</span> <span class="n">SOURCE_REPOSITORY</span> + <span class="p">}</span> +<span class="k">elif</span> <span class="n">ZIP_PATH</span><span class="p">:</span> + <span class="n">body</span><span class="p">[</span><span class="s1">'sourceUploadUrl'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">''</span> + <span class="n">default_args</span><span class="p">[</span><span class="s1">'zip_path'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ZIP_PATH</span> +<span class="k">elif</span> <span class="n">SOURCE_UPLOAD_URL</span><span class="p">:</span> + <span class="n">body</span><span class="p">[</span><span class="s1">'sourceUploadUrl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SOURCE_UPLOAD_URL</span> +<span class="k">else</span><span class="p">:</span> + <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s2">"Please provide one of the source_code parameters"</span><span class="p">)</span> +</pre></div> +</div> +<p>The code to create the operator:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span> <span class="n">deploy_task</span> <span class="o">=</span> <span class="n">GcfFunctionDeployOperator</span><span class="p">(</span> + <span class="n">task_id</span><span class="o">=</span><span class="s2">"gcf_deploy_task"</span><span class="p">,</span> + <span class="n">name</span><span class="o">=</span><span class="n">FUNCTION_NAME</span> + <span class="p">)</span> +</pre></div> +</div> +<div class="section" id="id3"> +<h4><a class="toc-backref" href="#id43">Troubleshooting</a><a class="headerlink" href="#id3" title="Permalink to this headline">¶</a></h4> +<p>If you want to run or deploy an operator using a service account and get âforbidden 403â +errors, it means that your service account does not have the correct +Cloud IAM permissions.</p> +<ol class="arabic simple"> +<li>Assign your Service Account the Cloud Functions Developer role.</li> +<li>Grant the user the Cloud IAM Service Account User role on the Cloud Functions runtime +service account.</li> +</ol> +<p>The typical way of assigning Cloud IAM permissions with <cite>gcloud</cite> is +shown below. Just replace PROJECT_ID with ID of your Google Cloud Platform project +and SERVICE_ACCOUNT_EMAIL with the email ID of your service account.</p> +<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>gcloud iam service-accounts add-iam-policy-binding <span class="se">\</span> + [email protected] <span class="se">\</span> + --member<span class="o">=</span><span class="s2">"serviceAccount:[SERVICE_ACCOUNT_EMAIL]"</span> <span class="se">\</span> + --role<span class="o">=</span><span class="s2">"roles/iam.serviceAccountUser"</span> +</pre></div> +</div> +<p>See <a class="reference external" href="https://cloud.google.com/functions/docs/reference/iam/roles#adding_the_iam_service_agent_user_role_to_the_runtime_service_account";>Adding the IAM service agent user role to the runtime service</a> for details</p> +<p>If the source code for your function is in Google Source Repository, make sure that +your service account has the Source Repository Viewer role so that the source code +can be downloaded if necessary.</p> +</div> +</div> +<div class="section" id="cloudsqlinstancedatabasecreateoperator"> +<h3><a class="toc-backref" href="#id44">CloudSqlInstanceDatabaseCreateOperator</a><a class="headerlink" href="#cloudsqlinstancedatabasecreateoperator" title="Permalink to this headline">¶</a></h3> +<p>Creates a new database inside a Cloud SQL instance.</p> +<p>For parameter definition take a look at +<a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDatabaseCreateOperator</span></code></a>.</p> +<div class="section" id="arguments"> +<h4><a class="toc-backref" href="#id45">Arguments</a><a class="headerlink" href="#arguments" title="Permalink to this headline">¶</a></h4> +<p>Some arguments in the example DAG are taken from environment variables:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> +<span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> +<span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="using-the-operator"> +<h4><a class="toc-backref" href="#id46">Using the operator</a><a class="headerlink" href="#using-the-operator" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_db_create_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabaseCreateOperator</span><span class="p">(</span> + <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="n">body</span><span class="o">=</span><span class="n">db_create_body</span><span class="p">,</span> + <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_create_task'</span> +<span class="p">)</span> +</pre></div> +</div> +<p>Example request body:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">db_create_body</span> <span class="o">=</span> <span class="p">{</span> + <span class="s2">"instance"</span><span class="p">:</span> <span class="n">INSTANCE_NAME</span><span class="p">,</span> + <span class="s2">"name"</span><span class="p">:</span> <span class="n">DB_NAME</span><span class="p">,</span> + <span class="s2">"project"</span><span class="p">:</span> <span class="n">PROJECT_ID</span> +<span class="p">}</span> +</pre></div> +</div> +</div> +<div class="section" id="id5"> +<h4><a class="toc-backref" href="#id47">Templating</a><a class="headerlink" href="#id5" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="more-information"> +<h4><a class="toc-backref" href="#id48">More information</a><a class="headerlink" href="#more-information" title="Permalink to this headline">¶</a></h4> +<p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/insert";>Google Cloud SQL API documentation for database insert</a>.</p> +</div> +</div> +<div class="section" id="cloudsqlinstancedatabasedeleteoperator"> +<h3><a class="toc-backref" href="#id49">CloudSqlInstanceDatabaseDeleteOperator</a><a class="headerlink" href="#cloudsqlinstancedatabasedeleteoperator" title="Permalink to this headline">¶</a></h3> +<p>Deletes a database from a Cloud SQL instance.</p> +<p>For parameter definition take a look at +<a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDatabaseDeleteOperator</span></code></a>.</p> +<div class="section" id="id6"> +<h4><a class="toc-backref" href="#id50">Arguments</a><a class="headerlink" href="#id6" title="Permalink to this headline">¶</a></h4> +<p>Some arguments in the example DAG are taken from environment variables:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> +<span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> +<span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id7"> +<h4><a class="toc-backref" href="#id51">Using the operator</a><a class="headerlink" href="#id7" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_db_delete_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabaseDeleteOperator</span><span class="p">(</span> + <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> + <span class="n">database</span><span class="o">=</span><span class="n">DB_NAME</span><span class="p">,</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_delete_task'</span> +<span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id8"> +<h4><a class="toc-backref" href="#id52">Templating</a><a class="headerlink" href="#id8" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'database'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> + <span class="s1">'api_version'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id9"> +<h4><a class="toc-backref" href="#id53">More information</a><a class="headerlink" href="#id9" title="Permalink to this headline">¶</a></h4> +<p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/delete";>Google Cloud SQL API documentation for database delete</a>.</p> +</div> +</div> +<div class="section" id="cloudsqlinstancedatabasepatchoperator"> +<h3><a class="toc-backref" href="#id54">CloudSqlInstanceDatabasePatchOperator</a><a class="headerlink" href="#cloudsqlinstancedatabasepatchoperator" title="Permalink to this headline">¶</a></h3> +<p>Updates a resource containing information about a database inside a Cloud SQL instance +using patch semantics. +See: <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch";>https://cloud.google.com/sql/docs/mysql/admin-api/how-tos/performance#patch</a></p> +<p>For parameter definition take a look at +<a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDatabasePatchOperator</span></code></a>.</p> +<div class="section" id="id10"> +<h4><a class="toc-backref" href="#id55">Arguments</a><a class="headerlink" href="#id10" title="Permalink to this headline">¶</a></h4> +<p>Some arguments in the example DAG are taken from environment variables:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> +<span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> +<span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id11"> +<h4><a class="toc-backref" href="#id56">Using the operator</a><a class="headerlink" href="#id11" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_db_patch_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDatabasePatchOperator</span><span class="p">(</span> + <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="n">body</span><span class="o">=</span><span class="n">db_patch_body</span><span class="p">,</span> + <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> + <span class="n">database</span><span class="o">=</span><span class="n">DB_NAME</span><span class="p">,</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_db_patch_task'</span> +<span class="p">)</span> +</pre></div> +</div> +<p>Example request body:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">db_patch_body</span> <span class="o">=</span> <span class="p">{</span> + <span class="s2">"charset"</span><span class="p">:</span> <span class="s2">"utf16"</span><span class="p">,</span> + <span class="s2">"collation"</span><span class="p">:</span> <span class="s2">"utf16_general_ci"</span> +<span class="p">}</span> +</pre></div> +</div> +</div> +<div class="section" id="id12"> +<h4><a class="toc-backref" href="#id57">Templating</a><a class="headerlink" href="#id12" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'database'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> + <span class="s1">'api_version'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id13"> +<h4><a class="toc-backref" href="#id58">More information</a><a class="headerlink" href="#id13" title="Permalink to this headline">¶</a></h4> +<p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases/patch";>Google Cloud SQL API documentation for database patch</a>.</p> +</div> +</div> +<div class="section" id="cloudsqlinstancedeleteoperator"> +<h3><a class="toc-backref" href="#id59">CloudSqlInstanceDeleteOperator</a><a class="headerlink" href="#cloudsqlinstancedeleteoperator" title="Permalink to this headline">¶</a></h3> +<p>Deletes a Cloud SQL instance in Google Cloud Platform.</p> +<p>For parameter definition take a look at +<a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceDeleteOperator</span></code></a>.</p> +<div class="section" id="id14"> +<h4><a class="toc-backref" href="#id60">Arguments</a><a class="headerlink" href="#id14" title="Permalink to this headline">¶</a></h4> +<p>Some arguments in the example DAG are taken from environment variables:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> +<span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> +<span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id15"> +<h4><a class="toc-backref" href="#id61">Using the operator</a><a class="headerlink" href="#id15" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_instance_delete_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceDeleteOperator</span><span class="p">(</span> + <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_delete_task'</span> +<span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id16"> +<h4><a class="toc-backref" href="#id62">Templating</a><a class="headerlink" href="#id16" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id17"> +<h4><a class="toc-backref" href="#id63">More information</a><a class="headerlink" href="#id17" title="Permalink to this headline">¶</a></h4> +<p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/delete";>Google Cloud SQL API documentation for delete</a>.</p> +</div> +</div> +<div class="section" id="cloudsqlinstancecreateoperator"> +<span id="id18"></span><h3><a class="toc-backref" href="#id64">CloudSqlInstanceCreateOperator</a><a class="headerlink" href="#cloudsqlinstancecreateoperator" title="Permalink to this headline">¶</a></h3> +<p>Creates a new Cloud SQL instance in Google Cloud Platform.</p> +<p>For parameter definition take a look at +<a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstanceCreateOperator</span></code></a>.</p> +<p>If an instance with the same name exists, no action will be taken and the operator +will succeed.</p> +<div class="section" id="id19"> +<h4><a class="toc-backref" href="#id65">Arguments</a><a class="headerlink" href="#id19" title="Permalink to this headline">¶</a></h4> +<p>Some arguments in the example DAG are taken from environment variables:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> +<span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> +<span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> +</pre></div> +</div> +<p>Example body defining the instance:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">body</span> <span class="o">=</span> <span class="p">{</span> + <span class="s2">"name"</span><span class="p">:</span> <span class="n">INSTANCE_NAME</span><span class="p">,</span> + <span class="s2">"settings"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"tier"</span><span class="p">:</span> <span class="s2">"db-n1-standard-1"</span><span class="p">,</span> + <span class="s2">"backupConfiguration"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"binaryLogEnabled"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> + <span class="s2">"enabled"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> + <span class="s2">"startTime"</span><span class="p">:</span> <span class="s2">"05:00"</span> + <span class="p">},</span> + <span class="s2">"activationPolicy"</span><span class="p">:</span> <span class="s2">"ALWAYS"</span><span class="p">,</span> + <span class="s2">"dataDiskSizeGb"</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span> + <span class="s2">"dataDiskType"</span><span class="p">:</span> <span class="s2">"PD_SSD"</span><span class="p">,</span> + <span class="s2">"databaseFlags"</span><span class="p">:</span> <span class="p">[],</span> + <span class="s2">"ipConfiguration"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"ipv4Enabled"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> + <span class="s2">"requireSsl"</span><span class="p">:</span> <span class="bp">True</span><span class="p">,</span> + <span class="p">},</span> + <span class="s2">"locationPreference"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"zone"</span><span class="p">:</span> <span class="s2">"europe-west4-a"</span> + <span class="p">},</span> + <span class="s2">"maintenanceWindow"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"hour"</span><span class="p">:</span> <span class="mi">5</span><span class="p">,</span> + <span class="s2">"day"</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> + <span class="s2">"updateTrack"</span><span class="p">:</span> <span class="s2">"canary"</span> + <span class="p">},</span> + <span class="s2">"pricingPlan"</span><span class="p">:</span> <span class="s2">"PER_USE"</span><span class="p">,</span> + <span class="s2">"replicationType"</span><span class="p">:</span> <span class="s2">"ASYNCHRONOUS"</span><span class="p">,</span> + <span class="s2">"storageAutoResize"</span><span class="p">:</span> <span class="bp">False</span><span class="p">,</span> + <span class="s2">"storageAutoResizeLimit"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> + <span class="s2">"userLabels"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"my-key"</span><span class="p">:</span> <span class="s2">"my-value"</span> + <span class="p">}</span> + <span class="p">},</span> + <span class="s2">"databaseVersion"</span><span class="p">:</span> <span class="s2">"MYSQL_5_7"</span><span class="p">,</span> + <span class="s2">"region"</span><span class="p">:</span> <span class="s2">"europe-west4"</span><span class="p">,</span> +<span class="p">}</span> +</pre></div> +</div> +</div> +<div class="section" id="id20"> +<h4><a class="toc-backref" href="#id66">Using the operator</a><a class="headerlink" href="#id20" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_instance_create_task</span> <span class="o">=</span> <span class="n">CloudSqlInstanceCreateOperator</span><span class="p">(</span> + <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">,</span> + <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_create_task'</span> +<span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id21"> +<h4><a class="toc-backref" href="#id67">Templating</a><a class="headerlink" href="#id21" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id22"> +<h4><a class="toc-backref" href="#id68">More information</a><a class="headerlink" href="#id22" title="Permalink to this headline">¶</a></h4> +<p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/insert";>Google Cloud SQL API documentation for insert</a>.</p> +</div> +</div> +<div class="section" id="cloudsqlinstancepatchoperator"> +<span id="id23"></span><h3><a class="toc-backref" href="#id69">CloudSqlInstancePatchOperator</a><a class="headerlink" href="#cloudsqlinstancepatchoperator" title="Permalink to this headline">¶</a></h3> +<p>Updates settings of a Cloud SQL instance in Google Cloud Platform (partial update).</p> +<p>For parameter definition take a look at +<a class="reference internal" href="../integration.html#airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator" title="airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">CloudSqlInstancePatchOperator</span></code></a>.</p> +<p>This is a partial update, so only values for the settings specified in the body +will be set / updated. The rest of the existing instanceâs configuration will remain +unchanged.</p> +<div class="section" id="id24"> +<h4><a class="toc-backref" href="#id70">Arguments</a><a class="headerlink" href="#id24" title="Permalink to this headline">¶</a></h4> +<p>Some arguments in the example DAG are taken from environment variables:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">PROJECT_ID</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'PROJECT_ID'</span><span class="p">,</span> <span class="s1">'example-project'</span><span class="p">)</span> +<span class="n">INSTANCE_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'INSTANCE_NAME'</span><span class="p">,</span> <span class="s1">'testinstance'</span><span class="p">)</span> +<span class="n">DB_NAME</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DB_NAME'</span><span class="p">,</span> <span class="s1">'testdb'</span><span class="p">)</span> +</pre></div> +</div> +<p>Example body defining the instance:</p> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">patch_body</span> <span class="o">=</span> <span class="p">{</span> + <span class="s2">"name"</span><span class="p">:</span> <span class="n">INSTANCE_NAME</span><span class="p">,</span> + <span class="s2">"settings"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"dataDiskSizeGb"</span><span class="p">:</span> <span class="mi">35</span><span class="p">,</span> + <span class="s2">"maintenanceWindow"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"hour"</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> + <span class="s2">"day"</span><span class="p">:</span> <span class="mi">6</span><span class="p">,</span> + <span class="s2">"updateTrack"</span><span class="p">:</span> <span class="s2">"canary"</span> + <span class="p">},</span> + <span class="s2">"userLabels"</span><span class="p">:</span> <span class="p">{</span> + <span class="s2">"my-key-patch"</span><span class="p">:</span> <span class="s2">"my-value-patch"</span> + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> +</div> +</div> +<div class="section" id="id25"> +<h4><a class="toc-backref" href="#id71">Using the operator</a><a class="headerlink" href="#id25" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">sql_instance_patch_task</span> <span class="o">=</span> <span class="n">CloudSqlInstancePatchOperator</span><span class="p">(</span> + <span class="n">project_id</span><span class="o">=</span><span class="n">PROJECT_ID</span><span class="p">,</span> + <span class="n">body</span><span class="o">=</span><span class="n">patch_body</span><span class="p">,</span> + <span class="n">instance</span><span class="o">=</span><span class="n">INSTANCE_NAME</span><span class="p">,</span> + <span class="n">task_id</span><span class="o">=</span><span class="s1">'sql_instance_patch_task'</span> +<span class="p">)</span> </pre></div> </div> </div> +<div class="section" id="id26"> +<h4><a class="toc-backref" href="#id72">Templating</a><a class="headerlink" href="#id26" title="Permalink to this headline">¶</a></h4> +<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'project_id'</span><span class="p">,</span> <span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'gcp_conn_id'</span><span class="p">,</span> <span class="s1">'api_version'</span><span class="p">)</span> +</pre></div> +</div> +</div> +<div class="section" id="id27"> +<h4><a class="toc-backref" href="#id73">More information</a><a class="headerlink" href="#id27" title="Permalink to this headline">¶</a></h4> +<p>See <a class="reference external" href="https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances/patch";>Google Cloud SQL API documentation for patch</a>.</p> +</div> +</div> </div> </div> @@ -371,20 +923,13 @@ to execute a BigQuery load job.</p> - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> +
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/run-with-systemd.html ---------------------------------------------------------------------- diff --git a/howto/run-with-systemd.html b/howto/run-with-systemd.html index 87e85ce..b6903d4 100644 --- a/howto/run-with-systemd.html +++ b/howto/run-with-systemd.html @@ -231,20 +231,13 @@ can also define here, for example, <code class="docutils literal notranslate"><s - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/run-with-upstart.html ---------------------------------------------------------------------- diff --git a/howto/run-with-upstart.html b/howto/run-with-upstart.html index add6b47..e72b1db 100644 --- a/howto/run-with-upstart.html +++ b/howto/run-with-upstart.html @@ -233,20 +233,13 @@ integrated with upstart</p> - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/secure-connections.html ---------------------------------------------------------------------- diff --git a/howto/secure-connections.html b/howto/secure-connections.html index 1b801e0..fd5f41b 100644 --- a/howto/secure-connections.html +++ b/howto/secure-connections.html @@ -183,12 +183,12 @@ <p>By default, Airflow will save the passwords for the connection in plain text within the metadata database. The <code class="docutils literal notranslate"><span class="pre">crypto</span></code> package is highly recommended during installation. The <code class="docutils literal notranslate"><span class="pre">crypto</span></code> package does require that your operating -system have libffi-dev installed.</p> -<p>If <code class="docutils literal notranslate"><span class="pre">crypto</span></code> package was not installed initially, you can still enable encryption for -connections by following steps below:</p> +system has <code class="docutils literal notranslate"><span class="pre">libffi-dev</span></code> installed.</p> +<p>If <code class="docutils literal notranslate"><span class="pre">crypto</span></code> package was not installed initially, it means that your Fernet key in <code class="docutils literal notranslate"><span class="pre">airflow.cfg</span></code> is empty.</p> +<p>You can still enable encryption for passwords within connections by following below steps:</p> <ol class="arabic simple"> <li>Install crypto package <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">apache-airflow[crypto]</span></code></li> -<li>Generate fernet_key, using this code snippet below. fernet_key must be a base64-encoded 32-byte key.</li> +<li>Generate fernet_key, using this code snippet below. <code class="docutils literal notranslate"><span class="pre">fernet_key</span></code> must be a base64-encoded 32-byte key.</li> </ol> <div class="code python highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">cryptography.fernet</span> <span class="k">import</span> <span class="n">Fernet</span> <span class="n">fernet_key</span><span class="o">=</span> <span class="n">Fernet</span><span class="o">.</span><span class="n">generate_key</span><span class="p">()</span> @@ -200,7 +200,7 @@ Alternatively, you can store your fernet_key in OS environment variable. You do not need to change <code class="docutils literal notranslate"><span class="pre">airflow.cfg</span></code> in this case as Airflow will use environment variable over the value in <code class="docutils literal notranslate"><span class="pre">airflow.cfg</span></code>:</p> <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Note the double underscores</span> -EXPORT <span class="nv">AIRFLOW__CORE__FERNET_KEY</span> <span class="o">=</span> your_fernet_key +<span class="nb">export</span> <span class="nv">AIRFLOW__CORE__FERNET_KEY</span><span class="o">=</span>your_fernet_key </pre></div> </div> <ol class="arabic simple" start="4"> @@ -247,20 +247,13 @@ EXPORT <span class="nv">AIRFLOW__CORE__FERNET_KEY</span> <span class="o">=</span - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/set-config.html ---------------------------------------------------------------------- diff --git a/howto/set-config.html b/howto/set-config.html index e278919..85a54bd 100644 --- a/howto/set-config.html +++ b/howto/set-config.html @@ -194,17 +194,30 @@ metadata database connection string can either be set in <code class="docutils l <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="nv">AIRFLOW__CORE__SQL_ALCHEMY_CONN</span><span class="o">=</span>my_conn_string </pre></div> </div> -<p>You can also derive the connection string at run time by appending <code class="docutils literal notranslate"><span class="pre">_cmd</span></code> to the key like this:</p> +<p>You can also derive the connection string at run time by appending <code class="docutils literal notranslate"><span class="pre">_cmd</span></code> to +the key like this:</p> <div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="o">[</span>core<span class="o">]</span> <span class="nv">sql_alchemy_conn_cmd</span> <span class="o">=</span> bash_command_to_run </pre></div> </div> -<p>-But only three such configuration elements namely sql_alchemy_conn, broker_url and result_backend can be fetched as a command. The idea behind this is to not store passwords on boxes in plain text files. The order of precedence is as follows -</p> +<p>The following config options support this <code class="docutils literal notranslate"><span class="pre">_cmd</span></code> version:</p> +<ul class="simple"> +<li><code class="docutils literal notranslate"><span class="pre">sql_alchemy_conn</span></code> in <code class="docutils literal notranslate"><span class="pre">[core]</span></code> section</li> +<li><code class="docutils literal notranslate"><span class="pre">fernet_key</span></code> in <code class="docutils literal notranslate"><span class="pre">[core]</span></code> section</li> +<li><code class="docutils literal notranslate"><span class="pre">broker_url</span></code> in <code class="docutils literal notranslate"><span class="pre">[celery]</span></code> section</li> +<li><code class="docutils literal notranslate"><span class="pre">result_backend</span></code> in <code class="docutils literal notranslate"><span class="pre">[celery]</span></code> section</li> +<li><code class="docutils literal notranslate"><span class="pre">password</span></code> in <code class="docutils literal notranslate"><span class="pre">[atlas]</span></code> section</li> +<li><code class="docutils literal notranslate"><span class="pre">smtp_password</span></code> in <code class="docutils literal notranslate"><span class="pre">[smtp]</span></code> section</li> +<li><code class="docutils literal notranslate"><span class="pre">bind_password</span></code> in <code class="docutils literal notranslate"><span class="pre">[ldap]</span></code> section</li> +<li><code class="docutils literal notranslate"><span class="pre">git_password</span></code> in <code class="docutils literal notranslate"><span class="pre">[kubernetes]</span></code> section</li> +</ul> +<p>The idea behind this is to not store passwords on boxes in plain text files.</p> +<p>The order of precedence for all connfig options is as follows -</p> <ol class="arabic simple"> <li>environment variable</li> <li>configuration in airflow.cfg</li> <li>command in airflow.cfg</li> -<li>default</li> +<li>Airflowâs built in defaults</li> </ol> </div> @@ -246,20 +259,13 @@ metadata database connection string can either be set in <code class="docutils l - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/howto/use-test-config.html ---------------------------------------------------------------------- diff --git a/howto/use-test-config.html b/howto/use-test-config.html index 9a6db88..71d1214 100644 --- a/howto/use-test-config.html +++ b/howto/use-test-config.html @@ -232,20 +232,13 @@ airflow.cfg.</p> - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../_static/jquery.js"></script> - <script type="text/javascript" src="../_static/underscore.js"></script> - <script type="text/javascript" src="../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> +
