http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/gcs_download_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/gcs_download_operator.html b/_modules/airflow/contrib/operators/gcs_download_operator.html index a02f0a2..3154d79 100644 --- a/_modules/airflow/contrib/operators/gcs_download_operator.html +++ b/_modules/airflow/contrib/operators/gcs_download_operator.html @@ -232,7 +232,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="GoogleCloudStorageDownloadOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span> @@ -249,7 +249,7 @@ <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span> <span class="s1">'The size of the downloaded file is too large to push to XCom!'</span> <span class="p">)</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div></div> </pre></div> </div> @@ -280,20 +280,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> +
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/gcs_list_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/gcs_list_operator.html b/_modules/airflow/contrib/operators/gcs_list_operator.html index bacdd21..5d2f44d 100644 --- a/_modules/airflow/contrib/operators/gcs_list_operator.html +++ b/_modules/airflow/contrib/operators/gcs_list_operator.html @@ -238,7 +238,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="GoogleCloudStorageListOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_list_operator.GoogleCloudStorageListOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span> <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span> @@ -250,7 +250,7 @@ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prefix</span><span class="p">,</span> - <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">)</span></div> + <span class="n">delimiter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delimiter</span><span class="p">)</span></div></div> </pre></div> </div> @@ -281,20 +281,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/gcs_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/gcs_operator.html b/_modules/airflow/contrib/operators/gcs_operator.html index dcdee63..e563104 100644 --- a/_modules/airflow/contrib/operators/gcs_operator.html +++ b/_modules/airflow/contrib/operators/gcs_operator.html @@ -265,7 +265,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="GoogleCloudStorageCreateBucketOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_operator.GoogleCloudStorageCreateBucketOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="o">.</span><span class="n">update</span><span class="p">(</span> <span class="p">{</span><span class="s1">'airflow-version'</span><span class="p">:</span> <span class="s1">'v'</span> <span class="o">+</span> <span class="n">version</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'+'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)}</span> @@ -280,7 +280,7 @@ <span class="n">storage_class</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">storage_class</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span> <span class="n">project_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> - <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">)</span></div> + <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">)</span></div></div> </pre></div> </div> @@ -311,20 +311,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/gcs_to_bq.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/gcs_to_bq.html b/_modules/airflow/contrib/operators/gcs_to_bq.html index 4dabc6a..34bafd1 100644 --- a/_modules/airflow/contrib/operators/gcs_to_bq.html +++ b/_modules/airflow/contrib/operators/gcs_to_bq.html @@ -341,7 +341,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">src_fmt_configs</span> <span class="o">=</span> <span class="n">src_fmt_configs</span> <span class="bp">self</span><span class="o">.</span><span class="n">time_partitioning</span> <span class="o">=</span> <span class="n">time_partitioning</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="GoogleCloudStorageToBigQueryOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">bq_hook</span> <span class="o">=</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> @@ -407,7 +407,7 @@ <span class="s1">'Loaded BQ data with max </span><span class="si">%s</span><span class="s1">.</span><span class="si">%s</span><span class="s1">=</span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_project_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_id_key</span><span class="p">,</span> <span class="n">max_id</span> <span class="p">)</span> - <span class="k">return</span> <span class="n">max_id</span></div> + <span class="k">return</span> <span class="n">max_id</span></div></div> </pre></div> </div> @@ -438,20 +438,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/gcs_to_gcs.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/gcs_to_gcs.html b/_modules/airflow/contrib/operators/gcs_to_gcs.html index 86acbc4..d2964fd 100644 --- a/_modules/airflow/contrib/operators/gcs_to_gcs.html +++ b/_modules/airflow/contrib/operators/gcs_to_gcs.html @@ -289,7 +289,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span> <span class="bp">self</span><span class="o">.</span><span class="n">wildcard</span> <span class="o">=</span> <span class="s1">'*'</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="GoogleCloudStorageToGoogleCloudStorageOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span> <span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span> @@ -327,7 +327,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">destination_bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_object</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">move_object</span><span class="p">:</span> - <span class="n">hook</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_object</span><span class="p">)</span></div> + <span class="n">hook</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">source_bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_object</span><span class="p">)</span></div></div> </pre></div> </div> @@ -358,20 +358,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/gcs_to_s3.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/gcs_to_s3.html b/_modules/airflow/contrib/operators/gcs_to_s3.html index 6a0b651..c656f0d 100644 --- a/_modules/airflow/contrib/operators/gcs_to_s3.html +++ b/_modules/airflow/contrib/operators/gcs_to_s3.html @@ -238,7 +238,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">dest_s3_key</span> <span class="o">=</span> <span class="n">dest_s3_key</span> <span class="bp">self</span><span class="o">.</span><span class="n">replace</span> <span class="o">=</span> <span class="n">replace</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="GoogleCloudStorageToS3Operator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.gcs_to_s3.GoogleCloudStorageToS3Operator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="c1"># use the super to list all files in an Google Cloud Storage bucket</span> <span class="n">files</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">GoogleCloudStorageToS3Operator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="p">)</span> <span class="n">s3_hook</span> <span class="o">=</span> <span class="n">S3Hook</span><span class="p">(</span><span class="n">aws_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dest_aws_conn_id</span><span class="p">)</span> @@ -271,7 +271,7 @@ <span class="k">else</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"In sync, no files needed to be uploaded to S3"</span><span class="p">)</span> - <span class="k">return</span> <span class="n">files</span></div> + <span class="k">return</span> <span class="n">files</span></div></div> </pre></div> </div> @@ -302,20 +302,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/hipchat_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/hipchat_operator.html b/_modules/airflow/contrib/operators/hipchat_operator.html index 09b1eba..eefdf9b 100644 --- a/_modules/airflow/contrib/operators/hipchat_operator.html +++ b/_modules/airflow/contrib/operators/hipchat_operator.html @@ -222,7 +222,7 @@ <span class="sd"> """</span> <span class="k">pass</span></div> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="HipChatAPIOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.hipchat_operator.HipChatAPIOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">prepare_request</span><span class="p">()</span> <span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">request</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> @@ -235,7 +235,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'HipChat API call failed: </span><span class="si">%s</span><span class="s1"> </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'HipChat API call failed: </span><span class="si">%s</span><span class="s1"> </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> - <span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">))</span></div> + <span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">))</span></div></div> <div class="viewcode-block" id="HipChatAPISendRoomNotificationOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.hipchat_operator.HipChatAPISendRoomNotificationOperator">[docs]</a><span class="k">class</span> <span class="nc">HipChatAPISendRoomNotificationOperator</span><span class="p">(</span><span class="n">HipChatAPIOperator</span><span class="p">):</span> @@ -324,20 +324,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/hive_to_dynamodb.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/hive_to_dynamodb.html b/_modules/airflow/contrib/operators/hive_to_dynamodb.html index 31cac1d..a6d4766 100644 --- a/_modules/airflow/contrib/operators/hive_to_dynamodb.html +++ b/_modules/airflow/contrib/operators/hive_to_dynamodb.html @@ -244,7 +244,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">hiveserver2_conn_id</span> <span class="o">=</span> <span class="n">hiveserver2_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">aws_conn_id</span> <span class="o">=</span> <span class="n">aws_conn_id</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="HiveToDynamoDBTransferOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.hive_to_dynamodb.HiveToDynamoDBTransferOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hive</span> <span class="o">=</span> <span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hiveserver2_conn_id</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Extracting data from Hive'</span><span class="p">)</span> @@ -267,7 +267,7 @@ <span class="n">args</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pre_process_args</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pre_process_kwargs</span><span class="p">))</span> - <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Done.'</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Done.'</span><span class="p">)</span></div></div> </pre></div> </div> @@ -298,20 +298,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/jenkins_job_trigger_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/jenkins_job_trigger_operator.html b/_modules/airflow/contrib/operators/jenkins_job_trigger_operator.html index 85c3651..422b86e 100644 --- a/_modules/airflow/contrib/operators/jenkins_job_trigger_operator.html +++ b/_modules/airflow/contrib/operators/jenkins_job_trigger_operator.html @@ -352,7 +352,7 @@ <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="k">return</span> <span class="n">JenkinsHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jenkins_connection_id</span><span class="p">)</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="JenkinsJobTriggerOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.jenkins_job_trigger_operator.JenkinsJobTriggerOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">jenkins_connection_id</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span> <span class="s1">'Please specify the jenkins connection id to use.'</span> @@ -407,7 +407,7 @@ <span class="k">if</span> <span class="n">build_info</span><span class="p">:</span> <span class="c1"># If we can we return the url of the job</span> <span class="c1"># for later use (like retrieving an artifact)</span> - <span class="k">return</span> <span class="n">build_info</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span></div> + <span class="k">return</span> <span class="n">build_info</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span></div></div> </pre></div> </div> @@ -438,20 +438,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/jira_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/jira_operator.html b/_modules/airflow/contrib/operators/jira_operator.html index 9248a74..719bbf4 100644 --- a/_modules/airflow/contrib/operators/jira_operator.html +++ b/_modules/airflow/contrib/operators/jira_operator.html @@ -222,7 +222,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">result_processor</span> <span class="o">=</span> <span class="n">result_processor</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_jira_resource_method</span> <span class="o">=</span> <span class="n">get_jira_resource_method</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="JiraOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.jira_operator.JiraOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="k">try</span><span class="p">:</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_jira_resource_method</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># if get_jira_resource_method is provided, jira_method will be executed on</span> @@ -252,7 +252,7 @@ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Failed to execute jiraOperator, error: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="nb">str</span><span class="p">(</span><span class="n">jira_error</span><span class="p">))</span> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> - <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Jira operator error: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div> + <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Jira operator error: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div></div> </pre></div> </div> @@ -283,20 +283,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/kubernetes_pod_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/kubernetes_pod_operator.html b/_modules/airflow/contrib/operators/kubernetes_pod_operator.html index 4fe38df..7a3999e 100644 --- a/_modules/airflow/contrib/operators/kubernetes_pod_operator.html +++ b/_modules/airflow/contrib/operators/kubernetes_pod_operator.html @@ -231,16 +231,20 @@ <span class="sd"> :type get_logs: bool</span> <span class="sd"> :param affinity: A dict containing a group of affinity scheduling rules</span> <span class="sd"> :type affinity: dict</span> +<span class="sd"> :param node_selectors: A dict containing a group of scheduling rules</span> +<span class="sd"> :type node_selectors: dict</span> <span class="sd"> :param config_file: The path to the Kubernetes config file</span> <span class="sd"> :type config_file: str</span> <span class="sd"> :param xcom_push: If xcom_push is True, the content of the file</span> <span class="sd"> /airflow/xcom/return.json in the container will also be pushed to an</span> <span class="sd"> XCom when the container completes.</span> <span class="sd"> :type xcom_push: bool</span> +<span class="sd"> :param tolerations: Kubernetes tolerations</span> +<span class="sd"> :type list of tolerations</span> <span class="sd"> """</span> <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'cmds'</span><span class="p">,</span> <span class="s1">'arguments'</span><span class="p">,</span> <span class="s1">'env_vars'</span><span class="p">,</span> <span class="s1">'config_file'</span><span class="p">)</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="KubernetesPodOperator.execute"><a class="viewcode-back" href="../../../../kubernetes.html#airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="k">try</span><span class="p">:</span> <span class="n">client</span> <span class="o">=</span> <span class="n">kube_client</span><span class="o">.</span><span class="n">get_kube_client</span><span class="p">(</span><span class="n">in_cluster</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">in_cluster</span><span class="p">,</span> <span class="n">cluster_context</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cluster_context</span><span class="p">,</span> @@ -261,12 +265,16 @@ <span class="n">labels</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">,</span> <span class="p">)</span> + <span class="n">pod</span><span class="o">.</span><span class="n">service_account_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">service_account_name</span> <span class="n">pod</span><span class="o">.</span><span class="n">secrets</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">secrets</span> <span class="n">pod</span><span class="o">.</span><span class="n">envs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">env_vars</span> <span class="n">pod</span><span class="o">.</span><span class="n">image_pull_policy</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span> <span class="n">pod</span><span class="o">.</span><span class="n">annotations</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span> <span class="n">pod</span><span class="o">.</span><span class="n">resources</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">resources</span> <span class="n">pod</span><span class="o">.</span><span class="n">affinity</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">affinity</span> + <span class="n">pod</span><span class="o">.</span><span class="n">node_selectors</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">node_selectors</span> + <span class="n">pod</span><span class="o">.</span><span class="n">hostnetwork</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hostnetwork</span> + <span class="n">pod</span><span class="o">.</span><span class="n">tolerations</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tolerations</span> <span class="n">launcher</span> <span class="o">=</span> <span class="n">pod_launcher</span><span class="o">.</span><span class="n">PodLauncher</span><span class="p">(</span><span class="n">kube_client</span><span class="o">=</span><span class="n">client</span><span class="p">,</span> <span class="n">extract_xcom</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">)</span> @@ -274,6 +282,10 @@ <span class="n">pod</span><span class="p">,</span> <span class="n">startup_timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">startup_timeout_seconds</span><span class="p">,</span> <span class="n">get_logs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">get_logs</span><span class="p">)</span> + + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_delete_operator_pod</span><span class="p">:</span> + <span class="n">launcher</span><span class="o">.</span><span class="n">delete_pod</span><span class="p">(</span><span class="n">pod</span><span class="p">)</span> + <span class="k">if</span> <span class="n">final_state</span> <span class="o">!=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">:</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span> <span class="s1">'Pod returned a failure: </span><span class="si">{state}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">state</span><span class="o">=</span><span class="n">final_state</span><span class="p">)</span> @@ -281,7 +293,7 @@ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">:</span> <span class="k">return</span> <span class="n">result</span> <span class="k">except</span> <span class="n">AirflowException</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span> - <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Pod Launching failed: </span><span class="si">{error}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">=</span><span class="n">ex</span><span class="p">))</span> + <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Pod Launching failed: </span><span class="si">{error}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">=</span><span class="n">ex</span><span class="p">))</span></div> <span class="nd">@apply_defaults</span> <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> @@ -305,6 +317,12 @@ <span class="n">affinity</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">config_file</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> + <span class="n">node_selectors</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">image_pull_secrets</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">service_account_name</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span> + <span class="n">is_delete_operator_pod</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> + <span class="n">hostnetwork</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> + <span class="n">tolerations</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="nb">super</span><span class="p">(</span><span class="n">KubernetesPodOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> @@ -323,11 +341,17 @@ <span class="bp">self</span><span class="o">.</span><span class="n">cluster_context</span> <span class="o">=</span> <span class="n">cluster_context</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_logs</span> <span class="o">=</span> <span class="n">get_logs</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_policy</span> <span class="o">=</span> <span class="n">image_pull_policy</span> + <span class="bp">self</span><span class="o">.</span><span class="n">node_selectors</span> <span class="o">=</span> <span class="n">node_selectors</span> <span class="ow">or</span> <span class="p">{}</span> <span class="bp">self</span><span class="o">.</span><span class="n">annotations</span> <span class="o">=</span> <span class="n">annotations</span> <span class="ow">or</span> <span class="p">{}</span> <span class="bp">self</span><span class="o">.</span><span class="n">affinity</span> <span class="o">=</span> <span class="n">affinity</span> <span class="ow">or</span> <span class="p">{}</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span> <span class="o">=</span> <span class="n">xcom_push</span> <span class="bp">self</span><span class="o">.</span><span class="n">resources</span> <span class="o">=</span> <span class="n">resources</span> <span class="ow">or</span> <span class="n">Resources</span><span class="p">()</span> - <span class="bp">self</span><span class="o">.</span><span class="n">config_file</span> <span class="o">=</span> <span class="n">config_file</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">config_file</span> <span class="o">=</span> <span class="n">config_file</span> + <span class="bp">self</span><span class="o">.</span><span class="n">image_pull_secrets</span> <span class="o">=</span> <span class="n">image_pull_secrets</span> + <span class="bp">self</span><span class="o">.</span><span class="n">service_account_name</span> <span class="o">=</span> <span class="n">service_account_name</span> + <span class="bp">self</span><span class="o">.</span><span class="n">is_delete_operator_pod</span> <span class="o">=</span> <span class="n">is_delete_operator_pod</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hostnetwork</span> <span class="o">=</span> <span class="n">hostnetwork</span> + <span class="bp">self</span><span class="o">.</span><span class="n">tolerations</span> <span class="o">=</span> <span class="n">tolerations</span> <span class="ow">or</span> <span class="p">[]</span></div> </pre></div> </div> @@ -358,20 +382,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/mlengine_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/mlengine_operator.html b/_modules/airflow/contrib/operators/mlengine_operator.html index 66bb281..c2043ca 100644 --- a/_modules/airflow/contrib/operators/mlengine_operator.html +++ b/_modules/airflow/contrib/operators/mlengine_operator.html @@ -382,7 +382,7 @@ <span class="s1">'Missing model origin: Batch prediction expects a model, '</span> <span class="s1">'a model & version combination, or a URI to a savedModel.'</span><span class="p">)</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="MLEngineBatchPredictionOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.mlengine_operator.MLEngineBatchPredictionOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">job_id</span> <span class="o">=</span> <span class="n">_normalize_mlengine_job_id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">)</span> <span class="n">prediction_request</span> <span class="o">=</span> <span class="p">{</span> <span class="s1">'jobId'</span><span class="p">:</span> <span class="n">job_id</span><span class="p">,</span> @@ -433,7 +433,7 @@ <span class="nb">str</span><span class="p">(</span><span class="n">finished_prediction_job</span><span class="p">)))</span> <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="n">finished_prediction_job</span><span class="p">[</span><span class="s1">'errorMessage'</span><span class="p">])</span> - <span class="k">return</span> <span class="n">finished_prediction_job</span><span class="p">[</span><span class="s1">'predictionOutput'</span><span class="p">]</span></div> + <span class="k">return</span> <span class="n">finished_prediction_job</span><span class="p">[</span><span class="s1">'predictionOutput'</span><span class="p">]</span></div></div> <div class="viewcode-block" id="MLEngineModelOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.mlengine_operator.MLEngineModelOperator">[docs]</a><span class="k">class</span> <span class="nc">MLEngineModelOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -486,7 +486,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">_gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">_delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="MLEngineModelOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.mlengine_operator.MLEngineModelOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">MLEngineHook</span><span class="p">(</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_delegate_to</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_operation</span> <span class="o">==</span> <span class="s1">'create'</span><span class="p">:</span> @@ -494,7 +494,7 @@ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_operation</span> <span class="o">==</span> <span class="s1">'get'</span><span class="p">:</span> <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">get_model</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model</span><span class="p">[</span><span class="s1">'name'</span><span class="p">])</span> <span class="k">else</span><span class="p">:</span> - <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Unknown operation: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_operation</span><span class="p">))</span></div> + <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Unknown operation: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_operation</span><span class="p">))</span></div></div> <div class="viewcode-block" id="MLEngineVersionOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.mlengine_operator.MLEngineVersionOperator">[docs]</a><span class="k">class</span> <span class="nc">MLEngineVersionOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -580,7 +580,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">_gcp_conn_id</span> <span class="o">=</span> <span class="n">gcp_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">_delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="MLEngineVersionOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.mlengine_operator.MLEngineVersionOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="k">if</span> <span class="s1">'name'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version_name</span> @@ -588,7 +588,9 @@ <span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_delegate_to</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_operation</span> <span class="o">==</span> <span class="s1">'create'</span><span class="p">:</span> - <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> + <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">:</span> + <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"version attribute of </span><span class="si">{}</span><span class="s2"> could not "</span> + <span class="s2">"be empty"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">))</span> <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_version</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">)</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_operation</span> <span class="o">==</span> <span class="s1">'set_default'</span><span class="p">:</span> @@ -600,7 +602,7 @@ <span class="k">return</span> <span class="n">hook</span><span class="o">.</span><span class="n">delete_version</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_project_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_model_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">[</span><span class="s1">'name'</span><span class="p">])</span> <span class="k">else</span><span class="p">:</span> - <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Unknown operation: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_operation</span><span class="p">))</span></div> + <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'Unknown operation: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_operation</span><span class="p">))</span></div></div> <div class="viewcode-block" id="MLEngineTrainingOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.mlengine_operator.MLEngineTrainingOperator">[docs]</a><span class="k">class</span> <span class="nc">MLEngineTrainingOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -723,7 +725,7 @@ <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_region</span><span class="p">:</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'Google Compute Engine region is required.'</span><span class="p">)</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="MLEngineTrainingOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.mlengine_operator.MLEngineTrainingOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">job_id</span> <span class="o">=</span> <span class="n">_normalize_mlengine_job_id</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_job_id</span><span class="p">)</span> <span class="n">training_request</span> <span class="o">=</span> <span class="p">{</span> <span class="s1">'jobId'</span><span class="p">:</span> <span class="n">job_id</span><span class="p">,</span> @@ -769,7 +771,7 @@ <span class="k">if</span> <span class="n">finished_training_job</span><span class="p">[</span><span class="s1">'state'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'SUCCEEDED'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'MLEngine training job failed: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span> <span class="nb">str</span><span class="p">(</span><span class="n">finished_training_job</span><span class="p">)))</span> - <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="n">finished_training_job</span><span class="p">[</span><span class="s1">'errorMessage'</span><span class="p">])</span></div> + <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="n">finished_training_job</span><span class="p">[</span><span class="s1">'errorMessage'</span><span class="p">])</span></div></div> </pre></div> </div> @@ -800,20 +802,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/mongo_to_s3.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/mongo_to_s3.html b/_modules/airflow/contrib/operators/mongo_to_s3.html index b57b2f8..db9d1f5 100644 --- a/_modules/airflow/contrib/operators/mongo_to_s3.html +++ b/_modules/airflow/contrib/operators/mongo_to_s3.html @@ -308,20 +308,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/mysql_to_gcs.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/mysql_to_gcs.html b/_modules/airflow/contrib/operators/mysql_to_gcs.html index 8c8e80e..583b5a6 100644 --- a/_modules/airflow/contrib/operators/mysql_to_gcs.html +++ b/_modules/airflow/contrib/operators/mysql_to_gcs.html @@ -262,7 +262,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="MySqlToGoogleCloudStorageOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.mysql_to_gcs.MySqlToGoogleCloudStorageOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_query_mysql</span><span class="p">()</span> <span class="n">files_to_upload</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_write_local_data_files</span><span class="p">(</span><span class="n">cursor</span><span class="p">)</span> @@ -278,7 +278,7 @@ <span class="c1"># Close all temp file handles.</span> <span class="k">for</span> <span class="n">file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">values</span><span class="p">():</span> - <span class="n">file_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> + <span class="n">file_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div> <span class="k">def</span> <span class="nf">_query_mysql</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="sd">"""</span> @@ -477,20 +477,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/postgres_to_gcs_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/postgres_to_gcs_operator.html b/_modules/airflow/contrib/operators/postgres_to_gcs_operator.html index 4f75bad..8f7e6c2 100644 --- a/_modules/airflow/contrib/operators/postgres_to_gcs_operator.html +++ b/_modules/airflow/contrib/operators/postgres_to_gcs_operator.html @@ -257,7 +257,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="PostgresToGoogleCloudStorageOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.postgres_to_gcs_operator.PostgresToGoogleCloudStorageOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_query_postgres</span><span class="p">()</span> <span class="n">files_to_upload</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_write_local_data_files</span><span class="p">(</span><span class="n">cursor</span><span class="p">)</span> @@ -273,7 +273,7 @@ <span class="c1"># Close all temp file handles.</span> <span class="k">for</span> <span class="n">file_handle</span> <span class="ow">in</span> <span class="n">files_to_upload</span><span class="o">.</span><span class="n">values</span><span class="p">():</span> - <span class="n">file_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> + <span class="n">file_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div> <span class="k">def</span> <span class="nf">_query_postgres</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="sd">"""</span> @@ -435,20 +435,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> +
