http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/bigquery_to_gcs.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html index 9052db6..87a2a29 100644 --- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html +++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html @@ -215,8 +215,12 @@ <span class="sd"> For this to work, the service account making the request must have domain-wide</span> <span class="sd"> delegation enabled.</span> <span class="sd"> :type delegate_to: string</span> +<span class="sd"> :param labels: a dictionary containing labels for the job/query,</span> +<span class="sd"> passed to BigQuery</span> +<span class="sd"> :type labels: dict</span> <span class="sd"> """</span> - <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'source_project_dataset_table'</span><span class="p">,</span> <span class="s1">'destination_cloud_storage_uris'</span><span class="p">)</span> + <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'source_project_dataset_table'</span><span class="p">,</span> + <span class="s1">'destination_cloud_storage_uris'</span><span class="p">,</span> <span class="s1">'labels'</span><span class="p">)</span> <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span> <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#e4e6f0'</span> @@ -230,6 +234,7 @@ <span class="n">print_header</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">bigquery_conn_id</span><span class="o">=</span><span class="s1">'bigquery_default'</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="nb">super</span><span class="p">(</span><span class="n">BigQueryToCloudStorageOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> @@ -241,8 +246,9 @@ <span class="bp">self</span><span class="o">.</span><span class="n">print_header</span> <span class="o">=</span> <span class="n">print_header</span> <span class="bp">self</span><span class="o">.</span><span class="n">bigquery_conn_id</span> <span class="o">=</span> <span class="n">bigquery_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span> + <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="BigQueryToCloudStorageOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing extract of </span><span class="si">%s</span><span class="s1"> into: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">source_project_dataset_table</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">destination_cloud_storage_uris</span><span class="p">)</span> @@ -256,7 +262,8 @@ <span class="bp">self</span><span class="o">.</span><span class="n">compression</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">export_format</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">field_delimiter</span><span class="p">,</span> - <span class="bp">self</span><span class="o">.</span><span class="n">print_header</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">print_header</span><span class="p">,</span> + <span class="bp">self</span><span class="o">.</span><span class="n">labels</span><span class="p">)</span></div></div> </pre></div> </div> @@ -287,20 +294,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> +
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/cassandra_to_gcs.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/cassandra_to_gcs.html b/_modules/airflow/contrib/operators/cassandra_to_gcs.html index a53bc3d..73b5b34 100644 --- a/_modules/airflow/contrib/operators/cassandra_to_gcs.html +++ b/_modules/airflow/contrib/operators/cassandra_to_gcs.html @@ -287,7 +287,7 @@ <span class="s1">'VarcharType'</span><span class="p">:</span> <span class="s1">'STRING'</span><span class="p">,</span> <span class="p">}</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="CassandraToGoogleCloudStorageOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.cassandra_to_gcs.CassandraToGoogleCloudStorageOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_query_cassandra</span><span class="p">()</span> <span class="n">files_to_upload</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_write_local_data_files</span><span class="p">(</span><span class="n">cursor</span><span class="p">)</span> @@ -306,7 +306,7 @@ <span class="n">file_handle</span><span class="o">.</span><span class="n">close</span><span class="p">()</span> <span class="c1"># Close all sessions and connection associated with this Cassandra cluster</span> - <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">shutdown_cluster</span><span class="p">()</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">shutdown_cluster</span><span class="p">()</span></div> <span class="k">def</span> <span class="nf">_query_cassandra</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="sd">"""</span> @@ -544,20 +544,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/databricks_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/databricks_operator.html b/_modules/airflow/contrib/operators/databricks_operator.html index f401636..51c13d7 100644 --- a/_modules/airflow/contrib/operators/databricks_operator.html +++ b/_modules/airflow/contrib/operators/databricks_operator.html @@ -307,6 +307,9 @@ <span class="sd"> :param databricks_retry_limit: Amount of times retry if the Databricks backend is</span> <span class="sd"> unreachable. Its value must be greater than or equal to 1.</span> <span class="sd"> :type databricks_retry_limit: int</span> +<span class="sd"> :param databricks_retry_delay: Number of seconds to wait between retries (it</span> +<span class="sd"> might be a floating point number).</span> +<span class="sd"> :type databricks_retry_delay: float</span> <span class="sd"> :param do_xcom_push: Whether we should push run_id and run_page_url to xcom.</span> <span class="sd"> :type do_xcom_push: boolean</span> <span class="sd"> """</span> @@ -329,6 +332,7 @@ <span class="n">databricks_conn_id</span><span class="o">=</span><span class="s1">'databricks_default'</span><span class="p">,</span> <span class="n">polling_period_seconds</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="n">databricks_retry_limit</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> + <span class="n">databricks_retry_delay</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">do_xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="sd">"""</span> @@ -339,6 +343,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span> <span class="o">=</span> <span class="n">databricks_conn_id</span> <span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span> <span class="o">=</span> <span class="n">polling_period_seconds</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_limit</span> <span class="o">=</span> <span class="n">databricks_retry_limit</span> + <span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_delay</span> <span class="o">=</span> <span class="n">databricks_retry_delay</span> <span class="k">if</span> <span class="n">spark_jar_task</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">[</span><span class="s1">'spark_jar_task'</span><span class="p">]</span> <span class="o">=</span> <span class="n">spark_jar_task</span> <span class="k">if</span> <span class="n">notebook_task</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> @@ -393,9 +398,10 @@ <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="k">return</span> <span class="n">DatabricksHook</span><span class="p">(</span> <span class="bp">self</span><span class="o">.</span><span class="n">databricks_conn_id</span><span class="p">,</span> - <span class="n">retry_limit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_limit</span><span class="p">)</span> + <span class="n">retry_limit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_limit</span><span class="p">,</span> + <span class="n">retry_delay</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">databricks_retry_delay</span><span class="p">)</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DatabricksSubmitRunOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">submit_run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">json</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">do_xcom_push</span><span class="p">:</span> @@ -421,15 +427,15 @@ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'</span><span class="si">%s</span><span class="s1"> in run state: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">run_state</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">_log_run_page_url</span><span class="p">(</span><span class="n">run_page_url</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Sleeping for </span><span class="si">%s</span><span class="s1"> seconds.'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span><span class="p">)</span> - <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span><span class="p">)</span> + <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">polling_period_seconds</span><span class="p">)</span></div> - <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> +<div class="viewcode-block" id="DatabricksSubmitRunOperator.on_kill"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator.on_kill">[docs]</a> <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span> <span class="n">hook</span><span class="o">.</span><span class="n">cancel_run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">run_id</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> <span class="s1">'Task: </span><span class="si">%s</span><span class="s1"> with run_id: </span><span class="si">%s</span><span class="s1"> was requested to be cancelled.'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_id</span> - <span class="p">)</span></div> + <span class="p">)</span></div></div> </pre></div> </div> @@ -460,20 +466,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/dataflow_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/dataflow_operator.html b/_modules/airflow/contrib/operators/dataflow_operator.html index 09cc5c1..59107e2 100644 --- a/_modules/airflow/contrib/operators/dataflow_operator.html +++ b/_modules/airflow/contrib/operators/dataflow_operator.html @@ -177,7 +177,7 @@ <span class="c1"># KIND, either express or implied. See the License for the</span> <span class="c1"># specific language governing permissions and limitations</span> <span class="c1"># under the License.</span> - +<span class="kn">import</span> <span class="nn">os</span> <span class="kn">import</span> <span class="nn">re</span> <span class="kn">import</span> <span class="nn">uuid</span> <span class="kn">import</span> <span class="nn">copy</span> @@ -291,7 +291,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span> <span class="o">=</span> <span class="n">poll_sleep</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_class</span> <span class="o">=</span> <span class="n">job_class</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataFlowJavaOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataFlowJavaOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">bucket_helper</span> <span class="o">=</span> <span class="n">GoogleCloudBucketHelper</span><span class="p">(</span> <span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">jar</span> <span class="o">=</span> <span class="n">bucket_helper</span><span class="o">.</span><span class="n">google_cloud_to_local</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">)</span> @@ -303,7 +303,7 @@ <span class="n">dataflow_options</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">options</span><span class="p">)</span> <span class="n">hook</span><span class="o">.</span><span class="n">start_java_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="n">dataflow_options</span><span class="p">,</span> - <span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_class</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">jar</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">job_class</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataflowTemplateOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataflowTemplateOperator">[docs]</a><span class="k">class</span> <span class="nc">DataflowTemplateOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -403,16 +403,48 @@ <span class="bp">self</span><span class="o">.</span><span class="n">template</span> <span class="o">=</span> <span class="n">template</span> <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataflowTemplateOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataflowTemplateOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataFlowHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">,</span> <span class="n">poll_sleep</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">poll_sleep</span><span class="p">)</span> <span class="n">hook</span><span class="o">.</span><span class="n">start_template_dataflow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataflow_default_options</span><span class="p">,</span> - <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">parameters</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataFlowPythonOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator">[docs]</a><span class="k">class</span> <span class="nc">DataFlowPythonOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Create a new DataFlowPythonOperator. Note that both</span> +<span class="sd"> dataflow_default_options and options will be merged to specify pipeline</span> +<span class="sd"> execution parameter, and dataflow_default_options is expected to save</span> +<span class="sd"> high-level options, for instances, project and zone information, which</span> +<span class="sd"> apply to all dataflow operators in the DAG.</span> + +<span class="sd"> .. seealso::</span> +<span class="sd"> For more detail on job submission have a look at the reference:</span> +<span class="sd"> https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span> + +<span class="sd"> :param py_file: Reference to the python dataflow pipleline file.py, e.g.,</span> +<span class="sd"> /some/local/file/path/to/your/python/pipeline/file.</span> +<span class="sd"> :type py_file: string</span> +<span class="sd"> :param py_options: Additional python options.</span> +<span class="sd"> :type pyt_options: list of strings, e.g., ["-m", "-v"].</span> +<span class="sd"> :param dataflow_default_options: Map of default job options.</span> +<span class="sd"> :type dataflow_default_options: dict</span> +<span class="sd"> :param options: Map of job specific options.</span> +<span class="sd"> :type options: dict</span> +<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud</span> +<span class="sd"> Platform.</span> +<span class="sd"> :type gcp_conn_id: string</span> +<span class="sd"> :param delegate_to: The account to impersonate, if any.</span> +<span class="sd"> For this to work, the service account making the request must have</span> +<span class="sd"> domain-wide delegation enabled.</span> +<span class="sd"> :type delegate_to: string</span> +<span class="sd"> :param poll_sleep: The time in seconds to sleep between polling Google</span> +<span class="sd"> Cloud Platform for the dataflow job status while the job is in the</span> +<span class="sd"> JOB_STATE_RUNNING state.</span> +<span class="sd"> :type poll_sleep: int</span> +<span class="sd"> """</span> <span class="n">template_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'options'</span><span class="p">,</span> <span class="s1">'dataflow_default_options'</span><span class="p">]</span> @@ -428,38 +460,7 @@ <span class="n">poll_sleep</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> - <span class="sd">"""</span> -<span class="sd"> Create a new DataFlowPythonOperator. Note that both</span> -<span class="sd"> dataflow_default_options and options will be merged to specify pipeline</span> -<span class="sd"> execution parameter, and dataflow_default_options is expected to save</span> -<span class="sd"> high-level options, for instances, project and zone information, which</span> -<span class="sd"> apply to all dataflow operators in the DAG.</span> - -<span class="sd"> .. seealso::</span> -<span class="sd"> For more detail on job submission have a look at the reference:</span> -<span class="sd"> https://cloud.google.com/dataflow/pipelines/specifying-exec-params</span> -<span class="sd"> :param py_file: Reference to the python dataflow pipleline file.py, e.g.,</span> -<span class="sd"> /some/local/file/path/to/your/python/pipeline/file.</span> -<span class="sd"> :type py_file: string</span> -<span class="sd"> :param py_options: Additional python options.</span> -<span class="sd"> :type pyt_options: list of strings, e.g., ["-m", "-v"].</span> -<span class="sd"> :param dataflow_default_options: Map of default job options.</span> -<span class="sd"> :type dataflow_default_options: dict</span> -<span class="sd"> :param options: Map of job specific options.</span> -<span class="sd"> :type options: dict</span> -<span class="sd"> :param gcp_conn_id: The connection ID to use connecting to Google Cloud</span> -<span class="sd"> Platform.</span> -<span class="sd"> :type gcp_conn_id: string</span> -<span class="sd"> :param delegate_to: The account to impersonate, if any.</span> -<span class="sd"> For this to work, the service account making the request must have</span> -<span class="sd"> domain-wide delegation enabled.</span> -<span class="sd"> :type delegate_to: string</span> -<span class="sd"> :param poll_sleep: The time in seconds to sleep between polling Google</span> -<span class="sd"> Cloud Platform for the dataflow job status while the job is in the</span> -<span class="sd"> JOB_STATE_RUNNING state.</span> -<span class="sd"> :type poll_sleep: int</span> -<span class="sd"> """</span> <span class="nb">super</span><span class="p">(</span><span class="n">DataFlowPythonOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="bp">self</span><span class="o">.</span><span class="n">py_file</span> <span class="o">=</span> <span class="n">py_file</span> @@ -519,21 +520,21 @@ <span class="c1"># Extracts bucket_id and object_id by first removing 'gs://' prefix and</span> <span class="c1"># then split the remaining by path delimiter '/'.</span> <span class="n">path_components</span> <span class="o">=</span> <span class="n">file_name</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">GCS_PREFIX_LENGTH</span><span class="p">:]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span> - <span class="k">if</span> <span class="n">path_components</span> <span class="o"><</span> <span class="mi">2</span><span class="p">:</span> + <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">path_components</span><span class="p">)</span> <span class="o"><</span> <span class="mi">2</span><span class="p">:</span> <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span> - <span class="s1">'Invalid Google Cloud Storage (GCS) object path: </span><span class="si">{}</span><span class="s1">.'</span> + <span class="s1">'Invalid Google Cloud Storage (GCS) object path: </span><span class="si">{}</span><span class="s1">'</span> <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_name</span><span class="p">))</span> <span class="n">bucket_id</span> <span class="o">=</span> <span class="n">path_components</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="n">object_id</span> <span class="o">=</span> <span class="s1">'/'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path_components</span><span class="p">[</span><span class="mi">1</span><span class="p">:])</span> <span class="n">local_file</span> <span class="o">=</span> <span class="s1">'/tmp/dataflow</span><span class="si">{}</span><span class="s1">-</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid1</span><span class="p">())[:</span><span class="mi">8</span><span class="p">],</span> <span class="n">path_components</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> - <span class="n">file_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_gcs_hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="n">bucket_id</span><span class="p">,</span> <span class="n">object_id</span><span class="p">,</span> <span class="n">local_file</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">_gcs_hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="n">bucket_id</span><span class="p">,</span> <span class="n">object_id</span><span class="p">,</span> <span class="n">local_file</span><span class="p">)</span> - <span class="k">if</span> <span class="n">file_size</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> + <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">local_file</span><span class="p">)</span><span class="o">.</span><span class="n">st_size</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span> <span class="k">return</span> <span class="n">local_file</span> <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span> - <span class="s1">'Failed to download Google Cloud Storage GCS object: </span><span class="si">{}</span><span class="s1">'</span> + <span class="s1">'Failed to download Google Cloud Storage (GCS) object: </span><span class="si">{}</span><span class="s1">'</span> <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">file_name</span><span class="p">))</span> </pre></div> @@ -565,20 +566,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> + http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/1f06fa0e/_modules/airflow/contrib/operators/dataproc_operator.html ---------------------------------------------------------------------- diff --git a/_modules/airflow/contrib/operators/dataproc_operator.html b/_modules/airflow/contrib/operators/dataproc_operator.html index 6bc5d70..ec9caec 100644 --- a/_modules/airflow/contrib/operators/dataproc_operator.html +++ b/_modules/airflow/contrib/operators/dataproc_operator.html @@ -212,8 +212,9 @@ <span class="sd"> :type cluster_name: string</span> <span class="sd"> :param project_id: The ID of the google cloud project in which</span> <span class="sd"> to create the cluster. (templated)</span> -<span class="sd"> :type project_id: string</span> -<span class="sd"> :param num_workers: The # of workers to spin up</span> +<span class="sd"> :type project_id: str</span> +<span class="sd"> :param num_workers: The # of workers to spin up. If set to zero will</span> +<span class="sd"> spin up cluster in a single node mode</span> <span class="sd"> :type num_workers: int</span> <span class="sd"> :param storage_bucket: The storage bucket to use, setting to None lets dataproc</span> <span class="sd"> generate a custom one for you</span> @@ -229,6 +230,9 @@ <span class="sd"> :type metadata: dict</span> <span class="sd"> :param image_version: the version of software inside the Dataproc cluster</span> <span class="sd"> :type image_version: string</span> +<span class="sd"> :param custom_image: custom Dataproc image for more info see</span> +<span class="sd"> https://cloud.google.com/dataproc/docs/guides/dataproc-images</span> +<span class="sd"> :type: custom_image: string</span> <span class="sd"> :param properties: dict of properties to set on</span> <span class="sd"> config files (e.g. spark-defaults.conf), see</span> <span class="sd"> https://cloud.google.com/dataproc/docs/reference/rest/v1/ \</span> @@ -236,10 +240,20 @@ <span class="sd"> :type properties: dict</span> <span class="sd"> :param master_machine_type: Compute engine machine type to use for the master node</span> <span class="sd"> :type master_machine_type: string</span> +<span class="sd"> :param master_disk_type: Type of the boot disk for the master node</span> +<span class="sd"> (default is ``pd-standard``).</span> +<span class="sd"> Valid values: ``pd-ssd`` (Persistent Disk Solid State Drive) or</span> +<span class="sd"> ``pd-standard`` (Persistent Disk Hard Disk Drive).</span> +<span class="sd"> :type master_disk_type: string</span> <span class="sd"> :param master_disk_size: Disk size for the master node</span> <span class="sd"> :type master_disk_size: int</span> <span class="sd"> :param worker_machine_type: Compute engine machine type to use for the worker nodes</span> <span class="sd"> :type worker_machine_type: string</span> +<span class="sd"> :param worker_disk_type: Type of the boot disk for the worker node</span> +<span class="sd"> (default is ``pd-standard``).</span> +<span class="sd"> Valid values: ``pd-ssd`` (Persistent Disk Solid State Drive) or</span> +<span class="sd"> ``pd-standard`` (Persistent Disk Hard Disk Drive).</span> +<span class="sd"> :type worker_disk_type: string</span> <span class="sd"> :param worker_disk_size: Disk size for the worker nodes</span> <span class="sd"> :type worker_disk_size: int</span> <span class="sd"> :param num_preemptible_workers: The # of preemptible worker nodes to spin up</span> @@ -299,11 +313,14 @@ <span class="n">init_actions_uris</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">init_action_timeout</span><span class="o">=</span><span class="s2">"10m"</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> + <span class="n">custom_image</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">image_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">properties</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">master_machine_type</span><span class="o">=</span><span class="s1">'n1-standard-4'</span><span class="p">,</span> + <span class="n">master_disk_type</span><span class="o">=</span><span class="s1">'pd-standard'</span><span class="p">,</span> <span class="n">master_disk_size</span><span class="o">=</span><span class="mi">500</span><span class="p">,</span> <span class="n">worker_machine_type</span><span class="o">=</span><span class="s1">'n1-standard-4'</span><span class="p">,</span> + <span class="n">worker_disk_type</span><span class="o">=</span><span class="s1">'pd-standard'</span><span class="p">,</span> <span class="n">worker_disk_size</span><span class="o">=</span><span class="mi">500</span><span class="p">,</span> <span class="n">num_preemptible_workers</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> @@ -329,11 +346,14 @@ <span class="bp">self</span><span class="o">.</span><span class="n">init_actions_uris</span> <span class="o">=</span> <span class="n">init_actions_uris</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_action_timeout</span> <span class="o">=</span> <span class="n">init_action_timeout</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">metadata</span> + <span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span> <span class="o">=</span> <span class="n">custom_image</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span> <span class="o">=</span> <span class="n">image_version</span> - <span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="o">=</span> <span class="n">properties</span> + <span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="o">=</span> <span class="n">properties</span> <span class="ow">or</span> <span class="nb">dict</span><span class="p">()</span> <span class="bp">self</span><span class="o">.</span><span class="n">master_machine_type</span> <span class="o">=</span> <span class="n">master_machine_type</span> + <span class="bp">self</span><span class="o">.</span><span class="n">master_disk_type</span> <span class="o">=</span> <span class="n">master_disk_type</span> <span class="bp">self</span><span class="o">.</span><span class="n">master_disk_size</span> <span class="o">=</span> <span class="n">master_disk_size</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_machine_type</span> <span class="o">=</span> <span class="n">worker_machine_type</span> + <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_type</span> <span class="o">=</span> <span class="n">worker_disk_type</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span> <span class="o">=</span> <span class="n">worker_disk_size</span> <span class="bp">self</span><span class="o">.</span><span class="n">labels</span> <span class="o">=</span> <span class="n">labels</span> <span class="bp">self</span><span class="o">.</span><span class="n">zone</span> <span class="o">=</span> <span class="n">zone</span> @@ -347,6 +367,19 @@ <span class="bp">self</span><span class="o">.</span><span class="n">idle_delete_ttl</span> <span class="o">=</span> <span class="n">idle_delete_ttl</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_time</span> <span class="o">=</span> <span class="n">auto_delete_time</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_delete_ttl</span> <span class="o">=</span> <span class="n">auto_delete_ttl</span> + <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span> <span class="o">=</span> <span class="n">num_workers</span> <span class="o">==</span> <span class="mi">0</span> + + <span class="k">assert</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span><span class="p">),</span> \ + <span class="s2">"custom_image and image_version can't be both set"</span> + + <span class="k">assert</span> <span class="p">(</span> + <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span> <span class="ow">or</span> <span class="p">(</span> + <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_preemptible_workers</span> <span class="o">==</span> <span class="mi">0</span> + <span class="p">)</span> + <span class="p">),</span> <span class="s2">"num_workers == 0 means single node mode - no preemptibles allowed"</span> + + <span class="k">assert</span> <span class="ow">not</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span><span class="p">),</span> \ + <span class="s2">"custom_image and image_version can't be both set"</span> <span class="k">def</span> <span class="nf">_get_cluster_list_for_project</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">service</span><span class="p">):</span> <span class="n">result</span> <span class="o">=</span> <span class="n">service</span><span class="o">.</span><span class="n">projects</span><span class="p">()</span><span class="o">.</span><span class="n">regions</span><span class="p">()</span><span class="o">.</span><span class="n">clusters</span><span class="p">()</span><span class="o">.</span><span class="n">list</span><span class="p">(</span> @@ -433,6 +466,7 @@ <span class="s1">'numInstances'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'machineTypeUri'</span><span class="p">:</span> <span class="n">master_type_uri</span><span class="p">,</span> <span class="s1">'diskConfig'</span><span class="p">:</span> <span class="p">{</span> + <span class="s1">'bootDiskType'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">master_disk_type</span><span class="p">,</span> <span class="s1">'bootDiskSizeGb'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">master_disk_size</span> <span class="p">}</span> <span class="p">},</span> @@ -440,6 +474,7 @@ <span class="s1">'numInstances'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span> <span class="s1">'machineTypeUri'</span><span class="p">:</span> <span class="n">worker_type_uri</span><span class="p">,</span> <span class="s1">'diskConfig'</span><span class="p">:</span> <span class="p">{</span> + <span class="s1">'bootDiskType'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_type</span><span class="p">,</span> <span class="s1">'bootDiskSizeGb'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span> <span class="p">}</span> <span class="p">},</span> @@ -453,6 +488,7 @@ <span class="s1">'numInstances'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_preemptible_workers</span><span class="p">,</span> <span class="s1">'machineTypeUri'</span><span class="p">:</span> <span class="n">worker_type_uri</span><span class="p">,</span> <span class="s1">'diskConfig'</span><span class="p">:</span> <span class="p">{</span> + <span class="s1">'bootDiskType'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_type</span><span class="p">,</span> <span class="s1">'bootDiskSizeGb'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">worker_disk_size</span> <span class="p">},</span> <span class="s1">'isPreemptible'</span><span class="p">:</span> <span class="kc">True</span> @@ -482,6 +518,17 @@ <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'gceClusterConfig'</span><span class="p">][</span><span class="s1">'tags'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tags</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span><span class="p">:</span> <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'softwareConfig'</span><span class="p">][</span><span class="s1">'imageVersion'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image_version</span> + <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span><span class="p">:</span> + <span class="n">custom_image_url</span> <span class="o">=</span> <span class="s1">'https://www.googleapis.com/compute/beta/projects/'</span> \ + <span class="s1">'</span><span class="si">{}</span><span class="s1">/global/images/</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> + <span class="bp">self</span><span class="o">.</span><span class="n">custom_image</span><span class="p">)</span> + <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'masterConfig'</span><span class="p">][</span><span class="s1">'imageUri'</span><span class="p">]</span> <span class="o">=</span> <span class="n">custom_image_url</span> + <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span><span class="p">:</span> + <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'workerConfig'</span><span class="p">][</span><span class="s1">'imageUri'</span><span class="p">]</span> <span class="o">=</span> <span class="n">custom_image_url</span> + + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">single_node</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="p">[</span><span class="s2">"dataproc:dataproc.allow.zero.workers"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"true"</span> + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="p">:</span> <span class="n">cluster_data</span><span class="p">[</span><span class="s1">'config'</span><span class="p">][</span><span class="s1">'softwareConfig'</span><span class="p">][</span><span class="s1">'properties'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">idle_delete_ttl</span><span class="p">:</span> @@ -510,7 +557,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">service_account_scopes</span> <span class="k">return</span> <span class="n">cluster_data</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataprocClusterCreateOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterCreateOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Creating cluster: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">)</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> @@ -546,7 +593,7 @@ <span class="k">else</span><span class="p">:</span> <span class="k">raise</span> <span class="n">e</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataprocClusterScaleOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterScaleOperator">[docs]</a><span class="k">class</span> <span class="nc">DataprocClusterScaleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -556,14 +603,14 @@ <span class="sd"> **Example**: ::</span> -<span class="sd"> t1 = DataprocClusterScaleOperator(</span> -<span class="sd"> task_id='dataproc_scale',</span> -<span class="sd"> project_id='my-project',</span> -<span class="sd"> cluster_name='cluster-1',</span> -<span class="sd"> num_workers=10,</span> -<span class="sd"> num_preemptible_workers=10,</span> -<span class="sd"> graceful_decommission_timeout='1h'</span> -<span class="sd"> dag=dag)</span> +<span class="sd"> t1 = DataprocClusterScaleOperator(</span> +<span class="sd"> task_id='dataproc_scale',</span> +<span class="sd"> project_id='my-project',</span> +<span class="sd"> cluster_name='cluster-1',</span> +<span class="sd"> num_workers=10,</span> +<span class="sd"> num_preemptible_workers=10,</span> +<span class="sd"> graceful_decommission_timeout='1h',</span> +<span class="sd"> dag=dag)</span> <span class="sd"> .. seealso::</span> <span class="sd"> For more detail on about scaling clusters have a look at the reference:</span> @@ -672,7 +719,7 @@ <span class="s2">" should be expressed in day, hours, minutes or seconds. "</span> <span class="s2">" i.e. 1d, 4h, 10m, 30s"</span><span class="p">)</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataprocClusterScaleOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterScaleOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Scaling cluster: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">)</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> @@ -694,7 +741,7 @@ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span> <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Cluster scale operation name: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataprocClusterDeleteOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterDeleteOperator">[docs]</a><span class="k">class</span> <span class="nc">DataprocClusterDeleteOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -750,7 +797,7 @@ <span class="k">return</span> <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">15</span><span class="p">)</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataprocClusterDeleteOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataprocClusterDeleteOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Deleting cluster: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">)</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> @@ -765,7 +812,7 @@ <span class="p">)</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span> <span class="n">operation_name</span> <span class="o">=</span> <span class="n">response</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Cluster delete operation name: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span> - <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div> + <span class="bp">self</span><span class="o">.</span><span class="n">_wait_for_done</span><span class="p">(</span><span class="n">service</span><span class="p">,</span> <span class="n">operation_name</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataProcPigOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPigOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcPigOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -863,7 +910,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_pig_jars</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataProcPigOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPigOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> <span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">"pigJob"</span><span class="p">,</span> @@ -877,7 +924,7 @@ <span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span> <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span> - <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div> + <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataProcHiveOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHiveOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcHiveOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -943,7 +990,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_hive_jars</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataProcHiveOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHiveOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> @@ -958,7 +1005,7 @@ <span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span> <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span> - <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div> + <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataProcSparkSqlOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcSparkSqlOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -1025,7 +1072,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_spark_jars</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataProcSparkSqlOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> @@ -1040,7 +1087,7 @@ <span class="n">job</span><span class="o">.</span><span class="n">add_jar_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span><span class="p">)</span> <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span> - <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div> + <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataProcSparkOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcSparkOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -1118,7 +1165,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_spark_jars</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataProcSparkOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcSparkOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> <span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">"sparkJob"</span><span class="p">,</span> @@ -1131,7 +1178,7 @@ <span class="n">job</span><span class="o">.</span><span class="n">add_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span> <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span> - <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div> + <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataProcHadoopOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcHadoopOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -1209,7 +1256,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_hadoop_jars</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataProcHadoopOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span><span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span> <span class="n">job</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">create_job_template</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cluster_name</span><span class="p">,</span> <span class="s2">"hadoopJob"</span><span class="p">,</span> @@ -1222,7 +1269,7 @@ <span class="n">job</span><span class="o">.</span><span class="n">add_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)</span> <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span> - <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div> + <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataProcPySparkOperator"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator">[docs]</a><span class="k">class</span> <span class="nc">DataProcPySparkOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -1327,7 +1374,7 @@ <span class="bp">self</span><span class="o">.</span><span class="n">dataproc_jars</span> <span class="o">=</span> <span class="n">dataproc_pyspark_jars</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span> <span class="o">=</span> <span class="n">region</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> +<div class="viewcode-block" id="DataProcPySparkOperator.execute"><a class="viewcode-back" href="../../../../integration.html#airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="n">hook</span> <span class="o">=</span> <span class="n">DataProcHook</span><span class="p">(</span> <span class="n">gcp_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">gcp_conn_id</span><span class="p">,</span> <span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> @@ -1353,7 +1400,7 @@ <span class="n">job</span><span class="o">.</span><span class="n">add_python_file_uris</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pyfiles</span><span class="p">)</span> <span class="n">job</span><span class="o">.</span><span class="n">set_job_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">job_name</span><span class="p">)</span> - <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div> + <span class="n">hook</span><span class="o">.</span><span class="n">submit</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">project_id</span><span class="p">,</span> <span class="n">job</span><span class="o">.</span><span class="n">build</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">region</span><span class="p">)</span></div></div> <div class="viewcode-block" id="DataprocWorkflowTemplateBaseOperator"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateBaseOperator">[docs]</a><span class="k">class</span> <span class="nc">DataprocWorkflowTemplateBaseOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> @@ -1376,8 +1423,8 @@ <span class="n">api_version</span><span class="o">=</span><span class="s1">'v1beta2'</span> <span class="p">)</span> - <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">start</span><span class="p">())</span> +<div class="viewcode-block" id="DataprocWorkflowTemplateBaseOperator.execute"><a class="viewcode-back" href="../../../../code.html#airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateBaseOperator.execute">[docs]</a> <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">start</span><span class="p">())</span></div> <span class="k">def</span> <span class="nf">start</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'plese start a workflow operation'</span><span class="p">)</span></div> @@ -1498,20 +1545,13 @@ - <script type="text/javascript"> - var DOCUMENTATION_OPTIONS = { - URL_ROOT:'../../../../', - VERSION:'', - LANGUAGE:'None', - COLLAPSE_INDEX:false, - FILE_SUFFIX:'.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' - }; - </script> - <script type="text/javascript" src="../../../../_static/jquery.js"></script> - <script type="text/javascript" src="../../../../_static/underscore.js"></script> - <script type="text/javascript" src="../../../../_static/doctools.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../../../_static/jquery.js"></script> + <script type="text/javascript" src="../../../../_static/underscore.js"></script> + <script type="text/javascript" src="../../../../_static/doctools.js"></script> +
