http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/gcs_hook.html ---------------------------------------------------------------------- diff --git a/_modules/gcs_hook.html b/_modules/gcs_hook.html index 012f4bf..c118b86 100644 --- a/_modules/gcs_hook.html +++ b/_modules/gcs_hook.html @@ -154,7 +154,7 @@ <span class="kn">import</span> <span class="nn">logging</span> -<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gc_base_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudBaseHook</span> +<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcp_api_base_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudBaseHook</span> <span class="kn">from</span> <span class="nn">apiclient.discovery</span> <span class="kn">import</span> <span class="n">build</span> <span class="kn">from</span> <span class="nn">apiclient.http</span> <span class="kn">import</span> <span class="n">MediaFileUpload</span> @@ -163,19 +163,8 @@ <div class="viewcode-block" id="GoogleCloudStorageHook"><a class="viewcode-back" href="../code.html#airflow.contrib.hooks.GoogleCloudStorageHook">[docs]</a><span class="k">class</span> <span class="nc">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">GoogleCloudBaseHook</span><span class="p">):</span> <span class="sd">"""</span> -<span class="sd"> Interact with Google Cloud Storage. Connections must be defined with an</span> -<span class="sd"> extras JSON field containing:</span> - -<span class="sd"> ::</span> -<span class="sd"> {</span> -<span class="sd"> "project": "<google project ID>",</span> -<span class="sd"> "service_account": "<google service account email>",</span> -<span class="sd"> "key_path": "<p12 key path>"</span> -<span class="sd"> }</span> - -<span class="sd"> If you have used ``gcloud auth`` to authenticate on the machine that's</span> -<span class="sd"> running Airflow, you can exclude the service_account and key_path</span> -<span class="sd"> parameters.</span> +<span class="sd"> Interact with Google Cloud Storage. This hook uses the Google Cloud Platform</span> +<span class="sd"> connection.</span> <span class="sd"> """</span> <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/index.html ---------------------------------------------------------------------- diff --git a/_modules/index.html b/_modules/index.html index 7fe6be6..d40ab64 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -135,6 +135,11 @@ <h1>All modules for which code is available</h1> <ul><li><a href="S3_hook.html">S3_hook</a></li> +<li><a href="airflow/contrib/executors/mesos_executor.html">airflow.contrib.executors.mesos_executor</a></li> +<li><a href="airflow/contrib/hooks/gcs_hook.html">airflow.contrib.hooks.gcs_hook</a></li> +<li><a href="airflow/contrib/operators/bigquery_operator.html">airflow.contrib.operators.bigquery_operator</a></li> +<li><a href="airflow/contrib/operators/bigquery_to_gcs.html">airflow.contrib.operators.bigquery_to_gcs</a></li> +<li><a href="airflow/contrib/operators/gcs_download_operator.html">airflow.contrib.operators.gcs_download_operator</a></li> <li><a href="airflow/contrib/operators/hipchat_operator.html">airflow.contrib.operators.hipchat_operator</a></li> <li><a href="airflow/executors/celery_executor.html">airflow.executors.celery_executor</a></li> <li><a href="airflow/executors/local_executor.html">airflow.executors.local_executor</a></li> @@ -145,6 +150,7 @@ <li><a href="airflow/operators/docker_operator.html">airflow.operators.docker_operator</a></li> <li><a href="airflow/operators/sensors.html">airflow.operators.sensors</a></li> <li><a href="bash_operator.html">bash_operator</a></li> +<li><a href="bigquery_hook.html">bigquery_hook</a></li> <li><a href="cloudant_hook.html">cloudant_hook</a></li> <li><a href="dagrun_operator.html">dagrun_operator</a></li> <li><a href="dbapi_hook.html">dbapi_hook</a></li> @@ -172,6 +178,7 @@ <li><a href="presto_check_operator.html">presto_check_operator</a></li> <li><a href="presto_hook.html">presto_hook</a></li> <li><a href="python_operator.html">python_operator</a></li> +<li><a href="qubole_operator.html">qubole_operator</a></li> <li><a href="s3_to_hive_operator.html">s3_to_hive_operator</a></li> <li><a href="sensors.html">sensors</a></li> <li><a href="slack_operator.html">slack_operator</a></li> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_modules/qubole_operator.html ---------------------------------------------------------------------- diff --git a/_modules/qubole_operator.html b/_modules/qubole_operator.html new file mode 100644 index 0000000..07ea626 --- /dev/null +++ b/_modules/qubole_operator.html @@ -0,0 +1,328 @@ + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>qubole_operator — Airflow Documentation</title> + + + + + + + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + + + + + + <link rel="top" title="Airflow Documentation" href="../index.html"/> + <link rel="up" title="Module code" href="index.html"/> + + + <script src="../_static/js/modernizr.min.js"></script> + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search"> + + + + <a href="../index.html" class="icon icon-home"> Airflow + + + + </a> + + + + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + <ul> +<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li> +<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li> +<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li> +<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../tutorial.html">Tutorial</a></li> +<li class="toctree-l1"><a class="reference internal" href="../configuration.html">Configuration</a></li> +<li class="toctree-l1"><a class="reference internal" href="../ui.html">UI / Screenshots</a></li> +<li class="toctree-l1"><a class="reference internal" href="../concepts.html">Concepts</a></li> +<li class="toctree-l1"><a class="reference internal" href="../profiling.html">Data Profiling</a></li> +<li class="toctree-l1"><a class="reference internal" href="../cli.html">Command Line Interface</a></li> +<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li> +<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li> +<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li> +<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li> +<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li> +</ul> + + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Airflow</a> + </nav> + + + + <div class="wy-nav-content"> + <div class="rst-content"> + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="index.html">Module code</a> »</li> + + <li>qubole_operator</li> + <li class="wy-breadcrumbs-aside"> + + + + </li> + </ul> + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> + <div itemprop="articleBody"> + + <h1>Source code for qubole_operator</h1><div class="highlight"><pre> +<span></span><span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span> +<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span> +<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">QuboleHook</span> + + +<div class="viewcode-block" id="QuboleOperator"><a class="viewcode-back" href="../code.html#airflow.contrib.operators.QuboleOperator">[docs]</a><span class="k">class</span> <span class="nc">QuboleOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span> + <span class="sd">"""</span> +<span class="sd"> Execute tasks (commands) on QDS (https://qubole.com).</span> + +<span class="sd"> :param qubole_conn_id: Connection id which consists of qds auth_token</span> +<span class="sd"> :type qubole_conn_id: str</span> + +<span class="sd"> kwargs:</span> +<span class="sd"> :command_type: type of command to be executed, e.g. hivecmd, shellcmd, hadoopcmd</span> +<span class="sd"> :tags: array of tags to be assigned with the command</span> +<span class="sd"> :cluster_label: cluster label on which the command will be executed</span> +<span class="sd"> :name: name to be given to command</span> + +<span class="sd"> **Arguments specific to command types**</span> + +<span class="sd"> hivecmd:</span> +<span class="sd"> :query: inline query statement</span> +<span class="sd"> :script_location: s3 location containing query statement</span> +<span class="sd"> :sample_size: size of sample in bytes on which to run query</span> +<span class="sd"> :macros: macro values which were used in query</span> +<span class="sd"> prestocmd:</span> +<span class="sd"> :query: inline query statement</span> +<span class="sd"> :script_location: s3 location containing query statement</span> +<span class="sd"> :macros: macro values which were used in query</span> +<span class="sd"> hadoopcmd:</span> +<span class="sd"> :sub_commnad: must be one these ["jar", "s3distcp", "streaming"] followed by 1 or more args</span> +<span class="sd"> shellcmd:</span> +<span class="sd"> :script: inline command with args</span> +<span class="sd"> :script_location: s3 location containing query statement</span> +<span class="sd"> :files: list of files in s3 bucket as file1,file2 format. These files will be copied into the working directory where the qubole command is being executed.</span> +<span class="sd"> :archives: list of archives in s3 bucket as archive1,archive2 format. These will be unarchived intothe working directory where the qubole command is being executed</span> +<span class="sd"> :parameters: any extra args which need to be passed to script (only when script_location is supplied)</span> +<span class="sd"> pigcmd:</span> +<span class="sd"> :script: inline query statement (latin_statements)</span> +<span class="sd"> :script_location: s3 location containing pig query</span> +<span class="sd"> :parameters: any extra args which need to be passed to script (only when script_location is supplied</span> +<span class="sd"> sparkcmd:</span> +<span class="sd"> :program: the complete Spark Program in Scala, SQL, Command, R, or Python</span> +<span class="sd"> :cmdline: spark-submit command line, all required information must be specify in cmdline itself.</span> +<span class="sd"> :sql: inline sql query</span> +<span class="sd"> :script_location: s3 location containing query statement</span> +<span class="sd"> :language: language of the program, Scala, SQL, Command, R, or Python</span> +<span class="sd"> :app_id: ID of an Spark job server app</span> +<span class="sd"> :arguments: spark-submit command line arguments</span> +<span class="sd"> :user_program_arguments: arguments that the user program takes in</span> +<span class="sd"> :macros: macro values which were used in query</span> +<span class="sd"> dbtapquerycmd:</span> +<span class="sd"> :db_tap_id: data store ID of the target database, in Qubole.</span> +<span class="sd"> :query: inline query statement</span> +<span class="sd"> :macros: macro values which were used in query</span> +<span class="sd"> dbexportcmd:</span> +<span class="sd"> :mode: 1 (simple), 2 (advance)</span> +<span class="sd"> :hive_table: Name of the hive table</span> +<span class="sd"> :partition_spec: partition specification for Hive table.</span> +<span class="sd"> :dbtap_id: data store ID of the target database, in Qubole.</span> +<span class="sd"> :db_table: name of the db table</span> +<span class="sd"> :db_update_mode: allowinsert or updateonly</span> +<span class="sd"> :db_update_keys: columns used to determine the uniqueness of rows</span> +<span class="sd"> :export_dir: HDFS/S3 location from which data will be exported.</span> +<span class="sd"> :fields_terminated_by: hex of the char used as column separator in the dataset.</span> +<span class="sd"> dbimportcmd:</span> +<span class="sd"> :mode: 1 (simple), 2 (advance)</span> +<span class="sd"> :hive_table: Name of the hive table</span> +<span class="sd"> :dbtap_id: data store ID of the target database, in Qubole.</span> +<span class="sd"> :db_table: name of the db table</span> +<span class="sd"> :where_clause: where clause, if any</span> +<span class="sd"> :parallelism: number of parallel db connections to use for extracting data</span> +<span class="sd"> :extract_query: SQL query to extract data from db. $CONDITIONS must be part of the where clause.</span> +<span class="sd"> :boundary_query: Query to be used get range of row IDs to be extracted</span> +<span class="sd"> :split_column: Column used as row ID to split data into ranges (mode 2)</span> + +<span class="sd"> .. note:: Following fields are template-supported : ``query``, ``script_location``, ``sub_command``, ``script``, ``files``,</span> +<span class="sd"> ``archives``, ``program``, ``cmdline``, ``sql``, ``where_clause``, ``extract_query``, ``boundary_query``, ``macros``, ``tags``,</span> +<span class="sd"> ``name``, ``parameters``. You can also use ``.txt`` files for template driven use cases.</span> +<span class="sd"> """</span> + + <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'query'</span><span class="p">,</span> <span class="s1">'script_location'</span><span class="p">,</span> <span class="s1">'sub_command'</span><span class="p">,</span> <span class="s1">'script'</span><span class="p">,</span> <span class="s1">'files'</span><span class="p">,</span> <span class="s1">'archives'</span><span class="p">,</span> <span class="s1">'program'</span><span class="p">,</span> <span class="s1">'cmdline'</span><span class="p">,</span> + <span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'where_clause'</span><span class="p">,</span> <span class="s1">'extract_query'</span><span class="p">,</span> <span class="s1">'boundary_query'</span><span class="p">,</span> <span class="s1">'macros'</span><span class="p">,</span> <span class="s1">'tags'</span><span class="p">,</span> <span class="s1">'name'</span><span class="p">,</span> <span class="s1">'parameters'</span><span class="p">)</span> + <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.txt'</span><span class="p">)</span> + <span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#3064A1'</span> + <span class="n">ui_fgcolor</span> <span class="o">=</span> <span class="s1">'#fff'</span> + + <span class="nd">@apply_defaults</span> + <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">qubole_conn_id</span><span class="o">=</span><span class="s2">"qubole_default"</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">args</span> <span class="o">=</span> <span class="n">args</span> + <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span> + <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="s1">'qubole_conn_id'</span><span class="p">]</span> <span class="o">=</span> <span class="n">qubole_conn_id</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">QuboleHook</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">)</span> + <span class="nb">super</span><span class="p">(</span><span class="n">QuboleOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span> + <span class="c1"># Reinitiating the hook, as some template fields might have changed</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="n">QuboleHook</span><span class="p">(</span><span class="o">*</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">)</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">context</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span> + <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">kill</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">get_results</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">fp</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">inline</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">delim</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">fetch</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_results</span><span class="p">(</span><span class="n">ti</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">inline</span><span class="p">,</span> <span class="n">delim</span><span class="p">,</span> <span class="n">fetch</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">get_log</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_log</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">get_jobs_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ti</span><span class="p">):</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">get_jobs_id</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span> + <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">QuboleOperator</span><span class="o">.</span><span class="n">template_fields</span><span class="p">:</span> + <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">:</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> + <span class="k">else</span><span class="p">:</span> + <span class="k">return</span> <span class="s1">''</span> + <span class="k">else</span><span class="p">:</span> + <span class="k">return</span> <span class="nb">object</span><span class="o">.</span><span class="n">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span> + <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">QuboleOperator</span><span class="o">.</span><span class="n">template_fields</span><span class="p">:</span> + <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span> + <span class="k">else</span><span class="p">:</span> + <span class="nb">object</span><span class="o">.</span><span class="n">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span></div> + + + +</pre></div> + + </div> + </div> + <footer> + + + <hr/> + + <div role="contentinfo"> + <p> + + </p> + </div> + Built with <a href="http://sphinx-doc.org/";>Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + + + <script type="text/javascript"> + var DOCUMENTATION_OPTIONS = { + URL_ROOT:'../', + VERSION:'', + COLLAPSE_INDEX:false, + FILE_SUFFIX:'.html', + HAS_SOURCE: true + }; + </script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.StickyNav.enable(); + }); + </script> + + +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_sources/code.txt ---------------------------------------------------------------------- diff --git a/_sources/code.txt b/_sources/code.txt index 8a48b81..2876876 100644 --- a/_sources/code.txt +++ b/_sources/code.txt @@ -90,13 +90,13 @@ Community-contributed Operators .. automodule:: airflow.contrib.operators :show-inheritance: :members: - BigQueryOperator, - BigQueryToCloudStorageOperator, - GoogleCloudStorageDownloadOperator, SSHExecuteOperator, VerticaOperator, VerticaToHiveTransfer +.. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryOperator +.. autoclass:: airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator +.. autoclass:: airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator .. autoclass:: airflow.contrib.operators.QuboleOperator .. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPIOperator .. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPISendRoomNotificationOperator @@ -226,6 +226,8 @@ Community contributed hooks SSHHook, CloudantHook +.. autoclass:: airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook + Executors --------- Executors are the mechanism by which task instances get run. @@ -237,7 +239,4 @@ Executors are the mechanism by which task instances get run. Community-contributed executors ''''''''''''''''''''''''''''''' -.. automodule:: airflow.contrib.executors - :show-inheritance: - :members: - MesosExecutor +.. autoclass:: airflow.contrib.executors.mesos_executor.MesosExecutor http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_sources/concepts.txt ---------------------------------------------------------------------- diff --git a/_sources/concepts.txt b/_sources/concepts.txt index 6e15ff8..811130d 100644 --- a/_sources/concepts.txt +++ b/_sources/concepts.txt @@ -664,6 +664,7 @@ configuration files, it allows you to expose the configuration that led to the related tasks in Airflow. .. code:: python + """ ### My great DAG """ http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/_sources/index.txt ---------------------------------------------------------------------- diff --git a/_sources/index.txt b/_sources/index.txt index 70f9355..c98d619 100644 --- a/_sources/index.txt +++ b/_sources/index.txt @@ -5,7 +5,19 @@ :width: 150 Apache Airflow (incubating) Documentation -================================ +========================================= + +.. note :: + + Apache Airflow is an effort undergoing incubation at The Apache + Software Foundation (ASF), sponsored by Chris Riccomini. + Incubation is required of all newly accepted projects until a further + review indicates that the infrastructure, communications, and + decision making process have stabilized in a manner consistent with + other successful ASF projects. While incubation status is not + necessarily a reflection of the completeness or stability of + the code, it does indicate that the project has yet to be fully + endorsed by the ASF. Airflow is a platform to programmatically author, schedule and monitor workflows. @@ -20,7 +32,6 @@ monitor progress, and troubleshoot issues when needed. When workflows are defined as code, they become more maintainable, versionable, testable, and collaborative. ------------- .. image:: img/airflow.gif http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/cli.html ---------------------------------------------------------------------- diff --git a/cli.html b/cli.html index c19ab2f..e8785c4 100644 --- a/cli.html +++ b/cli.html @@ -143,7 +143,7 @@ many types of operation on a DAG, starting services, and supporting development and testing.</p> <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> - <span class="p">{</span><span class="n">resetdb</span><span class="p">,</span><span class="n">render</span><span class="p">,</span><span class="n">variables</span><span class="p">,</span><span class="n">pause</span><span class="p">,</span><span class="n">version</span><span class="p">,</span><span class="n">initdb</span><span class="p">,</span><span class="n">test</span><span class="p">,</span><span class="n">unpause</span><span class="p">,</span><span class="n">run</span><span class="p">,</span><span class="n">list_tasks</span><span class="p">,</span><span class="n">backfill</span><span class="p">,</span><span class="n">list_dags</span><span class="p">,</span><span class="n">kerberos</span><span class="p">,</span><span class="n">worker</span><span class="p">,</span><span class="n">webserver</span><span class="p">,</span><span class="n">flower</span><span class="p">,</span><span class="n">scheduler</span><span class="p">,</span><span class="n">task_state</span><span c lass="p">,</span><span class="n">trigger_dag</span><span class="p">,</span><span class="n">serve_logs</span><span class="p">,</span><span class="n">clear</span><span class="p">,</span><span class="n">upgradedb</span><span class="p">}</span> + <span class="p">{</span><span class="n">resetdb</span><span class="p">,</span><span class="n">render</span><span class="p">,</span><span class="n">variables</span><span class="p">,</span><span class="n">pause</span><span class="p">,</span><span class="n">version</span><span class="p">,</span><span class="n">initdb</span><span class="p">,</span><span class="n">test</span><span class="p">,</span><span class="n">unpause</span><span class="p">,</span><span class="n">dag_state</span><span class="p">,</span><span class="n">run</span><span class="p">,</span><span class="n">list_tasks</span><span class="p">,</span><span class="n">backfill</span><span class="p">,</span><span class="n">list_dags</span><span class="p">,</span><span class="n">kerberos</span><span class="p">,</span><span class="n">worker</span><span class="p">,</span><span class="n">webserver</span><span class="p">,</span><span class="n">flower</span><span class="p">,</span><span class="n">scheduler</span><span cl ass="p">,</span><span class="n">task_state</span><span class="p">,</span><span class="n">trigger_dag</span><span class="p">,</span><span class="n">serve_logs</span><span class="p">,</span><span class="n">clear</span><span class="p">,</span><span class="n">upgradedb</span><span class="p">}</span> <span class="o">...</span> </pre></div> </div> @@ -330,7 +330,7 @@ development and testing.</p> </dl> </dd> <dt><strong>unpause</strong></dt> -<dd><p class="first">Pause a DAG</p> +<dd><p class="first">Resume a paused DAG</p> <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">unpause</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="n">dag_id</span> </pre></div> </div> @@ -360,6 +360,40 @@ development and testing.</p> </dd> </dl> </dd> +<dt><strong>dag_state</strong></dt> +<dd><p class="first">Get the status of a dag run</p> +<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">dag_state</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="n">dag_id</span> <span class="n">execution_date</span> +</pre></div> +</div> +<dl class="last docutils"> +<dt>Positional arguments:</dt> +<dd><table class="first last docutils option-list" frame="void" rules="none"> +<col class="option" /> +<col class="description" /> +<tbody valign="top"> +<tr><td class="option-group"> +<kbd>dag_id</kbd></td> +<td>The id of the dag</td></tr> +<tr><td class="option-group"> +<kbd>execution_date</kbd></td> +<td>The execution date of the DAG</td></tr> +</tbody> +</table> +</dd> +<dt>Options:</dt> +<dd><table class="first last docutils option-list" frame="void" rules="none"> +<col class="option" /> +<col class="description" /> +<tbody valign="top"> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">-sd=/Users/maxime_beauchemin/dags</span>, <span class="option">--subdir=/Users/maxime_beauchemin/dags</span></kbd></td> +</tr> +<tr><td> </td><td>File location or directory from which to look for the dag</td></tr> +</tbody> +</table> +</dd> +</dl> +</dd> <dt><strong>run</strong></dt> <dd><p class="first">Run a single task instance</p> <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">run</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">m</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">f</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">pool</span> <span class="n">POOL</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">i</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">I</span> <span class="p">]</span> @@ -401,7 +435,7 @@ development and testing.</p> <tr><td class="option-group" colspan="2"> <kbd><span class="option">-f=False</span>, <span class="option">--force=False</span></kbd></td> </tr> -<tr><td> </td><td>Force a run regardless or previous success</td></tr> +<tr><td> </td><td>Force a run regardless of previous success</td></tr> <tr><td class="option-group"> <kbd><span class="option">--pool</span></kbd></td> <td>Resource pool to use</td></tr> @@ -542,7 +576,7 @@ development and testing.</p> </dd> <dt><strong>list_dags</strong></dt> <dd><p class="first">List all the DAGs</p> -<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">list_dags</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> +<div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">list_dags</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">r</span><span class="p">]</span> </pre></div> </div> <dl class="last docutils"> @@ -555,6 +589,10 @@ development and testing.</p> <kbd><span class="option">-sd=/Users/maxime_beauchemin/dags</span>, <span class="option">--subdir=/Users/maxime_beauchemin/dags</span></kbd></td> </tr> <tr><td> </td><td>File location or directory from which to look for the dag</td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">-r=False</span>, <span class="option">--report=False</span></kbd></td> +</tr> +<tr><td> </td><td>Show DagBag loading report</td></tr> </tbody> </table> </dd> @@ -594,7 +632,7 @@ development and testing.</p> <tr><td class="option-group" colspan="2"> <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td> </tr> -<tr><td> </td><td>Daemonize instead of running on the foreground</td></tr> +<tr><td> </td><td>Daemonize instead of running in the foreground</td></tr> <tr><td class="option-group"> <kbd><span class="option">--stdout</span></kbd></td> <td>Redirect stdout to this file</td></tr> @@ -639,7 +677,7 @@ development and testing.</p> <tr><td class="option-group" colspan="2"> <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td> </tr> -<tr><td> </td><td>Daemonize instead of running on the foreground</td></tr> +<tr><td> </td><td>Daemonize instead of running in the foreground</td></tr> <tr><td class="option-group"> <kbd><span class="option">--stdout</span></kbd></td> <td>Redirect stdout to this file</td></tr> @@ -659,7 +697,8 @@ development and testing.</p> <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">webserver</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span> <span class="n">PORT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">w</span> <span class="n">WORKERS</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">k</span> <span class="p">{</span><span class="n">sync</span><span class="p">,</span><span class="n">eventlet</span><span class="p">,</span><span class="n">gevent</span><span class="p">,</span><span class="n">tornado</span><span class="p">}]</span> <span class="p">[</span><span class="o">-</span><span class="n">t</span> <span class="n">WORKER_TIMEOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">hn</span> <span class="n">HOSTNAME</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">pid</span> <span class="p">[</span><span class="n">PID</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">D</span><span class="p">]</span> - <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span> + <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span> + <span class="p">[</span><span class="o">-</span><span class="n">A</span> <span class="n">ACCESS_LOGFILE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">E</span> <span class="n">ERROR_LOGFILE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span><span class="p">]</span> </pre></div> </div> @@ -680,7 +719,7 @@ development and testing.</p> <tr><td class="option-group" colspan="2"> <kbd><span class="option">-k=sync</span>, <span class="option">--workerclass=sync</span></kbd></td> </tr> -<tr><td> </td><td><p class="first">The worker class to use for gunicorn</p> +<tr><td> </td><td><p class="first">The worker class to use for Gunicorn</p> <p class="last">Possible choices: sync, eventlet, gevent, tornado</p> </td></tr> <tr><td class="option-group" colspan="2"> @@ -697,13 +736,21 @@ development and testing.</p> <tr><td class="option-group" colspan="2"> <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td> </tr> -<tr><td> </td><td>Daemonize instead of running on the foreground</td></tr> +<tr><td> </td><td>Daemonize instead of running in the foreground</td></tr> <tr><td class="option-group"> <kbd><span class="option">--stdout</span></kbd></td> <td>Redirect stdout to this file</td></tr> <tr><td class="option-group"> <kbd><span class="option">--stderr</span></kbd></td> <td>Redirect stderr to this file</td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">-A</span>, <span class="option">--access_logfile</span></kbd></td> +</tr> +<tr><td> </td><td>The logfile to store the webserver access log. Use ‘-‘ to print to stderr.</td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">-E</span>, <span class="option">--error_logfile</span></kbd></td> +</tr> +<tr><td> </td><td>The logfile to store the webserver error log. Use ‘-‘ to print to stderr.</td></tr> <tr><td class="option-group"> <kbd><span class="option">-l</span>, <span class="option">--log-file</span></kbd></td> <td>Location of the log file</td></tr> @@ -742,7 +789,7 @@ development and testing.</p> <tr><td class="option-group" colspan="2"> <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td> </tr> -<tr><td> </td><td>Daemonize instead of running on the foreground</td></tr> +<tr><td> </td><td>Daemonize instead of running in the foreground</td></tr> <tr><td class="option-group"> <kbd><span class="option">--stdout</span></kbd></td> <td>Redirect stdout to this file</td></tr> @@ -758,7 +805,7 @@ development and testing.</p> </dl> </dd> <dt><strong>scheduler</strong></dt> -<dd><p class="first">Start a scheduler scheduler instance</p> +<dd><p class="first">Start a scheduler instance</p> <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">scheduler</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span> <span class="n">DAG_ID</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">n</span> <span class="n">NUM_RUNS</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">pid</span> <span class="p">[</span><span class="n">PID</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">D</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stdout</span> <span class="n">STDOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">stderr</span> <span class="n">STDERR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">l</span> <span class="n">LOG_FILE</span><span class="p">]</span> @@ -790,7 +837,7 @@ development and testing.</p> <tr><td class="option-group" colspan="2"> <kbd><span class="option">-D=False</span>, <span class="option">--daemon=False</span></kbd></td> </tr> -<tr><td> </td><td>Daemonize instead of running on the foreground</td></tr> +<tr><td> </td><td>Daemonize instead of running in the foreground</td></tr> <tr><td class="option-group"> <kbd><span class="option">--stdout</span></kbd></td> <td>Redirect stdout to this file</td></tr> @@ -870,10 +917,10 @@ development and testing.</p> <tr><td> </td><td>File location or directory from which to look for the dag</td></tr> <tr><td class="option-group"> <kbd><span class="option">-r</span>, <span class="option">--run_id</span></kbd></td> -<td>Helps to indentify this run</td></tr> +<td>Helps to identify this run</td></tr> <tr><td class="option-group"> <kbd><span class="option">-c</span>, <span class="option">--conf</span></kbd></td> -<td>json string that gets pickled into the DagRun’s conf attribute</td></tr> +<td>JSON string that gets pickled into the DagRun’s conf attribute</td></tr> </tbody> </table> </dd> @@ -890,7 +937,7 @@ development and testing.</p> <dt><strong>clear</strong></dt> <dd><p class="first">Clear a set of task instance, as if they never ran</p> <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">clear</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">t</span> <span class="n">TASK_REGEX</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">s</span> <span class="n">START_DATE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">e</span> <span class="n">END_DATE</span><span class="p">]</span> - <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">u</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">c</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">f</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">r</span><span class="p">]</span> + <span class="p">[</span><span class="o">-</span><span class="n">sd</span> <span class="n">SUBDIR</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">u</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">c</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">f</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">r</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">x</span><span class="p">]</span> <span class="n">dag_id</span> </pre></div> </div> @@ -946,13 +993,17 @@ development and testing.</p> <kbd><span class="option">-r=False</span>, <span class="option">--only_running=False</span></kbd></td> </tr> <tr><td> </td><td>Only running jobs</td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">-x=False</span>, <span class="option">--exclude_subdags=False</span></kbd></td> +</tr> +<tr><td> </td><td>Exclude subdags</td></tr> </tbody> </table> </dd> </dl> </dd> <dt><strong>upgradedb</strong></dt> -<dd><p class="first">Upgrade metadata database to latest version</p> +<dd><p class="first">Upgrade the metadata database to latest version</p> <div class="highlight-default"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">airflow</span> <span class="n">upgradedb</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> </pre></div> </div> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/code.html ---------------------------------------------------------------------- diff --git a/code.html b/code.html index 3bb26b1..06a0aab 100644 --- a/code.html +++ b/code.html @@ -96,7 +96,7 @@ <li class="toctree-l3"><a class="reference internal" href="#baseoperator">BaseOperator</a></li> <li class="toctree-l3"><a class="reference internal" href="#basesensoroperator">BaseSensorOperator</a></li> <li class="toctree-l3"><a class="reference internal" href="#module-airflow.operators">Operator API</a></li> -<li class="toctree-l3"><a class="reference internal" href="#community-contributed-operators">Community-contributed Operators</a></li> +<li class="toctree-l3"><a class="reference internal" href="#module-airflow.contrib.operators">Community-contributed Operators</a></li> </ul> </li> <li class="toctree-l2"><a class="reference internal" href="#macros">Macros</a><ul> @@ -106,7 +106,7 @@ </li> <li class="toctree-l2"><a class="reference internal" href="#models">Models</a></li> <li class="toctree-l2"><a class="reference internal" href="#module-airflow.hooks">Hooks</a><ul> -<li class="toctree-l3"><a class="reference internal" href="#community-contributed-hooks">Community contributed hooks</a></li> +<li class="toctree-l3"><a class="reference internal" href="#module-airflow.contrib.hooks">Community contributed hooks</a></li> </ul> </li> <li class="toctree-l2"><a class="reference internal" href="#executors">Executors</a><ul> @@ -1186,9 +1186,9 @@ The default is False.</li> </dd></dl> </div> -<div class="section" id="community-contributed-operators"> -<h3>Community-contributed Operators<a class="headerlink" href="#community-contributed-operators" title="Permalink to this headline">¶</a></h3> -<span class="target" id="module-airflow.contrib.operators"></span><dl class="class"> +<div class="section" id="module-airflow.contrib.operators"> +<span id="community-contributed-operators"></span><h3>Community-contributed Operators<a class="headerlink" href="#module-airflow.contrib.operators" title="Permalink to this headline">¶</a></h3> +<dl class="class"> <dt id="airflow.contrib.operators.SSHExecuteOperator"> <em class="property">class </em><code class="descclassname">airflow.contrib.operators.</code><code class="descname">SSHExecuteOperator</code><span class="sig-paren">(</span><em>ssh_hook</em>, <em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ssh_execute_operator.html#SSHExecuteOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.SSHExecuteOperator" title="Permalink to this definition">¶</a></dt> <dd><p>Bases: <a class="reference internal" href="#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal"><span class="pre">airflow.models.BaseOperator</span></code></a></p> @@ -1273,6 +1273,236 @@ specific database</li> </dd></dl> <dl class="class"> +<dt id="airflow.contrib.operators.bigquery_operator.BigQueryOperator"> +<em class="property">class </em><code class="descclassname">airflow.contrib.operators.bigquery_operator.</code><code class="descname">BigQueryOperator</code><span class="sig-paren">(</span><em>bql</em>, <em>destination_dataset_table=False</em>, <em>write_disposition='WRITE_EMPTY'</em>, <em>allow_large_results=False</em>, <em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em>, <em>udf_config=False</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/bigquery_operator.html#BigQueryOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.bigquery_operator.BigQueryOperator" title="Permalink to this definition">¶</a></dt> +<dd><p>Executes BigQuery SQL queries in a specific BigQuery database</p> +</dd></dl> + +<dl class="class"> +<dt id="airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator"> +<em class="property">class </em><code class="descclassname">airflow.contrib.operators.bigquery_to_gcs.</code><code class="descname">BigQueryToCloudStorageOperator</code><span class="sig-paren">(</span><em>source_project_dataset_table</em>, <em>destination_cloud_storage_uris</em>, <em>compression='NONE'</em>, <em>export_format='CSV'</em>, <em>field_delimiter='</em>, <em>'</em>, <em>print_header=True</em>, <em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/bigquery_to_gcs.html#BigQueryToCloudStorageOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator" title="Permalink to this definition">¶</a></dt> +<dd><p>Transfers a BigQuery table to a Google Cloud Storage bucket.</p> +</dd></dl> + +<dl class="class"> +<dt id="airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator"> +<em class="property">class </em><code class="descclassname">airflow.contrib.operators.gcs_download_operator.</code><code class="descname">GoogleCloudStorageDownloadOperator</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename</em>, <em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/gcs_download_operator.html#GoogleCloudStorageDownloadOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator" title="Permalink to this definition">¶</a></dt> +<dd><p>Downloads a file from Google Cloud Storage.</p> +</dd></dl> + +<dl class="class"> +<dt id="airflow.contrib.operators.QuboleOperator"> +<em class="property">class </em><code class="descclassname">airflow.contrib.operators.</code><code class="descname">QuboleOperator</code><span class="sig-paren">(</span><em>qubole_conn_id='qubole_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/qubole_operator.html#QuboleOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.QuboleOperator" title="Permalink to this definition">¶</a></dt> +<dd><p>Execute tasks (commands) on QDS (<a class="reference external" href="https://qubole.com";>https://qubole.com</a>).</p> +<table class="docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>qubole_conn_id</strong> (<em>str</em>) – Connection id which consists of qds auth_token</td> +</tr> +</tbody> +</table> +<dl class="docutils"> +<dt>kwargs:</dt> +<dd><table class="first docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">command_type:</th><td class="field-body">type of command to be executed, e.g. hivecmd, shellcmd, hadoopcmd</td> +</tr> +<tr class="field-even field"><th class="field-name">tags:</th><td class="field-body">array of tags to be assigned with the command</td> +</tr> +<tr class="field-odd field"><th class="field-name">cluster_label:</th><td class="field-body">cluster label on which the command will be executed</td> +</tr> +<tr class="field-even field"><th class="field-name">name:</th><td class="field-body">name to be given to command</td> +</tr> +</tbody> +</table> +<p><strong>Arguments specific to command types</strong></p> +<dl class="last docutils"> +<dt>hivecmd:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">query:</th><td class="field-body">inline query statement</td> +</tr> +<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr> +<tr class="field-even field"><td> </td><td class="field-body">s3 location containing query statement</td> +</tr> +<tr class="field-odd field"><th class="field-name">sample_size:</th><td class="field-body">size of sample in bytes on which to run query</td> +</tr> +<tr class="field-even field"><th class="field-name">macros:</th><td class="field-body">macro values which were used in query</td> +</tr> +</tbody> +</table> +</dd> +<dt>prestocmd:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">query:</th><td class="field-body">inline query statement</td> +</tr> +<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr> +<tr class="field-even field"><td> </td><td class="field-body">s3 location containing query statement</td> +</tr> +<tr class="field-odd field"><th class="field-name">macros:</th><td class="field-body">macro values which were used in query</td> +</tr> +</tbody> +</table> +</dd> +<dt>hadoopcmd:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">sub_commnad:</th><td class="field-body">must be one these [“jar”, “s3distcp”, “streaming”] followed by 1 or more args</td> +</tr> +</tbody> +</table> +</dd> +<dt>shellcmd:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">script:</th><td class="field-body">inline command with args</td> +</tr> +<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr> +<tr class="field-even field"><td> </td><td class="field-body">s3 location containing query statement</td> +</tr> +<tr class="field-odd field"><th class="field-name">files:</th><td class="field-body">list of files in s3 bucket as file1,file2 format. These files will be copied into the working directory where the qubole command is being executed.</td> +</tr> +<tr class="field-even field"><th class="field-name">archives:</th><td class="field-body">list of archives in s3 bucket as archive1,archive2 format. These will be unarchived intothe working directory where the qubole command is being executed</td> +</tr> +<tr class="field-odd field"><th class="field-name">parameters:</th><td class="field-body">any extra args which need to be passed to script (only when script_location is supplied)</td> +</tr> +</tbody> +</table> +</dd> +<dt>pigcmd:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">script:</th><td class="field-body">inline query statement (latin_statements)</td> +</tr> +<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr> +<tr class="field-even field"><td> </td><td class="field-body">s3 location containing pig query</td> +</tr> +<tr class="field-odd field"><th class="field-name">parameters:</th><td class="field-body">any extra args which need to be passed to script (only when script_location is supplied</td> +</tr> +</tbody> +</table> +</dd> +<dt>sparkcmd:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">program:</th><td class="field-body">the complete Spark Program in Scala, SQL, Command, R, or Python</td> +</tr> +<tr class="field-even field"><th class="field-name">cmdline:</th><td class="field-body">spark-submit command line, all required information must be specify in cmdline itself.</td> +</tr> +<tr class="field-odd field"><th class="field-name">sql:</th><td class="field-body">inline sql query</td> +</tr> +<tr class="field-even field"><th class="field-name" colspan="2">script_location:</th></tr> +<tr class="field-even field"><td> </td><td class="field-body">s3 location containing query statement</td> +</tr> +<tr class="field-odd field"><th class="field-name">language:</th><td class="field-body">language of the program, Scala, SQL, Command, R, or Python</td> +</tr> +<tr class="field-even field"><th class="field-name">app_id:</th><td class="field-body">ID of an Spark job server app</td> +</tr> +<tr class="field-odd field"><th class="field-name">arguments:</th><td class="field-body">spark-submit command line arguments</td> +</tr> +<tr class="field-even field"><th class="field-name" colspan="2">user_program_arguments:</th></tr> +<tr class="field-even field"><td> </td><td class="field-body">arguments that the user program takes in</td> +</tr> +<tr class="field-odd field"><th class="field-name">macros:</th><td class="field-body">macro values which were used in query</td> +</tr> +</tbody> +</table> +</dd> +<dt>dbtapquerycmd:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">db_tap_id:</th><td class="field-body">data store ID of the target database, in Qubole.</td> +</tr> +<tr class="field-even field"><th class="field-name">query:</th><td class="field-body">inline query statement</td> +</tr> +<tr class="field-odd field"><th class="field-name">macros:</th><td class="field-body">macro values which were used in query</td> +</tr> +</tbody> +</table> +</dd> +<dt>dbexportcmd:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">mode:</th><td class="field-body">1 (simple), 2 (advance)</td> +</tr> +<tr class="field-even field"><th class="field-name">hive_table:</th><td class="field-body">Name of the hive table</td> +</tr> +<tr class="field-odd field"><th class="field-name">partition_spec:</th><td class="field-body">partition specification for Hive table.</td> +</tr> +<tr class="field-even field"><th class="field-name">dbtap_id:</th><td class="field-body">data store ID of the target database, in Qubole.</td> +</tr> +<tr class="field-odd field"><th class="field-name">db_table:</th><td class="field-body">name of the db table</td> +</tr> +<tr class="field-even field"><th class="field-name">db_update_mode:</th><td class="field-body">allowinsert or updateonly</td> +</tr> +<tr class="field-odd field"><th class="field-name">db_update_keys:</th><td class="field-body">columns used to determine the uniqueness of rows</td> +</tr> +<tr class="field-even field"><th class="field-name">export_dir:</th><td class="field-body">HDFS/S3 location from which data will be exported.</td> +</tr> +<tr class="field-odd field"><th class="field-name" colspan="2">fields_terminated_by:</th></tr> +<tr class="field-odd field"><td> </td><td class="field-body">hex of the char used as column separator in the dataset.</td> +</tr> +</tbody> +</table> +</dd> +<dt>dbimportcmd:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">mode:</th><td class="field-body">1 (simple), 2 (advance)</td> +</tr> +<tr class="field-even field"><th class="field-name">hive_table:</th><td class="field-body">Name of the hive table</td> +</tr> +<tr class="field-odd field"><th class="field-name">dbtap_id:</th><td class="field-body">data store ID of the target database, in Qubole.</td> +</tr> +<tr class="field-even field"><th class="field-name">db_table:</th><td class="field-body">name of the db table</td> +</tr> +<tr class="field-odd field"><th class="field-name">where_clause:</th><td class="field-body">where clause, if any</td> +</tr> +<tr class="field-even field"><th class="field-name">parallelism:</th><td class="field-body">number of parallel db connections to use for extracting data</td> +</tr> +<tr class="field-odd field"><th class="field-name">extract_query:</th><td class="field-body">SQL query to extract data from db. $CONDITIONS must be part of the where clause.</td> +</tr> +<tr class="field-even field"><th class="field-name">boundary_query:</th><td class="field-body">Query to be used get range of row IDs to be extracted</td> +</tr> +<tr class="field-odd field"><th class="field-name">split_column:</th><td class="field-body">Column used as row ID to split data into ranges (mode 2)</td> +</tr> +</tbody> +</table> +</dd> +</dl> +</dd> +</dl> +<div class="admonition note"> +<p class="first admonition-title">Note</p> +<p class="last">Following fields are template-supported : <code class="docutils literal"><span class="pre">query</span></code>, <code class="docutils literal"><span class="pre">script_location</span></code>, <code class="docutils literal"><span class="pre">sub_command</span></code>, <code class="docutils literal"><span class="pre">script</span></code>, <code class="docutils literal"><span class="pre">files</span></code>, +<code class="docutils literal"><span class="pre">archives</span></code>, <code class="docutils literal"><span class="pre">program</span></code>, <code class="docutils literal"><span class="pre">cmdline</span></code>, <code class="docutils literal"><span class="pre">sql</span></code>, <code class="docutils literal"><span class="pre">where_clause</span></code>, <code class="docutils literal"><span class="pre">extract_query</span></code>, <code class="docutils literal"><span class="pre">boundary_query</span></code>, <code class="docutils literal"><span class="pre">macros</span></code>, <code class="docutils literal"><span class="pre">tags</span></code>, +<code class="docutils literal"><span class="pre">name</span></code>, <code class="docutils literal"><span class="pre">parameters</span></code>. You can also use <code class="docutils literal"><span class="pre">.txt</span></code> files for template driven use cases.</p> +</div> +</dd></dl> + +<dl class="class"> <dt id="airflow.contrib.operators.hipchat_operator.HipChatAPIOperator"> <em class="property">class </em><code class="descclassname">airflow.contrib.operators.hipchat_operator.</code><code class="descname">HipChatAPIOperator</code><span class="sig-paren">(</span><em>token</em>, <em>base_url='https://api.hipchat.com/v2'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/hipchat_operator.html#HipChatAPIOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.hipchat_operator.HipChatAPIOperator" title="Permalink to this definition">¶</a></dt> <dd><p>Base HipChat Operator. @@ -2223,12 +2453,12 @@ to be retried.</p> <col class="field-name" /> <col class="field-body" /> <tbody valign="top"> -<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>lock_for_update</strong> – if True, indicates that the database should</td> +<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>lock_for_update</strong> – if True, indicates that the database should +lock the TaskInstance (issuing a FOR UPDATE clause) until the +session is committed.</td> </tr> </tbody> </table> -<p>lock the TaskInstance (issuing a FOR UPDATE clause) until the session -is committed.</p> </dd></dl> <dl class="method"> @@ -3145,9 +3375,108 @@ directory, files will be uploaded inside.</li> </dd></dl> -<div class="section" id="community-contributed-hooks"> -<h3>Community contributed hooks<a class="headerlink" href="#community-contributed-hooks" title="Permalink to this headline">¶</a></h3> -<span class="target" id="module-airflow.contrib.hooks"></span><dl class="class"> +<div class="section" id="module-airflow.contrib.hooks"> +<span id="community-contributed-hooks"></span><h3>Community contributed hooks<a class="headerlink" href="#module-airflow.contrib.hooks" title="Permalink to this headline">¶</a></h3> +<dl class="class"> +<dt id="airflow.contrib.hooks.BigQueryHook"> +<em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">BigQueryHook</code><span class="sig-paren">(</span><em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook" title="Permalink to this definition">¶</a></dt> +<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook</span></code>, <code class="xref py py-class docutils literal"><span class="pre">airflow.hooks.dbapi_hook.DbApiHook</span></code></p> +<p>Interact with BigQuery. This hook uses the Google Cloud Platform +connection.</p> +<dl class="method"> +<dt id="airflow.contrib.hooks.BigQueryHook.get_conn"> +<code class="descname">get_conn</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook.get_conn"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook.get_conn" title="Permalink to this definition">¶</a></dt> +<dd><p>Returns a BigQuery PEP 249 connection object.</p> +</dd></dl> + +<dl class="method"> +<dt id="airflow.contrib.hooks.BigQueryHook.get_pandas_df"> +<code class="descname">get_pandas_df</code><span class="sig-paren">(</span><em>bql</em>, <em>parameters=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook.get_pandas_df"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook.get_pandas_df" title="Permalink to this definition">¶</a></dt> +<dd><p>Returns a Pandas DataFrame for the results produced by a BigQuery +query. The DbApiHook method must be overridden because Pandas +doesn’t support PEP 249 connections, except for SQLite. See:</p> +<p><a class="reference external" href="https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L447";>https://github.com/pydata/pandas/blob/master/pandas/io/sql.py#L447</a> +<a class="reference external" href="https://github.com/pydata/pandas/issues/6900";>https://github.com/pydata/pandas/issues/6900</a></p> +<table class="docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>bql</strong> (<em>string</em>) – The BigQuery SQL to execute.</td> +</tr> +</tbody> +</table> +</dd></dl> + +<dl class="method"> +<dt id="airflow.contrib.hooks.BigQueryHook.get_service"> +<code class="descname">get_service</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook.get_service"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook.get_service" title="Permalink to this definition">¶</a></dt> +<dd><p>Returns a BigQuery service object.</p> +</dd></dl> + +<dl class="method"> +<dt id="airflow.contrib.hooks.BigQueryHook.insert_rows"> +<code class="descname">insert_rows</code><span class="sig-paren">(</span><em>table</em>, <em>rows</em>, <em>target_fields=None</em>, <em>commit_every=1000</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook.insert_rows"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook.insert_rows" title="Permalink to this definition">¶</a></dt> +<dd><p>Insertion is currently unsupported. Theoretically, you could use +BigQuery’s streaming API to insert rows into a table, but this hasn’t +been implemented.</p> +</dd></dl> + +</dd></dl> + +<dl class="class"> +<dt id="airflow.contrib.hooks.GoogleCloudStorageHook"> +<em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">GoogleCloudStorageHook</code><span class="sig-paren">(</span><em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/gcs_hook.html#GoogleCloudStorageHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.GoogleCloudStorageHook" title="Permalink to this definition">¶</a></dt> +<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook</span></code></p> +<p>Interact with Google Cloud Storage. This hook uses the Google Cloud Platform +connection.</p> +<dl class="method"> +<dt id="airflow.contrib.hooks.GoogleCloudStorageHook.download"> +<code class="descname">download</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/gcs_hook.html#GoogleCloudStorageHook.download"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.GoogleCloudStorageHook.download" title="Permalink to this definition">¶</a></dt> +<dd><p>Get a file from Google Cloud Storage.</p> +<table class="docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple"> +<li><strong>bucket</strong> (<em>string</em>) – The bucket to fetch from.</li> +<li><strong>object</strong> (<em>string</em>) – The object to fetch.</li> +<li><strong>filename</strong> (<em>string</em>) – If set, a local file path where the file should be written to.</li> +</ul> +</td> +</tr> +</tbody> +</table> +</dd></dl> + +<dl class="method"> +<dt id="airflow.contrib.hooks.GoogleCloudStorageHook.get_conn"> +<code class="descname">get_conn</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/gcs_hook.html#GoogleCloudStorageHook.get_conn"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.GoogleCloudStorageHook.get_conn" title="Permalink to this definition">¶</a></dt> +<dd><p>Returns a Google Cloud Storage service object.</p> +</dd></dl> + +<dl class="method"> +<dt id="airflow.contrib.hooks.GoogleCloudStorageHook.upload"> +<code class="descname">upload</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename</em>, <em>mime_type='application/octet-stream'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/gcs_hook.html#GoogleCloudStorageHook.upload"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.GoogleCloudStorageHook.upload" title="Permalink to this definition">¶</a></dt> +<dd><p>Uploads a local file to Google Cloud Storage.</p> +<table class="docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple"> +<li><strong>bucket</strong> (<em>string</em>) – The bucket to upload to.</li> +<li><strong>object</strong> (<em>string</em>) – The object name to set when uploading the local file.</li> +<li><strong>filename</strong> (<em>string</em>) – The local file path to the file to be uploaded.</li> +<li><strong>mime_type</strong> (<em>string</em>) – The MIME type to set when uploading the file.</li> +</ul> +</td> +</tr> +</tbody> +</table> +</dd></dl> + +</dd></dl> + +<dl class="class"> <dt id="airflow.contrib.hooks.VerticaHook"> <em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">VerticaHook</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/vertica_hook.html#VerticaHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.VerticaHook" title="Permalink to this definition">¶</a></dt> <dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.hooks.dbapi_hook.DbApiHook</span></code></p> @@ -3406,6 +3735,13 @@ documentation <a class="reference external" href="https://github.com/cloudant-la </dd></dl> +<dl class="class"> +<dt id="airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook"> +<em class="property">class </em><code class="descclassname">airflow.contrib.hooks.gcs_hook.</code><code class="descname">GoogleCloudStorageHook</code><span class="sig-paren">(</span><em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/hooks/gcs_hook.html#GoogleCloudStorageHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook" title="Permalink to this definition">¶</a></dt> +<dd><p>Interact with Google Cloud Storage. This hook uses the Google Cloud Platform +connection.</p> +</dd></dl> + </div> </div> <div class="section" id="executors"> @@ -3444,7 +3780,19 @@ SequentialExecutor alongside sqlite as you first install it.</p> <div class="section" id="community-contributed-executors"> <h3>Community-contributed executors<a class="headerlink" href="#community-contributed-executors" title="Permalink to this headline">¶</a></h3> -<span class="target" id="module-airflow.contrib.executors"></span></div> +<dl class="class"> +<dt id="airflow.contrib.executors.mesos_executor.MesosExecutor"> +<em class="property">class </em><code class="descclassname">airflow.contrib.executors.mesos_executor.</code><code class="descname">MesosExecutor</code><span class="sig-paren">(</span><em>parallelism=16</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/executors/mesos_executor.html#MesosExecutor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.executors.mesos_executor.MesosExecutor" title="Permalink to this definition">¶</a></dt> +<dd><p>MesosExecutor allows distributing the execution of task +instances to multiple mesos workers.</p> +<p>Apache Mesos is a distributed systems kernel which abstracts +CPU, memory, storage, and other compute resources away from +machines (physical or virtual), enabling fault-tolerant and +elastic distributed systems to easily be built and run effectively. +See <a class="reference external" href="http://mesos.apache.org/";>http://mesos.apache.org/</a></p> +</dd></dl> + +</div> </div> </div> http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/502934ca/concepts.html ---------------------------------------------------------------------- diff --git a/concepts.html b/concepts.html index 5fda645..b41766f 100644 --- a/concepts.html +++ b/concepts.html @@ -748,6 +748,20 @@ content if defined:</p> <p>This is especially useful if your tasks are built dynamically from configuration files, it allows you to expose the configuration that led to the related tasks in Airflow.</p> +<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="sd">"""</span> +<span class="sd">### My great DAG</span> +<span class="sd">"""</span> + +<span class="n">dag</span> <span class="o">=</span> <span class="n">DAG</span><span class="p">(</span><span class="s1">'my_dag'</span><span class="p">,</span> <span class="n">default_args</span><span class="o">=</span><span class="n">default_args</span><span class="p">)</span> +<span class="n">dag</span><span class="o">.</span><span class="n">doc_md</span> <span class="o">=</span> <span class="n">__doc__</span> + +<span class="n">t</span> <span class="o">=</span> <span class="n">BashOperator</span><span class="p">(</span><span class="s2">"foo"</span><span class="p">,</span> <span class="n">dag</span><span class="o">=</span><span class="n">dag</span><span class="p">)</span> +<span class="n">t</span><span class="o">.</span><span class="n">doc_md</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span> +<span class="s2">#Title"</span> +<span class="s2">Here's a [url](www.airbnb.com)</span> +<span class="s2">"""</span> +</pre></div> +</div> <p>This content will get rendered as markdown respectively in the “Graph View” and “Task Details” pages.</p> </div>
