msumit closed pull request #3946: [AIRFLOW-3062] Add Qubole in integration docs
URL: https://github.com/apache/incubator-airflow/pull/3946
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/contrib/sensors/qubole_sensor.py
b/airflow/contrib/sensors/qubole_sensor.py
index f79f58746b..c0291b1521 100644
--- a/airflow/contrib/sensors/qubole_sensor.py
+++ b/airflow/contrib/sensors/qubole_sensor.py
@@ -29,14 +29,6 @@
class QuboleSensor(BaseSensorOperator):
"""
Base class for all Qubole Sensors
-
- :param qubole_conn_id: The qubole connection to run the sensor against
- :type qubole_conn_id: str
- :param data: a JSON object containing payload, whose presence needs to be
checked
- :type data: a JSON object
-
- .. note:: Both ``data`` and ``qubole_conn_id`` fields are
template-supported. You can
- also use ``.txt`` files for template driven use cases.
"""
template_fields = ('data', 'qubole_conn_id')
@@ -75,6 +67,22 @@ def poke(self, context):
class QuboleFileSensor(QuboleSensor):
+ """
+ Wait for a file or folder to be present in cloud storage
+ and check for its presence via QDS APIs
+
+ :param qubole_conn_id: Connection id which consists of qds auth_token
+ :type qubole_conn_id: str
+ :param data: a JSON object containing payload, whose presence needs to be
checked
+ Check this `example
<https://github.com/apache/incubator-airflow/blob/master\
+ /airflow/contrib/example_dags/example_qubole_sensor.py>`_ for sample
payload
+ structure.
+ :type data: a JSON object
+
+ .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating.
You can
+ also use ``.txt`` files for template-driven use cases.
+ """
+
@apply_defaults
def __init__(self, *args, **kwargs):
self.sensor_class = FileSensor
@@ -82,6 +90,22 @@ def __init__(self, *args, **kwargs):
class QubolePartitionSensor(QuboleSensor):
+ """
+ Wait for a Hive partition to show up in QHS (Qubole Hive Service)
+ and check for its presence via QDS APIs
+
+ :param qubole_conn_id: Connection id which consists of qds auth_token
+ :type qubole_conn_id: str
+ :param data: a JSON object containing payload, whose presence needs to be
checked.
+ Check this `example
<https://github.com/apache/incubator-airflow/blob/master\
+ /airflow/contrib/example_dags/example_qubole_sensor.py>`_ for sample
payload
+ structure.
+ :type data: a JSON object
+
+ .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating.
You can
+ also use ``.txt`` files for template-driven use cases.
+ """
+
@apply_defaults
def __init__(self, *args, **kwargs):
self.sensor_class = PartitionSensor
diff --git a/docs/integration.rst b/docs/integration.rst
index c4800d65ac..f1ad6c5cb6 100644
--- a/docs/integration.rst
+++ b/docs/integration.rst
@@ -6,6 +6,7 @@ Integration
- :ref:`AWS`
- :ref:`Databricks`
- :ref:`GCP`
+- :ref:`Qubole`
.. _ReverseProxy:
@@ -829,3 +830,28 @@ Google Kubernetes Engine Hook
.. autoclass:: airflow.contrib.hooks.gcp_container_hook.GKEClusterHook
:members:
+
+
+.. _Qubole:
+
+Qubole
+------
+
+Apache Airflow has a native operator and hooks to talk to `Qubole
<https://qubole.com/>`__,
+which lets you submit your big data jobs directly to Qubole from Apache
Airflow.
+
+QuboleOperator
+''''''''''''''
+
+.. autoclass:: airflow.contrib.operators.qubole_operator.QuboleOperator
+
+QubolePartitionSensor
+'''''''''''''''''''''
+
+.. autoclass:: airflow.contrib.sensors.qubole_sensor.QubolePartitionSensor
+
+
+QuboleFileSensor
+''''''''''''''''
+
+.. autoclass:: airflow.contrib.sensors.qubole_sensor.QuboleFileSensor
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
