andrewmchen commented on a change in pull request #3813: [AIRFLOW-1998]
Implemented DatabricksRunNowOperator for jobs/run-now …
URL: https://github.com/apache/incubator-airflow/pull/3813#discussion_r215474725
##########
File path: airflow/contrib/operators/databricks_operator.py
##########
@@ -30,6 +30,66 @@
XCOM_RUN_PAGE_URL_KEY = 'run_page_url'
+def _deep_string_coerce(content, json_path='json'):
+ """
+ Coerces content or all values of content if it is a dict to a string. The
+ function will throw if content contains non-string or non-numeric types.
+
+ The reason why we have this function is because the ``self.json`` field
must be a
+ dict with only string values. This is because ``render_template`` will
fail
+ for numerical values.
+ """
+ c = _deep_string_coerce
+ if isinstance(content, six.string_types):
+ return content
+ elif isinstance(content, six.integer_types + (float,)):
+ # Databricks can tolerate either numeric or string types in the API
backend.
+ return str(content)
+ elif isinstance(content, (list, tuple)):
+ return [c(e, '{0}[{1}]'.format(json_path, i)) for i, e in
enumerate(content)]
+ elif isinstance(content, dict):
+ return {k: c(v, '{0}[{1}]'.format(json_path, k))
+ for k, v in list(content.items())}
+ else:
+ param_type = type(content)
+ msg = 'Type {0} used for parameter {1} is not a number or a string' \
+ .format(param_type, json_path)
+ raise AirflowException(msg)
+
+
+def _handle_databricks_operator_execution(operator, hook, log, context):
+ """
+ Handles the Airflow + Databricks lifecycle logic for a data bricks operator
Review comment:
nit: `Databricks`
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
