[AIRFLOW-31] Use standard imports for hooks/operators
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/851adc55 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/851adc55 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/851adc55 Branch: refs/heads/master Commit: 851adc5547597ec51743be4bc47d634c77d6dc17 Parents: 06e70e2 Author: jlowin <[email protected]> Authored: Wed Jun 15 17:39:12 2016 -0400 Committer: jlowin <[email protected]> Committed: Thu Jun 16 14:55:07 2016 -0400 ---------------------------------------------------------------------- airflow/__init__.py | 6 +- .../contrib/example_dags/example_twitter_dag.py | 4 +- airflow/contrib/hooks/__init__.py | 33 +- airflow/contrib/operators/__init__.py | 42 ++- airflow/contrib/operators/fs_operator.py | 3 +- airflow/contrib/operators/mysql_to_gcs.py | 2 +- airflow/contrib/operators/vertica_to_hive.py | 2 +- .../contrib/plugins/metastore_browser/main.py | 4 +- airflow/example_dags/example_http_operator.py | 3 +- airflow/hooks/__init__.py | 73 +++- airflow/hooks/base_hook.py | 14 + airflow/hooks/dbapi_hook.py | 14 + airflow/hooks/druid_hook.py | 14 + airflow/hooks/hdfs_hook.py | 14 + airflow/hooks/http_hook.py | 2 +- airflow/hooks/jdbc_hook.py | 14 + airflow/hooks/mssql_hook.py | 14 + airflow/hooks/mysql_hook.py | 14 + airflow/hooks/oracle_hook.py | 14 + airflow/hooks/pig_hook.py | 14 + airflow/hooks/postgres_hook.py | 14 + airflow/hooks/presto_hook.py | 14 + airflow/hooks/samba_hook.py | 14 + airflow/hooks/sqlite_hook.py | 14 + airflow/hooks/webhdfs_hook.py | 14 + airflow/macros/__init__.py | 38 ++- airflow/macros/hive.py | 4 +- airflow/models.py | 38 ++- airflow/operators/__init__.py | 120 +++++-- airflow/operators/bash_operator.py | 14 + airflow/operators/check_operator.py | 14 + airflow/operators/dagrun_operator.py | 14 + airflow/operators/docker_operator.py | 14 + airflow/operators/dummy_operator.py | 14 + airflow/operators/email_operator.py | 14 + airflow/operators/generic_transfer.py | 14 + airflow/operators/hive_operator.py | 16 +- airflow/operators/hive_stats_operator.py | 18 +- airflow/operators/hive_to_druid.py | 17 +- airflow/operators/hive_to_mysql.py | 17 +- airflow/operators/hive_to_samba_operator.py | 17 +- airflow/operators/http_operator.py | 14 + airflow/operators/jdbc_operator.py | 14 + airflow/operators/mssql_operator.py | 16 +- airflow/operators/mssql_to_hive.py | 17 +- airflow/operators/mysql_operator.py | 16 +- airflow/operators/mysql_to_hive.py | 17 +- airflow/operators/oracle_operator.py | 2 +- airflow/operators/pig_operator.py | 16 +- airflow/operators/postgres_operator.py | 16 +- airflow/operators/presto_check_operator.py | 16 +- airflow/operators/presto_to_mysql.py | 17 +- airflow/operators/python_operator.py | 14 + airflow/operators/s3_file_transform_operator.py | 16 +- airflow/operators/s3_to_hive_operator.py | 17 +- airflow/operators/sensors.py | 29 +- airflow/operators/slack_operator.py | 14 + airflow/operators/sqlite_operator.py | 14 + airflow/plugins_manager.py | 30 +- airflow/utils/email.py | 18 +- airflow/utils/logging.py | 2 +- airflow/utils/tests.py | 23 ++ dags/testdruid.py | 2 +- run_unit_tests.sh | 3 + setup.py | 14 + tests/__init__.py | 2 +- tests/core.py | 340 +------------------ tests/operators/__init__.py | 17 + tests/operators/hive_operator.py | 209 ++++++++++++ tests/operators/operators.py | 174 ++++++++++ tests/operators/sensor.py | 39 --- tests/operators/sensors.py | 39 +++ 72 files changed, 1457 insertions(+), 473 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/__init__.py b/airflow/__init__.py index 9185c9e..03840ca 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -79,6 +79,6 @@ from airflow import executors from airflow import macros from airflow import contrib -operators.integrate_plugins() -hooks.integrate_plugins() -macros.integrate_plugins() +operators._integrate_plugins() +hooks._integrate_plugins() +macros._integrate_plugins() http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/contrib/example_dags/example_twitter_dag.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/example_dags/example_twitter_dag.py b/airflow/contrib/example_dags/example_twitter_dag.py index d7fffd8..af1978e 100644 --- a/airflow/contrib/example_dags/example_twitter_dag.py +++ b/airflow/contrib/example_dags/example_twitter_dag.py @@ -23,7 +23,8 @@ # -------------------------------------------------------------------------------- from airflow import DAG -from airflow.operators import BashOperator, HiveOperator, PythonOperator +from airflow.operators import BashOperator, PythonOperator +from airflow.operators.hive_operator import HiveOperator from datetime import datetime, date, timedelta # -------------------------------------------------------------------------------- @@ -180,4 +181,3 @@ for channel in from_channels: load_to_hive.set_upstream(load_to_hdfs) load_to_hive.set_downstream(hive_to_mysql) - http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/contrib/hooks/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/hooks/__init__.py b/airflow/contrib/hooks/__init__.py index 29bb44a..83b505d 100644 --- a/airflow/contrib/hooks/__init__.py +++ b/airflow/contrib/hooks/__init__.py @@ -11,7 +11,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + + +# Contrib hooks are not imported by default. They should be accessed +# directly: from airflow.contrib.hooks.hook_module import Hook + + + + + + + +# ------------------------------------------------------------------------ +# +# #TODO #FIXME Airflow 2.0 +# +# Old import machinary below. +# +# This is deprecated but should be kept until Airflow 2.0 +# for compatibility. # +# ------------------------------------------------------------------------ # Imports the hooks dynamically while keeping the package API clean, # abstracting the underlying modules @@ -31,4 +52,14 @@ _hooks = { 'fs_hook': ['FSHook'] } -_import_module_attrs(globals(), _hooks) +import os as _os +if not _os.environ.get('AIRFLOW_USE_NEW_IMPORTS', False): + from zope.deprecation import deprecated as _deprecated + _imported = _import_module_attrs(globals(), _hooks) + for _i in _imported: + _deprecated( + _i, + "Importing {i} directly from 'contrib.hooks' has been " + "deprecated. Please import from " + "'contrib.hooks.[hook_module]' instead. Support for direct imports " + "will be dropped entirely in Airflow 2.0.".format(i=_i)) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/contrib/operators/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/__init__.py b/airflow/contrib/operators/__init__.py index 9f758bf..5ac2d45 100644 --- a/airflow/contrib/operators/__init__.py +++ b/airflow/contrib/operators/__init__.py @@ -1,3 +1,33 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Contrib operators are not imported by default. They should be accessed +# directly: from airflow.contrib.operators.operator_module import Operator + + +# ------------------------------------------------------------------------ +# +# #TODO #FIXME Airflow 2.0 +# +# Old import machinary below. +# +# This is deprecated but should be kept until Airflow 2.0 +# for compatibility. +# +# ------------------------------------------------------------------------ + # Imports the operators dynamically while keeping the package API clean, # abstracting the underlying modules from airflow.utils.helpers import import_module_attrs as _import_module_attrs @@ -10,4 +40,14 @@ _operators = { 'fs': ['FileSensor'] } -_import_module_attrs(globals(), _operators) +import os as _os +if not _os.environ.get('AIRFLOW_USE_NEW_IMPORTS', False): + from zope.deprecation import deprecated as _deprecated + _imported = _import_module_attrs(globals(), _operators) + for _i in _imported: + _deprecated( + _i, + "Importing {i} directly from 'contrib.operators' has been " + "deprecated. Please import from " + "'contrib.operators.[operator_module]' instead. Support for direct " + "imports will be dropped entirely in Airflow 2.0.".format(i=_i)) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/contrib/operators/fs_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/fs_operator.py b/airflow/contrib/operators/fs_operator.py index c68eed2..98db70e 100644 --- a/airflow/contrib/operators/fs_operator.py +++ b/airflow/contrib/operators/fs_operator.py @@ -17,7 +17,7 @@ from os import walk import logging from airflow.operators.sensors import BaseSensorOperator -from airflow.contrib.hooks import FSHook +from airflow.contrib.hooks.fs_hook import FSHook from airflow.utils.decorators import apply_defaults class FileSensor(BaseSensorOperator): @@ -54,4 +54,3 @@ class FileSensor(BaseSensorOperator): except: return False return True - http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/contrib/operators/mysql_to_gcs.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/mysql_to_gcs.py b/airflow/contrib/operators/mysql_to_gcs.py index e740de2..89eaecd 100644 --- a/airflow/contrib/operators/mysql_to_gcs.py +++ b/airflow/contrib/operators/mysql_to_gcs.py @@ -3,7 +3,7 @@ import logging import time from airflow.contrib.hooks.gcs_hook import GoogleCloudStorageHook -from airflow.hooks import MySqlHook +from airflow.hooks.mysql_hook import MySqlHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults from collections import OrderedDict http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/contrib/operators/vertica_to_hive.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/vertica_to_hive.py b/airflow/contrib/operators/vertica_to_hive.py index 35a489a..e261e23 100644 --- a/airflow/contrib/operators/vertica_to_hive.py +++ b/airflow/contrib/operators/vertica_to_hive.py @@ -4,7 +4,7 @@ import unicodecsv as csv import logging from tempfile import NamedTemporaryFile -from airflow.hooks import HiveCliHook +from airflow.hooks.hive_hooks import HiveCliHook from airflow.contrib.hooks import VerticaHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/contrib/plugins/metastore_browser/main.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/plugins/metastore_browser/main.py b/airflow/contrib/plugins/metastore_browser/main.py index 4a9d383..eb0a60e 100644 --- a/airflow/contrib/plugins/metastore_browser/main.py +++ b/airflow/contrib/plugins/metastore_browser/main.py @@ -5,7 +5,9 @@ from flask import Blueprint, request from flask_admin import BaseView, expose import pandas as pd -from airflow.hooks import HiveMetastoreHook, MySqlHook, PrestoHook, HiveCliHook +from airflow.hooks.hive_hooks import HiveMetastoreHook, HiveCliHook +from airflow.hooks.mysql_hook import MySqlHook +from airflow.hooks.presto_hook import PrestoHook from airflow.plugins_manager import AirflowPlugin from airflow.www import utils as wwwutils http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/example_dags/example_http_operator.py ---------------------------------------------------------------------- diff --git a/airflow/example_dags/example_http_operator.py b/airflow/example_dags/example_http_operator.py index bbfc17a..4501825 100644 --- a/airflow/example_dags/example_http_operator.py +++ b/airflow/example_dags/example_http_operator.py @@ -2,7 +2,8 @@ ### Example HTTP operator and sensor """ from airflow import DAG -from airflow.operators import SimpleHttpOperator, HttpSensor +from airflow.operators import SimpleHttpOperator +from airflow.operators.sensors import HttpSensor from datetime import datetime, timedelta import json http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/__init__.py b/airflow/hooks/__init__.py index 58fac17..10fb5a7 100644 --- a/airflow/hooks/__init__.py +++ b/airflow/hooks/__init__.py @@ -1,3 +1,36 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Only import Core Airflow Operators that don't have extra requirements. +# All other operators must be imported directly. +from .base_hook import BaseHook +from .dbapi_hook import DbApiHook +from .http_hook import HttpHook +from .sqlite_hook import SqliteHook + +# ------------------------------------------------------------------------ +# +# #TODO #FIXME Airflow 2.0 +# +# Old import machinary below. +# +# This is deprecated but should be kept until Airflow 2.0 +# for compatibility. +# +# ------------------------------------------------------------------------ + # Imports the hooks dynamically while keeping the package API clean, # abstracting the underlying modules @@ -17,9 +50,9 @@ _hooks = { 'postgres_hook': ['PostgresHook'], 'presto_hook': ['PrestoHook'], 'samba_hook': ['SambaHook'], - 'sqlite_hook': ['SqliteHook'], + # 'sqlite_hook': ['SqliteHook'], 'S3_hook': ['S3Hook'], - 'http_hook': ['HttpHook'], + # 'http_hook': ['HttpHook'], 'druid_hook': ['DruidHook'], 'jdbc_hook': ['JdbcHook'], 'dbapi_hook': ['DbApiHook'], @@ -27,11 +60,39 @@ _hooks = { 'oracle_hook': ['OracleHook'], } -_import_module_attrs(globals(), _hooks) +import os as _os +if not _os.environ.get('AIRFLOW_USE_NEW_IMPORTS', False): + from zope.deprecation import deprecated as _deprecated + _imported = _import_module_attrs(globals(), _hooks) + for _i in _imported: + _deprecated( + _i, + "Importing {i} directly from 'airflow.hooks' has been " + "deprecated. Please import from " + "'airflow.hooks.[hook_module]' instead. Support for direct imports " + "will be dropped entirely in Airflow 2.0.".format(i=_i)) -def integrate_plugins(): + +def _integrate_plugins(): """Integrate plugins to the context""" + import sys from airflow.plugins_manager import hooks as _hooks - for _h in _hooks: - globals()[_h.__name__] = _h + for _hook_module in _hooks: + sys.modules[_hook_module.__name__] = _hook_module + globals()[_hook_module._name] = _hook_module + + + ########################################################## + # TODO FIXME Remove in Airflow 2.0 + + if not _os.environ.get('AIRFLOW_USE_NEW_IMPORTS', False): + from zope.deprecation import deprecated as _deprecated + for _hook in _hook_module._objects: + globals()[_hook.__name__] = _deprecated( + _hook, + "Importing plugin hook '{i}' directly from " + "'airflow.hooks' has been deprecated. Please " + "import from 'airflow.hooks.[plugin_module]' " + "instead. Support for direct imports will be dropped " + "entirely in Airflow 2.0.".format(i=_hook)) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/base_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/base_hook.py b/airflow/hooks/base_hook.py index 2a6cb73..e640b63 100644 --- a/airflow/hooks/base_hook.py +++ b/airflow/hooks/base_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import absolute_import from __future__ import division from __future__ import print_function http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/dbapi_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/dbapi_hook.py b/airflow/hooks/dbapi_hook.py index e5de92e..e48e138 100644 --- a/airflow/hooks/dbapi_hook.py +++ b/airflow/hooks/dbapi_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import str from past.builtins import basestring http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/druid_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/druid_hook.py b/airflow/hooks/druid_hook.py index bb6d9fa..4fe0020 100644 --- a/airflow/hooks/druid_hook.py +++ b/airflow/hooks/druid_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import logging import json http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/hdfs_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/hdfs_hook.py b/airflow/hooks/hdfs_hook.py index 98e5f97..e84595c 100644 --- a/airflow/hooks/hdfs_hook.py +++ b/airflow/hooks/hdfs_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from airflow.hooks.base_hook import BaseHook from airflow import configuration http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/http_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/http_hook.py b/airflow/hooks/http_hook.py index 5a257b0..7cf9a24 100644 --- a/airflow/hooks/http_hook.py +++ b/airflow/hooks/http_hook.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# + from builtins import str import logging http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/jdbc_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/jdbc_hook.py b/airflow/hooks/jdbc_hook.py index 1f9275f..510f386 100644 --- a/airflow/hooks/jdbc_hook.py +++ b/airflow/hooks/jdbc_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import str __author__ = 'janomar' http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/mssql_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/mssql_hook.py b/airflow/hooks/mssql_hook.py index 2b6610d..ae9d971 100644 --- a/airflow/hooks/mssql_hook.py +++ b/airflow/hooks/mssql_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pymssql from airflow.hooks.dbapi_hook import DbApiHook http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/mysql_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/mysql_hook.py b/airflow/hooks/mysql_hook.py index 9f2d951..e81d796 100644 --- a/airflow/hooks/mysql_hook.py +++ b/airflow/hooks/mysql_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import MySQLdb import MySQLdb.cursors http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/oracle_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/oracle_hook.py b/airflow/hooks/oracle_hook.py index 82ccddb..aa70a56 100644 --- a/airflow/hooks/oracle_hook.py +++ b/airflow/hooks/oracle_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import cx_Oracle from airflow.hooks.dbapi_hook import DbApiHook http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/pig_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/pig_hook.py b/airflow/hooks/pig_hook.py index 5b40e52..7201b9f 100644 --- a/airflow/hooks/pig_hook.py +++ b/airflow/hooks/pig_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function import logging import subprocess http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/postgres_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/postgres_hook.py b/airflow/hooks/postgres_hook.py index 0456416..627e97b 100644 --- a/airflow/hooks/postgres_hook.py +++ b/airflow/hooks/postgres_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import psycopg2 import psycopg2.extensions http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/presto_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/presto_hook.py b/airflow/hooks/presto_hook.py index da195bb..f8f6ac8 100644 --- a/airflow/hooks/presto_hook.py +++ b/airflow/hooks/presto_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import str import logging http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/samba_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/samba_hook.py b/airflow/hooks/samba_hook.py index 617c7fb..6a29982 100644 --- a/airflow/hooks/samba_hook.py +++ b/airflow/hooks/samba_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from smbclient import SambaClient import os http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/sqlite_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/sqlite_hook.py b/airflow/hooks/sqlite_hook.py index c6e8f21..c241c2d 100644 --- a/airflow/hooks/sqlite_hook.py +++ b/airflow/hooks/sqlite_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sqlite3 from airflow.hooks.dbapi_hook import DbApiHook http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/hooks/webhdfs_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/webhdfs_hook.py b/airflow/hooks/webhdfs_hook.py index 79a23bc..f808865 100644 --- a/airflow/hooks/webhdfs_hook.py +++ b/airflow/hooks/webhdfs_hook.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from airflow.hooks.base_hook import BaseHook from airflow import configuration import logging http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/macros/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/macros/__init__.py b/airflow/macros/__init__.py index 2634075..da4eb3d 100644 --- a/airflow/macros/__init__.py +++ b/airflow/macros/__init__.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import absolute_import from random import random from datetime import datetime, timedelta @@ -48,8 +62,26 @@ def ds_format(ds, input_format, output_format): return datetime.strptime(ds, input_format).strftime(output_format) -def integrate_plugins(): +def _integrate_plugins(): """Integrate plugins to the context""" + import sys from airflow.plugins_manager import macros as _macros - for _macro in _macros: - globals()[_macro.__name__] = _macro + for _macro_module in _macros: + sys.modules[_macro_module.__name__] = _macro_module + globals()[_macro_module._name] = _macro_module + + + ########################################################## + # TODO FIXME Remove in Airflow 2.0 + + import os as _os + if not _os.environ.get('AIRFLOW_USE_NEW_IMPORTS', False): + from zope.deprecation import deprecated as _deprecated + for _macro in _macro_module._objects: + globals()[_macro.__name__] = _deprecated( + _macro, + "Importing plugin macro '{i}' directly from " + "'airflow.macros' has been deprecated. Please " + "import from 'airflow.macros.[plugin_module]' " + "instead. Support for direct imports will be dropped " + "entirely in Airflow 2.0.".format(i=_macro)) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/macros/hive.py ---------------------------------------------------------------------- diff --git a/airflow/macros/hive.py b/airflow/macros/hive.py index 2f69d66..044fd5f 100644 --- a/airflow/macros/hive.py +++ b/airflow/macros/hive.py @@ -25,7 +25,7 @@ def max_partition( >>> max_partition('airflow.static_babynames_partitioned') '2015-01-01' ''' - from airflow.hooks import HiveMetastoreHook + from airflow.hooks.hive_hooks import HiveMetastoreHook if '.' in table: schema, table = table.split('.') hh = HiveMetastoreHook(metastore_conn_id=metastore_conn_id) @@ -78,7 +78,7 @@ def closest_ds_partition( >>> closest_ds_partition(tbl, '2015-01-02') '2015-01-01' ''' - from airflow.hooks import HiveMetastoreHook + from airflow.hooks.hive_hooks import HiveMetastoreHook if '.' in table: schema, table = table.split('.') hh = HiveMetastoreHook(metastore_conn_id=metastore_conn_id) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/models.py ---------------------------------------------------------------------- diff --git a/airflow/models.py b/airflow/models.py index b6b7987..d20def8 100644 --- a/airflow/models.py +++ b/airflow/models.py @@ -596,33 +596,43 @@ class Connection(Base): descriptor=property(cls.get_extra, cls.set_extra)) def get_hook(self): - from airflow import hooks - from airflow.contrib import hooks as contrib_hooks try: if self.conn_type == 'mysql': - return hooks.MySqlHook(mysql_conn_id=self.conn_id) + from airflow.hooks.mysql_hook import MySqlHook + return MySqlHook(mysql_conn_id=self.conn_id) elif self.conn_type == 'google_cloud_platform': - return contrib_hooks.BigQueryHook(bigquery_conn_id=self.conn_id) + from airflow.contrib.hooks.bigquery_hook import BigQueryHook + return BigQueryHook(bigquery_conn_id=self.conn_id) elif self.conn_type == 'postgres': - return hooks.PostgresHook(postgres_conn_id=self.conn_id) + from airflow.hooks.postgres_hook import PostgresHook + return PostgresHook(postgres_conn_id=self.conn_id) elif self.conn_type == 'hive_cli': - return hooks.HiveCliHook(hive_cli_conn_id=self.conn_id) + from airflow.hooks.hive_hooks import HiveCliHook + return HiveCliHook(hive_cli_conn_id=self.conn_id) elif self.conn_type == 'presto': - return hooks.PrestoHook(presto_conn_id=self.conn_id) + from airflow.hooks.presto_hook import PrestoHook + return PrestoHook(presto_conn_id=self.conn_id) elif self.conn_type == 'hiveserver2': - return hooks.HiveServer2Hook(hiveserver2_conn_id=self.conn_id) + from airflow.hooks.hive_hooks import HiveServer2Hook + return HiveServer2Hook(hiveserver2_conn_id=self.conn_id) elif self.conn_type == 'sqlite': - return hooks.SqliteHook(sqlite_conn_id=self.conn_id) + from airflow.hooks.sqlite_hook import SqliteHook + return SqliteHook(sqlite_conn_id=self.conn_id) elif self.conn_type == 'jdbc': - return hooks.JdbcHook(jdbc_conn_id=self.conn_id) + from airflow.hooks.jdbc_hook import JdbcHook + return JdbcHook(jdbc_conn_id=self.conn_id) elif self.conn_type == 'mssql': - return hooks.MsSqlHook(mssql_conn_id=self.conn_id) + from airflow.hooks.mssql_hook import MsSqlHook + return MsSqlHook(mssql_conn_id=self.conn_id) elif self.conn_type == 'oracle': - return hooks.OracleHook(oracle_conn_id=self.conn_id) + from airflow.hooks.oracle_hook import OracleHook + return OracleHook(oracle_conn_id=self.conn_id) elif self.conn_type == 'vertica': - return contrib_hooks.VerticaHook(vertica_conn_id=self.conn_id) + from airflow.contrib.hooks.vertica_hook import VerticaHook + return VerticaHook(vertica_conn_id=self.conn_id) elif self.conn_type == 'cloudant': - return contrib_hooks.CloudantHook(cloudant_conn_id=self.conn_id) + from airflow.contrib.hooks.cloudant_hook import CloudantHook + return CloudantHook(cloudant_conn_id=self.conn_id) except: return None http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/__init__.py ---------------------------------------------------------------------- diff --git a/airflow/operators/__init__.py b/airflow/operators/__init__.py index af63aee..0d2c403 100644 --- a/airflow/operators/__init__.py +++ b/airflow/operators/__init__.py @@ -1,23 +1,71 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Only import Core Airflow Operators that don't have extra requirements. +# All other operators must be imported directly. +from airflow.models import BaseOperator +from .bash_operator import BashOperator +from .python_operator import ( + BranchPythonOperator, + PythonOperator, + ShortCircuitOperator) +from .check_operator import ( + CheckOperator, + ValueCheckOperator, + IntervalCheckOperator) +from .dagrun_operator import TriggerDagRunOperator +from .dummy_operator import DummyOperator +from .email_operator import EmailOperator +from .http_operator import SimpleHttpOperator +import airflow.operators.sensors +from .subdag_operator import SubDagOperator + + + + +# ------------------------------------------------------------------------ +# +# #TODO #FIXME Airflow 2.0 +# +# Old import machinary below. +# +# This is deprecated but should be kept until Airflow 2.0 +# for compatibility. +# +# ------------------------------------------------------------------------ + # Imports operators dynamically while keeping the package API clean, # abstracting the underlying modules from airflow.utils.helpers import import_module_attrs as _import_module_attrs # These need to be integrated first as other operators depend on them -_import_module_attrs(globals(), { - 'check_operator': [ - 'CheckOperator', - 'ValueCheckOperator', - 'IntervalCheckOperator', - ], -}) +# _import_module_attrs(globals(), { +# 'check_operator': [ +# 'CheckOperator', +# 'ValueCheckOperator', +# 'IntervalCheckOperator', +# ], +# }) _operators = { - 'bash_operator': ['BashOperator'], - 'python_operator': [ - 'PythonOperator', - 'BranchPythonOperator', - 'ShortCircuitOperator', - ], + # 'bash_operator': ['BashOperator'], + # 'python_operator': [ + # 'PythonOperator', + # 'BranchPythonOperator', + # 'ShortCircuitOperator', + # ], 'hive_operator': ['HiveOperator'], 'pig_operator': ['PigOperator'], 'presto_check_operator': [ @@ -25,9 +73,9 @@ _operators = { 'PrestoValueCheckOperator', 'PrestoIntervalCheckOperator', ], - 'dagrun_operator': ['TriggerDagRunOperator'], - 'dummy_operator': ['DummyOperator'], - 'email_operator': ['EmailOperator'], + # 'dagrun_operator': ['TriggerDagRunOperator'], + # 'dummy_operator': ['DummyOperator'], + # 'email_operator': ['EmailOperator'], 'hive_to_samba_operator': ['Hive2SambaOperator'], 'mysql_operator': ['MySqlOperator'], 'sqlite_operator': ['SqliteOperator'], @@ -47,13 +95,13 @@ _operators = { 'TimeSensor', 'WebHdfsSensor', ], - 'subdag_operator': ['SubDagOperator'], + # 'subdag_operator': ['SubDagOperator'], 'hive_stats_operator': ['HiveStatsCollectionOperator'], 's3_to_hive_operator': ['S3ToHiveTransfer'], 'hive_to_mysql': ['HiveToMySqlTransfer'], 'presto_to_mysql': ['PrestoToMySqlTransfer'], 's3_file_transform_operator': ['S3FileTransformOperator'], - 'http_operator': ['SimpleHttpOperator'], + # 'http_operator': ['SimpleHttpOperator'], 'hive_to_druid': ['HiveToDruidTransfer'], 'jdbc_operator': ['JdbcOperator'], 'mssql_operator': ['MsSqlOperator'], @@ -63,12 +111,38 @@ _operators = { 'oracle_operator': ['OracleOperator'] } -_import_module_attrs(globals(), _operators) -from airflow.models import BaseOperator +import os as _os +if not _os.environ.get('AIRFLOW_USE_NEW_IMPORTS', False): + from zope.deprecation import deprecated as _deprecated + _imported = _import_module_attrs(globals(), _operators) + for _i in _imported: + _deprecated( + _i, + "Importing {i} directly from 'airflow.operators' has been " + "deprecated. Please import from " + "'airflow.operators.[operator_module]' instead. Support for direct " + "imports will be dropped entirely in Airflow 2.0.".format(i=_i)) -def integrate_plugins(): +def _integrate_plugins(): """Integrate plugins to the context""" + import sys from airflow.plugins_manager import operators as _operators - for _operator in _operators: - globals()[_operator.__name__] = _operator + for _operator_module in _operators: + sys.modules[_operator_module.__name__] = _operator_module + globals()[_operator_module._name] = _operator_module + + + ########################################################## + # TODO FIXME Remove in Airflow 2.0 + + if not _os.environ.get('AIRFLOW_USE_NEW_IMPORTS', False): + from zope.deprecation import deprecated as _deprecated + for _operator in _operator_module._objects: + globals()[_operator.__name__] = _deprecated( + _operator, + "Importing plugin operator '{i}' directly from " + "'airflow.operators' has been deprecated. Please " + "import from 'airflow.operators.[plugin_module]' " + "instead. Support for direct imports will be dropped " + "entirely in Airflow 2.0.".format(i=_operator)) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/bash_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/bash_operator.py b/airflow/operators/bash_operator.py index 3e9c7ef..a3b1b92 100644 --- a/airflow/operators/bash_operator.py +++ b/airflow/operators/bash_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import bytes import logging http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/check_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/check_operator.py b/airflow/operators/check_operator.py index 0624d91..e4c8262 100644 --- a/airflow/operators/check_operator.py +++ b/airflow/operators/check_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import zip from builtins import str import logging http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/dagrun_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/dagrun_operator.py b/airflow/operators/dagrun_operator.py index 7f8bb53..dc42d67 100644 --- a/airflow/operators/dagrun_operator.py +++ b/airflow/operators/dagrun_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from datetime import datetime import logging http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/docker_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py index b01d31a..d17d154 100644 --- a/airflow/operators/docker_operator.py +++ b/airflow/operators/docker_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import logging from airflow.exceptions import AirflowException http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/dummy_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/dummy_operator.py b/airflow/operators/dummy_operator.py index 1392e7d..4517a8a 100644 --- a/airflow/operators/dummy_operator.py +++ b/airflow/operators/dummy_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/email_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/email_operator.py b/airflow/operators/email_operator.py index 29b18ed..91a8d05 100644 --- a/airflow/operators/email_operator.py +++ b/airflow/operators/email_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from airflow.models import BaseOperator from airflow.utils.email import send_email from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/generic_transfer.py ---------------------------------------------------------------------- diff --git a/airflow/operators/generic_transfer.py b/airflow/operators/generic_transfer.py index eab9d61..de3bf73 100644 --- a/airflow/operators/generic_transfer.py +++ b/airflow/operators/generic_transfer.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from airflow.models import BaseOperator http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/hive_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_operator.py b/airflow/operators/hive_operator.py index 9a299e1..9849e9d 100644 --- a/airflow/operators/hive_operator.py +++ b/airflow/operators/hive_operator.py @@ -1,7 +1,21 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import re -from airflow.hooks import HiveCliHook +from airflow.hooks.hive_hooks import HiveCliHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/hive_stats_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_stats_operator.py b/airflow/operators/hive_stats_operator.py index aadca4d..b31c6b5 100644 --- a/airflow/operators/hive_stats_operator.py +++ b/airflow/operators/hive_stats_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import str from builtins import zip from collections import OrderedDict @@ -5,7 +19,9 @@ import json import logging from airflow.exceptions import AirflowException -from airflow.hooks import PrestoHook, HiveMetastoreHook, MySqlHook +from airflow.hooks.mysql_hook import MySqlHook +from airflow.hooks.presto_hook import PrestoHook +from airflow.hooks.hive_hooks import HiveMetastoreHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/hive_to_druid.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_to_druid.py b/airflow/operators/hive_to_druid.py index 420aeed..6d73e17 100644 --- a/airflow/operators/hive_to_druid.py +++ b/airflow/operators/hive_to_druid.py @@ -1,6 +1,21 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging -from airflow.hooks import HiveCliHook, DruidHook, HiveMetastoreHook +from airflow.hooks.hive_hooks import HiveCliHook, HiveMetastoreHook +from airflow.hooks.druid_hook import DruidHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/hive_to_mysql.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_to_mysql.py b/airflow/operators/hive_to_mysql.py index 9e27f38..36c91d4 100644 --- a/airflow/operators/hive_to_mysql.py +++ b/airflow/operators/hive_to_mysql.py @@ -1,6 +1,21 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging -from airflow.hooks import HiveServer2Hook, MySqlHook +from airflow.hooks.hive_hooks import HiveServer2Hook +from airflow.hooks.mysql_hook import MySqlHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/hive_to_samba_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_to_samba_operator.py b/airflow/operators/hive_to_samba_operator.py index 63881ab..8f18dd9 100644 --- a/airflow/operators/hive_to_samba_operator.py +++ b/airflow/operators/hive_to_samba_operator.py @@ -1,7 +1,22 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import tempfile -from airflow.hooks import HiveServer2Hook, SambaHook +from airflow.hooks.hive_hooks import HiveServer2Hook +from airflow.hooks.samba_hook import SambaHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/http_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/http_operator.py b/airflow/operators/http_operator.py index 87d1415..ad9bd4f 100644 --- a/airflow/operators/http_operator.py +++ b/airflow/operators/http_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from airflow.exceptions import AirflowException http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/jdbc_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/jdbc_operator.py b/airflow/operators/jdbc_operator.py index 5efdaf4..28977db 100644 --- a/airflow/operators/jdbc_operator.py +++ b/airflow/operators/jdbc_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + __author__ = 'janomar' import logging http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/mssql_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/mssql_operator.py b/airflow/operators/mssql_operator.py index 1d5273a..ed9bdf4 100644 --- a/airflow/operators/mssql_operator.py +++ b/airflow/operators/mssql_operator.py @@ -1,6 +1,20 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging -from airflow.hooks import MsSqlHook +from airflow.hooks.mssql_hook import MsSqlHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/mssql_to_hive.py ---------------------------------------------------------------------- diff --git a/airflow/operators/mssql_to_hive.py b/airflow/operators/mssql_to_hive.py index 6a981b4..6db0cba 100644 --- a/airflow/operators/mssql_to_hive.py +++ b/airflow/operators/mssql_to_hive.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import chr from collections import OrderedDict import unicodecsv as csv @@ -6,7 +20,8 @@ from tempfile import NamedTemporaryFile import pymssql -from airflow.hooks import HiveCliHook, MsSqlHook +from airflow.hooks.hive_hooks import HiveCliHook +from airflow.hooks.mssql_hook import MsSqlHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/mysql_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/mysql_operator.py b/airflow/operators/mysql_operator.py index ae6d36f..b3a3c73 100644 --- a/airflow/operators/mysql_operator.py +++ b/airflow/operators/mysql_operator.py @@ -1,6 +1,20 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging -from airflow.hooks import MySqlHook +from airflow.hooks.mysql_hook import MySqlHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/mysql_to_hive.py ---------------------------------------------------------------------- diff --git a/airflow/operators/mysql_to_hive.py b/airflow/operators/mysql_to_hive.py index 09ec190..2fa2541 100644 --- a/airflow/operators/mysql_to_hive.py +++ b/airflow/operators/mysql_to_hive.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import chr from collections import OrderedDict import unicodecsv as csv @@ -5,7 +19,8 @@ import logging from tempfile import NamedTemporaryFile import MySQLdb -from airflow.hooks import HiveCliHook, MySqlHook +from airflow.hooks.hive_hooks import HiveCliHook +from airflow.hooks.mysql_hook import MySqlHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/oracle_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/oracle_operator.py b/airflow/operators/oracle_operator.py index 28182cf..ab7bdb2 100644 --- a/airflow/operators/oracle_operator.py +++ b/airflow/operators/oracle_operator.py @@ -14,7 +14,7 @@ import logging -from airflow.hooks import OracleHook +from airflow.hooks.oracle_hook import OracleHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/pig_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/pig_operator.py b/airflow/operators/pig_operator.py index d25795d..4a21ecc 100644 --- a/airflow/operators/pig_operator.py +++ b/airflow/operators/pig_operator.py @@ -1,7 +1,21 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import re -from airflow.hooks import PigCliHook +from airflow.hooks.pig_hook import PigCliHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/postgres_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/postgres_operator.py b/airflow/operators/postgres_operator.py index 79fa5e7..c4f56a4 100644 --- a/airflow/operators/postgres_operator.py +++ b/airflow/operators/postgres_operator.py @@ -1,6 +1,20 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging -from airflow.hooks import PostgresHook +from airflow.hooks.postgres_hook import PostgresHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/presto_check_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/presto_check_operator.py b/airflow/operators/presto_check_operator.py index e857036..c1ac9cf 100644 --- a/airflow/operators/presto_check_operator.py +++ b/airflow/operators/presto_check_operator.py @@ -1,4 +1,18 @@ -from airflow.hooks import PrestoHook +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from airflow.hooks.presto_hook import PrestoHook from airflow.operators import CheckOperator, ValueCheckOperator, IntervalCheckOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/presto_to_mysql.py ---------------------------------------------------------------------- diff --git a/airflow/operators/presto_to_mysql.py b/airflow/operators/presto_to_mysql.py index 29de0c7..7ff2ad6 100644 --- a/airflow/operators/presto_to_mysql.py +++ b/airflow/operators/presto_to_mysql.py @@ -1,6 +1,21 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging -from airflow.hooks import PrestoHook, MySqlHook +from airflow.hooks.presto_hook import PrestoHook +from airflow.hooks.mysql_hook import MySqlHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/python_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/python_operator.py b/airflow/operators/python_operator.py index 290cc65..b5f6386 100644 --- a/airflow/operators/python_operator.py +++ b/airflow/operators/python_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import str from datetime import datetime import logging http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/s3_file_transform_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/s3_file_transform_operator.py b/airflow/operators/s3_file_transform_operator.py index 0641123..1cdd0e5 100644 --- a/airflow/operators/s3_file_transform_operator.py +++ b/airflow/operators/s3_file_transform_operator.py @@ -1,9 +1,23 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from tempfile import NamedTemporaryFile import subprocess from airflow.exceptions import AirflowException -from airflow.hooks import S3Hook +from airflow.hooks.S3_hook import S3Hook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/s3_to_hive_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/s3_to_hive_operator.py b/airflow/operators/s3_to_hive_operator.py index 3fc5327..3e01c29 100644 --- a/airflow/operators/s3_to_hive_operator.py +++ b/airflow/operators/s3_to_hive_operator.py @@ -1,10 +1,25 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from builtins import next from builtins import zip import logging from tempfile import NamedTemporaryFile from airflow.exceptions import AirflowException -from airflow.hooks import HiveCliHook, S3Hook +from airflow.hooks.S3_hook import S3Hook +from airflow.hooks.hive_hooks import HiveCliHook from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/sensors.py ---------------------------------------------------------------------- diff --git a/airflow/operators/sensors.py b/airflow/operators/sensors.py index 569b4d7..6d87b44 100644 --- a/airflow/operators/sensors.py +++ b/airflow/operators/sensors.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function from future import standard_library standard_library.install_aliases() @@ -7,6 +21,7 @@ import logging from urllib.parse import urlparse from time import sleep +import airflow from airflow import hooks, settings from airflow.exceptions import AirflowException, AirflowSensorTimeout, AirflowSkipException from airflow.models import BaseOperator, TaskInstance, Connection as DB @@ -249,7 +264,8 @@ class HivePartitionSensor(BaseSensorOperator): 'Poking for table {self.schema}.{self.table}, ' 'partition {self.partition}'.format(**locals())) if not hasattr(self, 'hook'): - self.hook = hooks.HiveMetastoreHook( + import airflow.hooks.hive_hooks + self.hook = airflow.hooks.hive_hooks.HiveMetastoreHook( metastore_conn_id=self.metastore_conn_id) return self.hook.check_for_partition( self.schema, self.table, self.partition) @@ -272,7 +288,8 @@ class HdfsSensor(BaseSensorOperator): self.hdfs_conn_id = hdfs_conn_id def poke(self, context): - sb = hooks.HDFSHook(self.hdfs_conn_id).get_conn() + import airflow.hooks.hdfs_hook + sb = airflow.hooks.hdfs_hook.HDFSHook(self.hdfs_conn_id).get_conn() logging.getLogger("snakebite").setLevel(logging.WARNING) logging.info( 'Poking for file {self.filepath} '.format(**locals())) @@ -300,7 +317,7 @@ class WebHdfsSensor(BaseSensorOperator): self.webhdfs_conn_id = webhdfs_conn_id def poke(self, context): - c = hooks.WebHDFSHook(self.webhdfs_conn_id) + c = airflow.hooks.webhdfs_hook.WebHDFSHook(self.webhdfs_conn_id) logging.info( 'Poking for file {self.filepath} '.format(**locals())) return c.check_for_path(hdfs_path=self.filepath) @@ -356,7 +373,8 @@ class S3KeySensor(BaseSensorOperator): session.close() def poke(self, context): - hook = hooks.S3Hook(s3_conn_id=self.s3_conn_id) + import airflow.hooks.S3_hook + hook = airflow.hooks.S3_hook.S3Hook(s3_conn_id=self.s3_conn_id) full_url = "s3://" + self.bucket_name + "/" + self.bucket_key logging.info('Poking for key : {full_url}'.format(**locals())) if self.wildcard_match: @@ -408,7 +426,8 @@ class S3PrefixSensor(BaseSensorOperator): def poke(self, context): logging.info('Poking for prefix : {self.prefix}\n' 'in bucket s3://{self.bucket_name}'.format(**locals())) - hook = hooks.S3Hook(s3_conn_id=self.s3_conn_id) + import airflow.hooks.S3_hook + hook = airflow.hooks.S3_hook.S3Hook(s3_conn_id=self.s3_conn_id) return hook.check_for_prefix( prefix=self.prefix, delimiter=self.delimiter, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/slack_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/slack_operator.py b/airflow/operators/slack_operator.py index f92eff1..2e6d426 100644 --- a/airflow/operators/slack_operator.py +++ b/airflow/operators/slack_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from slackclient import SlackClient from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/operators/sqlite_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/sqlite_operator.py b/airflow/operators/sqlite_operator.py index 700019d..52b3b4b 100644 --- a/airflow/operators/sqlite_operator.py +++ b/airflow/operators/sqlite_operator.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from airflow.hooks import SqliteHook http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/plugins_manager.py ---------------------------------------------------------------------- diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py index 25c65cd..b6108f4 100644 --- a/airflow/plugins_manager.py +++ b/airflow/plugins_manager.py @@ -89,10 +89,26 @@ for root, dirs, files in os.walk(plugins_folder, followlinks=True): logging.exception(e) logging.error('Failed to import plugin ' + filepath) -operators = merge([p.operators for p in plugins]) -hooks = merge([p.hooks for p in plugins]) -executors = merge([p.executors for p in plugins]) -macros = merge([p.macros for p in plugins]) -admin_views = merge([p.admin_views for p in plugins]) -flask_blueprints = merge([p.flask_blueprints for p in plugins]) -menu_links = merge([p.menu_links for p in plugins]) +def make_module(name, objects): + name = name.lower() + module = imp.new_module(name) + module._name = name.split('.')[-1] + module._objects = objects + module.__dict__.update((o.__name__, o) for o in objects) + return module + +operators, hooks, executors, macros, admin_views = [], [], [], [], [] +flask_blueprints, menu_links = [], [] + +for p in plugins: + operators.append(make_module('airflow.operators.' + p.name, p.operators)) + hooks.append(make_module('airflow.hooks.' + p.name, p.hooks)) + executors.append(make_module('airflow.executors.' + p.name, p.executors)) + macros.append(make_module('airflow.macros.' + p.name, p.macros)) + admin_views.append( + make_module('airflow.www.admin_views' + p.name, p.admin_views)) + flask_blueprints.append( + make_module( + 'airflow.www.flask_blueprints' + p.name, p.flask_blueprints)) + menu_links.append( + make_module('airflow.www.menu_links' + p.name, p.menu_links)) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/utils/email.py ---------------------------------------------------------------------- diff --git a/airflow/utils/email.py b/airflow/utils/email.py index 6877e47..c19bb89 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -1,11 +1,3 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from builtins import str -from past.builtins import basestring - # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +11,15 @@ from past.builtins import basestring # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from builtins import str +from past.builtins import basestring + import importlib import logging import os http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/utils/logging.py ---------------------------------------------------------------------- diff --git a/airflow/utils/logging.py b/airflow/utils/logging.py index 117587f..8f5fc51 100644 --- a/airflow/utils/logging.py +++ b/airflow/utils/logging.py @@ -47,7 +47,7 @@ class S3Log(object): def __init__(self): remote_conn_id = configuration.get('core', 'REMOTE_LOG_CONN_ID') try: - from airflow.hooks import S3Hook + from airflow.hooks.S3_hook import S3Hook self.hook = S3Hook(remote_conn_id) except: self.hook = None http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/airflow/utils/tests.py ---------------------------------------------------------------------- diff --git a/airflow/utils/tests.py b/airflow/utils/tests.py new file mode 100644 index 0000000..50490d3 --- /dev/null +++ b/airflow/utils/tests.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +def skipUnlessImported(module, obj): + import importlib + m = importlib.import_module(module) + return unittest.skipUnless( + obj in dir(m), + "Skipping test because {} could not be imported from {}".format( + obj, module)) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/dags/testdruid.py ---------------------------------------------------------------------- diff --git a/dags/testdruid.py b/dags/testdruid.py index ebfc607..c356fc9 100644 --- a/dags/testdruid.py +++ b/dags/testdruid.py @@ -1,4 +1,4 @@ -from airflow.operators import HiveToDruidTransfer +from airflow.operators.hive_to_druid import HiveToDruidTransfer from airflow import DAG from datetime import datetime http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/run_unit_tests.sh ---------------------------------------------------------------------- diff --git a/run_unit_tests.sh b/run_unit_tests.sh index 71df44c..8cb916d 100755 --- a/run_unit_tests.sh +++ b/run_unit_tests.sh @@ -7,6 +7,9 @@ export AIRFLOW_CONFIG=$AIRFLOW_HOME/unittests.cfg # configuration test export AIRFLOW__TESTSECTION__TESTKEY=testvalue +# use Airflow 2.0-style imports +export AIRFLOW_USE_NEW_IMPORTS=1 + # any argument received is overriding the default nose execution arguments: nose_args=$@ http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/851adc55/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 4741f20..411a699 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from setuptools import setup, find_packages, Command from setuptools.command.test import test as TestCommand
