ashb closed pull request #4014: [AIRFLOW-3168] More resillient database use in
CI setup.
URL: https://github.com/apache/incubator-airflow/pull/4014
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/scripts/ci/3-setup-databases.sh b/scripts/ci/3-setup-mysql.sh
similarity index 79%
rename from scripts/ci/3-setup-databases.sh
rename to scripts/ci/3-setup-mysql.sh
index 2a5cb682e0..ca5c1aa9f1 100755
--- a/scripts/ci/3-setup-databases.sh
+++ b/scripts/ci/3-setup-mysql.sh
@@ -20,4 +20,11 @@ set -exuo pipefail
MYSQL_HOST=mysql
-mysql -h ${MYSQL_HOST} -u root -e 'drop database if exists airflow; create
database airflow'
+retries=3
+for ((i=0; i<retries; i++)); do
+ mysql -h ${MYSQL_HOST} -u root -e 'drop database if exists airflow; create
database airflow' && exit 0
+ echo "mysql db creation did not succeed, waiting 5 seconds to retry"
+ sleep 5
+done
+
+echo "mysql db creation could not succeed" && exit 1
diff --git a/scripts/ci/airflow_travis.cfg b/scripts/ci/airflow_travis.cfg
index 6895c4d5ad..8adf50b928 100644
--- a/scripts/ci/airflow_travis.cfg
+++ b/scripts/ci/airflow_travis.cfg
@@ -21,7 +21,7 @@ airflow_home = ~/airflow
dags_folder = ~/airflow/dags
base_log_folder = ~/airflow/logs
executor = LocalExecutor
-sql_alchemy_conn = mysql://root@mysql/airflow
+sql_alchemy_conn = # overridden by tox.ini
unit_test_mode = True
load_examples = True
donot_pickle = False
@@ -56,7 +56,7 @@ celery_app_name = airflow.executors.celery_executor
worker_concurrency = 16
worker_log_server_port = 8793
broker_url = amqp://guest:guest@rabbitmq:5672/
-result_backend = db+mysql://root@mysql/airflow
+result_backend = # overridden by tox.ini
flower_port = 5555
default_queue = default
sync_parallelism = 0
diff --git a/tests/configuration.py b/tests/configuration.py
index b50a83259a..e94491ee4e 100644
--- a/tests/configuration.py
+++ b/tests/configuration.py
@@ -191,4 +191,9 @@ def test_deprecated_options_cmd(self):
conf.set('celery', 'celery_result_backend_cmd', '/bin/echo 99')
with self.assertWarns(DeprecationWarning):
+ tmp = None
+ if 'AIRFLOW__CELERY__RESULT_BACKEND' in os.environ:
+ tmp = os.environ.pop('AIRFLOW__CELERY__RESULT_BACKEND')
self.assertEquals(conf.getint('celery', 'result_backend'), 99)
+ if tmp:
+ os.environ['AIRFLOW__CELERY__RESULT_BACKEND'] = tmp
diff --git a/tests/executors/test_celery_executor.py
b/tests/executors/test_celery_executor.py
index 5c70b5bf20..f1b6a429fa 100644
--- a/tests/executors/test_celery_executor.py
+++ b/tests/executors/test_celery_executor.py
@@ -26,11 +26,16 @@
from airflow.executors.celery_executor import CELERY_FETCH_ERR_MSG_HEADER
from airflow.utils.state import State
+from airflow import configuration
+configuration.load_test_config()
+
# leave this it is used by the test worker
import celery.contrib.testing.tasks # noqa: F401
class CeleryExecutorTest(unittest.TestCase):
+ @unittest.skipIf('sqlite' in configuration.conf.get('core',
'sql_alchemy_conn'),
+ "sqlite is configured with SequentialExecutor")
def test_celery_integration(self):
executor = CeleryExecutor()
executor.start()
diff --git a/tests/sensors/test_sql_sensor.py b/tests/sensors/test_sql_sensor.py
index 433eaa61e9..ec909431f0 100644
--- a/tests/sensors/test_sql_sensor.py
+++ b/tests/sensors/test_sql_sensor.py
@@ -38,6 +38,8 @@ def setUp(self):
}
self.dag = DAG(TEST_DAG_ID, default_args=args)
+ @unittest.skipUnless('mysql' in configuration.conf.get('core',
'sql_alchemy_conn'),
+ "this is a mysql test")
def test_sql_sensor_mysql(self):
t = SqlSensor(
task_id='sql_sensor_check',
@@ -47,6 +49,8 @@ def test_sql_sensor_mysql(self):
)
t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE,
ignore_ti_state=True)
+ @unittest.skipUnless('postgresql' in configuration.conf.get('core',
'sql_alchemy_conn'),
+ "this is a postgres test")
def test_sql_sensor_postgres(self):
t = SqlSensor(
task_id='sql_sensor_check',
diff --git a/tox.ini b/tox.ini
index f07641f29b..17bfeb6ec2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -48,7 +48,9 @@ setenv =
KRB5_CONFIG=/etc/krb5.conf
KRB5_KTNAME=/etc/airflow.keytab
backend_mysql: AIRFLOW__CORE__SQL_ALCHEMY_CONN=mysql://root@mysql/airflow
+ backend_mysql: AIRFLOW__CELERY__RESULT_BACKEND=db+mysql://root@mysql/airflow
backend_postgres:
AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:airflow@postgres/airflow
+ backend_postgres:
AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://postgres:airflow@postgres/airflow
backend_sqlite:
AIRFLOW__CORE__SQL_ALCHEMY_CONN=sqlite:///{homedir}/airflow.db
backend_sqlite: AIRFLOW__CORE__EXECUTOR=SequentialExecutor
@@ -59,7 +61,7 @@ commands =
pip install --progress-bar off --find-links={homedir}/.wheelhouse --no-index
-e .[devel_ci]
env_docker: {toxinidir}/scripts/ci/1-setup-env.sh
env_docker: {toxinidir}/scripts/ci/2-setup-kdc.sh
- env_docker: {toxinidir}/scripts/ci/3-setup-databases.sh
+ backend_mysql: {toxinidir}/scripts/ci/3-setup-mysql.sh
{toxinidir}/scripts/ci/5-run-tests.sh []
{toxinidir}/scripts/ci/6-check-license.sh
codecov -e TOXENV
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
