Greetings ,
We are trying to extract common DAG creation code into a single source file
and create seperate file for differente environment imporing this "common
logic".
"cdna_daily_common.py" has the common DAG creation code. File for different
environments are like "cdna_daily_dev.py" ,"cdna_daily_stg.py" ..etc
However when I copy these files in the "dags" folder , they doesnt show up
in Airflow UI. How can I make these DAGs visible on Airflow UI following
this coding convention.
====================
cdna_daily_common.py
====================
global airflow_host_port
global env
global alert_email
global spdb_sync_prefix
global post_validate_prefix
global schedule_interval
global dag_name#PROD cdna_daily_prd
global job_execution_prefix
global dag
def initDAG():
# define SSHHook
sshHook = SSHHook()
global dag
dag = DAG(
dag_id=dag_name,
default_args=default_args,
schedule_interval="0 2 * * *"
)
..create rest of the DAG
def execDAG():
initDAG()
dag.cli()
==================
cdna_daily_dev.py
==================
import cdna_daily_common
cdna_daily_common.airflow_host_port='hostnme.jp.local:9553'
cdna_daily_common.env='dev'
cdna_daily_common.alert_email='[email protected]'
cdna_daily_common.spdb_sync_prefix='echo SPDBSync'
cdna_daily_common.post_validate_prefix='echo PostVal'
cdna_daily_common.schedule_interval='0 2 * * *'
cdna_daily_common.dag_name='cdna_daily_dev_v2'
cdna_daily_common.job_execution_prefix="python
/home/bisuser/bo_dir/repos/customer_dna/py/job_execution.py -j"
if __name__ == "__main__":
cdna_daily_common.execDAG();
