This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 3646de384e683ef2e70f05cf5f3919aabacd840b Author: Jarek Potiuk <[email protected]> AuthorDate: Wed Apr 10 16:37:41 2024 +0200 Add multi-team diagrams (#38861) (cherry picked from commit 16082319936de80a0a3023a4c4a52ce9f8b8872d) --- ..._team_airflow_architecture_with_grpc_api.md5sum | 1 + ...lti_team_airflow_architecture_with_grpc_api.png | Bin 0 -> 647436 bytes ...lti_team_airflow_architecture_with_grpc_api.py} | 38 ++++----- ...am_airflow_architecture_without_grpc_api.md5sum | 1 + ..._team_airflow_architecture_without_grpc_api.png | Bin 0 -> 597552 bytes ..._team_airflow_architecture_without_grpc_api.py} | 92 +++++++-------------- ...diagram_multitenant_airflow_architecture.md5sum | 1 - .../diagram_multitenant_airflow_architecture.png | Bin 601270 -> 0 bytes hatch_build.py | 1 + 9 files changed, 52 insertions(+), 82 deletions(-) diff --git a/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.md5sum b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.md5sum new file mode 100644 index 0000000000..e90c3fd8b6 --- /dev/null +++ b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.md5sum @@ -0,0 +1 @@ +b84a3673426377d3d7972315321dbb00 diff --git a/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.png b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.png new file mode 100644 index 0000000000..9b8dbfd1c1 Binary files /dev/null and b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.png differ diff --git a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.py b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.py similarity index 87% copy from docs/apache-airflow/img/diagram_multitenant_airflow_architecture.py copy to docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.py index daf68a1a9e..5ac38b3c71 100644 --- a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.py +++ b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.py @@ -60,8 +60,8 @@ def generate_dag_processor_airflow_diagram(): "Common Organization Airflow Deployment", graph_attr={"bgcolor": "lightgrey", "fontsize": "22"} ): with Cluster("Scheduling\n\n"): - executor_1 = Custom("Executor\nTenant 1", PYTHON_MULTIPROCESS_LOGO.as_posix()) - executor_2 = Custom("Executor\nTenant 2", PYTHON_MULTIPROCESS_LOGO.as_posix()) + executor_1 = Custom("Executor\nTeam 1", PYTHON_MULTIPROCESS_LOGO.as_posix()) + executor_2 = Custom("Executor\nTeam 2", PYTHON_MULTIPROCESS_LOGO.as_posix()) schedulers = Custom("Scheduler(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) executor_1 - Edge(color="black", style="dashed", reverse=True) - schedulers executor_2 - Edge(color="black", style="dashed", reverse=True) - schedulers @@ -96,58 +96,58 @@ def generate_dag_processor_airflow_diagram(): >> organization_admin ) - deployment_manager_1 = User("Deployment\nManager\nTenant 1") - dag_author_1 = User("DAG Author\nTenant 1") + deployment_manager_1 = User("Deployment\nManager\nTeam 1") + dag_author_1 = User("DAG Author\nTeamt 1") - with Cluster("Tenant 1 Airflow Deployment", graph_attr={"bgcolor": "#AAAABB", "fontsize": "22"}): + with Cluster("Team 1 Airflow Deployment", graph_attr={"bgcolor": "#AAAABB", "fontsize": "22"}): with Cluster("No DB access"): with Cluster("Execution"): workers_1 = Custom("Worker(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) triggerer_1 = Custom("Triggerer(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) with Cluster("Parsing"): dag_processors_1 = Custom("DAG\nProcessor(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) - dag_files_1 = Custom("DAGS/Tenant 1", MULTIPLE_FILES_IMAGE.as_posix()) + dag_files_1 = Custom("DAGS/Team 1", MULTIPLE_FILES_IMAGE.as_posix()) plugins_and_packages_1 = Custom("Plugins\n& Packages\nTenant 1", PACKAGES_IMAGE.as_posix()) - config_file_1 = Custom("Config\nFile\nTenant 1", CONFIG_FILE.as_posix()) + config_file_1 = Custom("Config\nFile\nTeam 1", CONFIG_FILE.as_posix()) with Cluster("DB access", graph_attr={"bgcolor": "#D0BBCC"}): - internal_api_1 = Custom("Internal API\nTenant 1\n", PYTHON_MULTIPROCESS_LOGO.as_posix()) + internal_api_1 = Custom("GRPC API\nTeam 1\n", PYTHON_MULTIPROCESS_LOGO.as_posix()) ( internal_api_1 >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n\n\n") >> metadata_db ) - operations_user_1 = User("Operations User\nTenant 1") + operations_user_1 = User("Operations User\nTeam 1") - deployment_manager_2 = User("Deployment\nManager\nTenant 2") - dag_author_2 = User("DAG Author\nTenant 2") + deployment_manager_2 = User("Deployment\nManager\nTeam 2") + dag_author_2 = User("DAG Author\nTeam 2") - with Cluster("Tenant 2 Airflow Deployment", graph_attr={"fontsize": "22"}): + with Cluster("Team 2 Airflow Deployment", graph_attr={"fontsize": "22"}): with Cluster("No DB access"): with Cluster("Execution"): workers_2 = Custom("Worker(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) triggerer_2 = Custom("Triggerer(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) with Cluster("Parsing"): dag_processors_2 = Custom("DAG\nProcessor(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) - dag_files_2 = Custom("DAGS/Tenant 2", MULTIPLE_FILES_IMAGE.as_posix()) - plugins_and_packages_2 = Custom("Plugins\n& Packages\nTenant 2", PACKAGES_IMAGE.as_posix()) - config_file_2 = Custom("Config\nFile\nTenant 2", CONFIG_FILE.as_posix()) + dag_files_2 = Custom("DAGS/Team 2", MULTIPLE_FILES_IMAGE.as_posix()) + plugins_and_packages_2 = Custom("Plugins\n& Packages\nTeam 2", PACKAGES_IMAGE.as_posix()) + config_file_2 = Custom("Config\nFile\nTeam 2", CONFIG_FILE.as_posix()) with Cluster("DB access", graph_attr={"bgcolor": "#D0BBCC"}): - internal_api_2 = Custom("Internal API\nTenant 2", PYTHON_MULTIPROCESS_LOGO.as_posix()) + internal_api_2 = Custom("GRPC API\nTeam 2", PYTHON_MULTIPROCESS_LOGO.as_posix()) ( internal_api_2 >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n\n\n") >> metadata_db ) - operations_user_2 = User("Operations User\nTenant 2") + operations_user_2 = User("Operations User\nTeam 2") ( operations_user_1 - >> Edge(color="black", style="solid", reverse=True, label="operates\nTenant 1 Only\n\n") + >> Edge(color="black", style="solid", reverse=True, label="operates\nTeam 1 Only\n\n") >> auth_manager ) ( operations_user_2 - >> Edge(color="black", style="solid", reverse=True, label="operates\nTenant 2 Only\n\n") + >> Edge(color="black", style="solid", reverse=True, label="operates\nTeam 2 Only\n\n") >> auth_manager ) diff --git a/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.md5sum b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.md5sum new file mode 100644 index 0000000000..93a4bb637f --- /dev/null +++ b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.md5sum @@ -0,0 +1 @@ +7050a02a0a0f382db0556ac43430f807 diff --git a/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.png b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.png new file mode 100644 index 0000000000..6bc5679bdb Binary files /dev/null and b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.png differ diff --git a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.py b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.py similarity index 73% rename from docs/apache-airflow/img/diagram_multitenant_airflow_architecture.py rename to docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.py index daf68a1a9e..53a5d3c053 100644 --- a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.py +++ b/docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.py @@ -56,19 +56,19 @@ def generate_dag_processor_airflow_diagram(): graph_attr=graph_attr, edge_attr=edge_attr, ): + with Cluster("Organization DB", graph_attr={"bgcolor": "#D0BBCC", "fontsize": "22"}): + metadata_db = Custom("Metadata DB", DATABASE_IMAGE.as_posix()) + with Cluster( "Common Organization Airflow Deployment", graph_attr={"bgcolor": "lightgrey", "fontsize": "22"} ): with Cluster("Scheduling\n\n"): - executor_1 = Custom("Executor\nTenant 1", PYTHON_MULTIPROCESS_LOGO.as_posix()) - executor_2 = Custom("Executor\nTenant 2", PYTHON_MULTIPROCESS_LOGO.as_posix()) + executor_1 = Custom("Executor\nTeam 1", PYTHON_MULTIPROCESS_LOGO.as_posix()) + executor_2 = Custom("Executor\nTeam 2", PYTHON_MULTIPROCESS_LOGO.as_posix()) schedulers = Custom("Scheduler(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) executor_1 - Edge(color="black", style="dashed", reverse=True) - schedulers executor_2 - Edge(color="black", style="dashed", reverse=True) - schedulers - with Cluster("Organization DB", graph_attr={"bgcolor": "#D0BBCC", "fontsize": "22"}): - metadata_db = Custom("Metadata DB", DATABASE_IMAGE.as_posix()) - with Cluster("UI"): webservers = Custom("Webserver(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) auth_manager = Custom("Auth\nManager", PYTHON_MULTIPROCESS_LOGO.as_posix()) @@ -96,58 +96,44 @@ def generate_dag_processor_airflow_diagram(): >> organization_admin ) - deployment_manager_1 = User("Deployment\nManager\nTenant 1") - dag_author_1 = User("DAG Author\nTenant 1") + deployment_manager_1 = User("Deployment\nManager\nTeam 1") + dag_author_1 = User("DAG Author\nTeamt 1") - with Cluster("Tenant 1 Airflow Deployment", graph_attr={"bgcolor": "#AAAABB", "fontsize": "22"}): + with Cluster("Team 1 Airflow Deployment", graph_attr={"bgcolor": "#AAAABB", "fontsize": "22"}): with Cluster("No DB access"): with Cluster("Execution"): workers_1 = Custom("Worker(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) triggerer_1 = Custom("Triggerer(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) with Cluster("Parsing"): dag_processors_1 = Custom("DAG\nProcessor(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) - dag_files_1 = Custom("DAGS/Tenant 1", MULTIPLE_FILES_IMAGE.as_posix()) + dag_files_1 = Custom("DAGS/Team 1", MULTIPLE_FILES_IMAGE.as_posix()) plugins_and_packages_1 = Custom("Plugins\n& Packages\nTenant 1", PACKAGES_IMAGE.as_posix()) - config_file_1 = Custom("Config\nFile\nTenant 1", CONFIG_FILE.as_posix()) - with Cluster("DB access", graph_attr={"bgcolor": "#D0BBCC"}): - internal_api_1 = Custom("Internal API\nTenant 1\n", PYTHON_MULTIPROCESS_LOGO.as_posix()) - ( - internal_api_1 - >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n\n\n") - >> metadata_db - ) - operations_user_1 = User("Operations User\nTenant 1") - - deployment_manager_2 = User("Deployment\nManager\nTenant 2") - dag_author_2 = User("DAG Author\nTenant 2") - - with Cluster("Tenant 2 Airflow Deployment", graph_attr={"fontsize": "22"}): + config_file_1 = Custom("Config\nFile\nTeam 1", CONFIG_FILE.as_posix()) + operations_user_1 = User("Operations User\nTeam 1") + + deployment_manager_2 = User("Deployment\nManager\nTeam 2") + dag_author_2 = User("DAG Author\nTeam 2") + + with Cluster("Team 2 Airflow Deployment", graph_attr={"fontsize": "22"}): with Cluster("No DB access"): with Cluster("Execution"): workers_2 = Custom("Worker(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) triggerer_2 = Custom("Triggerer(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) with Cluster("Parsing"): dag_processors_2 = Custom("DAG\nProcessor(s)", PYTHON_MULTIPROCESS_LOGO.as_posix()) - dag_files_2 = Custom("DAGS/Tenant 2", MULTIPLE_FILES_IMAGE.as_posix()) - plugins_and_packages_2 = Custom("Plugins\n& Packages\nTenant 2", PACKAGES_IMAGE.as_posix()) - config_file_2 = Custom("Config\nFile\nTenant 2", CONFIG_FILE.as_posix()) - with Cluster("DB access", graph_attr={"bgcolor": "#D0BBCC"}): - internal_api_2 = Custom("Internal API\nTenant 2", PYTHON_MULTIPROCESS_LOGO.as_posix()) - ( - internal_api_2 - >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n\n\n") - >> metadata_db - ) - operations_user_2 = User("Operations User\nTenant 2") + dag_files_2 = Custom("DAGS/Team 2", MULTIPLE_FILES_IMAGE.as_posix()) + plugins_and_packages_2 = Custom("Plugins\n& Packages\nTeam 2", PACKAGES_IMAGE.as_posix()) + config_file_2 = Custom("Config\nFile\nTeam 2", CONFIG_FILE.as_posix()) + operations_user_2 = User("Operations User\nTeam 2") ( operations_user_1 - >> Edge(color="black", style="solid", reverse=True, label="operates\nTenant 1 Only\n\n") + >> Edge(color="black", style="solid", reverse=True, label="operates\nTeam 1 Only\n\n") >> auth_manager ) ( operations_user_2 - >> Edge(color="black", style="solid", reverse=True, label="operates\nTenant 2 Only\n\n") + >> Edge(color="black", style="solid", reverse=True, label="operates\nTeam 2 Only\n\n") >> auth_manager ) @@ -205,35 +191,19 @@ def generate_dag_processor_airflow_diagram(): ( dag_processors_1 - >> Edge(color="red", style="dotted", reverse=True, label="GRPC\nHTTPS\n\n") - >> internal_api_1 - ) - ( - workers_1 - >> Edge(color="red", style="dotted", reverse=True, label="GRPC\nHTTPS\n\n") - >> internal_api_1 - ) - ( - triggerer_1 - >> Edge(color="red", style="dotted", reverse=True, label="GRPC\nHTTPS\n\n") - >> internal_api_1 + >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n") + >> metadata_db ) + (workers_1 >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n") >> metadata_db) + (triggerer_1 >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n") >> metadata_db) ( dag_processors_2 - >> Edge(color="red", style="dotted", reverse=True, label="GRPC\nHTTPS\n\n") - >> internal_api_2 - ) - ( - workers_2 - >> Edge(color="red", style="dotted", reverse=True, label="GRPC\nHTTPS\n\n") - >> internal_api_2 - ) - ( - triggerer_2 - >> Edge(color="red", style="dotted", reverse=True, label="GRPC\nHTTPS\n\n") - >> internal_api_2 + >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n") + >> metadata_db ) + (workers_2 >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n") >> metadata_db) + (triggerer_2 >> Edge(color="red", style="dotted", reverse=True, label="DB Access\n") >> metadata_db) dag_files_1 >> Edge(color="brown", style="solid", label="sync\n\n") >> workers_1 dag_files_1 >> Edge(color="brown", style="solid", label="sync\n\n") >> dag_processors_1 @@ -248,8 +218,6 @@ def generate_dag_processor_airflow_diagram(): schedulers - Edge(style="invis") - organization_plugins_and_packages metadata_db - Edge(style="invis") - executor_1 metadata_db - Edge(style="invis") - executor_2 - workers_1 - Edge(style="invis") - operations_user_1 - workers_2 - Edge(style="invis") - operations_user_2 external_organization_identity_system - Edge(style="invis") - organization_admin diff --git a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.md5sum b/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.md5sum deleted file mode 100644 index e66f4190da..0000000000 --- a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.md5sum +++ /dev/null @@ -1 +0,0 @@ -4f47971fc98a3427953fd0d8d93abdc2 diff --git a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.png b/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.png deleted file mode 100644 index 37fc516741..0000000000 Binary files a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.png and /dev/null differ diff --git a/hatch_build.py b/hatch_build.py index 705cbf9f65..f6899ed590 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -182,6 +182,7 @@ DOC_EXTRAS: dict[str, list[str]] = { ], "doc-gen": [ "apache-airflow[doc]", + "diagrams>=0.23.4", "eralchemy2>=1.3.8", ], # END OF doc extras
