This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new fca8aa582e Add Multi-Tenant architecture diagram (#37931)
fca8aa582e is described below
commit fca8aa582e1b27b86fd519e3d66d0100d5fe0dbf
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Mar 8 21:29:36 2024 +0100
Add Multi-Tenant architecture diagram (#37931)
This image is not yet connected anywhere, I just want to keep it in
Airflow while we are discussing "AIP-67 - Multi-tenant Deployment of
Airflow components" and while we are iterating it - it will be easy
to collaborate on the diagrams.
---
...diagram_multitenant_airflow_architecture.md5sum | 1 +
.../diagram_multitenant_airflow_architecture.png | Bin 0 -> 568294 bytes
.../diagram_multitenant_airflow_architecture.py | 247 +++++++++++++++++++++
.../img/diagram_multitenant_architecture.md5sum | 1 +
docs/diagrams/config_file.png | Bin 0 -> 116921 bytes
5 files changed, 249 insertions(+)
diff --git
a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.md5sum
b/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.md5sum
new file mode 100644
index 0000000000..da3da1c44e
--- /dev/null
+++ b/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.md5sum
@@ -0,0 +1 @@
+f2dfcb3537301f006d2943e8ee9dd2b2
diff --git
a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.png
b/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.png
new file mode 100644
index 0000000000..d86f132585
Binary files /dev/null and
b/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.png differ
diff --git
a/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.py
b/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.py
new file mode 100644
index 0000000000..75524bbe91
--- /dev/null
+++ b/docs/apache-airflow/img/diagram_multitenant_airflow_architecture.py
@@ -0,0 +1,247 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from pathlib import Path
+
+from diagrams import Cluster, Diagram, Edge
+from diagrams.custom import Custom
+from diagrams.onprem.client import User
+from diagrams.onprem.identity import Dex
+from rich.console import Console
+
+MY_DIR = Path(__file__).parent
+MY_FILENAME = Path(__file__).with_suffix("").name
+PYTHON_MULTIPROCESS_LOGO = MY_DIR.parents[1] / "diagrams" /
"python_multiprocess_logo.png"
+PACKAGES_IMAGE = MY_DIR.parents[1] / "diagrams" / "packages.png"
+DATABASE_IMAGE = MY_DIR.parents[1] / "diagrams" / "database.png"
+MULTIPLE_FILES_IMAGE = MY_DIR.parents[1] / "diagrams" / "multiple_files.png"
+CONFIG_FILE = MY_DIR.parents[1] / "diagrams" / "config_file.png"
+
+console = Console(width=400, color_system="standard")
+
+graph_attr = {
+ "concentrate": "false",
+ "splines": "line",
+}
+
+edge_attr = {
+ "minlen": "2",
+}
+
+
+def generate_dag_processor_airflow_diagram():
+ dag_processor_architecture_image_file = (MY_DIR /
MY_FILENAME).with_suffix(".png")
+ console.print(f"[bright_blue]Generating architecture image
{dag_processor_architecture_image_file}")
+ with Diagram(
+ name="",
+ show=False,
+ direction="LR",
+ filename=MY_FILENAME,
+ outformat="png",
+ graph_attr=graph_attr,
+ edge_attr=edge_attr,
+ ):
+ operations_user_1 = User("Operations User\nTenant 1")
+ operations_user_2 = User("Operations User\nTenant 2")
+ organization_admin = User("Organization Admin")
+
+ with Cluster(
+ "Shared Organization Airflow Deployment", graph_attr={"bgcolor":
"lightgrey", "fontsize": "22"}
+ ):
+ with Cluster("Scheduling\n\n"):
+ schedulers = Custom("Scheduler(s)",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+
+ with Cluster("UI"):
+ webservers = Custom("Webserver(s)",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+ auth_manager = Custom("Auth\nManager",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+
+ metadata_db = Custom("Metadata DB", DATABASE_IMAGE.as_posix())
+
+ organization_plugins_and_packages = Custom(
+ "Shared\nOrganization\nPlugins &\nPackages",
PACKAGES_IMAGE.as_posix()
+ )
+
+ organization_config_file =
Custom("Config\nFile\nShared\nOrganization", CONFIG_FILE.as_posix())
+
+ organization_deployment_manager = User("Deployment
Manager\nOrganization")
+ external_organization_identity_system = Dex("Organization Identity
System")
+ external_organization_identity_system >> auth_manager
+
+ webservers >> Edge(color="black", style="solid", reverse=True) >>
auth_manager
+
+ (
+ auth_manager
+ >> Edge(color="black", style="solid", reverse=True,
label="operates\nTenant 1 Only\n\n")
+ >> operations_user_1
+ )
+ (
+ auth_manager
+ >> Edge(color="black", style="solid", reverse=True,
label="operates\nTenant 2 Only\n\n")
+ >> operations_user_2
+ )
+ (
+ auth_manager
+ >> Edge(color="black", style="solid", reverse=True,
label="manages\n\n")
+ >> organization_admin
+ )
+
+ deployment_manager_1 = User("Deployment\nManager\nTenant 1")
+ dag_author_1 = User("DAG Author\nTenant 1")
+
+ with Cluster("Tenant 1 Airflow Deployment", graph_attr={"bgcolor":
"#AAAABB", "fontsize": "22"}):
+ with Cluster("No DB access"):
+ with Cluster("Execution"):
+ workers_1 = Custom("Worker(s)",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+ triggerer_1 = Custom("Triggerer(s)",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+ with Cluster("Parsing"):
+ dag_processors_1 = Custom("DAG\nProcessor(s)",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+ dag_files_1 = Custom("DAGS/Tenant 1",
MULTIPLE_FILES_IMAGE.as_posix())
+ plugins_and_packages_1 = Custom("Plugins\n& Packages\nTenant
1", PACKAGES_IMAGE.as_posix())
+ config_file_1 = Custom("Config\nFile\nTenant 1",
CONFIG_FILE.as_posix())
+ with Cluster("DB access"):
+ internal_api_1 = Custom("Internal API\nTenant 1\n",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+ (
+ internal_api_1
+ >> Edge(color="red", style="dotted", reverse=True,
label="DB Access\n\n\n")
+ >> metadata_db
+ )
+
+ deployment_manager_2 = User("Deployment\nManager\nTenant 2")
+ dag_author_2 = User("DAG Author\nTenant 2")
+
+ with Cluster("Tenant 2 Airflow Deployment", graph_attr={"fontsize":
"22"}):
+ with Cluster("No DB access"):
+ with Cluster("Execution"):
+ workers_2 = Custom("Worker(s)",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+ triggerer_2 = Custom("Triggerer(s)",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+ with Cluster("Parsing"):
+ dag_processors_2 = Custom("DAG\nProcessor(s)",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+ dag_files_2 = Custom("DAGS/Tenant 2",
MULTIPLE_FILES_IMAGE.as_posix())
+ plugins_and_packages_2 = Custom("Plugins\n& Packages\nTenant
2", PACKAGES_IMAGE.as_posix())
+ config_file_2 = Custom("Config\nFile\nTenant 2",
CONFIG_FILE.as_posix())
+ with Cluster("DB access"):
+ internal_api_2 = Custom("Internal API\nTenant 2",
PYTHON_MULTIPROCESS_LOGO.as_posix())
+ (
+ internal_api_2
+ >> Edge(color="red", style="dotted", reverse=True,
label="DB Access\n\n\n")
+ >> metadata_db
+ )
+
+ dag_author_1 >> Edge(color="brown", style="dashed", reverse=False,
label="author\n\n") >> dag_files_1
+ (
+ deployment_manager_1
+ >> Edge(color="blue", style="solid", reverse=False,
label="install\n\n")
+ >> plugins_and_packages_1
+ )
+
+ (
+ deployment_manager_1
+ >> Edge(color="blue", style="solid", reverse=False,
label="configure\n\n")
+ >> config_file_1
+ )
+
+ dag_author_2 >> Edge(color="brown", style="dashed", reverse=False,
label="author\n\n") >> dag_files_2
+ (
+ deployment_manager_2
+ >> Edge(color="blue", style="solid", reverse=False,
label="install\n\n")
+ >> plugins_and_packages_2
+ )
+
+ (
+ deployment_manager_2
+ >> Edge(color="blue", style="solid", reverse=False,
label="configure\n\n")
+ >> config_file_2
+ )
+
+ (
+ organization_deployment_manager
+ >> Edge(color="blue", style="solid", reverse=False,
label="install\n\n")
+ >> organization_plugins_and_packages
+ )
+
+ (
+ organization_deployment_manager
+ >> Edge(color="blue", style="solid", reverse=False,
label="configure\n\n")
+ >> organization_config_file
+ )
+
+ plugins_and_packages_1 >> Edge(style="invis") >> workers_1
+ plugins_and_packages_1 >> Edge(style="invis") >> dag_processors_1
+ plugins_and_packages_1 >> Edge(style="invis") >> triggerer_1
+
+ plugins_and_packages_2 >> Edge(style="invis") >> workers_2
+ plugins_and_packages_2 >> Edge(style="invis") >> dag_processors_2
+ plugins_and_packages_2 >> Edge(style="invis") >> triggerer_2
+
+ organization_plugins_and_packages >> Edge(style="invis") >> schedulers
+ organization_plugins_and_packages >> Edge(style="invis") >> webservers
+
+ metadata_db >> Edge(color="red", style="dotted", reverse=True,
label="DB Access") >> webservers
+ metadata_db >> Edge(color="red", style="dotted", reverse=True,
label="DB Access") >> schedulers
+
+ (
+ dag_processors_1
+ >> Edge(color="red", style="dotted", reverse=True,
label="GRPC\nHTTPS\n\n")
+ >> internal_api_1
+ )
+ (
+ workers_1
+ >> Edge(color="red", style="dotted", reverse=True,
label="GRPC\nHTTPS\n\n")
+ >> internal_api_1
+ )
+ (
+ triggerer_1
+ >> Edge(color="red", style="dotted", reverse=True,
label="GRPC\nHTTPS\n\n")
+ >> internal_api_1
+ )
+
+ (
+ dag_processors_2
+ >> Edge(color="red", style="dotted", reverse=True,
label="GRPC\nHTTPS\n\n")
+ >> internal_api_2
+ )
+ (
+ workers_2
+ >> Edge(color="red", style="dotted", reverse=True,
label="GRPC\nHTTPS\n\n")
+ >> internal_api_2
+ )
+ (
+ triggerer_2
+ >> Edge(color="red", style="dotted", reverse=True,
label="GRPC\nHTTPS\n\n")
+ >> internal_api_2
+ )
+
+ dag_files_1 >> Edge(color="brown", style="solid", label="sync\n\n") >>
workers_1
+ dag_files_1 >> Edge(color="brown", style="solid", label="sync\n\n") >>
dag_processors_1
+ dag_files_1 >> Edge(color="brown", style="solid", label="sync\n\n") >>
triggerer_1
+
+ dag_files_2 >> Edge(color="brown", style="solid", label="sync\n\n") >>
workers_2
+ dag_files_2 >> Edge(color="brown", style="solid", label="sync\n\n") >>
dag_processors_2
+ dag_files_2 >> Edge(color="brown", style="solid", label="sync\n\n") >>
triggerer_2
+
+ # This is for better layout. Invisible edges are used to align the
nodes better
+ webservers - Edge(style="invis") -
external_organization_identity_system
+ schedulers - Edge(style="invis") -
external_organization_identity_system
+ internal_api_1 - Edge(style="invis") - organization_deployment_manager
+ internal_api_2 - Edge(style="invis") - organization_deployment_manager
+
+ console.print(f"[green]Generating architecture image
{dag_processor_architecture_image_file}")
+
+
+if __name__ == "__main__":
+ generate_dag_processor_airflow_diagram()
diff --git a/docs/apache-airflow/img/diagram_multitenant_architecture.md5sum
b/docs/apache-airflow/img/diagram_multitenant_architecture.md5sum
new file mode 100644
index 0000000000..9b9ffcdc2a
--- /dev/null
+++ b/docs/apache-airflow/img/diagram_multitenant_architecture.md5sum
@@ -0,0 +1 @@
+00f67a1e0cd073ba521da168dc80ccaa
diff --git a/docs/diagrams/config_file.png b/docs/diagrams/config_file.png
new file mode 100644
index 0000000000..0b6b343a2f
Binary files /dev/null and b/docs/diagrams/config_file.png differ
