This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-7-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7af9938a924f66b749c63fe302fcf865789b6b26 Author: Jarek Potiuk <[email protected]> AuthorDate: Fri Aug 4 21:50:44 2023 +0200 Move openlineage configuration to provider (#33124) (cherry picked from commit bdc10a5ff6fea0fd968345fd4a9b732be49b9761) --- airflow/config_templates/config.yml | 59 --------------------- .../providers/openlineage/plugins/openlineage.py | 6 +-- airflow/providers/openlineage/provider.yaml | 61 ++++++++++++++++++++++ .../configurations-ref.rst | 18 +++++++ .../apache-airflow-providers-openlineage/index.rst | 1 + docs/apache-airflow/configurations-ref.rst | 1 + 6 files changed, 84 insertions(+), 62 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 8b3188c062..3ec37a42e1 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -1230,65 +1230,6 @@ lineage: type: string example: ~ default: "" -openlineage: - description: | - This section applies settings for OpenLineage integration. - For backwards compatibility with `openlineage-python` one can still use - `openlineage.yml` file or `OPENLINEAGE_` environment variables. However, below - configuration takes precedence over those. - More in documentation - https://openlineage.io/docs/client/python#configuration. - options: - disabled: - description: | - Set this to true if you don't want OpenLineage to emit events. - type: boolean - example: ~ - default: "False" - version_added: ~ - namespace: - description: | - OpenLineage namespace - version_added: ~ - type: string - example: "food_delivery" - default: ~ - extractors: - description: | - Comma-separated paths to custom OpenLineage extractors. - type: string - example: full.path.to.ExtractorClass;full.path.to.AnotherExtractorClass - default: '' - version_added: ~ - config_path: - description: | - Path to YAML config. This provides backwards compatibility to pass config as - `openlineage.yml` file. - version_added: ~ - type: string - example: ~ - default: '' - transport: - description: | - OpenLineage Client transport configuration. It should contain type - and additional options per each type. - - Currently supported types are: - - * HTTP - * Kafka - * Console - type: string - example: '{"type": "http", "url": "http://localhost:5000"}' - default: '' - version_added: ~ - disable_source_code: - description: | - If disabled, OpenLineage events do not contain source code of particular - operators, like PythonOperator. - default: ~ - example: ~ - type: boolean - version_added: ~ operators: description: ~ options: diff --git a/airflow/providers/openlineage/plugins/openlineage.py b/airflow/providers/openlineage/plugins/openlineage.py index 318d2d5b17..34498c2f05 100644 --- a/airflow/providers/openlineage/plugins/openlineage.py +++ b/airflow/providers/openlineage/plugins/openlineage.py @@ -25,11 +25,11 @@ from airflow.providers.openlineage.plugins.macros import lineage_parent_id, line def _is_disabled() -> bool: return ( - conf.getboolean("openlineage", "disabled") + conf.getboolean("openlineage", "disabled", fallback=False) or os.getenv("OPENLINEAGE_DISABLED", "false").lower() == "true" or ( - conf.get("openlineage", "transport") == "" - and conf.get("openlineage", "config_path") == "" + conf.get("openlineage", "transport", fallback="") == "" + and conf.get("openlineage", "config_path", fallback="") == "" and os.getenv("OPENLINEAGE_URL", "") == "" and os.getenv("OPENLINEAGE_CONFIG", "") == "" ) diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml index 2a2c1a6669..5c64c9f6c3 100644 --- a/airflow/providers/openlineage/provider.yaml +++ b/airflow/providers/openlineage/provider.yaml @@ -41,3 +41,64 @@ integrations: plugins: - name: openlineage plugin-class: airflow.providers.openlineage.plugins.openlineage.OpenLineageProviderPlugin + +config: + openlineage: + description: | + This section applies settings for OpenLineage integration. + For backwards compatibility with `openlineage-python` one can still use + `openlineage.yml` file or `OPENLINEAGE_` environment variables. However, below + configuration takes precedence over those. + More in documentation - https://openlineage.io/docs/client/python#configuration. + options: + disabled: + description: | + Set this to true if you don't want OpenLineage to emit events. + type: boolean + example: ~ + default: "False" + version_added: ~ + namespace: + description: | + OpenLineage namespace + version_added: ~ + type: string + example: "food_delivery" + default: ~ + extractors: + description: | + Comma-separated paths to custom OpenLineage extractors. + type: string + example: full.path.to.ExtractorClass;full.path.to.AnotherExtractorClass + default: '' + version_added: ~ + config_path: + description: | + Path to YAML config. This provides backwards compatibility to pass config as + `openlineage.yml` file. + version_added: ~ + type: string + example: ~ + default: '' + transport: + description: | + OpenLineage Client transport configuration. It should contain type + and additional options per each type. + + Currently supported types are: + + * HTTP + * Kafka + * Console + type: string + example: '{"type": "http", "url": "http://localhost:5000"}' + default: '' + version_added: ~ + disable_source_code: + description: | + If disabled, OpenLineage events do not contain source code of particular + operators, like PythonOperator. + default: ~ + example: ~ + type: boolean + version_added: ~ diff --git a/docs/apache-airflow-providers-openlineage/configurations-ref.rst b/docs/apache-airflow-providers-openlineage/configurations-ref.rst new file mode 100644 index 0000000000..5885c9d91b --- /dev/null +++ b/docs/apache-airflow-providers-openlineage/configurations-ref.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../exts/includes/providers-configurations-ref.rst diff --git a/docs/apache-airflow-providers-openlineage/index.rst b/docs/apache-airflow-providers-openlineage/index.rst index a0548a9117..221fd99df0 100644 --- a/docs/apache-airflow-providers-openlineage/index.rst +++ b/docs/apache-airflow-providers-openlineage/index.rst @@ -62,6 +62,7 @@ :maxdepth: 1 :caption: References + Configuration <configurations-ref> Python API <_api/airflow/providers/openlineage/index> .. toctree:: diff --git a/docs/apache-airflow/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst index 43173fe6ac..f323ea31f0 100644 --- a/docs/apache-airflow/configurations-ref.rst +++ b/docs/apache-airflow/configurations-ref.rst @@ -40,6 +40,7 @@ in the provider's documentation. The pre-installed providers that you may want t * :doc:`Configuration Reference for CNCF Kubernetes Provider <apache-airflow-providers-cncf-kubernetes:configurations-ref>` * :doc:`Configuration Reference for SMTP Provider <apache-airflow-providers-smtp:configurations-ref>` * :doc:`Configuration Reference for IMAP Provider <apache-airflow-providers-imap:configurations-ref>` +* :doc:`Configuration Reference for OpenLineage Provider <apache-airflow-providers-openlineage:configurations-ref>` .. note:: For more information see :doc:`/howto/set-config`.
