This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f1e1cdcc3b Add option for storing configuration description in
providers (#32629)
f1e1cdcc3b is described below
commit f1e1cdcc3b2826e68ba133f350300b5065bbca33
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Jul 16 17:09:09 2023 +0200
Add option for storing configuration description in providers (#32629)
This is mostly for the capacity of adding configuration to
provider.yaml files.
This change adds capavilty of adding "config" option to provider
yaml and provider_info. This is not yet used, but it has been
extracted as separate PR from #32604 in an attempt to make it more
reviewable and readable.
---
airflow/provider.yaml.schema.json | 710 +++++++++++++++++++++-----------------
airflow/provider_info.schema.json | 285 +++++++++------
airflow/providers_manager.py | 45 ++-
3 files changed, 630 insertions(+), 410 deletions(-)
diff --git a/airflow/provider.yaml.schema.json
b/airflow/provider.yaml.schema.json
index a1d0822af6..509e5dfce6 100644
--- a/airflow/provider.yaml.schema.json
+++ b/airflow/provider.yaml.schema.json
@@ -1,345 +1,437 @@
{
- "$schema": "http://json-schema.org/draft-07/schema#";,
- "type": "object",
- "properties": {
- "package-name": {
- "description": "Package name available under which the package is
available in the PyPI repository.",
- "type": "string"
- },
- "name": {
- "description": "Provider name",
- "type": "string"
- },
- "description": {
- "description": "Information about the package in RST format",
- "type": "string"
- },
- "versions": {
- "description": "List of available versions in PyPI. Sorted descending
according to release date.",
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "suspended": {
- "description": "If set to true, the provider is suspended and it's not a
candidate for release nor contributes dependencies to constraint
calculations/CI image. Tests are excluded.",
- "type:": "boolean"
- },
- "dependencies": {
- "description": "Dependencies that should be added to the provider",
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "excluded-python-versions": {
- "description": "List of python versions excluded for that provider",
- "type": "array",
- "items": {
- "type": "string"
- }
- },
- "integrations": {
- "description": "List of integrations supported by the provider.",
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "integration-name": {
- "type": "string",
- "description": "Name of the integration."
- },
- "external-doc-url": {
- "type": "string",
- "description": "URL to external documentation for the integration."
- },
- "how-to-guide": {
- "description": "List of paths to how-to-guide for the integration.
The path must start with '/docs/'",
+ "$schema": "http://json-schema.org/draft-07/schema#";,
+ "type": "object",
+ "properties": {
+ "package-name": {
+ "description": "Package name available under which the package is
available in the PyPI repository.",
+ "type": "string"
+ },
+ "name": {
+ "description": "Provider name",
+ "type": "string"
+ },
+ "description": {
+ "description": "Information about the package in RST format",
+ "type": "string"
+ },
+ "versions": {
+ "description": "List of available versions in PyPI. Sorted
descending according to release date.",
"type": "array",
"items": {
- "type": "string"
+ "type": "string"
}
- },
- "logo": {
- "description": "Path to the logo for the integration. The path
must start with '/integration-logos/'",
- "type": "string"
- },
- "tags": {
- "description": "List of tags describing the integration. While
we're using RST, only one tag is supported per integration.",
+ },
+ "suspended": {
+ "description": "If set to true, the provider is suspended and it's
not a candidate for release nor contributes dependencies to constraint
calculations/CI image. Tests are excluded.",
+ "type:": "boolean"
+ },
+ "dependencies": {
+ "description": "Dependencies that should be added to the provider",
"type": "array",
"items": {
- "type": "string",
- "enum": [
- "alibaba",
- "apache",
- "aws",
- "azure",
- "dbt",
- "gcp",
- "gmp",
- "google",
- "kafka",
- "protocol",
- "service",
- "software",
- "yandex"
- ]
- },
- "minItems": 1,
- "maxItems": 1
- }
+ "type": "string"
+ }
},
- "additionalProperties": false,
- "required": [
- "integration-name",
- "external-doc-url",
- "tags"
- ]
- }
- },
- "operators": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "integration-name": {
- "type": "string",
- "description": "Integration name. It must have a matching item in
the 'integration' section of any provider."
- },
- "python-modules": {
- "description": "List of python modules containing the operators.",
+ "excluded-python-versions": {
+ "description": "List of python versions excluded for that
provider",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "integrations": {
+ "description": "List of integrations supported by the provider.",
"type": "array",
"items": {
- "type": "string"
+ "type": "object",
+ "properties": {
+ "integration-name": {
+ "type": "string",
+ "description": "Name of the integration."
+ },
+ "external-doc-url": {
+ "type": "string",
+ "description": "URL to external documentation for the
integration."
+ },
+ "how-to-guide": {
+ "description": "List of paths to how-to-guide for the
integration. The path must start with '/docs/'",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "logo": {
+ "description": "Path to the logo for the integration.
The path must start with '/integration-logos/'",
+ "type": "string"
+ },
+ "tags": {
+ "description": "List of tags describing the
integration. While we're using RST, only one tag is supported per integration.",
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": [
+ "alibaba",
+ "apache",
+ "aws",
+ "azure",
+ "dbt",
+ "gcp",
+ "gmp",
+ "google",
+ "kafka",
+ "protocol",
+ "service",
+ "software",
+ "yandex"
+ ]
+ },
+ "minItems": 1,
+ "maxItems": 1
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "integration-name",
+ "external-doc-url",
+ "tags"
+ ]
}
- }
},
- "additionalProperties": false,
- "required": [
- "integration-name",
- "python-modules"
- ]
- }
- },
- "sensors": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "integration-name": {
- "type": "string",
- "description": "Integration name. It must have a matching item in
the 'integration' section of any provider."
- },
- "python-modules": {
- "description": "List of python modules containing the sensors.",
+ "operators": {
"type": "array",
"items": {
- "type": "string"
+ "type": "object",
+ "properties": {
+ "integration-name": {
+ "type": "string",
+ "description": "Integration name. It must have a
matching item in the 'integration' section of any provider."
+ },
+ "python-modules": {
+ "description": "List of python modules containing the
operators.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "integration-name",
+ "python-modules"
+ ]
}
- }
},
- "required": [
- "integration-name",
- "python-modules"
- ],
- "additionalProperties": true
- }
- },
- "hooks": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "integration-name": {
- "type": "string",
- "description": "Integration name. It must have a matching item in
the 'integration' section of any provider."
- },
- "python-modules": {
- "description": "List of python modules containing the hooks.",
+ "sensors": {
"type": "array",
"items": {
- "type": "string"
+ "type": "object",
+ "properties": {
+ "integration-name": {
+ "type": "string",
+ "description": "Integration name. It must have a
matching item in the 'integration' section of any provider."
+ },
+ "python-modules": {
+ "description": "List of python modules containing the
sensors.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "integration-name",
+ "python-modules"
+ ],
+ "additionalProperties": true
}
- }
},
- "additionalProperties": false,
- "required": [
- "integration-name",
- "python-modules"
- ]
- }
- },
- "transfers": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "how-to-guide": {
- "description": "Path to how-to-guide for the transfer. The path
must start with '/docs/'",
- "type": "string"
- },
- "source-integration-name": {
- "type": "string",
- "description": "Integration name. It must have a matching item in
the 'integration' section of any provider."
- },
- "target-integration-name": {
- "type": "string",
- "description": "Target integration name. It must have a matching
item in the 'integration' section of any provider."
- },
- "python-module": {
- "type": "string",
- "description": "List of python modules containing the transfers."
- }
+ "hooks": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "integration-name": {
+ "type": "string",
+ "description": "Integration name. It must have a
matching item in the 'integration' section of any provider."
+ },
+ "python-modules": {
+ "description": "List of python modules containing the
hooks.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "integration-name",
+ "python-modules"
+ ]
+ }
},
- "additionalProperties": false,
- "required": [
- "source-integration-name",
- "target-integration-name",
- "python-module"
- ]
- }
- },
- "triggers": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "integration-name": {
- "type": "string",
- "description": "Integration name. It must have a matching item in
the 'integration' section of any provider."
- },
- "python-modules": {
- "description": "List of Python modules containing the triggers.",
+ "transfers": {
"type": "array",
"items": {
- "type": "string"
+ "type": "object",
+ "properties": {
+ "how-to-guide": {
+ "description": "Path to how-to-guide for the transfer.
The path must start with '/docs/'",
+ "type": "string"
+ },
+ "source-integration-name": {
+ "type": "string",
+ "description": "Integration name. It must have a
matching item in the 'integration' section of any provider."
+ },
+ "target-integration-name": {
+ "type": "string",
+ "description": "Target integration name. It must have
a matching item in the 'integration' section of any provider."
+ },
+ "python-module": {
+ "type": "string",
+ "description": "List of python modules containing the
transfers."
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "source-integration-name",
+ "target-integration-name",
+ "python-module"
+ ]
}
- }
},
- "additionalProperties": false,
- "required": [
- "integration-name",
- "python-modules"
- ]
- }
- },
- "connection-types": {
- "type": "array",
- "description": "Array of connection types mapped to hook class names",
- "items": {
- "type": "object",
- "properties": {
- "connection-type": {
- "description": "Type of connection defined by the provider",
- "type": "string"
- },
- "hook-class-name": {
- "description": "Hook class name that implements the
connection type",
- "type": "string"
- }
- },
- "required": [
- "connection-type",
- "hook-class-name"
- ]
- }
- },
- "extra-links": {
- "type": "array",
- "description": "Operator class names that provide extra link
functionality",
- "items": {
- "type": "string"
- }
- },
- "additional-extras": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "name": {
- "description": "Name of the extra",
- "type": "string"
- },
- "dependencies": {
- "description": "Dependencies that should be added for the extra",
+ "triggers": {
"type": "array",
"items": {
- "type": "string"
+ "type": "object",
+ "properties": {
+ "integration-name": {
+ "type": "string",
+ "description": "Integration name. It must have a
matching item in the 'integration' section of any provider."
+ },
+ "python-modules": {
+ "description": "List of Python modules containing the
triggers.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "integration-name",
+ "python-modules"
+ ]
}
- }
},
- "required": [ "name", "dependencies"]
- },
-
- "description": "Additional extras that the provider should have.
Replaces auto-generated cross-provider extras, if matching the same prefix, so
that you can specify boundaries for existing dependencies."
- },
- "task-decorators": {
- "type": "array",
- "description": "Decorators to use with the TaskFlow API. Can be
accessed by users via '@task.<name>'",
- "items": {
- "name": {
+ "connection-types": {
+ "type": "array",
+ "description": "Array of connection types mapped to hook class
names",
+ "items": {
+ "type": "object",
+ "properties": {
+ "connection-type": {
+ "description": "Type of connection defined by the
provider",
+ "type": "string"
+ },
+ "hook-class-name": {
+ "description": "Hook class name that implements the
connection type",
+ "type": "string"
+ }
+ },
+ "required": [
+ "connection-type",
+ "hook-class-name"
+ ]
+ }
+ },
+ "extra-links": {
+ "type": "array",
+ "description": "Operator class names that provide extra link
functionality",
+ "items": {
"type": "string"
+ }
+ },
+ "additional-extras": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "description": "Name of the extra",
+ "type": "string"
+ },
+ "dependencies": {
+ "description": "Dependencies that should be added for
the extra",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "name",
+ "dependencies"
+ ]
},
- "path": {
+ "description": "Additional extras that the provider should have.
Replaces auto-generated cross-provider extras, if matching the same prefix, so
that you can specify boundaries for existing dependencies."
+ },
+ "task-decorators": {
+ "type": "array",
+ "description": "Decorators to use with the TaskFlow API. Can be
accessed by users via '@task.<name>'",
+ "items": {
+ "name": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ }
+ }
+ },
+ "secrets-backends": {
+ "type": "array",
+ "description": "Secrets Backend class names",
+ "items": {
"type": "string"
}
+ },
+ "logging": {
+ "type": "array",
+ "description": "Logging Task Handlers class names",
+ "items": {
+ "type": "string"
+ }
+ },
+ "auth-backends": {
+ "type": "array",
+ "description": "API Auth Backend module names",
+ "items": {
+ "type": "string"
+ }
+ },
+ "notifications": {
+ "type": "array",
+ "description": "Notification class names",
+ "items": {
+ "type": "string"
+ }
+ },
+ "executors": {
+ "type": "array",
+ "description": "Executor class names",
+ "items": {
+ "type": "string"
+ }
+ },
+ "config": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "options": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/option"
+ }
+ },
+ "renamed": {
+ "type": "object",
+ "properties": {
+ "previous_name": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "required": [
+ "description",
+ "options"
+ ],
+ "additionalProperties": false
+ }
+ },
+ "plugins": {
+ "type": "array",
+ "description": "Plugins exposed by the provider",
+ "items": {
+ "name": {
+ "type": "string"
+ },
+ "plugin-class": {
+ "type": "string"
+ }
+ }
}
},
- "secrets-backends": {
- "type": "array",
- "description": "Secrets Backend class names",
- "items": {
- "type": "string"
- }
- },
- "logging": {
- "type": "array",
- "description": "Logging Task Handlers class names",
- "items": {
- "type": "string"
- }
- },
- "auth-backends": {
- "type": "array",
- "description": "API Auth Backend module names",
- "items": {
- "type": "string"
- }
- },
- "notifications": {
- "type": "array",
- "description": "Notification class names",
- "items": {
- "type": "string"
- }
- },
- "executors": {
- "type": "array",
- "description": "Executor class names",
- "items": {
- "type": "string"
- }
- },
- "plugins": {
- "type": "array",
- "description": "Plugins exposed by the provider",
- "items": {
- "name": {
- "type": "string"
- },
- "plugin-class": {
- "type": "string"
+ "additionalProperties": false,
+ "definitions": {
+ "option": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "version_added": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "string",
+ "boolean",
+ "integer",
+ "float"
+ ]
+ },
+ "example": {
+ "type": [
+ "string",
+ "null",
+ "number"
+ ]
+ },
+ "default": {
+ "type": [
+ "string",
+ "null",
+ "number"
+ ]
+ },
+ "sensitive": {
+ "type": "boolean",
+ "description": "When true, this option is sensitive and
can be specified using AIRFLOW__{section}___{name}__SECRET or
AIRFLOW__{section}___{name}_CMD environment variables. See:
airflow.configuration.AirflowConfigParser.sensitive_config_values"
+ }
+ },
+ "required": [
+ "description",
+ "version_added",
+ "type",
+ "example",
+ "default"
+ ],
+ "additional_properties": false
}
- }
- }
- },
- "additionalProperties": false,
- "required": [
- "name",
- "package-name",
- "description",
- "suspended",
- "dependencies",
- "versions"
- ]
+ },
+ "required": [
+ "name",
+ "package-name",
+ "description",
+ "suspended",
+ "dependencies",
+ "versions"
+ ]
}
diff --git a/airflow/provider_info.schema.json
b/airflow/provider_info.schema.json
index 86b0430cc3..9a5dcaefbe 100644
--- a/airflow/provider_info.schema.json
+++ b/airflow/provider_info.schema.json
@@ -1,105 +1,198 @@
{
- "$schema": "http://json-schema.org/draft-07/schema#";,
- "type": "object",
- "properties": {
- "package-name": {
- "description": "Package name available under which the package is
available in the PyPI repository.",
- "type": "string"
- },
- "name": {
- "description": "Provider name",
- "type": "string"
- },
- "description": {
- "description": "Information about the package in RST format",
- "type": "string"
- },
- "hook-class-names": {
- "type": "array",
- "description": "Hook class names that provide connection types to core
(deprecated by connection-types)",
- "items": {
- "type": "string"
- },
- "deprecated": {
- "description": "The hook-class-names property has been deprecated in
favour of connection-types which is more performant version allowing to only
import individual Hooks rather than all hooks at once",
- "deprecatedVersion": "2.2.0"
- }
- },
- "connection-types": {
- "type": "array",
- "description": "Map of connection types mapped to hook class names.",
- "items": {
- "type": "object",
- "properties": {
- "connection-type": {
- "description": "Type of connection defined by the provider",
+ "$schema": "http://json-schema.org/draft-07/schema#";,
+ "type": "object",
+ "properties": {
+ "package-name": {
+ "description": "Package name available under which the package is
available in the PyPI repository.",
"type": "string"
- },
- "hook-class-name": {
- "description": "Hook class name that implements the connection
type",
+ },
+ "name": {
+ "description": "Provider name",
"type": "string"
- }
- }
- },
- "required": ["connection-type", "hook-class-name"]
- },
- "extra-links": {
- "type": "array",
- "description": "Operator class names that provide extra link
functionality",
- "items": {
- "type": "string"
- }
- },
- "secrets-backends": {
- "type": "array",
- "description": "Secrets Backend class names",
- "items": {
- "type": "string"
- }
- },
- "logging": {
- "type": "array",
- "description": "Logging Task Handlers class names",
- "items": {
- "type": "string"
- }
- },
- "auth-backends": {
- "type": "array",
- "description": "API Auth Backend module names",
- "items": {
- "type": "string"
- }
- },
- "notifications": {
- "type": "array",
- "description": "Notification class names",
- "items": {
- "type": "string"
- }
- },
- "executors": {
- "type": "array",
- "description": "Executor class names",
- "items": {
- "type": "string"
- }
- },
- "task-decorators": {
- "type": "array",
- "description": "Apply custom decorators to the TaskFlow API. Can be
accessed by users via '@task.<name>'",
- "items": {
- "name": {
+ },
+ "description": {
+ "description": "Information about the package in RST format",
+ "type": "string"
+ },
+ "hook-class-names": {
+ "type": "array",
+ "description": "Hook class names that provide connection types to
core (deprecated by connection-types)",
+ "items": {
"type": "string"
},
- "path": {
+ "deprecated": {
+ "description": "The hook-class-names property has been
deprecated in favour of connection-types which is more performant version
allowing to only import individual Hooks rather than all hooks at once",
+ "deprecatedVersion": "2.2.0"
+ }
+ },
+ "connection-types": {
+ "type": "array",
+ "description": "Map of connection types mapped to hook class
names.",
+ "items": {
+ "type": "object",
+ "properties": {
+ "connection-type": {
+ "description": "Type of connection defined by the
provider",
+ "type": "string"
+ },
+ "hook-class-name": {
+ "description": "Hook class name that implements the
connection type",
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "connection-type",
+ "hook-class-name"
+ ]
+ },
+ "extra-links": {
+ "type": "array",
+ "description": "Operator class names that provide extra link
functionality",
+ "items": {
"type": "string"
}
+ },
+ "secrets-backends": {
+ "type": "array",
+ "description": "Secrets Backend class names",
+ "items": {
+ "type": "string"
+ }
+ },
+ "logging": {
+ "type": "array",
+ "description": "Logging Task Handlers class names",
+ "items": {
+ "type": "string"
+ }
+ },
+ "auth-backends": {
+ "type": "array",
+ "description": "API Auth Backend module names",
+ "items": {
+ "type": "string"
+ }
+ },
+ "notifications": {
+ "type": "array",
+ "description": "Notification class names",
+ "items": {
+ "type": "string"
+ }
+ },
+ "executors": {
+ "type": "array",
+ "description": "Executor class names",
+ "items": {
+ "type": "string"
+ }
+ },
+ "config": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "options": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/option"
+ }
+ },
+ "renamed": {
+ "type": "object",
+ "properties": {
+ "previous_name": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "required": [
+ "description",
+ "options"
+ ],
+ "additionalProperties": false
+ }
+ },
+ "task-decorators": {
+ "type": "array",
+ "description": "Apply custom decorators to the TaskFlow API. Can
be accessed by users via '@task.<name>'",
+ "items": {
+ "name": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ }
+ }
}
- }
- },
- "required": [
- "name",
- "description"
- ]
+ },
+ "definitions": {
+ "option": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "version_added": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "string",
+ "boolean",
+ "integer",
+ "float"
+ ]
+ },
+ "example": {
+ "type": [
+ "string",
+ "null",
+ "number"
+ ]
+ },
+ "default": {
+ "type": [
+ "string",
+ "null",
+ "number"
+ ]
+ },
+ "sensitive": {
+ "type": "boolean",
+ "description": "When true, this option is sensitive and
can be specified using AIRFLOW__{section}___{name}__SECRET or
AIRFLOW__{section}___{name}_CMD environment variables. See:
airflow.configuration.AirflowConfigParser.sensitive_config_values"
+ }
+ },
+ "required": [
+ "description",
+ "version_added",
+ "type",
+ "example",
+ "default"
+ ],
+ "additional_properties": false
+ }
+ },
+ "required": [
+ "name",
+ "description"
+ ]
}
diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py
index abd14d27b5..4f2f063599 100644
--- a/airflow/providers_manager.py
+++ b/airflow/providers_manager.py
@@ -130,12 +130,23 @@ class LazyDictWithCache(MutableMapping):
return key in self._raw_dict
+def _read_schema_from_resources_or_local_file(filename: str) -> dict:
+ try:
+ with resource_files("airflow").joinpath(filename).open("rb") as f:
+ schema = json.load(f)
+ except FileNotFoundError:
+ import pathlib
+
+ with (pathlib.Path(__file__).parent / filename).open("rb") as f:
+ schema = json.load(f)
+ return schema
+
+
def _create_provider_info_schema_validator():
"""Creates JSON schema validator from the provider_info.schema.json."""
import jsonschema
- with
resource_files("airflow").joinpath("provider_info.schema.json").open("rb") as f:
- schema = json.load(f)
+ schema =
_read_schema_from_resources_or_local_file("provider_info.schema.json")
cls = jsonschema.validators.validator_for(schema)
validator = cls(schema)
return validator
@@ -145,8 +156,7 @@ def
_create_customized_form_field_behaviours_schema_validator():
"""Creates JSON schema validator from the
customized_form_field_behaviours.schema.json."""
import jsonschema
- with
resource_files("airflow").joinpath("customized_form_field_behaviours.schema.json").open("rb")
as f:
- schema = json.load(f)
+ schema =
_read_schema_from_resources_or_local_file("customized_form_field_behaviours.schema.json")
cls = jsonschema.validators.validator_for(schema)
validator = cls(schema)
return validator
@@ -390,6 +400,7 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
self._logging_class_name_set: set[str] = set()
self._secrets_backend_class_name_set: set[str] = set()
self._executor_class_name_set: set[str] = set()
+ self._provider_configs: dict[str, dict[str, Any]] = {}
self._api_auth_backend_module_names: set[str] = set()
self._trigger_info_set: set[TriggerInfo] = set()
self._provider_schema_validator =
_create_provider_info_schema_validator()
@@ -462,6 +473,16 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
self.initialize_providers_list()
self._discover_executors()
+ @provider_info_cache("config")
+ def initialize_providers_configuration(self):
+ """Lazy initialization of providers configuration information."""
+ self.initialize_providers_list()
+ self._discover_config()
+ # Now update conf with the new provider configuration from providers
+ from airflow.configuration import conf
+
+ conf.load_provider_configuration()
+
@provider_info_cache("auth_backends")
def initialize_providers_auth_backends(self):
"""Lazy initialization of providers API auth_backends information."""
@@ -554,7 +575,6 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
with open(path) as provider_yaml_file:
provider_info = yaml.safe_load(provider_yaml_file)
self._provider_schema_validator.validate(provider_info)
-
version = provider_info["versions"][0]
if package_name not in self._provider_dict:
self._provider_dict[package_name] = ProviderInfo(version,
provider_info, "source")
@@ -965,6 +985,12 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
if _correctness_check(provider_package,
executors_class_name, provider):
self._executor_class_name_set.add(executors_class_name)
+ def _discover_config(self) -> None:
+ """Retrieve all configs defined in the providers."""
+ for provider_package, provider in self._provider_dict.items():
+ if provider.data.get("config"):
+ self._provider_configs[provider_package] =
provider.data.get("config")
+
@provider_info_cache("triggers")
def initialize_providers_triggers(self):
"""Initialization of providers triggers."""
@@ -1053,3 +1079,12 @@ class ProvidersManager(LoggingMixin,
metaclass=Singleton):
def executor_class_names(self) -> list[str]:
self.initialize_providers_executors()
return sorted(self._executor_class_name_set)
+
+ @property
+ def provider_configs(self) -> list[tuple[str, dict[str, Any]]]:
+ self.initialize_providers_configuration()
+ return sorted(self._provider_configs.items(), key=lambda x: x[0])
+
+ @property
+ def already_initialized_provider_configs(self) -> list[tuple[str,
dict[str, Any]]]:
+ return sorted(self._provider_configs.items(), key=lambda x: x[0])
