This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 9eb92e7 Support installing providers with no dependencies via extras
(#12497)
9eb92e7 is described below
commit 9eb92e734326fe60c1213cd0b8d18fb31328d84f
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Sun Nov 22 00:19:08 2020 -0700
Support installing providers with no dependencies via extras (#12497)
---
CONTRIBUTING.rst | 15 ++++-----
INSTALL | 15 ++++-----
docs/extra-packages-ref.rst | 36 +++++++++++++++++++---
docs/spelling_wordlist.txt | 1 +
.../pre_commit_check_setup_extra_packages_ref.py | 5 ++-
setup.py | 30 +++++++++++++++++-
6 files changed, 82 insertions(+), 20 deletions(-)
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 92d7d3a..3964789 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -541,14 +541,15 @@ This is the full list of those extras:
.. START EXTRAS HERE
all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid,
apache.hdfs,
-apache.hive, apache.kylin, apache.pinot, apache.spark, apache.webhdfs, async,
atlas, aws, azure,
-cassandra, celery, cgroups, cloudant, cncf.kubernetes, dask, databricks,
datadog, devel,
-devel_hadoop, doc, docker, druid, elasticsearch, exasol, facebook, gcp,
gcp_api, github_enterprise,
-google, google_auth, grpc, hashicorp, hdfs, hive, jdbc, jira, kerberos,
kubernetes, ldap,
-microsoft.azure, microsoft.mssql, microsoft.winrm, mongo, mssql, mysql, odbc,
oracle, pagerduty,
+apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot,
apache.spark, apache.sqoop,
+apache.webhdfs, async, atlas, aws, azure, cassandra, celery, cgroups,
cloudant, cncf.kubernetes,
+dask, databricks, datadog, devel, devel_hadoop, dingding, discord, doc,
docker, druid,
+elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github_enterprise, google,
google_auth, grpc,
+hashicorp, hdfs, hive, http, imap, jdbc, jenkins, jira, kerberos, kubernetes,
ldap, microsoft.azure,
+microsoft.mssql, microsoft.winrm, mongo, mssql, mysql, odbc, openfaas,
opsgenie, oracle, pagerduty,
papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq,
redis, salesforce,
-samba, segment, sendgrid, sentry, singularity, slack, snowflake, spark, ssh,
statsd, tableau,
-vertica, virtualenv, webhdfs, winrm, yandex, yandexcloud, all, devel_ci
+samba, segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark,
sqlite, ssh, statsd,
+tableau, vertica, virtualenv, webhdfs, winrm, yandex, yandexcloud, zendesk,
all, devel_ci
.. END EXTRAS HERE
diff --git a/INSTALL b/INSTALL
index 4fb8c6c..c22d050 100644
--- a/INSTALL
+++ b/INSTALL
@@ -64,14 +64,15 @@ pip install -e . \
# START EXTRAS HERE
all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.druid,
apache.hdfs,
-apache.hive, apache.kylin, apache.pinot, apache.spark, apache.webhdfs, async,
atlas, aws, azure,
-cassandra, celery, cgroups, cloudant, cncf.kubernetes, dask, databricks,
datadog, devel,
-devel_hadoop, doc, docker, druid, elasticsearch, exasol, facebook, gcp,
gcp_api, github_enterprise,
-google, google_auth, grpc, hashicorp, hdfs, hive, jdbc, jira, kerberos,
kubernetes, ldap,
-microsoft.azure, microsoft.mssql, microsoft.winrm, mongo, mssql, mysql, odbc,
oracle, pagerduty,
+apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot,
apache.spark, apache.sqoop,
+apache.webhdfs, async, atlas, aws, azure, cassandra, celery, cgroups,
cloudant, cncf.kubernetes,
+dask, databricks, datadog, devel, devel_hadoop, dingding, discord, doc,
docker, druid,
+elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github_enterprise, google,
google_auth, grpc,
+hashicorp, hdfs, hive, http, imap, jdbc, jenkins, jira, kerberos, kubernetes,
ldap, microsoft.azure,
+microsoft.mssql, microsoft.winrm, mongo, mssql, mysql, odbc, openfaas,
opsgenie, oracle, pagerduty,
papermill, password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq,
redis, salesforce,
-samba, segment, sendgrid, sentry, singularity, slack, snowflake, spark, ssh,
statsd, tableau,
-vertica, virtualenv, webhdfs, winrm, yandex, yandexcloud, all, devel_ci
+samba, segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark,
sqlite, ssh, statsd,
+tableau, vertica, virtualenv, webhdfs, winrm, yandex, yandexcloud, zendesk,
all, devel_ci
# END EXTRAS HERE
diff --git a/docs/extra-packages-ref.rst b/docs/extra-packages-ref.rst
index ffd02de..5f500bf 100644
--- a/docs/extra-packages-ref.rst
+++ b/docs/extra-packages-ref.rst
@@ -59,9 +59,17 @@ Here's the list of the :ref:`subpackages
<installation:extra_packages>` and what
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| kylin | ``pip install 'apache-airflow[apache.kylin]'`` |
All Kylin related operators & hooks |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| livy | ``pip install 'apache-airflow[apache.livy]'`` |
All Livy related operators, hooks & sensors |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| pig | ``pip install 'apache-airflow[apache.pig]'`` |
All Pig related operators & hooks |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| pinot | ``pip install 'apache-airflow[apache.pinot]'`` |
All Pinot related hooks |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| spark | ``pip install 'apache-airflow[apache.spark]'`` |
All Spark related operators & hooks |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| webhdfs | ``pip install 'apache-airflow[webhdfs]'`` |
HDFS hooks and operators |
+| sqoop | ``pip install 'apache-airflow[apache.sqoop]'`` |
All Sqoop related operators & hooks |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| webhdfs | ``pip install 'apache-airflow[apache.webhdfs]'`` |
HDFS hooks and operators |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
@@ -80,6 +88,10 @@ Here's the list of the :ref:`subpackages
<installation:extra_packages>` and what
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| datadog | ``pip install 'apache-airflow[datadog]'`` |
Datadog hooks and sensors |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| dingding | ``pip install 'apache-airflow[dingding]'`` |
Dingding hooks and sensors |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| discord | ``pip install 'apache-airflow[discord]'`` |
Discord hooks and sensors |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| facebook | ``pip install 'apache-airflow[facebook]'`` |
Facebook Social |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| gcp | ``pip install 'apache-airflow[google]'`` |
Google Cloud |
@@ -92,6 +104,8 @@ Here's the list of the :ref:`subpackages
<installation:extra_packages>` and what
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| jira | ``pip install 'apache-airflow[jira]'`` |
Jira hooks and operators |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| opsgenie | ``pip install 'apache-airflow[opsgenie]'`` |
OpsGenie hooks and operators |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| pagerduty | ``pip install 'apache-airflow[pagerduty]'`` |
Pagerduty hook |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| plexus | ``pip install 'apache-airflow[plexus]'`` |
Plexus service of CoreScientific.com AI platform |
@@ -114,6 +128,8 @@ Here's the list of the :ref:`subpackages
<installation:extra_packages>` and what
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| yandexcloud | ``pip install 'apache-airflow[yandexcloud]'`` |
Yandex.Cloud hooks and operators |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| zendesk | ``pip install 'apache-airflow[zendesk]'`` |
Zendesk hooks |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
**Software:**
@@ -133,6 +149,8 @@ Here's the list of the :ref:`subpackages
<installation:extra_packages>` and what
+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
| exasol | ``pip install 'apache-airflow[exasol]'`` |
Exasol hooks and operators
|
+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
+| jenkins | ``pip install 'apache-airflow[jenkins]'`` |
Jenkins hooks and operators
|
++---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
| kubernetes | ``pip install 'apache-airflow[cncf.kubernetes]'`` |
Kubernetes Executor and operator
|
+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
| mongo | ``pip install 'apache-airflow[mongo]'`` |
Mongo hooks and operators
|
@@ -152,9 +170,9 @@ Here's the list of the :ref:`subpackages
<installation:extra_packages>` and what
| | |
or as metastore database backend. Uses pyodbc.
|
| | |
See :ref:`howto/connection/odbc` for more info.
|
+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| oracle | ``pip install 'apache-airflow[oracle]'`` |
Oracle hooks and operators
|
+| openfaas | ``pip install 'apache-airflow[openfaas]'`` |
OpenFaaS hooks
|
+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
-| pinot | ``pip install 'apache-airflow[pinot]'`` |
Pinot DB hook
|
+| oracle | ``pip install 'apache-airflow[oracle]'`` |
Oracle hooks and operators
|
+---------------------+-----------------------------------------------------+------------------------------------------------------------------------------------+
| postgres | ``pip install 'apache-airflow[postgres]'`` |
PostgreSQL operators and hook, support as an
|
| | |
Airflow backend
|
@@ -184,8 +202,14 @@ Here's the list of the :ref:`subpackages
<installation:extra_packages>` and what
+=====================+=====================================================+======================================================================+
| cgroups | ``pip install 'apache-airflow[cgroups]'`` |
Needed To use CgroupTaskRunner |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| ftp | ``pip install 'apache-airflow[ftp]'`` |
FTP hooks and operators |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| grpc | ``pip install 'apache-airflow[grpc]'`` |
Grpc hooks and operators |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| http | ``pip install 'apache-airflow[http]'`` |
HTTP hooks, operators and sensors |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| imap | ``pip install 'apache-airflow[imap]'`` |
IMAP hooks and sensors |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| jdbc | ``pip install 'apache-airflow[jdbc]'`` |
JDBC hooks and operators |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| kerberos | ``pip install 'apache-airflow[kerberos]'`` |
Kerberos integration for Kerberized Hadoop |
@@ -194,7 +218,11 @@ Here's the list of the :ref:`subpackages
<installation:extra_packages>` and what
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| papermill | ``pip install 'apache-airflow[papermill]'`` |
Papermill hooks and operators |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
-| ssh | ``pip install 'apache-airflow[ssh]'`` |
SSH hooks and Operator |
+| sftp | ``pip install 'apache-airflow[sftp]'`` |
SFTP hooks, operators and sensors |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| sqlite | ``pip install 'apache-airflow[sqlite]'`` |
SQLite hooks and operators |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
+| ssh | ``pip install 'apache-airflow[ssh]'`` |
SSH hooks and operators |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
| winrm | ``pip install 'apache-airflow[microsoft.winrm]'`` |
WinRM hooks and operators |
+---------------------+-----------------------------------------------------+----------------------------------------------------------------------+
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index f3a2c7d..c4fadc8 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -997,6 +997,7 @@ onboarded
onkill
openfaas
oper
+opsgenie
optimise
ora
orm
diff --git a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
index b26e833..d8d9bf2 100755
--- a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
+++ b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
@@ -53,13 +53,16 @@ def get_extras_from_setup() -> Dict[str, List[str]]:
extras_section = extras_section_regex.findall(setup_content)[0]
extras_regex = re.compile(
-
rf'^\s+[\"\']({PY_IDENTIFIER})[\"\']:\s*({PY_IDENTIFIER})[^#\n]*(#\s*TODO.*)?$',
re.MULTILINE
+
rf'^\s+[\"\']({PY_IDENTIFIER})[\"\']:\s*({PY_IDENTIFIER}|\[\])[^#\n]*(#\s*TODO.*)?$',
re.MULTILINE
)
extras_dict: Dict[str, List[str]] = {}
for extras in extras_regex.findall(extras_section):
package = extras[1]
alias = extras[0]
+ # if there are no packages, use the extras alias itself
+ if package == '[]':
+ package = alias
if not extras_dict.get(package):
extras_dict[package] = []
extras_dict[package].append(alias)
diff --git a/setup.py b/setup.py
index c2a3d8b..c3b74a8 100644
--- a/setup.py
+++ b/setup.py
@@ -513,7 +513,7 @@ PROVIDERS_REQUIREMENTS: Dict[str, Iterable[str]] = {
"dingding": [],
"discord": [],
"docker": docker,
- "elasticsearch": [],
+ "elasticsearch": elasticsearch,
"exasol": exasol,
"facebook": facebook,
"ftp": [],
@@ -566,8 +566,11 @@ EXTRAS_REQUIREMENTS: Dict[str, List[str]] = {
"apache.hdfs": hdfs,
"apache.hive": hive,
"apache.kylin": kylin,
+ "apache.livy": [],
+ "apache.pig": [],
"apache.pinot": pinot,
"apache.spark": spark,
+ "apache.sqoop": [],
"apache.webhdfs": webhdfs,
'async': async_packages,
'atlas': atlas, # TODO: remove this in Airflow 2.1
@@ -583,12 +586,15 @@ EXTRAS_REQUIREMENTS: Dict[str, List[str]] = {
'datadog': datadog,
'devel': devel_minreq,
'devel_hadoop': devel_hadoop,
+ 'dingding': [],
+ 'discord': [],
'doc': doc,
'docker': docker,
'druid': druid, # TODO: remove this in Airflow 2.1
'elasticsearch': elasticsearch,
'exasol': exasol,
'facebook': facebook,
+ 'ftp': [],
'gcp': google, # TODO: remove this in Airflow 2.1
'gcp_api': google, # TODO: remove this in Airflow 2.1
'github_enterprise': flask_oauth,
@@ -598,7 +604,10 @@ EXTRAS_REQUIREMENTS: Dict[str, List[str]] = {
'hashicorp': hashicorp,
'hdfs': hdfs, # TODO: remove this in Airflow 2.1
'hive': hive, # TODO: remove this in Airflow 2.1
+ 'http': [],
+ 'imap': [],
'jdbc': jdbc,
+ 'jenkins': [],
'jira': jira,
'kerberos': kerberos,
'kubernetes': kubernetes, # TODO: remove this in Airflow 2.1
@@ -610,6 +619,8 @@ EXTRAS_REQUIREMENTS: Dict[str, List[str]] = {
'mssql': mssql, # TODO: remove this in Airflow 2.1
'mysql': mysql,
'odbc': odbc,
+ 'openfaas': [],
+ 'opsgenie': [],
'oracle': oracle,
'pagerduty': pagerduty,
'papermill': papermill,
@@ -627,10 +638,12 @@ EXTRAS_REQUIREMENTS: Dict[str, List[str]] = {
'segment': segment,
'sendgrid': sendgrid,
'sentry': sentry,
+ 'sftp': [],
'singularity': singularity,
'slack': slack,
'snowflake': snowflake,
'spark': spark,
+ 'sqlite': [],
'ssh': ssh,
'statsd': statsd,
'tableau': tableau,
@@ -640,6 +653,7 @@ EXTRAS_REQUIREMENTS: Dict[str, List[str]] = {
'winrm': winrm, # TODO: remove this in Airflow 2.1
'yandex': yandexcloud, # TODO: remove this in Airflow 2.1
'yandexcloud': yandexcloud,
+ 'zendesk': [],
}
EXTRAS_PROVIDERS_PACKAGES: Dict[str, Iterable[str]] = {
@@ -670,8 +684,11 @@ EXTRAS_PROVIDERS_PACKAGES: Dict[str, Iterable[str]] = {
"apache.hdfs": ["apache.hdfs"],
"apache.hive": ["apache.hive"],
"apache.kylin": ["apache.kylin"],
+ "apache.livy": ["apache.livy"],
+ "apache.pig": ["apache.pig"],
"apache.pinot": ["apache.pinot"],
"apache.spark": ["apache.spark"],
+ "apache.sqoop": ["apache.sqoop"],
"apache.webhdfs": ["apache.hdfs"],
'async': [],
'atlas': [], # TODO: remove this in Airflow 2.1
@@ -687,12 +704,15 @@ EXTRAS_PROVIDERS_PACKAGES: Dict[str, Iterable[str]] = {
'datadog': ["datadog"],
'devel': ["cncf.kubernetes", "mysql"],
'devel_hadoop': ["apache.hdfs", "apache.hive", "presto"],
+ 'dingding': ["dingding"],
+ 'discord': ["discord"],
'doc': [],
'docker': ["docker"],
'druid': ["apache.druid"], # TODO: remove this in Airflow 2.1
'elasticsearch': ["elasticsearch"],
'exasol': ["exasol"],
'facebook': ["facebook"],
+ 'ftp': ["ftp"],
'gcp': ["google"], # TODO: remove this in Airflow 2.1
'gcp_api': ["google"], # TODO: remove this in Airflow 2.1
'github_enterprise': [],
@@ -702,7 +722,10 @@ EXTRAS_PROVIDERS_PACKAGES: Dict[str, Iterable[str]] = {
'hashicorp': ["hashicorp"],
'hdfs': ["apache.hdfs"], # TODO: remove this in Airflow 2.1
'hive': ["apache.hive"], # TODO: remove this in Airflow 2.1
+ 'http': ["http"],
+ 'imap': ["imap"],
'jdbc': ["jdbc"],
+ 'jenkins': ["jenkins"],
'jira': ["jira"],
'kerberos': [],
'kubernetes': ["cncf.kubernetes"], # TODO: remove this in Airflow 2.1
@@ -714,6 +737,8 @@ EXTRAS_PROVIDERS_PACKAGES: Dict[str, Iterable[str]] = {
'mssql': ["microsoft.mssql"], # TODO: remove this in Airflow 2.1
'mysql': ["mysql"],
'odbc': ["odbc"],
+ 'openfaas': ["openfaas"],
+ 'opsgenie': ["opsgenie"],
'oracle': ["oracle"],
'pagerduty': ["pagerduty"],
'papermill': ["papermill"],
@@ -731,10 +756,12 @@ EXTRAS_PROVIDERS_PACKAGES: Dict[str, Iterable[str]] = {
'segment': ["segment"],
'sendgrid': ["sendgrid"],
'sentry': [],
+ 'sftp': ["sftp"],
'singularity': ["singularity"],
'slack': ["slack"],
'snowflake': ["snowflake"],
'spark': ["apache.spark"],
+ 'sqlite': ["sqlite"],
'ssh': ["ssh"],
'statsd': [],
'tableau': [],
@@ -744,6 +771,7 @@ EXTRAS_PROVIDERS_PACKAGES: Dict[str, Iterable[str]] = {
'winrm': ["microsoft.winrm"], # TODO: remove this in Airflow 2.1
'yandexcloud': ["yandex"], # TODO: remove this in Airflow 2.1
'yandex': ["yandex"],
+ 'zendesk': ["zendesk"],
}
