This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c67884bd3d Re-enable cassandra for Python 3.12 after the driver got
libev support (#38314)
c67884bd3d is described below
commit c67884bd3da2ac39d7b6c059d399005707cc06ff
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Mar 20 09:18:26 2024 +0100
Re-enable cassandra for Python 3.12 after the driver got libev support
(#38314)
Cassandra has been disabled for Python 3.12 because the binary
driver from cassandra has been compiled without libev support and
it failed to run on Python 3.12 (released in cassandra-driver
3.29.1 today as result of the issue we raised:
https://datastax-oss.atlassian.net/browse/PYTHON-1378
This PR add libev to our images and re-enables Python 3.12 support
for cassandra.
---
Dockerfile | 4 ++--
Dockerfile.ci | 4 ++--
airflow/providers/apache/cassandra/provider.yaml | 22 +---------------------
docs/docker-stack/changelog.rst | 4 ++++
generated/provider_dependencies.json | 6 ++----
pyproject.toml | 2 +-
scripts/docker/install_os_dependencies.sh | 4 ++--
7 files changed, 14 insertions(+), 32 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index bd6f5a56cc..9e207938b8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -106,7 +106,7 @@ fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
DEV_APT_DEPS="apt-transport-https apt-utils build-essential
ca-certificates dirmngr \
-freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils
libffi-dev libgeos-dev \
+freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev"
@@ -133,7 +133,7 @@ function get_runtime_apt_deps() {
echo
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \
-curl dumb-init freetds-bin krb5-user libgeos-dev \
+curl dumb-init freetds-bin krb5-user libev4 libgeos-dev \
ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc"
export RUNTIME_APT_DEPS
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 8c1227795b..992d3c26ed 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -53,7 +53,7 @@ fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
DEV_APT_DEPS="apt-transport-https apt-utils build-essential
ca-certificates dirmngr \
-freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils
libffi-dev libgeos-dev \
+freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev"
@@ -80,7 +80,7 @@ function get_runtime_apt_deps() {
echo
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \
-curl dumb-init freetds-bin krb5-user libgeos-dev \
+curl dumb-init freetds-bin krb5-user libev4 libgeos-dev \
ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc"
export RUNTIME_APT_DEPS
diff --git a/airflow/providers/apache/cassandra/provider.yaml
b/airflow/providers/apache/cassandra/provider.yaml
index a1b09ef22c..f061619e0b 100644
--- a/airflow/providers/apache/cassandra/provider.yaml
+++ b/airflow/providers/apache/cassandra/provider.yaml
@@ -44,27 +44,7 @@ versions:
dependencies:
- apache-airflow>=2.6.0
- - cassandra-driver>=3.13.0
-
-# Cassandra provider is not yet compatible with Python 3.12
-# The main issue is that python cassandra driver by default uses asyncore
which has been deprecated since
-# Python 3.6 and removed in Python 3.12
(https://docs.python.org/3.11/library/asyncore.html)
-#
-# Currently the "wheel" package 3.29.0 distributed for manylinux platform is
build without libev support (
-# which could be a viable asyncore replacement), and cassandra driver works in
Python 3.12 if it is built
-# with libev support (using sdist, having gcc, libev4 and libev-dev
installed). But it would be too much to
-# expect our users to build the driver from sources to use it with Python
3.12, so we are waiting for the
-# next release of cassandra-driver which will have libev support in the wheel
package.
-# The issue is tracked here
https://datastax-oss.atlassian.net/browse/PYTHON-1378
-#
-# Another option to get cassandra drive back is to have asyncio support in the
driver instead of asyncore:
-# The issue is tracked here:
https://datastax-oss.atlassian.net/browse/PYTHON-1375 and is scheduled
-# to be fixed in cassandra-driver 3.30.0.
-#
-# All Cassandra tests are automatically skipped if cassandra package is not
present, so once you remove the
-# exclusion, they will start running for Python 3.12.
-#
-excluded-python-versions: ['3.12']
+ - cassandra-driver>=3.29.1
integrations:
- integration-name: Apache Cassandra
diff --git a/docs/docker-stack/changelog.rst b/docs/docker-stack/changelog.rst
index 192fbd0f81..b632f6b8d1 100644
--- a/docs/docker-stack/changelog.rst
+++ b/docs/docker-stack/changelog.rst
@@ -61,6 +61,10 @@ Airflow 2.9
* Constraints used to install the image are available in
"${HOME}/constraints.txt" now - you can use them
to install additional packages in the image without having to find out
which constraints you should use.
+ * The image adds ``libev`` library to the image as it is required by
cassandra driver for Python 3.12, also
+ ``libev`` will be used in other Python versions as a more robust and
faster way for cassandra driver
+ to handle events.
+
Airflow 2.8
~~~~~~~~~~~
* 2.8.3
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index e29dea0a78..785fb817c9 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -86,13 +86,11 @@
"apache.cassandra": {
"deps": [
"apache-airflow>=2.6.0",
- "cassandra-driver>=3.13.0"
+ "cassandra-driver>=3.29.1"
],
"devel-deps": [],
"cross-providers-deps": [],
- "excluded-python-versions": [
- "3.12"
- ],
+ "excluded-python-versions": [],
"state": "ready"
},
"apache.drill": {
diff --git a/pyproject.toml b/pyproject.toml
index d15e9f2a79..b05e2edc5c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -581,7 +581,7 @@ apache-beam = [ # source:
airflow/providers/apache/beam/provider.yaml
"pyarrow>=14.0.1;python_version != \"3.12\"",
]
apache-cassandra = [ # source: airflow/providers/apache/cassandra/provider.yaml
- "cassandra-driver>=3.13.0;python_version != \"3.12\"",
+ "cassandra-driver>=3.29.1",
]
apache-drill = [ # source: airflow/providers/apache/drill/provider.yaml
"apache-airflow[common_sql]",
diff --git a/scripts/docker/install_os_dependencies.sh
b/scripts/docker/install_os_dependencies.sh
index 283895f1da..3bb7773352 100644
--- a/scripts/docker/install_os_dependencies.sh
+++ b/scripts/docker/install_os_dependencies.sh
@@ -35,7 +35,7 @@ fi
function get_dev_apt_deps() {
if [[ "${DEV_APT_DEPS=}" == "" ]]; then
DEV_APT_DEPS="apt-transport-https apt-utils build-essential
ca-certificates dirmngr \
-freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils
libffi-dev libgeos-dev \
+freetds-bin freetds-dev git graphviz graphviz-dev krb5-user ldap-utils libev4
libev-dev libffi-dev libgeos-dev \
libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev
libsasl2-modules \
libssl-dev libxmlsec1 libxmlsec1-dev locales lsb-release openssh-client
pkgconf sasl2-bin \
software-properties-common sqlite3 sudo unixodbc unixodbc-dev zlib1g-dev"
@@ -62,7 +62,7 @@ function get_runtime_apt_deps() {
echo
if [[ "${RUNTIME_APT_DEPS=}" == "" ]]; then
RUNTIME_APT_DEPS="apt-transport-https apt-utils ca-certificates \
-curl dumb-init freetds-bin krb5-user libgeos-dev \
+curl dumb-init freetds-bin krb5-user libev4 libgeos-dev \
ldap-utils libsasl2-2 libsasl2-modules libxmlsec1 locales
${debian_version_apt_deps} \
lsb-release openssh-client python3-selinux rsync sasl2-bin sqlite3 sudo
unixodbc"
export RUNTIME_APT_DEPS
