This is an automated email from the ASF dual-hosted git repository.
pankaj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 078e7ed135 Mysql change xcom value col type for MySQL backend (#38401)
078e7ed135 is described below
commit 078e7ed135cb580bca9ec0be7322a5261446470b
Author: Pankaj Singh <98807258+pankajas...@users.noreply.github.com>
AuthorDate: Sat Mar 23 18:49:43 2024 +0530
Mysql change xcom value col type for MySQL backend (#38401)
* Change table Xcom value column value type to longblob from blob
Xcom value we can store Airflow metadata is a bit smaller
when we use MySQL as a database backend since by default,
the Sqlalchemy map LargeBinary to MySQL blob type
which can store up to 65,535 bytes. In this PR,
I'm proposing to change the value column type to
longblob for xcom table if a user using MySQL database backend
https://dev.mysql.com/doc/refman/8.0/en/string-type-syntax.html
---
..._2_9_0_make_xcom_value_to_longblob_for_mysql.py | 52 ++++++++++++++++++++++
airflow/models/xcom.py | 3 +-
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/img/airflow_erd.svg | 8 ++--
docs/apache-airflow/migrations-ref.rst | 4 +-
docs/spelling_wordlist.txt | 1 +
newsfragments/38401.significant.rst | 5 +++
7 files changed, 68 insertions(+), 7 deletions(-)
diff --git
a/airflow/migrations/versions/0138_2_9_0_make_xcom_value_to_longblob_for_mysql.py
b/airflow/migrations/versions/0138_2_9_0_make_xcom_value_to_longblob_for_mysql.py
new file mode 100644
index 0000000000..b1cfa451c1
--- /dev/null
+++
b/airflow/migrations/versions/0138_2_9_0_make_xcom_value_to_longblob_for_mysql.py
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Change value column type to longblob in xcom table for mysql
+
+Revision ID: b4078ac230a1
+Revises: 8e1c784a4fc7
+Create Date: 2024-03-22 14:06:51.185268
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects.mysql import LONGBLOB
+
+# revision identifiers, used by Alembic.
+revision = 'b4078ac230a1'
+down_revision = '8e1c784a4fc7'
+branch_labels = None
+depends_on = None
+airflow_version = '2.9.0'
+
+
+def upgrade():
+ """Apply Change value column type to longblob in xcom table for mysql"""
+ conn = op.get_bind()
+ if conn.dialect.name == "mysql":
+ with op.batch_alter_table("xcom", schema=None) as batch_op:
+ batch_op.alter_column("value",
type_=sa.LargeBinary().with_variant(LONGBLOB, "mysql"))
+
+
+def downgrade():
+ """Unapply Change value column type to longblob in xcom table for mysql"""
+ conn = op.get_bind()
+ if conn.dialect.name == "mysql":
+ with op.batch_alter_table("xcom", schema=None) as batch_op:
+ batch_op.alter_column("value", type_=sa.LargeBinary)
diff --git a/airflow/models/xcom.py b/airflow/models/xcom.py
index f36f3ba3f7..be55837770 100644
--- a/airflow/models/xcom.py
+++ b/airflow/models/xcom.py
@@ -41,6 +41,7 @@ from sqlalchemy import (
delete,
text,
)
+from sqlalchemy.dialects.mysql import LONGBLOB
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.orm import Query, reconstructor, relationship
from sqlalchemy.orm.exc import NoResultFound
@@ -89,7 +90,7 @@ class BaseXCom(TaskInstanceDependencies, LoggingMixin):
dag_id = Column(String(ID_LEN, **COLLATION_ARGS), nullable=False)
run_id = Column(String(ID_LEN, **COLLATION_ARGS), nullable=False)
- value = Column(LargeBinary)
+ value = Column(LargeBinary().with_variant(LONGBLOB, "mysql"))
timestamp = Column(UtcDateTime, default=timezone.utcnow, nullable=False)
__table_args__ = (
diff --git a/docs/apache-airflow/img/airflow_erd.sha256
b/docs/apache-airflow/img/airflow_erd.sha256
index 7ba41e14de..3aea124fd8 100644
--- a/docs/apache-airflow/img/airflow_erd.sha256
+++ b/docs/apache-airflow/img/airflow_erd.sha256
@@ -1 +1 @@
-77ff6ecf7296a99e3b2721011605e6151001ca6c8a6c2058b2035daae9f23426
\ No newline at end of file
+1f9b9955efc927a319bb8c79df50f0f23a59e19b4f8379f95af346c19428c444
\ No newline at end of file
diff --git a/docs/apache-airflow/img/airflow_erd.svg
b/docs/apache-airflow/img/airflow_erd.svg
index d8d658028d..8ae1b83685 100644
--- a/docs/apache-airflow/img/airflow_erd.svg
+++ b/docs/apache-airflow/img/airflow_erd.svg
@@ -1369,14 +1369,14 @@
<g id="edge43" class="edge">
<title>task_instance--xcom</title>
<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M1198.1,-829.9C1228.72,-825.88 1260.55,-821.46 1290.36,-817.08"/>
-<text text-anchor="start" x="1280.36" y="-820.88" font-family="Times,serif"
font-size="14.00">1</text>
+<text text-anchor="start" x="1259.36" y="-820.88" font-family="Times,serif"
font-size="14.00">0..N</text>
<text text-anchor="start" x="1198.1" y="-833.7" font-family="Times,serif"
font-size="14.00">1</text>
</g>
<!-- task_instance--xcom -->
<g id="edge44" class="edge">
<title>task_instance--xcom</title>
<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M1198.1,-843.21C1228.72,-839.53 1260.55,-834.99 1290.36,-830.03"/>
-<text text-anchor="start" x="1259.36" y="-833.83" font-family="Times,serif"
font-size="14.00">0..N</text>
+<text text-anchor="start" x="1280.36" y="-833.83" font-family="Times,serif"
font-size="14.00">1</text>
<text text-anchor="start" x="1198.1" y="-847.01" font-family="Times,serif"
font-size="14.00">1</text>
</g>
<!-- rendered_task_instance_fields -->
@@ -1605,8 +1605,8 @@
<text text-anchor="start" x="697" y="-817.8"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
<polygon fill="none" stroke="black" points="506,-783 506,-808 789,-808
789,-783 506,-783"/>
<text text-anchor="start" x="511" y="-792.8"
font-family="Helvetica,sans-Serif" font-size="14.00">kwargs</text>
-<text text-anchor="start" x="561" y="-792.8"
font-family="Helvetica,sans-Serif" font-size="14.00"> [JSON]</text>
-<text text-anchor="start" x="612" y="-792.8"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
+<text text-anchor="start" x="561" y="-792.8"
font-family="Helvetica,sans-Serif" font-size="14.00"> [TEXT]</text>
+<text text-anchor="start" x="611" y="-792.8"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
<polygon fill="none" stroke="black" points="506,-758 506,-783 789,-783
789,-758 506,-758"/>
<text text-anchor="start" x="511" y="-767.8"
font-family="Helvetica,sans-Serif" font-size="14.00">triggerer_id</text>
<text text-anchor="start" x="592" y="-767.8"
font-family="Helvetica,sans-Serif" font-size="14.00"> [INTEGER]</text>
diff --git a/docs/apache-airflow/migrations-ref.rst
b/docs/apache-airflow/migrations-ref.rst
index cb72790575..63096ce8b7 100644
--- a/docs/apache-airflow/migrations-ref.rst
+++ b/docs/apache-airflow/migrations-ref.rst
@@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are
executed via when you ru
+---------------------------------+-------------------+-------------------+--------------------------------------------------------------+
| Revision ID | Revises ID | Airflow Version |
Description |
+=================================+===================+===================+==============================================================+
-| ``8e1c784a4fc7`` (head) | ``ab34f260b71c`` | ``2.9.0`` |
Adding max_consecutive_failed_dag_runs column to dag_model |
+| ``b4078ac230a1`` (head) | ``8e1c784a4fc7`` | ``2.9.0`` |
Change value column type to longblob in xcom table for mysql |
++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+
+| ``8e1c784a4fc7`` | ``ab34f260b71c`` | ``2.9.0`` |
Adding max_consecutive_failed_dag_runs column to dag_model |
| | | |
table |
+---------------------------------+-------------------+-------------------+--------------------------------------------------------------+
| ``ab34f260b71c`` | ``d75389605139`` | ``2.9.0`` |
add dataset_expression in DagModel |
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index f985451620..ebff77ea37 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -961,6 +961,7 @@ logformat
loglevel
Logstash
logstash
+longblob
Lowin
lshift
lxml
diff --git a/newsfragments/38401.significant.rst
b/newsfragments/38401.significant.rst
new file mode 100644
index 0000000000..7b57207eaf
--- /dev/null
+++ b/newsfragments/38401.significant.rst
@@ -0,0 +1,5 @@
+Change xcom table column value type to longblob for MySQL backend.
+
+Xcom table column ``value`` type has changed from ``blob`` to ``longblob``.
This will allow you to store relatively big data in Xcom but process can take a
significant amount of time if you have a lot of large data stored in Xcom.
+
+To downgrade from revision: ``b4078ac230a1``, ensure that you don't have Xcom
values larger than 65,535 bytes. Otherwise, you'll need to clean those rows or
run ``airflow db clean xcom`` to clean the Xcom table.
