This is an automated email from the ASF dual-hosted git repository.
dstandish pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new dd2e3d2a368 [v3-0-test] Skip empty dag run conf rows and set statement
timeout (#50788) (#53619)
dd2e3d2a368 is described below
commit dd2e3d2a368e2506dcaf4baf068425a12453f301
Author: Daniel Standish <[email protected]>
AuthorDate: Tue Jul 22 11:17:45 2025 -0700
[v3-0-test] Skip empty dag run conf rows and set statement timeout (#50788)
(#53619)
Don't bother depickling `{}` object in dag run conf
Also, set the statement timeout a little higher for xcom migration.
(cherry picked from commit f75b60740e59d47a223b350f506d9100e76c584d)
Co-authored-by: Jed Cunningham
<[email protected]>
---
airflow-core/docs/img/airflow_erd.sha256 | 2 +-
airflow-core/docs/img/airflow_erd.svg | 108 ++++++++++-----------
...3_0_0_add_uuid_primary_key_to_task_instance_.py | 3 +
...49_3_0_0_remove_pickled_data_from_xcom_table.py | 19 +++-
..._3_0_0_remove_pickled_data_from_dagrun_table.py | 37 +++++--
5 files changed, 106 insertions(+), 63 deletions(-)
diff --git a/airflow-core/docs/img/airflow_erd.sha256
b/airflow-core/docs/img/airflow_erd.sha256
index 2d2c19551b3..076b590dedd 100644
--- a/airflow-core/docs/img/airflow_erd.sha256
+++ b/airflow-core/docs/img/airflow_erd.sha256
@@ -1 +1 @@
-968bf5c974c4a9663b6be095837d255a2cc8e25ce80209904e672b36edd14148
\ No newline at end of file
+47bc0fb61cdb68be06d02c7f7d88bf39c0d454bd23e9ad924e218999c40fc121
\ No newline at end of file
diff --git a/airflow-core/docs/img/airflow_erd.svg
b/airflow-core/docs/img/airflow_erd.svg
index 8c66ac9da76..c69dfc2771d 100644
--- a/airflow-core/docs/img/airflow_erd.svg
+++ b/airflow-core/docs/img/airflow_erd.svg
@@ -1482,81 +1482,35 @@
<text text-anchor="start" x="302.2" y="-1915.3" font-family="Times,serif"
font-size="14.00">{0,1}</text>
</g>
<!-- dag--dag_schedule_asset_alias_reference -->
-<g id="edge19" class="edge">
+<g id="edge18" class="edge">
<title>dag--dag_schedule_asset_alias_reference</title>
<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M717.95,-2244.65C753.67,-2362.49 793.89,-2497.38 829,-2620.5 863.21,-2740.45
820.19,-2797.34 902,-2891.5 904.68,-2894.59 907.51,-2897.55 910.46,-2900.4"/>
<text text-anchor="start" x="879.46" y="-2889.2" font-family="Times,serif"
font-size="14.00">0..N</text>
<text text-anchor="start" x="707.95" y="-2248.45" font-family="Times,serif"
font-size="14.00">1</text>
</g>
<!-- dag--dag_schedule_asset_reference -->
-<g id="edge20" class="edge">
+<g id="edge19" class="edge">
<title>dag--dag_schedule_asset_reference</title>
<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M802.43,-2244.52C832.55,-2282.49 865.87,-2318.39 902,-2348.5 911.97,-2356.81
922.95,-2364.36 934.43,-2371.21"/>
<text text-anchor="start" x="903.43" y="-2360.01" font-family="Times,serif"
font-size="14.00">0..N</text>
<text text-anchor="start" x="792.43" y="-2248.32" font-family="Times,serif"
font-size="14.00">1</text>
</g>
<!-- dag--task_outlet_asset_reference -->
-<g id="edge21" class="edge">
+<g id="edge20" class="edge">
<title>dag--task_outlet_asset_reference</title>
<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M730.32,-2244.59C773.32,-2336.19 829.95,-2430.88 902,-2503.5 913.84,-2515.43
927.58,-2526.05 942.1,-2535.44"/>
<text text-anchor="start" x="911.1" y="-2524.24" font-family="Times,serif"
font-size="14.00">0..N</text>
<text text-anchor="start" x="720.32" y="-2248.39" font-family="Times,serif"
font-size="14.00">1</text>
</g>
<!-- dag--asset_dag_run_queue -->
-<g id="edge22" class="edge">
+<g id="edge21" class="edge">
<title>dag--asset_dag_run_queue</title>
<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M829.14,-2164.26C852.52,-2184.43 876.97,-2203.28 902,-2219.5 908.43,-2223.67
915.16,-2227.61 922.08,-2231.33"/>
<text text-anchor="start" x="891.08" y="-2220.13" font-family="Times,serif"
font-size="14.00">0..N</text>
<text text-anchor="start" x="829.14" y="-2153.06" font-family="Times,serif"
font-size="14.00">1</text>
</g>
-<!-- dag_version -->
-<g id="node25" class="node">
-<title>dag_version</title>
-<polygon fill="none" stroke="black" points="943.5,-2178.5 943.5,-2206.5
1224.5,-2206.5 1224.5,-2178.5 943.5,-2178.5"/>
-<text text-anchor="start" x="1029.5" y="-2189.7"
font-family="Helvetica,sans-Serif" font-weight="bold"
font-size="16.00">dag_version</text>
-<polygon fill="none" stroke="black" points="943.5,-2153.5 943.5,-2178.5
1224.5,-2178.5 1224.5,-2153.5 943.5,-2153.5"/>
-<text text-anchor="start" x="948.5" y="-2163.3"
font-family="Helvetica,sans-Serif" text-decoration="underline"
font-size="14.00">id</text>
-<text text-anchor="start" x="961.5" y="-2163.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
-<text text-anchor="start" x="966.5" y="-2163.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [UUID]</text>
-<text text-anchor="start" x="1018.5" y="-2163.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
-<polygon fill="none" stroke="black" points="943.5,-2128.5 943.5,-2153.5
1224.5,-2153.5 1224.5,-2128.5 943.5,-2128.5"/>
-<text text-anchor="start" x="948.5" y="-2138.3"
font-family="Helvetica,sans-Serif" font-size="14.00">bundle_name</text>
-<text text-anchor="start" x="1042.5" y="-2138.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
-<text text-anchor="start" x="1047.5" y="-2138.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [VARCHAR(250)]</text>
-<polygon fill="none" stroke="black" points="943.5,-2103.5 943.5,-2128.5
1224.5,-2128.5 1224.5,-2103.5 943.5,-2103.5"/>
-<text text-anchor="start" x="948.5" y="-2113.3"
font-family="Helvetica,sans-Serif" font-size="14.00">bundle_version</text>
-<text text-anchor="start" x="1054.5" y="-2113.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
-<text text-anchor="start" x="1059.5" y="-2113.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [VARCHAR(250)]</text>
-<polygon fill="none" stroke="black" points="943.5,-2078.5 943.5,-2103.5
1224.5,-2103.5 1224.5,-2078.5 943.5,-2078.5"/>
-<text text-anchor="start" x="948.5" y="-2088.3"
font-family="Helvetica,sans-Serif" font-size="14.00">created_at</text>
-<text text-anchor="start" x="1021.5" y="-2088.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
-<text text-anchor="start" x="1026.5" y="-2088.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [TIMESTAMP]</text>
-<text text-anchor="start" x="1122.5" y="-2088.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
-<polygon fill="none" stroke="black" points="943.5,-2053.5 943.5,-2078.5
1224.5,-2078.5 1224.5,-2053.5 943.5,-2053.5"/>
-<text text-anchor="start" x="948.5" y="-2063.3"
font-family="Helvetica,sans-Serif" font-size="14.00">dag_id</text>
-<text text-anchor="start" x="994.5" y="-2063.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
-<text text-anchor="start" x="999.5" y="-2063.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [VARCHAR(250)]</text>
-<text text-anchor="start" x="1120.5" y="-2063.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
-<polygon fill="none" stroke="black" points="943.5,-2028.5 943.5,-2053.5
1224.5,-2053.5 1224.5,-2028.5 943.5,-2028.5"/>
-<text text-anchor="start" x="948.5" y="-2038.3"
font-family="Helvetica,sans-Serif" font-size="14.00">last_updated</text>
-<text text-anchor="start" x="1038.5" y="-2038.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
-<text text-anchor="start" x="1043.5" y="-2038.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [TIMESTAMP]</text>
-<text text-anchor="start" x="1139.5" y="-2038.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
-<polygon fill="none" stroke="black" points="943.5,-2003.5 943.5,-2028.5
1224.5,-2028.5 1224.5,-2003.5 943.5,-2003.5"/>
-<text text-anchor="start" x="948.5" y="-2013.3"
font-family="Helvetica,sans-Serif" font-size="14.00">version_number</text>
-<text text-anchor="start" x="1061.5" y="-2013.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
-<text text-anchor="start" x="1066.5" y="-2013.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [INTEGER]</text>
-<text text-anchor="start" x="1143.5" y="-2013.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
-</g>
-<!-- dag--dag_version -->
-<g id="edge16" class="edge">
-<title>dag--dag_version</title>
-<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M829.26,-2007.49C864.9,-2021.28 901.24,-2035.35 934.86,-2048.36"/>
-<text text-anchor="start" x="903.86" y="-2037.16" font-family="Times,serif"
font-size="14.00">0..N</text>
-<text text-anchor="start" x="829.26" y="-1996.29" font-family="Times,serif"
font-size="14.00">1</text>
-</g>
<!-- dag_schedule_asset_name_reference -->
-<g id="node26" class="node">
+<g id="node25" class="node">
<title>dag_schedule_asset_name_reference</title>
<polygon fill="none" stroke="black" points="914.5,-1393.5 914.5,-1421.5
1253.5,-1421.5 1253.5,-1393.5 914.5,-1393.5"/>
<text text-anchor="start" x="919.5" y="-1404.7"
font-family="Helvetica,sans-Serif" font-weight="bold"
font-size="16.00">dag_schedule_asset_name_reference</text>
@@ -1577,14 +1531,14 @@
<text text-anchor="start" x="1093.5" y="-1328.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
</g>
<!-- dag--dag_schedule_asset_name_reference -->
-<g id="edge17" class="edge">
+<g id="edge16" class="edge">
<title>dag--dag_schedule_asset_name_reference</title>
<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M758.83,-1608.26C797.89,-1544.61 845.49,-1482.86 902,-1435.5 905.96,-1432.18
910.1,-1429.03 914.38,-1426.03"/>
<text text-anchor="start" x="883.38" y="-1429.83" font-family="Times,serif"
font-size="14.00">0..N</text>
<text text-anchor="start" x="748.83" y="-1597.06" font-family="Times,serif"
font-size="14.00">1</text>
</g>
<!-- dag_schedule_asset_uri_reference -->
-<g id="node27" class="node">
+<g id="node26" class="node">
<title>dag_schedule_asset_uri_reference</title>
<polygon fill="none" stroke="black" points="926.5,-1949.5 926.5,-1977.5
1241.5,-1977.5 1241.5,-1949.5 926.5,-1949.5"/>
<text text-anchor="start" x="931.5" y="-1960.7"
font-family="Helvetica,sans-Serif" font-weight="bold"
font-size="16.00">dag_schedule_asset_uri_reference</text>
@@ -1605,12 +1559,58 @@
<text text-anchor="start" x="1105.5" y="-1884.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
</g>
<!-- dag--dag_schedule_asset_uri_reference -->
-<g id="edge18" class="edge">
+<g id="edge17" class="edge">
<title>dag--dag_schedule_asset_uri_reference</title>
<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M829.26,-1926.5C858.92,-1926.5 889.07,-1926.5 917.73,-1926.5"/>
<text text-anchor="start" x="886.73" y="-1915.3" font-family="Times,serif"
font-size="14.00">0..N</text>
<text text-anchor="start" x="829.26" y="-1915.3" font-family="Times,serif"
font-size="14.00">1</text>
</g>
+<!-- dag_version -->
+<g id="node27" class="node">
+<title>dag_version</title>
+<polygon fill="none" stroke="black" points="943.5,-2178.5 943.5,-2206.5
1224.5,-2206.5 1224.5,-2178.5 943.5,-2178.5"/>
+<text text-anchor="start" x="1029.5" y="-2189.7"
font-family="Helvetica,sans-Serif" font-weight="bold"
font-size="16.00">dag_version</text>
+<polygon fill="none" stroke="black" points="943.5,-2153.5 943.5,-2178.5
1224.5,-2178.5 1224.5,-2153.5 943.5,-2153.5"/>
+<text text-anchor="start" x="948.5" y="-2163.3"
font-family="Helvetica,sans-Serif" text-decoration="underline"
font-size="14.00">id</text>
+<text text-anchor="start" x="961.5" y="-2163.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
+<text text-anchor="start" x="966.5" y="-2163.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [UUID]</text>
+<text text-anchor="start" x="1018.5" y="-2163.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
+<polygon fill="none" stroke="black" points="943.5,-2128.5 943.5,-2153.5
1224.5,-2153.5 1224.5,-2128.5 943.5,-2128.5"/>
+<text text-anchor="start" x="948.5" y="-2138.3"
font-family="Helvetica,sans-Serif" font-size="14.00">bundle_name</text>
+<text text-anchor="start" x="1042.5" y="-2138.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
+<text text-anchor="start" x="1047.5" y="-2138.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [VARCHAR(250)]</text>
+<polygon fill="none" stroke="black" points="943.5,-2103.5 943.5,-2128.5
1224.5,-2128.5 1224.5,-2103.5 943.5,-2103.5"/>
+<text text-anchor="start" x="948.5" y="-2113.3"
font-family="Helvetica,sans-Serif" font-size="14.00">bundle_version</text>
+<text text-anchor="start" x="1054.5" y="-2113.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
+<text text-anchor="start" x="1059.5" y="-2113.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [VARCHAR(250)]</text>
+<polygon fill="none" stroke="black" points="943.5,-2078.5 943.5,-2103.5
1224.5,-2103.5 1224.5,-2078.5 943.5,-2078.5"/>
+<text text-anchor="start" x="948.5" y="-2088.3"
font-family="Helvetica,sans-Serif" font-size="14.00">created_at</text>
+<text text-anchor="start" x="1021.5" y="-2088.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
+<text text-anchor="start" x="1026.5" y="-2088.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [TIMESTAMP]</text>
+<text text-anchor="start" x="1122.5" y="-2088.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
+<polygon fill="none" stroke="black" points="943.5,-2053.5 943.5,-2078.5
1224.5,-2078.5 1224.5,-2053.5 943.5,-2053.5"/>
+<text text-anchor="start" x="948.5" y="-2063.3"
font-family="Helvetica,sans-Serif" font-size="14.00">dag_id</text>
+<text text-anchor="start" x="994.5" y="-2063.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
+<text text-anchor="start" x="999.5" y="-2063.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [VARCHAR(250)]</text>
+<text text-anchor="start" x="1120.5" y="-2063.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
+<polygon fill="none" stroke="black" points="943.5,-2028.5 943.5,-2053.5
1224.5,-2053.5 1224.5,-2028.5 943.5,-2028.5"/>
+<text text-anchor="start" x="948.5" y="-2038.3"
font-family="Helvetica,sans-Serif" font-size="14.00">last_updated</text>
+<text text-anchor="start" x="1038.5" y="-2038.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
+<text text-anchor="start" x="1043.5" y="-2038.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [TIMESTAMP]</text>
+<text text-anchor="start" x="1139.5" y="-2038.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
+<polygon fill="none" stroke="black" points="943.5,-2003.5 943.5,-2028.5
1224.5,-2028.5 1224.5,-2003.5 943.5,-2003.5"/>
+<text text-anchor="start" x="948.5" y="-2013.3"
font-family="Helvetica,sans-Serif" font-size="14.00">version_number</text>
+<text text-anchor="start" x="1061.5" y="-2013.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> </text>
+<text text-anchor="start" x="1066.5" y="-2013.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> [INTEGER]</text>
+<text text-anchor="start" x="1143.5" y="-2013.3"
font-family="Helvetica,sans-Serif" font-size="14.00"> NOT NULL</text>
+</g>
+<!-- dag--dag_version -->
+<g id="edge22" class="edge">
+<title>dag--dag_version</title>
+<path fill="none" stroke="#7f7f7f" stroke-dasharray="5,2"
d="M829.26,-2007.49C864.9,-2021.28 901.24,-2035.35 934.86,-2048.36"/>
+<text text-anchor="start" x="903.86" y="-2037.16" font-family="Times,serif"
font-size="14.00">0..N</text>
+<text text-anchor="start" x="829.26" y="-1996.29" font-family="Times,serif"
font-size="14.00">1</text>
+</g>
<!-- dag_tag -->
<g id="node28" class="node">
<title>dag_tag</title>
diff --git
a/airflow-core/src/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py
b/airflow-core/src/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py
index ccd0697ff63..df7965d38e9 100644
---
a/airflow-core/src/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py
+++
b/airflow-core/src/airflow/migrations/versions/0042_3_0_0_add_uuid_primary_key_to_task_instance_.py
@@ -206,7 +206,10 @@ def upgrade():
op.execute(pg_uuid7_fn)
# Migrate existing rows with UUID v7 using a timestamp-based generation
+ batch_num = 0
while True:
+ batch_num += 1
+ print(f"processing batch {batch_num}")
result = conn.execute(
text(
"""
diff --git
a/airflow-core/src/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py
b/airflow-core/src/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py
index c3972edbd12..fed37837829 100644
---
a/airflow-core/src/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py
+++
b/airflow-core/src/airflow/migrations/versions/0049_3_0_0_remove_pickled_data_from_xcom_table.py
@@ -28,7 +28,7 @@ Create Date: 2024-11-18 18:41:50.849514
from __future__ import annotations
import sqlalchemy as sa
-from alembic import op
+from alembic import context, op
from sqlalchemy import text
from sqlalchemy.dialects.mysql import LONGBLOB
@@ -77,9 +77,24 @@ def upgrade():
condition = condition_templates.get(dialect)
if not condition:
raise RuntimeError(f"Unsupported dialect: {dialect}")
-
# Key is a reserved keyword in MySQL, so we need to quote it
quoted_key = conn.dialect.identifier_preparer.quote("key")
+ if dialect == "postgresql" and not context.is_offline_mode():
+ curr_timeout = (
+ int(
+ conn.execute(
+ text("""
+ SELECT setting
+ FROM pg_settings
+ WHERE name = 'statement_timeout'
+ """)
+ ).scalar_one()
+ )
+ / 1000
+ )
+ if curr_timeout > 0 and curr_timeout < 1800:
+ print("setting local statement timeout to 1800s")
+ conn.execute(text("SET LOCAL statement_timeout='1800s'"))
# Archive pickled data using the condition
conn.execute(
diff --git
a/airflow-core/src/airflow/migrations/versions/0055_3_0_0_remove_pickled_data_from_dagrun_table.py
b/airflow-core/src/airflow/migrations/versions/0055_3_0_0_remove_pickled_data_from_dagrun_table.py
index 07d012ddf57..31b71e2bedc 100644
---
a/airflow-core/src/airflow/migrations/versions/0055_3_0_0_remove_pickled_data_from_dagrun_table.py
+++
b/airflow-core/src/airflow/migrations/versions/0055_3_0_0_remove_pickled_data_from_dagrun_table.py
@@ -47,6 +47,17 @@ airflow_version = "3.0.0"
def upgrade():
"""Apply remove pickled data from dagrun table."""
conn = op.get_bind()
+ empty_vals = {
+ "mysql": "X'80057D942E'",
+ "postgresql": r"'\x80057D942E'",
+ "sqlite": "X'80057D942E'",
+ }
+ dialect = conn.dialect.name
+ try:
+ empty_val = empty_vals[dialect]
+ except KeyError:
+ raise RuntimeError(f"Dialect {dialect} not supported.")
+
conf_type = sa.JSON().with_variant(postgresql.JSONB, "postgresql")
op.add_column("dag_run", sa.Column("conf_json", conf_type, nullable=True))
@@ -61,12 +72,20 @@ def upgrade():
""")
)
else:
- BATCH_SIZE = 100
+ BATCH_SIZE = 1000
offset = 0
while True:
+ err_count = 0
+ batch_num = offset + 1
+ print(f"converting dag run conf. batch={batch_num}")
rows = conn.execute(
text(
- f"SELECT id,conf FROM dag_run WHERE conf IS not NULL order
by id LIMIT {BATCH_SIZE} OFFSET {offset}"
+ "SELECT id, conf "
+ "FROM dag_run "
+ "WHERE conf IS not NULL "
+ f"AND conf != {empty_val}"
+ f"ORDER BY id LIMIT {BATCH_SIZE} "
+ f"OFFSET {offset}"
)
).fetchall()
if not rows:
@@ -85,9 +104,11 @@ def upgrade():
"""),
{"json_data": json_data, "id": row_id},
)
- except Exception as e:
- print(f"Error converting dagrun conf to json for dagrun ID
{row_id}: {e}")
+ except Exception:
+ err_count += 1
continue
+ if err_count:
+ print(f"could not convert dag run conf for {err_count}
records. batch={batch_num}")
offset += BATCH_SIZE
op.drop_column("dag_run", "conf")
@@ -112,12 +133,16 @@ def downgrade():
)
else:
- BATCH_SIZE = 100
+ BATCH_SIZE = 1000
offset = 0
while True:
rows = conn.execute(
text(
- f"SELECT id,conf FROM dag_run WHERE conf IS not NULL order
by id LIMIT {BATCH_SIZE} OFFSET {offset}"
+ "SELECT id,conf "
+ "FROM dag_run "
+ "WHERE conf IS NOT NULL "
+ f"ORDER BY id LIMIT {BATCH_SIZE} "
+ f"OFFSET {offset}"
)
).fetchall()
if not rows:
