This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new eca7ad467f0 Include tables when `DocumentLoaderOperator` parses DOCX
files (#73926)
eca7ad467f0 is described below
commit eca7ad467f02c804c9832ddc05cc884bb937fb76
Author: Xiaolong Song <[email protected]>
AuthorDate: Wed Nov 18 23:51:17 2026 +0100
Include tables when `DocumentLoaderOperator` parses DOCX files (#73926)
Tables in Word documents often hold the facts a RAG pipeline needs
(specs, prices, parameters), but the DOCX parser dropped them silently,
so their content never reached the embeddings. The PDF parser already
keeps table text.
python-docx's row.cells walks up to the start of a vertical merge for
every merged cell, so a merge of a few hundred rows took minutes, one
of about 1000 rows overflowed the stack, and a continuation narrower
than the cell above still produced extra cells. Merges nested inside
merged cells repeat their text 8^depth times, so a small file could
expand to tens of millions of characters.
The column count came from w:tblGrid, which Apache POI does not write,
so every table in a POI-generated file was skipped, and because the
count came from the file it bounded nothing.
---
providers/common/ai/README.rst | 2 +-
providers/common/ai/docs/index.rst | 2 +-
.../common/ai/docs/operators/document_loader.rst | 31 +-
providers/common/ai/pyproject.toml | 6 +-
.../common/ai/operators/document_loader.py | 186 +++++++++-
.../common/ai/operators/test_document_loader.py | 403 +++++++++++++++++++--
uv.lock | 4 +-
7 files changed, 581 insertions(+), 53 deletions(-)
diff --git a/providers/common/ai/README.rst b/providers/common/ai/README.rst
index 249f6303836..9c8b22689b5 100644
--- a/providers/common/ai/README.rst
+++ b/providers/common/ai/README.rst
@@ -109,7 +109,7 @@ Extra Dependencies
``langchain`` ``langchain>=1.0.0``
``llamaindex`` ``dataclasses-json>=0.6.7``,
``llama-index-core>=0.14.5``, ``llama-index-embeddings-openai>=0.6.0``,
``llama-index-llms-openai>=0.6.8``
``pdf`` ``pypdf>=4.0.0``
-``docx`` ``python-docx>=1.0.0``
+``docx`` ``python-docx>=1.1.2``
``git`` ``apache-airflow-providers-git``
``claude-agent-sdk`` ``claude-agent-sdk>=0.2.160``
====================
=======================================================================================================================================
diff --git a/providers/common/ai/docs/index.rst
b/providers/common/ai/docs/index.rst
index f9aaea9c01c..ec4f01cedfb 100644
--- a/providers/common/ai/docs/index.rst
+++ b/providers/common/ai/docs/index.rst
@@ -249,7 +249,7 @@ Extra Dependencies
``langchain`` ``langchain>=1.0.0``
``llamaindex`` ``dataclasses-json>=0.6.7``,
``llama-index-core>=0.14.5``, ``llama-index-embeddings-openai>=0.6.0``,
``llama-index-llms-openai>=0.6.8``
``pdf`` ``pypdf>=4.0.0``
-``docx`` ``python-docx>=1.0.0``
+``docx`` ``python-docx>=1.1.2``
``git`` ``apache-airflow-providers-git``
``claude-agent-sdk`` ``claude-agent-sdk>=0.2.160``
====================
=======================================================================================================================================
diff --git a/providers/common/ai/docs/operators/document_loader.rst
b/providers/common/ai/docs/operators/document_loader.rst
index b86f1f0d7ab..583612699c7 100644
--- a/providers/common/ai/docs/operators/document_loader.rst
+++ b/providers/common/ai/docs/operators/document_loader.rst
@@ -73,14 +73,35 @@ Install the ``docx`` extra to parse Word documents via
pip install "apache-airflow-providers-common-ai[docx]"
-All non-empty paragraphs are concatenated into a single document per file.
+Paragraphs and tables in the document body are concatenated, in document
+order, into a single document per file. Empty paragraphs and table rows are
+skipped. Each table row becomes one line:
+
+.. code-block:: text
+
+ Quarterly results
+
+ | Region | Revenue |
+ | EMEA | 1.2M |
+
+A cell merged across columns appears once, with an empty cell for each further
+column it spans, so later values stay under their headers. A cell merged down
+several rows is repeated on each of those rows, so every row still reads on
its own.
+A table nested inside a cell is flattened into that cell, with ``/`` between
its
+cells and ``;`` between its rows. A table is skipped with a warning when it is
+malformed, such as one with a broken vertical merge or a cell whose column span
+does not fit the table, when it has more than 1,000 columns, or when its
output,
+including any tables nested in it, would be longer than about 1,000,000
+characters. If a skipped table is nested inside a cell, only that table is left
+out.
.. note::
- DOCX extraction reads paragraph text only. Tables, headers, footers, and
- footnotes are not included. For richer DOCX parsing, use a dedicated
- extraction tool (``Unstructured``, ``docling``) as a custom parser
- backend.
+ Only paragraphs and tables in the document body are read. Headers, footers,
+ footnotes, content controls, text boxes, and pending tracked insertions are
+ not included. For richer DOCX parsing, use a dedicated extraction tool
+ (``Unstructured``, ``docling``) in a ``@task`` that returns the same
+ ``list[dict(text, metadata)]`` shape.
Directory mode and filtering
----------------------------
diff --git a/providers/common/ai/pyproject.toml
b/providers/common/ai/pyproject.toml
index 8b6d67f05a7..4723562da47 100644
--- a/providers/common/ai/pyproject.toml
+++ b/providers/common/ai/pyproject.toml
@@ -176,7 +176,9 @@ dependencies = [
"llama-index-llms-openai>=0.6.8",
]
"pdf" = ["pypdf>=4.0.0"]
-"docx" = ["python-docx>=1.0.0"]
+# python-docx 1.1.1 added grid_cols_before/grid_cols_after but pins
lxml<=4.9.2, which does
+# not build on 3.12, so 1.1.2 is the first release that works.
+"docx" = ["python-docx>=1.1.2"]
"git" = [
"apache-airflow-providers-git"
]
@@ -212,6 +214,8 @@ dev = [
"grpcio>=1.78.0",
"protobuf>=6.31.1",
"claude-agent-sdk>=0.2.160",
+ # The DOCX parser tests build real documents in memory instead of mocking
python-docx.
+ "python-docx>=1.1.2",
]
# To build docs:
diff --git
a/providers/common/ai/src/airflow/providers/common/ai/operators/document_loader.py
b/providers/common/ai/src/airflow/providers/common/ai/operators/document_loader.py
index a226b89ef76..0e4b1be16cb 100644
---
a/providers/common/ai/src/airflow/providers/common/ai/operators/document_loader.py
+++
b/providers/common/ai/src/airflow/providers/common/ai/operators/document_loader.py
@@ -30,6 +30,10 @@ from airflow.providers.common.compat.sdk import (
)
if TYPE_CHECKING:
+ from docx.oxml.exceptions import InvalidXmlError
+ from docx.table import Table, _Cell
+ from docx.text.paragraph import Paragraph
+
from airflow.sdk import Context
@@ -40,6 +44,18 @@ if TYPE_CHECKING:
# polymorphic.
FilePathT = Any # Path | ObjectStoragePath
+# Word allows at most 63 cells in a row, but rows whose borders do not line up
add grid
+# columns (a 40-cell row above a 41-cell row saves with 80), so the limit sits
well above
+# any real table. It bounds the width a file sets through w:tblGrid or
w:gridSpan, and
+# wider tables are skipped.
+_DOCX_TABLE_MAX_COLUMNS = 1000
+
+# A vertically merged cell repeats its text on every row it spans, and merges
nested inside
+# merged cells multiply that with each level, so a small file can expand to
millions of
+# characters. The count includes about 3 characters of separators per cell, so
padded empty
+# cells count too. 1M characters is about 6000 rows of 10 columns; longer
tables are skipped.
+_DOCX_TABLE_MAX_CHARS = 1_000_000
+
class DocumentLoaderOperator(BaseOperator):
"""
@@ -297,13 +313,13 @@ class DocumentLoaderOperator(BaseOperator):
def _parse_bytes(self, raw: bytes, file_type: str) -> list[dict[str, Any]]:
ext = file_type if file_type.startswith(".") else f".{file_type}"
backend = self._resolve_backend(ext)
+ source_hint = f"<bytes:{ext}>"
if backend == "pypdf":
return self._parse_pdf_stream(io.BytesIO(raw))
if backend == "python-docx":
- return self._parse_docx_stream(io.BytesIO(raw))
+ return self._parse_docx_stream(io.BytesIO(raw),
source_hint=source_hint)
- source_hint = f"<bytes:{ext}>"
text = self._decode(raw, source_hint=source_hint)
if backend == "csv":
return self._parse_csv_text(text)
@@ -329,7 +345,7 @@ class DocumentLoaderOperator(BaseOperator):
return self._parse_pdf_stream(fh)
if backend == "python-docx":
with file_path.open("rb") as fh:
- return self._parse_docx_stream(fh)
+ return self._parse_docx_stream(fh, source_hint=str(file_path))
raise ValueError(f"No parser found for backend '{backend}'.")
@@ -450,21 +466,169 @@ class DocumentLoaderOperator(BaseOperator):
documents.append({"text": text, "metadata": {"page_number":
page_num + 1}})
return documents
- def _parse_docx_stream(self, stream: BinaryIO) -> list[dict[str, Any]]:
+ def _parse_docx_stream(self, stream: BinaryIO, *, source_hint: str) ->
list[dict[str, Any]]:
"""
- Parse a DOCX stream into documents.
+ Parse a DOCX stream into a single document.
- Extracts paragraph text only. Tables, headers, footers, and footnotes
- are not included. For richer DOCX parsing, plug in a dedicated
- extraction tool (``Unstructured``, ``docling``) as a custom parser
- backend.
+ Paragraphs and tables in the document body are extracted in document
+ order. Each table row becomes one "| cell | cell |" line, and a nested
+ table is flattened into its cell. Headers, footers, footnotes, content
+ controls, text boxes, and pending tracked insertions are not included.
"""
try:
from docx import Document
+ from docx.oxml.exceptions import InvalidXmlError
+ from docx.table import Table, _Cell
except ImportError as e:
raise AirflowOptionalProviderFeatureException(e)
doc = Document(stream)
- paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
- text = "\n\n".join(paragraphs)
+ blocks = []
+ for block in doc.iter_inner_content():
+ if isinstance(block, Table):
+ rows = self._get_docx_table_rows_or_none(
+ block, Table, _Cell, InvalidXmlError,
source_hint=source_hint
+ )
+ if rows is None:
+ continue
+ text = "\n".join(f"| {' | '.join(cells)} |" for cells in rows)
+ else:
+ text = block.text
+ if text.strip():
+ blocks.append(text)
+ text = "\n\n".join(blocks)
return [{"text": text, "metadata": {}}]
+
+ def _get_docx_table_rows_or_none(
+ self,
+ table: Table,
+ table_cls: type[Table],
+ cell_cls: type[_Cell],
+ invalid_xml_error_cls: type[InvalidXmlError],
+ *,
+ source_hint: str,
+ ) -> list[list[str]] | None:
+ try:
+ return self._get_docx_table_rows(
+ table, table_cls, cell_cls, invalid_xml_error_cls,
source_hint=source_hint
+ )
+ except (ValueError, invalid_xml_error_cls) as e:
+ # ValueError covers a w:vMerge continuation that does not line up
with a cell above,
+ # a w:gridSpan that does not fit the table, a table wider than
_DOCX_TABLE_MAX_COLUMNS,
+ # and a table whose text exceeds _DOCX_TABLE_MAX_CHARS;
InvalidXmlError covers a
+ # missing required attribute. Nested tables are read recursively,
but python-docx
+ # parses without huge_tree, so libxml2 refuses a file nested more
than 256 elements
+ # deep (83 tables) and the recursion stays far below Python's
limit.
+ self.log.warning("Skipping a table in %s: %r", source_hint, e)
+ return None
+
+ def _get_docx_table_rows(
+ self,
+ table: Table,
+ table_cls: type[Table],
+ cell_cls: type[_Cell],
+ invalid_xml_error_cls: type[InvalidXmlError],
+ *,
+ source_hint: str,
+ ) -> list[list[str]]:
+ # w:gridBefore and w:gridAfter come straight from the XML and can be
any integer, so the
+ # empty cells added for them only fill the row up to the column count.
+ try:
+ column_count = len(table.columns)
+ except invalid_xml_error_cls:
+ # Apache POI writes tables without w:tblGrid, so the widest row
sets the column count.
+ column_count = max(
+ (
+ max(row.grid_cols_before, 0)
+ + sum(tc.grid_span for tc in row._tr.tc_lst)
+ + max(row.grid_cols_after, 0)
+ for row in table.rows
+ ),
+ default=0,
+ )
+ if column_count > _DOCX_TABLE_MAX_COLUMNS:
+ raise ValueError(
+ f"the table has {column_count} columns, which exceeds the
maximum of {_DOCX_TABLE_MAX_COLUMNS}"
+ )
+
+ rows = []
+ char_count = 0
+ # row.cells is not used: it finds the text of each w:vMerge
continuation by recursing up
+ # to the merge start, which is quadratic in the merge height and
overflows the stack at
+ # about 1000 rows. Instead the text and w:gridSpan of each cell is
carried down from the
+ # row above, keyed by the grid column the cell starts at.
+ above: dict[int, tuple[str, int]] = {}
+ for row in table.rows:
+ tcs = row._tr.tc_lst
+ free_columns = column_count - sum(tc.grid_span for tc in tcs)
+ grid_before = max(0, min(row.grid_cols_before, free_columns))
+ grid_after = max(0, min(row.grid_cols_after, free_columns -
grid_before))
+
+ current: dict[int, tuple[str, int]] = {}
+ cells = [""] * grid_before
+ offset = grid_before
+ for index, tc in enumerate(tcs, start=1):
+ span = tc.grid_span
+ # Each cell fills one column for every grid column its
w:gridSpan covers, so a
+ # span below 1, or one that runs past the table width, is
rejected before any
+ # column is built.
+ if span < 1 or offset + span > column_count:
+ raise ValueError(
+ f"w:gridSpan {span} of cell {index} does not fit the
{column_count} table columns"
+ )
+ if tc.vMerge == "continue":
+ if offset not in above or above[offset][1] != span:
+ raise ValueError(
+ f"w:vMerge continuation of cell {index} does not
line up with a cell above it"
+ )
+ text = above[offset][0]
+ else:
+ text = self._get_docx_cell_text(
+ cell_cls(tc, table),
+ table_cls,
+ cell_cls,
+ invalid_xml_error_cls,
+ source_hint=source_hint,
+ )
+ current[offset] = (text, span)
+ # A horizontal merge keeps its text once and leaves its other
columns empty so
+ # later cells stay under their headers. A vertical merge
repeats on each row.
+ cells += [text] + [""] * (span - 1)
+ offset += span
+ cells += [""] * grid_after
+ # Merges starting in a row that is left out for being empty still
continue below it.
+ above = current
+ if any(cells):
+ char_count += sum(map(len, cells)) + 3 * len(cells)
+ if char_count > _DOCX_TABLE_MAX_CHARS:
+ raise ValueError(
+ f"the table text exceeds the maximum of
{_DOCX_TABLE_MAX_CHARS} characters"
+ )
+ rows.append(cells)
+ return rows
+
+ def _get_docx_cell_text(
+ self,
+ cell: _Cell,
+ table_cls: type[Table],
+ cell_cls: type[_Cell],
+ invalid_xml_error_cls: type[InvalidXmlError],
+ *,
+ source_hint: str,
+ ) -> str:
+ parts = []
+ for item in cell.iter_inner_content():
+ if isinstance(item, table_cls):
+ rows = self._get_docx_table_rows_or_none(
+ item, table_cls, cell_cls, invalid_xml_error_cls,
source_hint=source_hint
+ )
+ if rows is None:
+ continue
+ text = "; ".join(" / ".join(cells) for cells in rows)
+ else:
+ if TYPE_CHECKING:
+ assert isinstance(item, Paragraph)
+ text = " ".join(item.text.split())
+ if text:
+ parts.append(text)
+ return " ".join(parts)
diff --git
a/providers/common/ai/tests/unit/common/ai/operators/test_document_loader.py
b/providers/common/ai/tests/unit/common/ai/operators/test_document_loader.py
index dcd0ab8dac9..6a459d460c1 100644
--- a/providers/common/ai/tests/unit/common/ai/operators/test_document_loader.py
+++ b/providers/common/ai/tests/unit/common/ai/operators/test_document_loader.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import annotations
+import io
import json
import logging
from unittest.mock import MagicMock, patch
@@ -289,11 +290,37 @@ def _make_mock_pypdf_module(mock_reader):
return mock_module
-def _make_mock_docx_module(mock_doc):
- """Create a fake docx module with a Document that returns mock_doc."""
- mock_module = MagicMock()
- mock_module.Document = MagicMock(return_value=mock_doc)
- return mock_module
+def _create_docx():
+ return pytest.importorskip("docx", reason="needs the 'docx'
extra").Document()
+
+
+def _get_docx_bytes(doc) -> bytes:
+ buffer = io.BytesIO()
+ doc.save(buffer)
+ return buffer.getvalue()
+
+
+def _fill_docx_table(table, rows: list[list[str]]) -> None:
+ for r, values in enumerate(rows):
+ for c, value in enumerate(values):
+ table.cell(r, c).text = value
+
+
+def _build_docx_bytes(*blocks: str | list[list[str]]) -> bytes:
+ """Build a DOCX in memory: a ``str`` adds a paragraph, a list of rows adds
a table."""
+ doc = _create_docx()
+ for block in blocks:
+ if isinstance(block, str):
+ doc.add_paragraph(block)
+ else:
+ _fill_docx_table(doc.add_table(rows=len(block),
cols=len(block[0])), block)
+ return _get_docx_bytes(doc)
+
+
+def _load_docx_text(raw: bytes) -> str:
+ op = DocumentLoaderOperator(task_id="test", source_bytes=raw,
file_type=".docx")
+ (document,) = op.execute(context=MagicMock())
+ return document["text"]
class TestPdfParser:
@@ -371,45 +398,28 @@ class TestPdfParser:
class TestDocxParser:
def test_docx_parsing(self, tmp_path):
- mock_para_1 = MagicMock()
- mock_para_1.text = "First paragraph"
- mock_para_2 = MagicMock()
- mock_para_2.text = "Second paragraph"
- mock_para_empty = MagicMock()
- mock_para_empty.text = " "
-
- mock_doc_obj = MagicMock()
- mock_doc_obj.paragraphs = [mock_para_1, mock_para_empty, mock_para_2]
-
f = tmp_path / "doc.docx"
- f.write_bytes(b"fake docx")
+ f.write_bytes(_build_docx_bytes("First paragraph", " ", "Second
paragraph"))
- mock_docx = _make_mock_docx_module(mock_doc_obj)
- with patch.dict("sys.modules", {"docx": mock_docx}):
- op = DocumentLoaderOperator(task_id="test", source_path=str(f))
- result = op.execute(context=MagicMock())
+ op = DocumentLoaderOperator(task_id="test", source_path=str(f))
+ result = op.execute(context=MagicMock())
assert len(result) == 1
assert "First paragraph" in result[0]["text"]
assert "Second paragraph" in result[0]["text"]
def test_docx_from_bytes_uses_stream_no_tempfile(self):
- mock_para = MagicMock()
- mock_para.text = "Stream paragraph"
- mock_doc_obj = MagicMock()
- mock_doc_obj.paragraphs = [mock_para]
-
- mock_docx = _make_mock_docx_module(mock_doc_obj)
- with patch.dict("sys.modules", {"docx": mock_docx}):
- op = DocumentLoaderOperator(task_id="test", source_bytes=b"fake
docx", file_type=".docx")
+ docx = pytest.importorskip("docx", reason="needs the 'docx' extra")
+ raw = _build_docx_bytes("Stream paragraph")
+
+ with patch("docx.Document", autospec=True, side_effect=docx.Document)
as mock_document:
+ op = DocumentLoaderOperator(task_id="test", source_bytes=raw,
file_type=".docx")
result = op.execute(context=MagicMock())
assert "Stream paragraph" in result[0]["text"]
- mock_docx.Document.assert_called_once()
- (call_arg,) = mock_docx.Document.call_args.args
- import io as _io
-
- assert isinstance(call_arg, _io.BytesIO)
+ mock_document.assert_called_once()
+ (call_arg,) = mock_document.call_args.args
+ assert isinstance(call_arg, io.BytesIO)
def test_docx_missing_raises_optional_feature_exception(self, tmp_path):
from airflow.providers.common.compat.sdk import
AirflowOptionalProviderFeatureException
@@ -422,6 +432,333 @@ class TestDocxParser:
with pytest.raises(AirflowOptionalProviderFeatureException):
op.execute(context=MagicMock())
+ @pytest.mark.parametrize(
+ ("blocks", "expected"),
+ [
+ pytest.param(
+ ("Intro", " ", [["Name", "Qty"], ["Apple", "3"]], "Outro"),
+ "Intro\n\n| Name | Qty |\n| Apple | 3 |\n\nOutro",
+ id="tables-in-document-order",
+ ),
+ pytest.param(([["a", "", "c"]],), "| a | | c |",
id="empty-cell-keeps-its-column"),
+ pytest.param(
+ ([["h1", "h2"], ["", ""], ["x", "y"]],),
+ "| h1 | h2 |\n| x | y |",
+ id="empty-row-skipped",
+ ),
+ pytest.param(
+ ("Before", [["", ""]], "After", [["Name", "Qty"]]),
+ "Before\n\nAfter\n\n| Name | Qty |",
+ id="empty-table-skipped",
+ ),
+ pytest.param(
+ ([["line1\nline2", "b"]],),
+ "| line1 line2 | b |",
+ id="cell-line-break-stays-on-row",
+ ),
+ pytest.param(
+ ([["0", "0"], ["x", "x"]],),
+ "| 0 | 0 |\n| x | x |",
+ id="equal-adjacent-unmerged-cells-kept",
+ ),
+ ],
+ )
+ def test_docx_tables(self, blocks, expected):
+ assert _load_docx_text(_build_docx_bytes(*blocks)) == expected
+
+ def test_docx_merged_cells_horizontal_once_vertical_repeated(self):
+ doc = _create_docx()
+ table = doc.add_table(rows=3, cols=3)
+ _fill_docx_table(table, [["", "", "Note"], ["a", "", "c"], ["d", "",
"f"]])
+ table.cell(0, 0).merge(table.cell(0, 1)).text = "Title"
+ table.cell(1, 1).merge(table.cell(2, 1)).text = "Shared"
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == (
+ "| Title | | Note |\n| a | Shared | c |\n| d | Shared | f |"
+ )
+
+ def test_docx_long_vertical_merge_read_in_full(self):
+ # python-docx's row.cells recurses once per merged row above, so it
fails before 1000 rows.
+ # The merge starts in an empty row, which is left out, and the rows
below still continue it.
+ row_count = 1100
+ doc = _create_docx()
+ table = doc.add_table(rows=row_count, cols=2)
+ for index, row in enumerate(table.rows):
+ if index:
+ row.cells[1].text = f"r{index}"
+ row._tr.tc_lst[0].vMerge = "continue" if index else "restart"
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == "\n".join(
+ f"| | r{index} |" for index in range(1, row_count)
+ )
+
+ @patch.object(
+ DocumentLoaderOperator,
+ "_get_docx_cell_text",
+ autospec=True,
+ side_effect=DocumentLoaderOperator._get_docx_cell_text,
+ )
+ def test_docx_vertical_merge_text_extracted_once(self, mock_get_cell_text):
+ doc = _create_docx()
+ table = doc.add_table(rows=3, cols=2)
+ _fill_docx_table(table, [["", "r0"], ["", "r1"], ["", "r2"]])
+ table.cell(0, 0).merge(table.cell(2, 0)).text = "Dept"
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == "| Dept | r0 |\n| Dept
| r1 |\n| Dept | r2 |"
+ assert [call.args[1].text for call in
mock_get_cell_text.call_args_list] == ["Dept", "r0", "r1", "r2"]
+
+ def test_docx_grid_before_and_after_keep_columns(self):
+ doc = _create_docx()
+ table = doc.add_table(rows=3, cols=3)
+ _fill_docx_table(table, [["Product", "Revenue", "Cost"], ["", "100",
"80"], ["Widget", "5", ""]])
+ starts_late, ends_early = table.rows[1]._tr, table.rows[2]._tr
+ starts_late.remove(starts_late.tc_lst[0])
+ starts_late.get_or_add_trPr().get_or_add_gridBefore().val = 1
+ ends_early.remove(ends_early.tc_lst[-1])
+ ends_early.get_or_add_trPr().get_or_add_gridAfter().val = 1
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == (
+ "| Product | Revenue | Cost |\n| | 100 | 80 |\n| Widget | 5 | |"
+ )
+
+ @pytest.mark.parametrize(
+ ("grid_before", "grid_after", "expected_row"),
+ [
+ pytest.param(1000, 0, "| | x |", id="grid-before"),
+ pytest.param(0, 1000, "| x | |", id="grid-after"),
+ pytest.param(1000, 1000, "| | x |", id="grid-before-and-after"),
+ pytest.param(-5, 1000, "| x | |", id="negative-grid-before"),
+ ],
+ )
+ def test_docx_grid_values_capped_at_column_count(self, grid_before,
grid_after, expected_row):
+ doc = _create_docx()
+ table = doc.add_table(rows=2, cols=2)
+ _fill_docx_table(table, [["a", "b"], ["x", ""]])
+ row = table.rows[1]._tr
+ row.remove(row.tc_lst[1])
+ tr_pr = row.get_or_add_trPr()
+ tr_pr.get_or_add_gridBefore().val = grid_before
+ tr_pr.get_or_add_gridAfter().val = grid_after
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == f"| a | b
|\n{expected_row}"
+
+ @pytest.mark.parametrize(
+ ("grid_before", "grid_after", "expected_row"),
+ [
+ pytest.param(1, 0, "| | c | d |", id="grid-before"),
+ pytest.param(0, 1, "| c | d | |", id="grid-after"),
+ ],
+ )
+ def test_docx_table_without_grid_sized_by_widest_row(self, grid_before,
grid_after, expected_row):
+ doc = _create_docx()
+ table = doc.add_table(rows=2, cols=2)
+ _fill_docx_table(table, [["a", "b"], ["c", "d"]])
+ tr_pr = table.rows[1]._tr.get_or_add_trPr()
+ tr_pr.get_or_add_gridBefore().val = grid_before
+ tr_pr.get_or_add_gridAfter().val = grid_after
+ table._tbl.remove(table._tbl.tblGrid)
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == f"| a | b
|\n{expected_row}"
+
+ @pytest.mark.parametrize(
+ ("grid_before", "grid_after"),
+ [
+ pytest.param(-1, 0, id="negative-grid-before"),
+ pytest.param(0, -1, id="negative-grid-after"),
+ ],
+ )
+ def test_docx_table_without_grid_ignores_negative_grid_values(self,
grid_before, grid_after):
+ doc = _create_docx()
+ table = doc.add_table(rows=1, cols=2)
+ _fill_docx_table(table, [["c", "d"]])
+ tr_pr = table.rows[0]._tr.get_or_add_trPr()
+ tr_pr.get_or_add_gridBefore().val = grid_before
+ tr_pr.get_or_add_gridAfter().val = grid_after
+ table._tbl.remove(table._tbl.tblGrid)
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == "| c | d |"
+
+ def test_docx_table_without_grid_or_rows_left_out_without_warning(self,
caplog):
+ doc = _create_docx()
+ tbl = doc.add_table(rows=1, cols=2)._tbl
+ tbl.remove(tbl.tr_lst[0])
+ tbl.remove(tbl.tblGrid)
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == ""
+ assert not [message for message in caplog.messages if
message.startswith("Skipping a table")]
+
+ @pytest.mark.parametrize(
+ "remove_grid",
+ [
+ pytest.param(False, id="with-grid"),
+ pytest.param(True, id="without-grid"),
+ ],
+ )
+ def test_docx_table_wider_than_limit_skipped_with_warning(self,
remove_grid, caplog):
+ doc = _create_docx()
+ for column_count in (1000, 1001):
+ table = doc.add_table(rows=1, cols=column_count)
+ table.cell(0, 0).text = "x"
+ if remove_grid:
+ table._tbl.remove(table._tbl.tblGrid)
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == f"| {' | '.join(['x']
+ [''] * 999)} |"
+ assert (
+ "Skipping a table in <bytes:.docx>: "
+ "ValueError('the table has 1001 columns, which exceeds the maximum
of 1000')"
+ ) in caplog.messages
+
+
@patch("airflow.providers.common.ai.operators.document_loader._DOCX_TABLE_MAX_CHARS",
12)
+ def test_docx_table_text_over_limit_skipped_with_warning(self, caplog):
+ # A row counts its text plus 3 characters per cell, so 6 + 6 fits the
limit and 6 + 7 does not.
+ raw = _build_docx_bytes([["abc"], ["abc"]], [["abc"], ["abcd"]])
+
+ assert _load_docx_text(raw) == "| abc |\n| abc |"
+ assert (
+ "Skipping a table in <bytes:.docx>: "
+ "ValueError('the table text exceeds the maximum of 12 characters')"
+ ) in caplog.messages
+
+
@patch("airflow.providers.common.ai.operators.document_loader._DOCX_TABLE_MAX_CHARS",
12)
+ def
test_docx_padding_and_repeated_merge_text_count_toward_text_limit(self, caplog):
+ doc = _create_docx()
+ # "x" padded with w:gridAfter to 4 columns counts 1 + 4 * 3.
+ padded = doc.add_table(rows=1, cols=4)
+ padded.cell(0, 0).text = "x"
+ tr = padded.rows[0]._tr
+ for tc in tr.tc_lst[1:]:
+ tr.remove(tc)
+ tr.get_or_add_trPr().get_or_add_gridAfter().val = 3
+ # A vertical merge repeats its text on each row it spans: 7 + 7.
+ merged = doc.add_table(rows=2, cols=1)
+ merged.cell(0, 0).merge(merged.cell(1, 0)).text = "abcd"
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == ""
+ assert (
+ caplog.messages.count(
+ "Skipping a table in <bytes:.docx>: "
+ "ValueError('the table text exceeds the maximum of 12
characters')"
+ )
+ == 2
+ )
+
+
@patch("airflow.providers.common.ai.operators.document_loader._DOCX_TABLE_MAX_CHARS",
20)
+ def test_docx_nested_table_text_counts_toward_parent_table_limit(self,
caplog):
+ doc = _create_docx()
+ outer = doc.add_table(rows=1, cols=2)
+ outer.cell(0, 0).text = "keep"
+ # The inner table fits (8 + 3), but its parent repeats it on both
merged rows: 11 + 11.
+ parent = outer.cell(0, 1).add_table(rows=2, cols=1)
+ parent.cell(0, 0).merge(parent.cell(1, 0)).add_table(rows=1,
cols=1).cell(0, 0).text = "aaaaaaaa"
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == "| keep | |"
+ assert (
+ "Skipping a table in <bytes:.docx>: "
+ "ValueError('the table text exceeds the maximum of 20 characters')"
+ ) in caplog.messages
+
+ def test_docx_vmerge_continuation_lines_up_after_capped_grid_before(self):
+ doc = _create_docx()
+ table = doc.add_table(rows=2, cols=2)
+ _fill_docx_table(table, [["a", "b"], ["", ""]])
+ row = table.rows[1]._tr
+ row.remove(row.tc_lst[0])
+ row.get_or_add_trPr().get_or_add_gridBefore().val = 1000
+ table.rows[0]._tr.tc_lst[1].vMerge = "restart"
+ row.tc_lst[0].vMerge = "continue"
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == "| a | b |\n| | b |"
+
+ @pytest.mark.parametrize(
+ ("break_first_cell", "error"),
+ [
+ pytest.param(
+ # A vMerge continuation in the first row has no cell above it
to continue.
+ lambda tc_pr: tc_pr.get_or_add_vMerge(),
+ "ValueError('w:vMerge continuation of cell 1 does not line up
with a cell above it')",
+ id="first-row-vmerge-continuation",
+ ),
+ pytest.param(
+ lambda tc_pr: tc_pr.get_or_add_gridSpan(),
+ "InvalidXmlError(\"required 'w:val' attribute not present on
element "
+
'{http://schemas.openxmlformats.org/wordprocessingml/2006/main}gridSpan")',
+ id="gridspan-without-val",
+ ),
+ pytest.param(
+ lambda tc_pr: setattr(tc_pr.get_or_add_gridSpan(), "val",
100_000_000),
+ "ValueError('w:gridSpan 100000000 of cell 1 does not fit the 2
table columns')",
+ id="gridspan-wider-than-table",
+ ),
+ pytest.param(
+ lambda tc_pr: setattr(tc_pr.get_or_add_gridSpan(), "val", 0),
+ "ValueError('w:gridSpan 0 of cell 1 does not fit the 2 table
columns')",
+ id="gridspan-zero",
+ ),
+ pytest.param(
+ # The first cell spans both columns on its own, so the second
cell is one too many.
+ lambda tc_pr: setattr(tc_pr.get_or_add_gridSpan(), "val", 2),
+ "ValueError('w:gridSpan 1 of cell 2 does not fit the 2 table
columns')",
+ id="gridspans-together-wider-than-table",
+ ),
+ ],
+ )
+ def test_docx_unreadable_table_skipped_with_warning(self,
break_first_cell, error, caplog):
+ doc = _create_docx()
+ doc.add_paragraph("Before")
+ unreadable = doc.add_table(rows=2, cols=2)
+ break_first_cell(unreadable.cell(0, 0)._tc.get_or_add_tcPr())
+ doc.add_paragraph("After")
+ _fill_docx_table(doc.add_table(rows=1, cols=2), [["Name", "Qty"]])
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == "Before\n\nAfter\n\n|
Name | Qty |"
+ assert f"Skipping a table in <bytes:.docx>: {error}" in caplog.messages
+
+ @pytest.mark.parametrize(
+ "merged_row",
+ [
+ pytest.param(0, id="continuation-narrower-than-cell-above"),
+ pytest.param(1, id="continuation-wider-than-cell-above"),
+ ],
+ )
+ def test_docx_vmerge_continuation_not_lining_up_skipped_with_warning(self,
merged_row, caplog):
+ doc = _create_docx()
+ doc.add_paragraph("Before")
+ unreadable = doc.add_table(rows=2, cols=3)
+ _fill_docx_table(unreadable, [["a", "b", "c"], ["d", "e", "f"]])
+ # The merged row's first cell spans 2 columns while the other row's
first cell spans 1.
+ unreadable.cell(merged_row, 0).merge(unreadable.cell(merged_row, 1))
+ unreadable.rows[1]._tr.tc_lst[0].vMerge = "continue"
+ doc.add_paragraph("After")
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == "Before\n\nAfter"
+ assert (
+ "Skipping a table in <bytes:.docx>: "
+ "ValueError('w:vMerge continuation of cell 1 does not line up with
a cell above it')"
+ ) in caplog.messages
+
+ def test_docx_unreadable_nested_table_skipped_with_warning(self, caplog):
+ doc = _create_docx()
+ doc.add_paragraph("Before")
+ outer = doc.add_table(rows=2, cols=2)
+ _fill_docx_table(outer, [["Name", "Qty"], ["Apple", ""]])
+ nested = outer.cell(1, 1).add_table(rows=2, cols=2)
+ nested.cell(0, 0)._tc.get_or_add_tcPr().get_or_add_vMerge()
+ doc.add_paragraph("After")
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == "Before\n\n| Name |
Qty |\n| Apple | |\n\nAfter"
+ assert (
+ "Skipping a table in <bytes:.docx>: "
+ "ValueError('w:vMerge continuation of cell 1 does not line up with
a cell above it')"
+ ) in caplog.messages
+
+ def test_docx_nested_table_flattened_into_its_cell(self):
+ doc = _create_docx()
+ table = doc.add_table(rows=1, cols=2)
+ _fill_docx_table(table, [["Outer", "Right"]])
+ _fill_docx_table(table.cell(0, 0).add_table(rows=2, cols=2), [["n1",
"n2"], ["n3", "n4"]])
+
+ assert _load_docx_text(_get_docx_bytes(doc)) == "| Outer n1 / n2; n3 /
n4 | Right |"
+
class TestFileDiscovery:
def test_glob_multiple_files(self, tmp_path):
diff --git a/uv.lock b/uv.lock
index 9c2467d7de1..5e6aaf1c9b9 100644
--- a/uv.lock
+++ b/uv.lock
@@ -4938,6 +4938,7 @@ dev = [
{ name = "pydantic-ai-harness", extra = ["codemode"] },
{ name = "pydantic-ai-skills" },
{ name = "pydantic-ai-slim", extra = ["mcp"] },
+ { name = "python-docx" },
{ name = "sqlglot" },
]
docs = [
@@ -4984,7 +4985,7 @@ requires-dist = [
{ name = "pydantic-ai-slim", extras = ["mcp"], marker = "extra == 'mcp'",
specifier = ">=2.46.0" },
{ name = "pydantic-ai-slim", extras = ["openai"], marker = "extra ==
'openai'", specifier = ">=2.46.0" },
{ name = "pypdf", marker = "extra == 'pdf'", specifier = ">=4.0.0" },
- { name = "python-docx", marker = "extra == 'docx'", specifier = ">=1.0.0"
},
+ { name = "python-docx", marker = "extra == 'docx'", specifier = ">=1.1.2"
},
{ name = "sqlglot", marker = "extra == 'sql'", specifier = ">=30.0.0" },
{ name = "structlog", specifier = ">=24.2.0" },
{ name = "typesafe-sdk", marker = "extra == 'typesafe'", specifier =
">=0.6.0" },
@@ -5015,6 +5016,7 @@ dev = [
{ name = "pydantic-ai-harness", extras = ["codemode"], specifier =
">=0.35.0" },
{ name = "pydantic-ai-skills", specifier = ">=1.2.0,<2" },
{ name = "pydantic-ai-slim", extras = ["mcp"], specifier = ">=2.46.0" },
+ { name = "python-docx", specifier = ">=1.1.2" },
{ name = "sqlglot", specifier = ">=30.0.0" },
]
docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable =
"devel-common" }]