xlsongc commented on code in PR #73926:
URL: https://github.com/apache/airflow/pull/73926#discussion_r4291201551
##########
providers/common/ai/src/airflow/providers/common/ai/operators/document_loader.py:
##########
@@ -450,21 +453,77 @@ def _parse_pdf_stream(self, stream: BinaryIO) ->
list[dict[str, Any]]:
documents.append({"text": text, "metadata": {"page_number":
page_num + 1}})
return documents
- def _parse_docx_stream(self, stream: BinaryIO) -> list[dict[str, Any]]:
+ def _parse_docx_stream(self, stream: BinaryIO, *, source_hint: str) ->
list[dict[str, Any]]:
"""
- Parse a DOCX stream into documents.
+ Parse a DOCX stream into a single document.
- Extracts paragraph text only. Tables, headers, footers, and footnotes
- are not included. For richer DOCX parsing, plug in a dedicated
- extraction tool (``Unstructured``, ``docling``) as a custom parser
- backend.
+ Paragraphs and tables in the document body are extracted in document
+ order. Each table row becomes one "| cell | cell |" line, and a nested
+ table is flattened into its cell. Headers, footers, footnotes, content
+ controls, text boxes, and pending tracked insertions are not included.
"""
try:
from docx import Document
+ from docx.oxml.exceptions import InvalidXmlError
+ from docx.table import Table
except ImportError as e:
raise AirflowOptionalProviderFeatureException(e)
doc = Document(stream)
- paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
- text = "\n\n".join(paragraphs)
+ blocks = []
+ for block in doc.iter_inner_content():
+ if isinstance(block, Table):
+ try:
+ rows = self._get_docx_table_rows(block, Table)
+ except (ValueError, InvalidXmlError, RecursionError) as e:
+ # Malformed table XML raises ValueError (a vMerge with no
cell above) or
+ # InvalidXmlError (a missing required attribute).
python-docx walks vertical
+ # merges up recursively, so one spanning about 1000 rows
raises RecursionError.
+ self.log.warning(
+ "Skipping a table in %s that python-docx could not
read: %r", source_hint, e
+ )
+ continue
+ text = "\n".join(f"| {' | '.join(cells)} |" for cells in rows)
+ else:
+ text = block.text
+ if text.strip():
+ blocks.append(text)
+ text = "\n\n".join(blocks)
return [{"text": text, "metadata": {}}]
+
+ def _get_docx_table_rows(self, table: Table, table_cls: type[Table]) ->
list[list[str]]:
+ # w:gridBefore, w:gridAfter and w:gridSpan come straight from the XML,
so the empty
+ # cells added for them are capped at the table's column count.
+ column_count = len(table.columns)
+
+ rows = []
+ for row in table.rows:
+ cells: list[str] = [""] * min(row.grid_cols_before, column_count)
+ previous_cell = None
+ for cell in row.cells:
+ # python-docx repeats the same _Cell object for every grid
column a horizontal
+ # merge spans: keep its text once and leave the other columns
empty so later
+ # cells stay under their headers. A vertical merge repeats on
each row it spans.
+ if cell is previous_cell:
+ if len(cells) < column_count:
+ cells.append("")
+ continue
+ previous_cell = cell
+ cells.append(self._get_docx_cell_text(cell, table_cls))
+ cells.extend([""] * min(row.grid_cols_after, column_count))
+ if any(cells):
+ rows.append(cells)
+ return rows
+
+ def _get_docx_cell_text(self, cell: _Cell, table_cls: type[Table]) -> str:
+ parts = []
+ for item in cell.iter_inner_content():
+ if isinstance(item, table_cls):
+ text = "; ".join(" / ".join(cells) for cells in
self._get_docx_table_rows(item, table_cls))
Review Comment:
Done. The try/except is now a helper, `_get_docx_table_rows_or_none`, used
for both top-level and nested tables, so an unreadable nested table is left out
of its cell and the outer rows stay.
`test_docx_unreadable_nested_table_skipped_with_warning` uses your example and
expects `| Name | Qty |` and `| Apple | |` plus the warning; it fails without
the change. `InvalidXmlError` is passed down as `invalid_xml_error_cls`, like
`table_cls`, and the docs now say only the nested table is left out.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
