This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 84a2037827b Keep CSV quoting in LLMFileAnalysisOperator sample rows
(#76389)
84a2037827b is described below
commit 84a2037827b7da4db45a4899b607fa4b1e4f809e
Author: Aarmen Sidhu <[email protected]>
AuthorDate: Sat Nov 7 19:05:14 2026 -0700
Keep CSV quoting in LLMFileAnalysisOperator sample rows (#76389)
* Keep CSV quoting in LLMFileAnalysisOperator sample rows
* Test each CSV quoting trigger on its own
The single value with both a quote and a line break meant dropping either
character from the quoting set still passed. One case per character, plus a
plain value that must stay unquoted.
---------
Co-authored-by: aarmens702-hub
<[email protected]>
Co-authored-by: Kaxil Naik <[email protected]>
---
.../providers/common/ai/utils/file_analysis.py | 13 ++++++++--
.../unit/common/ai/utils/test_file_analysis.py | 28 ++++++++++++++++++++++
2 files changed, 39 insertions(+), 2 deletions(-)
diff --git
a/providers/common/ai/src/airflow/providers/common/ai/utils/file_analysis.py
b/providers/common/ai/src/airflow/providers/common/ai/utils/file_analysis.py
index eb402dfcd9f..a89eee34422 100644
--- a/providers/common/ai/src/airflow/providers/common/ai/utils/file_analysis.py
+++ b/providers/common/ai/src/airflow/providers/common/ai/utils/file_analysis.py
@@ -482,10 +482,10 @@ def _render_csv(
return _RenderResult(text="", estimated_rows=0,
content_size_bytes=len(raw_bytes))
header, rows = reader[0], reader[1:]
sampled_rows = rows[:sample_rows]
- payload = ["Header: " + ", ".join(header)]
+ payload = ["Header: " + _join_csv_values(header)]
if sampled_rows:
payload.append("Sample rows:")
- payload += [", ".join(str(value) for value in row) for row in
sampled_rows]
+ payload += [_join_csv_values(row) for row in sampled_rows]
return _RenderResult(
text=_truncate_text("\n".join(payload)),
estimated_rows=len(rows),
@@ -493,6 +493,15 @@ def _render_csv(
)
+def _join_csv_values(values: list[str]) -> str:
+ # Quoted the way CSV quotes them, or a value holding a comma or line break
reads as several.
+ quoted = [
+ '"' + value.replace('"', '""') + '"' if any(char in value for char in
',"\r\n') else value
+ for value in values
+ ]
+ return ", ".join(quoted)
+
+
def sample_columnar_file(
path: ObjectStoragePath, *, file_format: Literal["parquet", "avro"],
sample_rows: int, max_bytes: int
) -> ColumnarSample:
diff --git
a/providers/common/ai/tests/unit/common/ai/utils/test_file_analysis.py
b/providers/common/ai/tests/unit/common/ai/utils/test_file_analysis.py
index c7bfb7d9389..71240fe3958 100644
--- a/providers/common/ai/tests/unit/common/ai/utils/test_file_analysis.py
+++ b/providers/common/ai/tests/unit/common/ai/utils/test_file_analysis.py
@@ -397,6 +397,34 @@ class TestBuildFileAnalysisRequest:
assert "beta, 2" not in request.user_content
assert "gamma, 3" not in request.user_content
+ @pytest.mark.parametrize(
+ ("csv_value", "expected_row"),
+ [
+ pytest.param('"Smith, John"', '"Smith, John"', id="comma"),
+ pytest.param('"5"" pipe"', '"5"" pipe"', id="quote"),
+ pytest.param('"said hi\nthen left"', '"said hi\nthen left"',
id="line-feed"),
+ pytest.param('"said hi\rthen left"', '"said hi\rthen left"',
id="carriage-return"),
+ pytest.param("plain value", "plain value",
id="no-special-character"),
+ ],
+ )
+ def test_csv_values_keep_their_quotes(self, tmp_path, csv_value,
expected_row):
+ path = tmp_path / "people.csv"
+ path.write_bytes(f"name,note\nSmith,{csv_value}\n".encode())
+
+ request = build_file_analysis_request(
+ file_path=str(path),
+ file_conn_id=None,
+ prompt="Analyze",
+ multi_modal=False,
+ max_files=1,
+ max_file_size_bytes=1024,
+ max_total_size_bytes=1024,
+ max_text_chars=500,
+ sample_rows=10,
+ )
+
+ assert f"Header: name, note\nSample rows:\nSmith, {expected_row}" in
request.user_content
+
class TestFileAnalysisHelpers:
@pytest.mark.parametrize(
