This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d26160dbf39 Parse HookToolset :param descriptions by indentation
(#76701)
d26160dbf39 is described below
commit d26160dbf39474687378a7906586a78253fdc5ba
Author: Aarmen Sidhu <[email protected]>
AuthorDate: Mon Nov 9 12:42:19 2026 -0700
Parse HookToolset :param descriptions by indentation (#76701)
HookToolset matched each :param description with a regex that ran until the
next line starting with ":" or the end of the docstring. So the last
parameter absorbed everything after it (601 characters, including a code
block, for apply_wildcard in S3Hook.list_keys, the docs' own example), a
continuation line starting with a role such as :class: cut a description
short, and a parameter with no description took the next field as its own.
Follow reStructuredText field lists instead: a description continues while
lines are indented deeper than its :param line.
Generated-by: Claude Code following the guidelines
* Keep Google-style Args when a :param has no description
An empty :param field still left an entry in params, so the Google-style
fallback was skipped and then the empty entry was filtered out, returning
nothing. Drop empty descriptions before deciding whether to fall back.
Reported by @himanshuchawla0807-gif.
Generated-by: Claude Code following the guidelines
* Fix erf_uri param docs and test a description continuing after a blank
line
GoogleDisplayVideo360Hook.erf_uri's entity_type continuation line sat at
the same indent as its :param line, so the field ended after the first
line, and partner_id was missing its colon. A scan of every provider hook
found no other docstring with either problem.
The new parser case covers a description whose second paragraph follows a
blank line, shaped like S3Hook.copy_object's source_bucket_key.
Generated-by: Claude Code following the guidelines
---------
Co-authored-by: aarmens702-hub
<[email protected]>
---
.../airflow/providers/common/ai/toolsets/hook.py | 22 +++++--
.../ai/tests/unit/common/ai/toolsets/test_hook.py | 69 ++++++++++++++++++++++
.../marketing_platform/hooks/display_video.py | 4 +-
3 files changed, 87 insertions(+), 8 deletions(-)
diff --git
a/providers/common/ai/src/airflow/providers/common/ai/toolsets/hook.py
b/providers/common/ai/src/airflow/providers/common/ai/toolsets/hook.py
index aa35dcc80b6..7c7f57c6c4b 100644
--- a/providers/common/ai/src/airflow/providers/common/ai/toolsets/hook.py
+++ b/providers/common/ai/src/airflow/providers/common/ai/toolsets/hook.py
@@ -502,7 +502,7 @@ def _extract_description(method: Callable[..., Any]) -> str:
# Matches Sphinx-style `:param name:` and Google-style `name:` under an
``Args:`` block.
-_SPHINX_PARAM_RE = re.compile(r":param\s+(\w+):\s*(.+?)(?=\n\s*:|$)",
re.DOTALL)
+_SPHINX_PARAM_RE = re.compile(r"^(\s*):param\s+(\w+):(.*)$")
_GOOGLE_ARGS_RE = re.compile(r"^\s{2,}(\w+)\s*(?:\(.+?\))?:\s*(.+)",
re.MULTILINE)
@@ -510,12 +510,22 @@ def _parse_param_docs(docstring: str) -> dict[str, str]:
"""Parse parameter descriptions from Sphinx or Google-style docstrings."""
params: dict[str, str] = {}
- # Try Sphinx style first.
- for match in _SPHINX_PARAM_RE.finditer(docstring):
- name = match.group(1)
- desc = " ".join(match.group(2).split())
- params[name] = desc
+ # Try Sphinx style first. As in a reStructuredText field list, a
description runs on
+ # through the lines indented deeper than its ``:param`` line, so neither a
continuation
+ # line starting with a role such as ``:class:`` nor an example block after
the last
+ # parameter changes where it ends.
+ name: str | None = None
+ indent = 0
+ for line in docstring.splitlines():
+ if match := _SPHINX_PARAM_RE.match(line):
+ name, indent = match.group(2), len(match.group(1))
+ params[name] = match.group(3)
+ elif name and (not line.strip() or len(line) - len(line.lstrip()) >
indent):
+ params[name] += " " + line
+ else:
+ name = None
+ params = {param: desc for param, text in params.items() if (desc := "
".join(text.split()))}
if params:
return params
diff --git a/providers/common/ai/tests/unit/common/ai/toolsets/test_hook.py
b/providers/common/ai/tests/unit/common/ai/toolsets/test_hook.py
index 9722a8a5167..2a781988c91 100644
--- a/providers/common/ai/tests/unit/common/ai/toolsets/test_hook.py
+++ b/providers/common/ai/tests/unit/common/ai/toolsets/test_hook.py
@@ -782,6 +782,75 @@ class TestParseParamDocs:
assert result["name"] == "The name of the thing."
assert result["count"] == "How many items."
+ def test_sphinx_last_param_stops_at_dedented_text(self):
+ # Shaped like S3Hook.list_keys, whose example block followed
apply_wildcard into its description.
+ docstring = """List keys.
+
+ :param prefix: a key prefix
+ :param apply_wildcard: whether to treat '*' as a wildcard.
+
+ **Example**: Returns the matching keys:
+
+ .. code-block:: python
+
+ hook.list_keys(prefix="data/*", apply_wildcard=True)
+
+ :return: a list of matched keys
+ """
+ result = _parse_param_docs(docstring)
+ assert result == {"prefix": "a key prefix", "apply_wildcard": "whether
to treat '*' as a wildcard."}
+
+ def test_sphinx_continuation_line_starting_with_a_role(self):
+ # Shaped like BigQueryHook.create_routine, whose description was cut
off at "Either a".
+ docstring = """Create a routine.
+
+ :param routine: The routine to create. Either a
+ :class:`~google.cloud.bigquery.routine.Routine` instance or a dict.
+ :param routine_id: The routine identifier.
+ """
+ result = _parse_param_docs(docstring)
+ assert result["routine"] == (
+ "The routine to create. Either a
:class:`~google.cloud.bigquery.routine.Routine` instance or a dict."
+ )
+ assert result["routine_id"] == "The routine identifier."
+
+ def test_sphinx_description_continues_after_a_blank_line(self):
+ # Shaped like S3Hook.copy_object, whose source_bucket_key has an
indented second paragraph.
+ docstring = """Copy an object.
+
+ :param source_bucket_key: The key of the source object.
+
+ It can be either full s3:// style url or relative path from root
level.
+ :param dest_bucket_key: The key of the object to copy to.
+ """
+ assert _parse_param_docs(docstring) == {
+ "source_bucket_key": (
+ "The key of the source object. "
+ "It can be either full s3:// style url or relative path from
root level."
+ ),
+ "dest_bucket_key": "The key of the object to copy to.",
+ }
+
+ def test_sphinx_param_without_description_is_left_out(self):
+ # Shaped like GKEHook._append_label, whose val was described as
":return: The cluster proto ...".
+ docstring = """Append a label.
+
+ :param key: The key label
+ :param val:
+ :return: The cluster proto updated with new label
+ """
+ assert _parse_param_docs(docstring) == {"key": "The key label"}
+
+ def test_empty_sphinx_param_does_not_hide_google_args(self):
+ docstring = """Read records.
+
+ Args:
+ path: Path to read.
+
+ :param undocumented:
+ """
+ assert _parse_param_docs(docstring) == {"path": "Path to read."}
+
def test_google_style(self):
docstring = """Do something.
diff --git
a/providers/google/src/airflow/providers/google/marketing_platform/hooks/display_video.py
b/providers/google/src/airflow/providers/google/marketing_platform/hooks/display_video.py
index 6ae63441342..bb9d33428e6 100644
---
a/providers/google/src/airflow/providers/google/marketing_platform/hooks/display_video.py
+++
b/providers/google/src/airflow/providers/google/marketing_platform/hooks/display_video.py
@@ -72,9 +72,9 @@ class GoogleDisplayVideo360Hook(GoogleBaseHook):
More information:
https://developers.google.com/bid-manager/guides/entity-read/overview
- :param partner_id The numeric ID of your Partner.
+ :param partner_id: The numeric ID of your Partner.
:param entity_type: The type of file Partner, Advertiser,
InsertionOrder,
- LineItem, Creative, Pixel, InventorySource, UserList,
UniversalChannel, and summary.
+ LineItem, Creative, Pixel, InventorySource, UserList,
UniversalChannel, and summary.
"""
return [f"gdbm-{partner_id}/entity/{{{{ ds_nodash
}}}}.*.{entity_type}.json"]
