This is an automated email from the ASF dual-hosted git repository.
phanikumv pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 11bdfe4c12 Work around typing issue in examples and providers (#35494)
11bdfe4c12 is described below
commit 11bdfe4c12efa2f5d256cc49916a20beaa5487eb
Author: Tzu-ping Chung <[email protected]>
AuthorDate: Tue Nov 7 16:32:53 2023 +0900
Work around typing issue in examples and providers (#35494)
* Work around typing issue in workday examples
Since this is just an example, we'll just ignore this and not worry
too much about it.
* Rewrite Pandas column-renaming
The assignment syntax doesn't seem to be type-safe (although working at
runtime), let's use a more semantic method.
* Fix commin.sql arg type to Pandas
Some of the parameters were too lax for Pandas and need to be tightened
up.
* More column rename fixes
* Work around strict Numpy typing
This seems to be a definition issue between Numpy and Pandas? Not sure,
but this works at runtime so let's not dig too deep. Simply annotating
the variable as Any is enough to work around the problem.
* Work around complex Pandas typing
This is just too much. Let's cast and not worry about it.
* More parameters type fixes
---
airflow/example_dags/plugins/workday.py | 2 +-
airflow/providers/amazon/aws/transfers/sql_to_s3.py | 16 +++++++++-------
airflow/providers/common/sql/hooks/sql.py | 21 +++++++++++++++++----
airflow/providers/presto/hooks/presto.py | 2 +-
airflow/providers/salesforce/hooks/salesforce.py | 3 +--
.../providers/slack/transfers/base_sql_to_slack.py | 4 ++--
airflow/providers/slack/transfers/sql_to_slack.py | 4 ++--
.../slack/transfers/sql_to_slack_webhook.py | 2 +-
airflow/providers/trino/hooks/trino.py | 2 +-
tests/plugins/workday.py | 2 +-
10 files changed, 36 insertions(+), 22 deletions(-)
diff --git a/airflow/example_dags/plugins/workday.py
b/airflow/example_dags/plugins/workday.py
index 2c3299c960..848ef14051 100644
--- a/airflow/example_dags/plugins/workday.py
+++ b/airflow/example_dags/plugins/workday.py
@@ -38,7 +38,7 @@ try:
holiday_calendar = USFederalHolidayCalendar()
except ImportError:
log.warning("Could not import pandas. Holidays will not be considered.")
- holiday_calendar = None
+ holiday_calendar = None # type: ignore[assignment]
class AfterWorkdayTimetable(Timetable):
diff --git a/airflow/providers/amazon/aws/transfers/sql_to_s3.py
b/airflow/providers/amazon/aws/transfers/sql_to_s3.py
index b71554b2a3..c00784ad4a 100644
--- a/airflow/providers/amazon/aws/transfers/sql_to_s3.py
+++ b/airflow/providers/amazon/aws/transfers/sql_to_s3.py
@@ -20,7 +20,7 @@ from __future__ import annotations
import enum
from collections import namedtuple
from tempfile import NamedTemporaryFile
-from typing import TYPE_CHECKING, Iterable, Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence, cast
from typing_extensions import Literal
@@ -105,7 +105,7 @@ class SqlToS3Operator(BaseOperator):
s3_key: str,
sql_conn_id: str,
sql_hook_params: dict | None = None,
- parameters: None | Mapping | Iterable = None,
+ parameters: None | Mapping[str, Any] | list | tuple = None,
replace: bool = False,
aws_conn_id: str = "aws_default",
verify: bool | str | None = None,
@@ -158,7 +158,7 @@ class SqlToS3Operator(BaseOperator):
if "float" in df[col].dtype.name and df[col].hasnans:
# inspect values to determine if dtype of non-null values is
int or float
- notna_series = df[col].dropna().values
+ notna_series: Any = df[col].dropna().values
if np.equal(notna_series, notna_series.astype(int)).all():
# set to dtype that retains integers and supports NaNs
# The type ignore can be removed here if
https://github.com/numpy/numpy/pull/23690
@@ -196,10 +196,12 @@ class SqlToS3Operator(BaseOperator):
"""Partition dataframe using pandas groupby() method."""
if not self.groupby_kwargs:
yield "", df
- else:
- grouped_df = df.groupby(**self.groupby_kwargs)
- for group_label in grouped_df.groups:
- yield group_label,
grouped_df.get_group(group_label).reset_index(drop=True)
+ return
+ for group_label in (grouped_df :=
df.groupby(**self.groupby_kwargs)).groups:
+ yield (
+ cast(str, group_label),
+ cast("pd.DataFrame",
grouped_df.get_group(group_label).reset_index(drop=True)),
+ )
def _get_hook(self) -> DbApiHook:
self.log.debug("Get connection for %s", self.sql_conn_id)
diff --git a/airflow/providers/common/sql/hooks/sql.py
b/airflow/providers/common/sql/hooks/sql.py
index adbb95d784..950ac7ee19 100644
--- a/airflow/providers/common/sql/hooks/sql.py
+++ b/airflow/providers/common/sql/hooks/sql.py
@@ -22,6 +22,7 @@ from typing import (
TYPE_CHECKING,
Any,
Callable,
+ Generator,
Iterable,
Mapping,
Protocol,
@@ -41,6 +42,8 @@ from airflow.hooks.base import BaseHook
from airflow.version import version
if TYPE_CHECKING:
+ from pandas import DataFrame
+
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.sqlparser import DatabaseInfo
@@ -198,7 +201,12 @@ class DbApiHook(BaseForDbApiHook):
engine_kwargs = {}
return create_engine(self.get_uri(), **engine_kwargs)
- def get_pandas_df(self, sql, parameters: Iterable | Mapping[str, Any] |
None = None, **kwargs):
+ def get_pandas_df(
+ self,
+ sql,
+ parameters: list | tuple | Mapping[str, Any] | None = None,
+ **kwargs,
+ ) -> DataFrame:
"""
Execute the sql and returns a pandas dataframe.
@@ -218,14 +226,19 @@ class DbApiHook(BaseForDbApiHook):
return psql.read_sql(sql, con=conn, params=parameters, **kwargs)
def get_pandas_df_by_chunks(
- self, sql, parameters: Iterable | Mapping[str, Any] | None = None, *,
chunksize: int | None, **kwargs
- ):
+ self,
+ sql,
+ parameters: list | tuple | Mapping[str, Any] | None = None,
+ *,
+ chunksize: int,
+ **kwargs,
+ ) -> Generator[DataFrame, None, None]:
"""
Execute the sql and return a generator.
:param sql: the sql statement to be executed (str) or a list of sql
statements to execute
:param parameters: The parameters to render the SQL query with
- :param chunksize: number of rows to include in each chunk
+ :param chunksize: number of rows to include in each chunk
:param kwargs: (optional) passed into pandas.io.sql.read_sql method
"""
try:
diff --git a/airflow/providers/presto/hooks/presto.py
b/airflow/providers/presto/hooks/presto.py
index 87dd0c68bc..4cfb2b7601 100644
--- a/airflow/providers/presto/hooks/presto.py
+++ b/airflow/providers/presto/hooks/presto.py
@@ -171,7 +171,7 @@ class PrestoHook(DbApiHook):
column_descriptions = cursor.description
if data:
df = pd.DataFrame(data, **kwargs)
- df.columns = [c[0] for c in column_descriptions]
+ df.rename(columns={n: c[0] for n, c in zip(df.columns,
column_descriptions)}, inplace=True)
else:
df = pd.DataFrame(**kwargs)
return df
diff --git a/airflow/providers/salesforce/hooks/salesforce.py
b/airflow/providers/salesforce/hooks/salesforce.py
index 5bdbb6eaa8..b8dc1af10f 100644
--- a/airflow/providers/salesforce/hooks/salesforce.py
+++ b/airflow/providers/salesforce/hooks/salesforce.py
@@ -367,8 +367,7 @@ class SalesforceHook(BaseHook):
# that's because None/np.nan cannot exist in an integer column
# we should write all of our timestamps as FLOATS in our final schema
df = pd.DataFrame.from_records(query_results, exclude=["attributes"])
-
- df.columns = [column.lower() for column in df.columns]
+ df.rename(columns=str.lower, inplace=True)
# convert columns with datetime strings to datetimes
# not all strings will be datetimes, so we ignore any errors that occur
diff --git a/airflow/providers/slack/transfers/base_sql_to_slack.py
b/airflow/providers/slack/transfers/base_sql_to_slack.py
index cdc5b4cc7a..70e48e72bf 100644
--- a/airflow/providers/slack/transfers/base_sql_to_slack.py
+++ b/airflow/providers/slack/transfers/base_sql_to_slack.py
@@ -16,7 +16,7 @@
# under the License.
from __future__ import annotations
-from typing import TYPE_CHECKING, Any, Iterable, Mapping
+from typing import TYPE_CHECKING, Any, Mapping
from airflow.exceptions import AirflowException
from airflow.hooks.base import BaseHook
@@ -50,7 +50,7 @@ class BaseSqlToSlackOperator(BaseOperator):
sql: str,
sql_conn_id: str,
sql_hook_params: dict | None = None,
- parameters: Iterable | Mapping[str, Any] | None = None,
+ parameters: list | tuple | Mapping[str, Any] | None = None,
slack_proxy: str | None = None,
slack_timeout: int | None = None,
slack_retry_handlers: list[RetryHandler] | None = None,
diff --git a/airflow/providers/slack/transfers/sql_to_slack.py
b/airflow/providers/slack/transfers/sql_to_slack.py
index 2bc249c7c1..6f2ab24f85 100644
--- a/airflow/providers/slack/transfers/sql_to_slack.py
+++ b/airflow/providers/slack/transfers/sql_to_slack.py
@@ -18,7 +18,7 @@ from __future__ import annotations
import warnings
from tempfile import NamedTemporaryFile
-from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Mapping, Sequence
from airflow.exceptions import AirflowException,
AirflowProviderDeprecationWarning
from airflow.providers.slack.hooks.slack import SlackHook
@@ -74,7 +74,7 @@ class SqlToSlackApiFileOperator(BaseSqlToSlackOperator):
sql: str,
sql_conn_id: str,
sql_hook_params: dict | None = None,
- parameters: Iterable | Mapping[str, Any] | None = None,
+ parameters: list | tuple | Mapping[str, Any] | None = None,
slack_conn_id: str = SlackHook.default_conn_name,
slack_filename: str,
slack_channels: str | Sequence[str] | None = None,
diff --git a/airflow/providers/slack/transfers/sql_to_slack_webhook.py
b/airflow/providers/slack/transfers/sql_to_slack_webhook.py
index 31700ed8b0..0293b684ae 100644
--- a/airflow/providers/slack/transfers/sql_to_slack_webhook.py
+++ b/airflow/providers/slack/transfers/sql_to_slack_webhook.py
@@ -85,7 +85,7 @@ class SqlToSlackWebhookOperator(BaseSqlToSlackOperator):
slack_channel: str | None = None,
slack_message: str,
results_df_name: str = "results_df",
- parameters: Iterable | Mapping[str, Any] | None = None,
+ parameters: list | tuple | Mapping[str, Any] | None = None,
**kwargs,
) -> None:
if slack_conn_id := kwargs.pop("slack_conn_id", None):
diff --git a/airflow/providers/trino/hooks/trino.py
b/airflow/providers/trino/hooks/trino.py
index 1d1bbd1cd2..03195fe452 100644
--- a/airflow/providers/trino/hooks/trino.py
+++ b/airflow/providers/trino/hooks/trino.py
@@ -189,7 +189,7 @@ class TrinoHook(DbApiHook):
column_descriptions = cursor.description
if data:
df = pd.DataFrame(data, **kwargs)
- df.columns = [c[0] for c in column_descriptions]
+ df.rename(columns={n: c[0] for n, c in zip(df.columns,
column_descriptions)}, inplace=True)
else:
df = pd.DataFrame(**kwargs)
return df
diff --git a/tests/plugins/workday.py b/tests/plugins/workday.py
index 20363a69e7..72202fcc5c 100644
--- a/tests/plugins/workday.py
+++ b/tests/plugins/workday.py
@@ -34,7 +34,7 @@ try:
holiday_calendar = USFederalHolidayCalendar()
except ImportError:
log.warning("Could not import pandas. Holidays will not be considered.")
- holiday_calendar = None
+ holiday_calendar = None # type: ignore[assignment]
class AfterWorkdayTimetable(Timetable):