guotongfei commented on a change in pull request #18755:
URL: https://github.com/apache/airflow/pull/18755#discussion_r723416681
##########
File path: airflow/providers/amazon/aws/transfers/mysql_to_s3.py
##########
@@ -92,15 +108,27 @@ def __init__(
self.aws_conn_id = aws_conn_id
self.verify = verify
- self.pd_csv_kwargs = pd_csv_kwargs or {}
- if "path_or_buf" in self.pd_csv_kwargs:
- raise AirflowException('The argument path_or_buf is not allowed,
please remove it')
- if "index" not in self.pd_csv_kwargs:
- self.pd_csv_kwargs["index"] = index
- if "header" not in self.pd_csv_kwargs:
- self.pd_csv_kwargs["header"] = header
+ if file_format == "csv":
+ self.file_format = FILE_FORMAT.CSV
+ else:
+ self.file_format = FILE_FORMAT.PARQUET
- def _fix_int_dtypes(self, df: pd.DataFrame) -> None:
+ self.pd_kwargs = pd_kwargs or pd_csv_kwargs or {}
+ if self.file_format == FILE_FORMAT.CSV:
+ if "path_or_buf" in self.pd_kwargs:
+ raise AirflowException('The argument path_or_buf is not
allowed, please remove it')
+ if "index" not in self.pd_kwargs:
+ self.pd_kwargs["index"] = index
+ if "header" not in self.pd_kwargs:
+ self.pd_kwargs["header"] = header
+ else:
+ if pd_csv_kwargs is not None:
+ raise AirflowException(
+ f"The destination file format is parquet so pd_csv_kwargs
shouldn't be set."
+ )
Review comment:
done.
##########
File path: airflow/providers/amazon/aws/transfers/mysql_to_s3.py
##########
@@ -54,12 +59,17 @@ class MySQLToS3Operator(BaseOperator):
You can specify this argument if you want to use a different
CA cert bundle than the one used by botocore.
:type verify: bool or str
- :param pd_csv_kwargs: arguments to include in pd.to_csv (header, index,
columns...)
+ :param pd_csv_kwargs: deprecated. Use pd_kwargs instead.
+ Arguments to include in pd.to_csv (header, index, columns...)
Review comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
