uranusjr commented on a change in pull request #16792:
URL: https://github.com/apache/airflow/pull/16792#discussion_r698291886
##########
File path: airflow/providers/sftp/operators/sftp.py
##########
@@ -134,29 +176,81 @@ def execute(self, context: Any) -> str:
with self.ssh_hook.get_conn() as ssh_client:
sftp_client = ssh_client.open_sftp()
- if self.operation.lower() == SFTPOperation.GET:
- local_folder = os.path.dirname(self.local_filepath)
- if self.create_intermediate_dirs:
- Path(local_folder).mkdir(parents=True, exist_ok=True)
- file_msg = f"from {self.remote_filepath} to
{self.local_filepath}"
- self.log.info("Starting to transfer %s", file_msg)
- sftp_client.get(self.remote_filepath, self.local_filepath)
- else:
- remote_folder = os.path.dirname(self.remote_filepath)
- if self.create_intermediate_dirs:
- _make_intermediate_dirs(
- sftp_client=sftp_client,
- remote_directory=remote_folder,
+ if self.local_filepath and self.remote_filepath:
+ if isinstance(self.local_filepath, list) and
isinstance(self.remote_filepath, str):
+ for file_path in self.local_filepath:
+ local_folder = os.path.dirname(file_path)
+ local_file = os.path.basename(file_path)
+ file_msg = file_path
+ self._transfer(sftp_client, local_folder,
local_file, self.remote_filepath)
+ elif isinstance(self.remote_filepath, list) and
isinstance(self.local_filepath, str):
+ for file_path in self.remote_filepath:
+ remote_folder = os.path.dirname(file_path)
+ remote_file = os.path.basename(file_path)
+ file_msg = file_path
+ self._transfer(sftp_client, self.local_filepath,
remote_file, remote_folder)
+ elif isinstance(self.remote_filepath, str) and
isinstance(self.local_filepath, str):
+ local_folder = os.path.dirname(self.local_filepath)
+ file_msg = self.local_filepath
+ self._transfer(
+ sftp_client,
+ local_folder,
+ self.local_filepath,
+ self.remote_filepath,
+ only_file=True,
)
- file_msg = f"from {self.local_filepath} to
{self.remote_filepath}"
- self.log.info("Starting to transfer file %s", file_msg)
- sftp_client.put(self.local_filepath, self.remote_filepath,
confirm=self.confirm)
+ elif self.local_folder and self.remote_folder:
+ if self.operation.lower() == SFTPOperation.PUT:
+ files_list =
self._search_files(os.listdir(self.local_folder))
+ for file in files_list:
+ local_file = os.path.basename(file)
+ file_msg = file
+ self._transfer(sftp_client, self.local_folder,
local_file, self.remote_folder)
+ elif self.operation.lower() == SFTPOperation.GET:
+ files_list =
self._search_files(sftp_client.listdir(self.remote_folder))
+ for file in files_list:
+ remote_file = ntpath.basename(file)
Review comment:
I think this should be `os.path.basename` like in the `PUT` block.
##########
File path: airflow/providers/sftp/operators/sftp.py
##########
@@ -88,6 +124,9 @@ def __init__(
remote_host=None,
local_filepath=None,
remote_filepath=None,
+ local_folder=None,
+ remote_folder=None,
+ regexp_mask=None,
Review comment:
I am really not fond of this argument list since it’s too easy to pass
in nonsensical combinations e.g.
```python
SFTPOperator(
local_filepath="/my/file",
local_folder="/my/dir",
)
```
But I understand this is not easy to change the arguments due to
compatibility considerations.
Perhapes what we should do is to introduce a separate operator (say
`SFTPBatchOperator`) to handle the new feature, instead of bolting it onto the
existing one.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
