kunaljubce commented on code in PR #44126:
URL: https://github.com/apache/airflow/pull/44126#discussion_r1857895095
##########
providers/src/airflow/providers/sftp/hooks/sftp.py:
##########
@@ -276,6 +277,49 @@ def delete_file(self, path: str) -> None:
conn = self.get_conn()
conn.remove(path)
+ def retrieve_directory(self, remote_full_path: str, local_full_path: str,
prefetch: bool = True) -> None:
+ """
+ Transfer the remote directory to a local location.
+
+ If local_full_path is a string path, the directory will be put
+ at that location.
+
+ :param remote_full_path: full path to the remote directory
+ :param local_full_path: full path to the local directory
+ :param prefetch: controls whether prefetch is performed (default: True)
+ """
+ if Path(local_full_path).is_dir():
+ raise AirflowException(f"{local_full_path} already exists")
+ Path(local_full_path).mkdir(parents=True)
+ files, dirs, _ = self.get_tree_map(remote_full_path)
+ for dir_path in dirs:
+ new_local_path = os.path.join(local_full_path,
os.path.relpath(dir_path, remote_full_path))
+ Path(new_local_path).mkdir(parents=True, exist_ok=True)
+ for file_path in files:
+ new_local_path = os.path.join(local_full_path,
os.path.relpath(file_path, remote_full_path))
+ self.retrieve_file(file_path, new_local_path, prefetch)
+
+ def store_directory(self, remote_full_path: str, local_full_path: str,
confirm: bool = True) -> None:
+ """
+ Transfer a local directory to the remote location.
+
+ If local_full_path is a string path, the directory will be read
+ from that location.
+
+ :param remote_full_path: full path to the remote directory
+ :param local_full_path: full path to the local directory
+ """
+ self.create_directory(remote_full_path)
Review Comment:
@Dawnpool What happens if this `remote_full_path` is present in the remote
location? Did you test this? Maybe add another check here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
