caishunfeng commented on code in PR #12076:
URL:
https://github.com/apache/dolphinscheduler/pull/12076#discussion_r1005130559
##########
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java:
##########
@@ -749,4 +899,67 @@ public void deleteTenant(String tenantCode) throws
Exception {
public ResUploadType returnStorageType() {
return ResUploadType.HDFS;
}
+
+ @Override
+ public List<StorageEntity> listFilesStatusRecursively(String path, String
defaultPath, String tenantCode,
+ ResourceType type) {
+ List<StorageEntity> storageEntityList = new ArrayList<>();
+
+ LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
+
+ do {
+ String pathToExplore = "";
+ if (foldersToFetch.size() == 0) {
+ pathToExplore = path;
+ } else {
+ pathToExplore = foldersToFetch.pop().getFullName();
+ }
+
+ try {
+ List<StorageEntity> tempList = listFilesStatus(pathToExplore,
defaultPath, tenantCode, type);
+
+ for (StorageEntity temp : tempList) {
+ if (temp.isDirectory()) {
+ foldersToFetch.add(temp);
+ }
+ }
+
+ storageEntityList.addAll(tempList);
+ } catch (FileNotFoundException e) {
+ logger.error(e.getMessage() + "Resource path:" +
pathToExplore, e);
+ // return the resources fetched before error occurs.
+ return storageEntityList;
+ } catch (IOException e) {
+ logger.error(e.getMessage() + "Resource path:" +
pathToExplore, e);
Review Comment:
```suggestion
logger.error("Resource path: {}", pathToExplore, e);
```
##########
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java:
##########
@@ -384,4 +433,197 @@ private void deleteDirectory(String directoryName) {
public ResUploadType returnStorageType() {
return ResUploadType.S3;
}
+
+ @Override
+ public List<StorageEntity> listFilesStatusRecursively(String path, String
defaultPath, String tenantCode,
+ ResourceType type) {
+ List<StorageEntity> storageEntityList = new ArrayList<>();
+
+ LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
+
+ do {
+ String pathToExplore = "";
+ if (foldersToFetch.size() == 0) {
+ pathToExplore = path;
+ } else {
+ pathToExplore = foldersToFetch.pop().getFullName();
+ }
+
+ try {
+ List<StorageEntity> tempList = listFilesStatus(pathToExplore,
defaultPath, tenantCode, type);
+
+ for (StorageEntity temp : tempList) {
+ if (temp.isDirectory()) {
+ foldersToFetch.add(temp);
+ }
+ }
+
+ storageEntityList.addAll(tempList);
+ } catch (AmazonServiceException e) {
+ logger.error(e.getMessage() + "Resource path:" +
pathToExplore, e);
Review Comment:
```suggestion
logger.error("Resource path: {}", pathToExplore, e);
```
##########
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java:
##########
@@ -171,6 +183,26 @@ public String getResourceFileName(String tenantCode,
String fileName) {
return String.format(FORMAT_S_S, getS3ResDir(tenantCode), fileName);
}
+ @Override
+ public String getResourceFileName(String fullName) {
Review Comment:
It seems the same method `getResourceFileName` in `HadoopUtils`?
##########
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java:
##########
@@ -206,21 +238,38 @@ public void download(String tenantCode, String
srcFilePath, String dstFilePath,
}
@Override
- public boolean exists(String tenantCode, String fileName) throws
IOException {
- return s3Client.doesObjectExist(BUCKET_NAME, fileName);
+ public boolean exists(String fullName) throws IOException {
+ return s3Client.doesObjectExist(BUCKET_NAME, fullName);
}
@Override
- public boolean delete(String tenantCode, String filePath, boolean
recursive) throws IOException {
+ public boolean delete(String fullName, boolean recursive) throws
IOException {
try {
- s3Client.deleteObject(BUCKET_NAME, filePath);
+ s3Client.deleteObject(BUCKET_NAME, fullName);
return true;
} catch (AmazonServiceException e) {
- logger.error("delete the object error,the resource path is {}",
filePath);
+ logger.error("delete the object error,the resource path is {}",
fullName);
return false;
}
}
+ @Override
+ public boolean delete(String fullName, List<String> childrenPathList,
boolean recursive) throws IOException {
+ // append the resource fullName to the list for deletion.
+ childrenPathList.add(fullName);
+
+ DeleteObjectsRequest deleteObjectsRequest = new
DeleteObjectsRequest(BUCKET_NAME)
+ .withKeys(childrenPathList.stream().toArray(String[]::new));
+ try {
+ s3Client.deleteObjects(deleteObjectsRequest);
+ } catch (AmazonServiceException e) {
+ System.out.println(e.getMessage());
Review Comment:
logger.error
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
