rickchengx commented on code in PR #16141:
URL:
https://github.com/apache/dolphinscheduler/pull/16141#discussion_r1640634827
##########
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java:
##########
@@ -17,1286 +17,404 @@
package org.apache.dolphinscheduler.api.service.impl;
-import static org.apache.dolphinscheduler.common.constants.Constants.ALIAS;
-import static org.apache.dolphinscheduler.common.constants.Constants.CONTENT;
-import static
org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING;
-import static
org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
-import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_SS;
-import static org.apache.dolphinscheduler.common.constants.Constants.JAR;
-import static org.apache.dolphinscheduler.common.constants.Constants.PERIOD;
-
-import
org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
-import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter;
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
+import
org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
+import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceDto;
+import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest;
+import org.apache.dolphinscheduler.api.dto.resources.DownloadFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest;
+import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto;
+import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest;
+import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
+import org.apache.dolphinscheduler.api.dto.resources.QueryResourceDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
+import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto;
+import
org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
import
org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor;
import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
-import org.apache.dolphinscheduler.api.utils.RegexUtils;
-import org.apache.dolphinscheduler.api.utils.Result;
-import org.apache.dolphinscheduler.common.constants.Constants;
-import org.apache.dolphinscheduler.common.enums.ProgramType;
-import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import
org.apache.dolphinscheduler.api.validator.resource.CreateDirectoryDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.CreateDirectoryRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.CreateFileDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.CreateFileFromContentDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.DeleteResourceDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.DownloadFileDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.FetchFileContentDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.FileFromContentRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.FileRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.PagingResourceItemRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.RenameDirectoryDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.RenameDirectoryRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.RenameFileDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.RenameFileRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.UpdateFileDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.UpdateFileFromContentDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.UpdateFileFromContentRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.UpdateFileRequestTransformer;
+import org.apache.dolphinscheduler.api.vo.ResourceItemVO;
+import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse;
import org.apache.dolphinscheduler.common.utils.FileUtils;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
-import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
-import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
import java.io.File;
-import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Paths;
-import java.text.MessageFormat;
-import java.time.LocalDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.UUID;
import java.util.stream.Collectors;
+import javax.servlet.http.HttpServletResponse;
+
+import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
-import com.google.common.io.Files;
-
@Service
@Slf4j
public class ResourcesServiceImpl extends BaseServiceImpl implements
ResourcesService {
@Autowired
- private UdfFuncMapper udfFunctionMapper;
-
- @Autowired
- private TenantMapper tenantMapper;
+ private TenantDao tenantDao;
@Autowired
private UserMapper userMapper;
- @Autowired(required = false)
- private StorageOperate storageOperate;
-
- /**
- * create directory
- *
- * @param loginUser login user
- * @param name alias
- * @param type type
- * @param pid parent id
- * @param currentDir current directory
- * @return create directory result
- */
- @Override
- @Transactional
- public Result<Object> createDirectory(User loginUser, String name,
ResourceType type, int pid, String currentDir) {
- Result<Object> result = new Result<>();
- if (FileUtils.directoryTraversal(name)) {
- log.warn("Parameter name is invalid, name:{}.",
RegexUtils.escapeNRT(name));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, currentDir);
-
- String userResRootPath = ResourceType.UDF.equals(type) ?
storageOperate.getUdfDir(tenantCode)
- : storageOperate.getResDir(tenantCode);
- String fullName = !currentDir.contains(userResRootPath) ?
userResRootPath + name : currentDir + name;
-
- try {
- if (checkResourceExists(fullName)) {
- log.error("resource directory {} has exist, can't recreate",
fullName);
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- } catch (Exception e) {
- log.warn("Resource exists, can not create again, fullName:{}.",
fullName, e);
- throw new ServiceException("resource already exists, can't
recreate");
- }
-
- // create directory in storage
- createDirectory(loginUser, fullName, type, result);
- return result;
- }
-
- /**
- * create resource
- *
- * @param loginUser login user
- * @param name alias
- * @param type type
- * @param file file
- * @param currentDir current directory
- * @return create result code
- */
- @Override
- @Transactional
- public Result<Object> uploadResource(User loginUser, String name,
ResourceType type, MultipartFile file,
- String currentDir) {
- Result<Object> result = new Result<>();
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, currentDir);
-
- result = verifyFile(name, type, file);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- return result;
- }
-
- // check resource name exists
- String userResRootPath = ResourceType.UDF.equals(type) ?
storageOperate.getUdfDir(tenantCode)
- : storageOperate.getResDir(tenantCode);
- String currDirNFileName = !currentDir.contains(userResRootPath) ?
userResRootPath + name : currentDir + name;
-
- try {
- if (checkResourceExists(currDirNFileName)) {
- log.error("resource {} has exist, can't recreate",
RegexUtils.escapeNRT(name));
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- } catch (Exception e) {
- throw new ServiceException("resource already exists, can't
recreate");
- }
- if (currDirNFileName.length() >
Constants.RESOURCE_FULL_NAME_MAX_LENGTH) {
- log.error(
- "Resource file's name is longer than max full name length,
fullName:{}, "
- + "fullNameSize:{}, maxFullNameSize:{}",
- RegexUtils.escapeNRT(name), currDirNFileName.length(),
Constants.RESOURCE_FULL_NAME_MAX_LENGTH);
- putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR);
- return result;
- }
-
- // fail upload
- if (!upload(loginUser, currDirNFileName, file, type)) {
- log.error("upload resource: {} file: {} failed.",
RegexUtils.escapeNRT(name),
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.STORE_OPERATE_CREATE_ERROR);
- throw new ServiceException(
- String.format("upload resource: %s file: %s failed.",
name, file.getOriginalFilename()));
- } else
- ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
- log.info("Upload resource file complete, resourceName:{},
fileName:{}.", RegexUtils.escapeNRT(name),
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.SUCCESS);
- return result;
- }
-
- /**
- * check resource is exists
- *
- * @param fullName fullName
- * @return true if resource exists
- */
- private boolean checkResourceExists(String fullName) {
- Boolean existResource = false;
- try {
- existResource = storageOperate.exists(fullName);
- } catch (IOException e) {
- log.error("error occurred when checking resource: " + fullName, e);
- }
- return Boolean.TRUE.equals(existResource);
- }
-
- /**
- * update resource
- *
- * @param loginUser login user
- * @param resourceFullName resource full name
- * @param resTenantCode tenantCode in the request field "resTenantCode"
for tenant code owning the resource,
- * can be different from the login user in the
case of logging in as admin users.
- * @param name name
- * @param type resource type
- * @param file resource file
- * @return update result code
- */
- @Override
- @Transactional
- public Result<Object> updateResource(User loginUser, String
resourceFullName, String resTenantCode, String name,
- ResourceType type, MultipartFile
file) {
- Result<Object> result = new Result<>();
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, resourceFullName);
+ @Autowired
+ private StorageOperator storageOperator;
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode))
{
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
+ @Autowired
+ private CreateDirectoryRequestTransformer
createDirectoryRequestTransformer;
- String defaultPath = storageOperate.getDir(type, tenantCode);
+ @Autowired
+ private CreateDirectoryDtoValidator createDirectoryDtoValidator;
- StorageEntity resource;
- try {
- resource = storageOperate.getFileStatus(resourceFullName,
defaultPath, resTenantCode, type);
- } catch (Exception e) {
- log.error("Get file status fail, resource path: {}",
resourceFullName, e);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- throw new ServiceException((String.format("Get file status fail,
resource path: %s", resourceFullName)));
- }
+ @Autowired
+ private RenameDirectoryRequestTransformer
renameDirectoryRequestTransformer;
- // TODO: deal with OSS
- if (resource.isDirectory() &&
storageOperate.returnStorageType().equals(ResUploadType.S3)
- && !resource.getFileName().equals(name)) {
- log.warn("Directory in S3 storage can not be renamed.");
- putMsg(result, Status.S3_CANNOT_RENAME);
- return result;
- }
+ @Autowired
+ private RenameDirectoryDtoValidator renameDirectoryDtoValidator;
- // check if updated name of the resource already exists
- String originFullName = resource.getFullName();
- String originResourceName = resource.getAlias();
-
- // the format of hdfs folders in the implementation has a "/" at the
very end, we need to remove it.
- originFullName = originFullName.endsWith("/") ?
StringUtils.chop(originFullName) : originFullName;
- name = name.endsWith("/") ? StringUtils.chop(name) : name;
- // updated fullName
- String fullName = String.format(FORMAT_SS,
- originFullName.substring(0,
originFullName.lastIndexOf(FOLDER_SEPARATOR) + 1), name);
- if (!originResourceName.equals(name)) {
- try {
- if (checkResourceExists(fullName)) {
- log.error("resource {} already exists, can't recreate",
fullName);
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- } catch (Exception e) {
- throw new ServiceException(String.format("error occurs while
querying resource: %s", fullName));
- }
+ @Autowired
+ private RenameFileRequestTransformer renameFileRequestTransformer;
- }
+ @Autowired
+ private RenameFileDtoValidator renameFileDtoValidator;
- result = verifyFile(name, type, file);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- return result;
- }
+ @Autowired
+ private FileFromContentRequestTransformer
createFileFromContentRequestTransformer;
- Date now = new Date();
+ @Autowired
+ private CreateFileFromContentDtoValidator
createFileFromContentDtoValidator;
- resource.setAlias(name);
- resource.setFileName(name);
- resource.setFullName(fullName);
- resource.setUpdateTime(now);
- if (file != null) {
- resource.setSize(file.getSize());
- }
+ @Autowired
+ private FetchFileContentDtoValidator fetchFileContentDtoValidator;
- // if name unchanged, return directly without moving on HDFS
- if (originResourceName.equals(name) && file == null) {
- return result;
- }
+ @Autowired
+ private UpdateFileFromContentRequestTransformer
updateFileFromContentRequestTransformer;
- if (file != null) {
- // fail upload
- if (!upload(loginUser, fullName, file, type)) {
- log.error("Storage operation error, resourceName:{},
originFileName:{}.", name,
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.HDFS_OPERATION_ERROR);
- throw new ServiceException(
- String.format("upload resource: %s file: %s failed.",
name, file.getOriginalFilename()));
- }
- if (!fullName.equals(originFullName)) {
- try {
- storageOperate.delete(originFullName, false);
- } catch (IOException e) {
- log.error("Resource delete error, resourceFullName:{}.",
originFullName, e);
- throw new ServiceException(String.format("delete resource:
%s failed.", originFullName));
- }
- }
+ @Autowired
+ private UpdateFileFromContentDtoValidator
updateFileFromContentDtoValidator;
- ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
- return result;
- }
+ @Autowired
+ private FileRequestTransformer createFileRequestTransformer;
- // get the path of dest file in hdfs
- String destHdfsFileName = fullName;
- try {
- log.info("start copy {} -> {}", originFullName, destHdfsFileName);
- storageOperate.copy(originFullName, destHdfsFileName, true, true);
- putMsg(result, Status.SUCCESS);
- } catch (Exception e) {
- log.error(MessageFormat.format(" copy {0} -> {1} fail",
originFullName, destHdfsFileName), e);
- putMsg(result, Status.HDFS_COPY_FAIL);
- throw new ServiceException(
- MessageFormat.format(Status.HDFS_COPY_FAIL.getMsg(),
originFullName, destHdfsFileName));
- }
+ @Autowired
+ private CreateFileDtoValidator createFileDtoValidator;
- return result;
- }
+ @Autowired
+ private UpdateFileRequestTransformer updateFileRequestTransformer;
- private Result<Object> verifyFile(String name, ResourceType type,
MultipartFile file) {
- Result<Object> result = new Result<>();
- putMsg(result, Status.SUCCESS);
+ @Autowired
+ private UpdateFileDtoValidator updateFileDtoValidator;
- if (FileUtils.directoryTraversal(name)) {
- log.warn("Parameter file alias name verify failed,
fileAliasName:{}.", RegexUtils.escapeNRT(name));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
+ @Autowired
+ private DeleteResourceDtoValidator deleteResourceDtoValidator;
- if (file != null &&
FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename())))
{
- log.warn("File original name verify failed, fileOriginalName:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
+ @Autowired
+ private DownloadFileDtoValidator downloadFileDtoValidator;
- if (file != null) {
- // file is empty
- if (file.isEmpty()) {
- log.warn("Parameter file is empty, fileOriginalName:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.RESOURCE_FILE_IS_EMPTY);
- return result;
- }
-
- // file suffix
- String fileSuffix =
Files.getFileExtension(file.getOriginalFilename());
- String nameSuffix = Files.getFileExtension(name);
-
- // determine file suffix
- if (!fileSuffix.equalsIgnoreCase(nameSuffix)) {
- // rename file suffix and original suffix must be consistent
- log.warn("Rename file suffix and original suffix must be
consistent, fileOriginalName:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.RESOURCE_SUFFIX_FORBID_CHANGE);
- return result;
- }
-
- // If resource type is UDF, only jar packages are allowed to be
uploaded, and the suffix must be .jar
- if (Constants.UDF.equals(type.name()) &&
!JAR.equalsIgnoreCase(fileSuffix)) {
- log.warn(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg());
- putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR);
- return result;
- }
- if (file.getSize() > Constants.MAX_FILE_SIZE) {
- log.warn(
- "Resource file size is larger than max file size,
fileOriginalName:{}, fileSize:{}, maxFileSize:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()),
file.getSize(), Constants.MAX_FILE_SIZE);
- putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT);
- return result;
- }
- }
- return result;
- }
+ @Autowired
+ private PagingResourceItemRequestTransformer
pagingResourceItemRequestTransformer;
- /**
- * query resources list paging
- *
- * @param loginUser login user
- * @param fullName resource full name
- * @param resTenantCode tenantCode in the request field "resTenantCode"
for tenant code owning the resource,
- * can be different from the login user in the case
of logging in as admin users.
- * @param type resource type
- * @param searchVal search value
- * @param pageNo page number
- * @param pageSize page size
- * @return resource list page
- */
@Override
- public Result<PageInfo<StorageEntity>> queryResourceListPaging(User
loginUser, String fullName,
- String
resTenantCode, ResourceType type,
- String
searchVal, Integer pageNo, Integer pageSize) {
- Result<PageInfo<StorageEntity>> result = new Result<>();
- PageInfo<StorageEntity> pageInfo = new PageInfo<>(pageNo, pageSize);
- if (storageOperate == null) {
- log.warn("The resource storage is not opened.");
- return Result.success(pageInfo);
- }
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, fullName);
+ public void createDirectory(CreateDirectoryRequest createDirectoryRequest)
{
+ CreateDirectoryDto createDirectoryDto =
createDirectoryRequestTransformer.transform(createDirectoryRequest);
+ createDirectoryDtoValidator.validate(createDirectoryDto);
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode))
{
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
-
- List<StorageEntity> resourcesList;
- try {
- resourcesList = queryStorageEntityList(loginUser, fullName, type,
tenantCode, false);
- } catch (ServiceException e) {
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
-
- // remove leading and trailing spaces in searchVal
- String trimmedSearchVal = searchVal != null ? searchVal.trim() : "";
- // filter based on trimmed searchVal
- List<StorageEntity> filteredResourceList = resourcesList.stream()
- .filter(x ->
x.getFileName().contains(trimmedSearchVal)).collect(Collectors.toList());
- // inefficient pagination
- List<StorageEntity> slicedResourcesList =
filteredResourceList.stream().skip((long) (pageNo - 1) * pageSize)
- .limit(pageSize).collect(Collectors.toList());
-
- pageInfo.setTotal(filteredResourceList.size());
- pageInfo.setTotalList(slicedResourcesList);
- result.setData(pageInfo);
- putMsg(result, Status.SUCCESS);
- return result;
- }
-
- private List<StorageEntity> queryStorageEntityList(User loginUser, String
fullName, ResourceType type,
- String tenantCode,
boolean recursive) {
- String defaultPath = "";
- List<StorageEntity> resourcesList = new ArrayList<>();
- String resourceStorageType =
- PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE,
ResUploadType.LOCAL.name());
- if (isAdmin(loginUser) && StringUtils.isBlank(fullName)) {
- // list all tenants' resources to admin users in the root directory
- List<User> userList = userMapper.selectList(null);
- Set<String> visitedTenantEntityCode = new HashSet<>();
- for (User userEntity : userList) {
- String tenantEntityCode = getTenantCode(userEntity);
- if (!visitedTenantEntityCode.contains(tenantEntityCode)) {
- defaultPath = storageOperate.getResDir(tenantEntityCode);
- if (type.equals(ResourceType.UDF)) {
- defaultPath =
storageOperate.getUdfDir(tenantEntityCode);
- }
- try {
- resourcesList.addAll(recursive
- ?
storageOperate.listFilesStatusRecursively(defaultPath, defaultPath,
tenantEntityCode,
- type)
- : storageOperate.listFilesStatus(defaultPath,
defaultPath, tenantEntityCode, type));
-
- visitedTenantEntityCode.add(tenantEntityCode);
- } catch (Exception e) {
- log.error(e.getMessage() + " Resource path: {}",
defaultPath, e);
- throw new ServiceException(
- String.format(e.getMessage() + " make sure
resource path: %s exists in %s", defaultPath,
- resourceStorageType));
- }
- }
- }
- } else {
- defaultPath = storageOperate.getResDir(tenantCode);
- if (type.equals(ResourceType.UDF)) {
- defaultPath = storageOperate.getUdfDir(tenantCode);
- }
-
- try {
- if (StringUtils.isBlank(fullName)) {
- fullName = defaultPath;
- }
- resourcesList =
- recursive ?
storageOperate.listFilesStatusRecursively(fullName, defaultPath, tenantCode,
type)
- : storageOperate.listFilesStatus(fullName,
defaultPath, tenantCode, type);
- } catch (Exception e) {
- log.error(e.getMessage() + " Resource path: {}", fullName, e);
- throw new ServiceException(String.format(e.getMessage() + "
make sure resource path: %s exists in %s",
- defaultPath, resourceStorageType));
- }
- }
-
- return resourcesList;
+
storageOperator.createStorageDir(createDirectoryDto.getDirectoryAbsolutePath());
+ log.info("Success create directory: {}",
createDirectoryRequest.getParentAbsoluteDirectory());
}
- /**
- * create directory
- * xxx The steps to verify resources are cumbersome and can be optimized
- *
- * @param loginUser login user
- * @param fullName full name
- * @param type resource type
- * @param result Result
- */
- private void createDirectory(User loginUser, String fullName, ResourceType
type, Result<Object> result) {
- String tenantCode =
tenantMapper.queryById(loginUser.getTenantId()).getTenantCode();
- // String directoryName = storageOperate.getFileName(type, tenantCode,
fullName);
- String resourceRootPath = storageOperate.getDir(type, tenantCode);
+ @Override
+ public void createFile(CreateFileRequest createFileRequest) {
+ CreateFileDto createFileDto =
createFileRequestTransformer.transform(createFileRequest);
+ createFileDtoValidator.validate(createFileDto);
+
+ // todo: use storage proxy
+ MultipartFile file = createFileDto.getFile();
+ String fileAbsolutePath = createFileDto.getFileAbsolutePath();
+ String srcLocalTmpFileAbsolutePath = copyFileToLocal(file);
try {
- if (!storageOperate.exists(resourceRootPath)) {
- storageOperate.createTenantDirIfNotExists(tenantCode);
- }
-
- if (!storageOperate.mkdir(tenantCode, fullName)) {
- throw new ServiceException(String.format("Create resource
directory: %s failed.", fullName));
- }
- putMsg(result, Status.SUCCESS);
- } catch (Exception e) {
- throw new ServiceException(String.format("create resource
directory: %s failed.", fullName));
+ storageOperator.upload(srcLocalTmpFileAbsolutePath,
fileAbsolutePath, true, false);
+ ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
+ log.info("Success upload resource file: {} complete.",
fileAbsolutePath);
+ } catch (Exception ex) {
+ // If exception, clear the tmp path
+ FileUtils.deleteFile(srcLocalTmpFileAbsolutePath);
+ throw ex;
}
}
- /**
- * upload file to hdfs
- *
- * @param loginUser login user
- * @param fullName full name
- * @param file file
- * @param type resource type
- * @return upload success return true, otherwise false
- */
- private boolean upload(User loginUser, String fullName, MultipartFile
file, ResourceType type) {
- // save to local
- String fileSuffix = Files.getFileExtension(file.getOriginalFilename());
- String nameSuffix = Files.getFileExtension(fullName);
-
- // determine file suffix
- if (!fileSuffix.equalsIgnoreCase(nameSuffix)) {
- return false;
- }
- // query tenant
- String tenantCode = getTenantCode(loginUser);
- // random file name
- String localFilename = FileUtils.getUploadFilename(tenantCode,
UUID.randomUUID().toString());
-
- // save file to hdfs, and delete original file
- String resourcePath = storageOperate.getDir(type, tenantCode);
+ @Override
+ public void createFileFromContent(CreateFileFromContentRequest
createFileFromContentRequest) {
+ CreateFileFromContentDto createFileFromContentDto =
+
createFileFromContentRequestTransformer.transform(createFileFromContentRequest);
+ createFileFromContentDtoValidator.validate(createFileFromContentDto);
Review Comment:
DTO validation logic is used in many places. How about using AOP to reduce
duplicate code through annotations?
##########
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java:
##########
@@ -17,1286 +17,404 @@
package org.apache.dolphinscheduler.api.service.impl;
-import static org.apache.dolphinscheduler.common.constants.Constants.ALIAS;
-import static org.apache.dolphinscheduler.common.constants.Constants.CONTENT;
-import static
org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING;
-import static
org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR;
-import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_SS;
-import static org.apache.dolphinscheduler.common.constants.Constants.JAR;
-import static org.apache.dolphinscheduler.common.constants.Constants.PERIOD;
-
-import
org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
-import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter;
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
+import
org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
+import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
+import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceDto;
+import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest;
+import org.apache.dolphinscheduler.api.dto.resources.DownloadFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest;
+import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto;
+import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest;
+import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
+import org.apache.dolphinscheduler.api.dto.resources.QueryResourceDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
+import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto;
+import
org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
+import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
import
org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor;
import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
-import org.apache.dolphinscheduler.api.utils.RegexUtils;
-import org.apache.dolphinscheduler.api.utils.Result;
-import org.apache.dolphinscheduler.common.constants.Constants;
-import org.apache.dolphinscheduler.common.enums.ProgramType;
-import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import
org.apache.dolphinscheduler.api.validator.resource.CreateDirectoryDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.CreateDirectoryRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.CreateFileDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.CreateFileFromContentDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.DeleteResourceDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.DownloadFileDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.FetchFileContentDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.FileFromContentRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.FileRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.PagingResourceItemRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.RenameDirectoryDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.RenameDirectoryRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.RenameFileDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.RenameFileRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.UpdateFileDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.UpdateFileFromContentDtoValidator;
+import
org.apache.dolphinscheduler.api.validator.resource.UpdateFileFromContentRequestTransformer;
+import
org.apache.dolphinscheduler.api.validator.resource.UpdateFileRequestTransformer;
+import org.apache.dolphinscheduler.api.vo.ResourceItemVO;
+import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse;
import org.apache.dolphinscheduler.common.utils.FileUtils;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
-import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
-import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
-import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
+import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
import java.io.File;
-import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Paths;
-import java.text.MessageFormat;
-import java.time.LocalDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.UUID;
import java.util.stream.Collectors;
+import javax.servlet.http.HttpServletResponse;
+
+import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
-import com.google.common.io.Files;
-
@Service
@Slf4j
public class ResourcesServiceImpl extends BaseServiceImpl implements
ResourcesService {
@Autowired
- private UdfFuncMapper udfFunctionMapper;
-
- @Autowired
- private TenantMapper tenantMapper;
+ private TenantDao tenantDao;
@Autowired
private UserMapper userMapper;
- @Autowired(required = false)
- private StorageOperate storageOperate;
-
- /**
- * create directory
- *
- * @param loginUser login user
- * @param name alias
- * @param type type
- * @param pid parent id
- * @param currentDir current directory
- * @return create directory result
- */
- @Override
- @Transactional
- public Result<Object> createDirectory(User loginUser, String name,
ResourceType type, int pid, String currentDir) {
- Result<Object> result = new Result<>();
- if (FileUtils.directoryTraversal(name)) {
- log.warn("Parameter name is invalid, name:{}.",
RegexUtils.escapeNRT(name));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, currentDir);
-
- String userResRootPath = ResourceType.UDF.equals(type) ?
storageOperate.getUdfDir(tenantCode)
- : storageOperate.getResDir(tenantCode);
- String fullName = !currentDir.contains(userResRootPath) ?
userResRootPath + name : currentDir + name;
-
- try {
- if (checkResourceExists(fullName)) {
- log.error("resource directory {} has exist, can't recreate",
fullName);
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- } catch (Exception e) {
- log.warn("Resource exists, can not create again, fullName:{}.",
fullName, e);
- throw new ServiceException("resource already exists, can't
recreate");
- }
-
- // create directory in storage
- createDirectory(loginUser, fullName, type, result);
- return result;
- }
-
- /**
- * create resource
- *
- * @param loginUser login user
- * @param name alias
- * @param type type
- * @param file file
- * @param currentDir current directory
- * @return create result code
- */
- @Override
- @Transactional
- public Result<Object> uploadResource(User loginUser, String name,
ResourceType type, MultipartFile file,
- String currentDir) {
- Result<Object> result = new Result<>();
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, currentDir);
-
- result = verifyFile(name, type, file);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- return result;
- }
-
- // check resource name exists
- String userResRootPath = ResourceType.UDF.equals(type) ?
storageOperate.getUdfDir(tenantCode)
- : storageOperate.getResDir(tenantCode);
- String currDirNFileName = !currentDir.contains(userResRootPath) ?
userResRootPath + name : currentDir + name;
-
- try {
- if (checkResourceExists(currDirNFileName)) {
- log.error("resource {} has exist, can't recreate",
RegexUtils.escapeNRT(name));
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- } catch (Exception e) {
- throw new ServiceException("resource already exists, can't
recreate");
- }
- if (currDirNFileName.length() >
Constants.RESOURCE_FULL_NAME_MAX_LENGTH) {
- log.error(
- "Resource file's name is longer than max full name length,
fullName:{}, "
- + "fullNameSize:{}, maxFullNameSize:{}",
- RegexUtils.escapeNRT(name), currDirNFileName.length(),
Constants.RESOURCE_FULL_NAME_MAX_LENGTH);
- putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR);
- return result;
- }
-
- // fail upload
- if (!upload(loginUser, currDirNFileName, file, type)) {
- log.error("upload resource: {} file: {} failed.",
RegexUtils.escapeNRT(name),
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.STORE_OPERATE_CREATE_ERROR);
- throw new ServiceException(
- String.format("upload resource: %s file: %s failed.",
name, file.getOriginalFilename()));
- } else
- ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
- log.info("Upload resource file complete, resourceName:{},
fileName:{}.", RegexUtils.escapeNRT(name),
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.SUCCESS);
- return result;
- }
-
- /**
- * check resource is exists
- *
- * @param fullName fullName
- * @return true if resource exists
- */
- private boolean checkResourceExists(String fullName) {
- Boolean existResource = false;
- try {
- existResource = storageOperate.exists(fullName);
- } catch (IOException e) {
- log.error("error occurred when checking resource: " + fullName, e);
- }
- return Boolean.TRUE.equals(existResource);
- }
-
- /**
- * update resource
- *
- * @param loginUser login user
- * @param resourceFullName resource full name
- * @param resTenantCode tenantCode in the request field "resTenantCode"
for tenant code owning the resource,
- * can be different from the login user in the
case of logging in as admin users.
- * @param name name
- * @param type resource type
- * @param file resource file
- * @return update result code
- */
- @Override
- @Transactional
- public Result<Object> updateResource(User loginUser, String
resourceFullName, String resTenantCode, String name,
- ResourceType type, MultipartFile
file) {
- Result<Object> result = new Result<>();
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, resourceFullName);
+ @Autowired
+ private StorageOperator storageOperator;
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode))
{
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
+ @Autowired
+ private CreateDirectoryRequestTransformer
createDirectoryRequestTransformer;
- String defaultPath = storageOperate.getDir(type, tenantCode);
+ @Autowired
+ private CreateDirectoryDtoValidator createDirectoryDtoValidator;
- StorageEntity resource;
- try {
- resource = storageOperate.getFileStatus(resourceFullName,
defaultPath, resTenantCode, type);
- } catch (Exception e) {
- log.error("Get file status fail, resource path: {}",
resourceFullName, e);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- throw new ServiceException((String.format("Get file status fail,
resource path: %s", resourceFullName)));
- }
+ @Autowired
+ private RenameDirectoryRequestTransformer
renameDirectoryRequestTransformer;
- // TODO: deal with OSS
- if (resource.isDirectory() &&
storageOperate.returnStorageType().equals(ResUploadType.S3)
- && !resource.getFileName().equals(name)) {
- log.warn("Directory in S3 storage can not be renamed.");
- putMsg(result, Status.S3_CANNOT_RENAME);
- return result;
- }
+ @Autowired
+ private RenameDirectoryDtoValidator renameDirectoryDtoValidator;
- // check if updated name of the resource already exists
- String originFullName = resource.getFullName();
- String originResourceName = resource.getAlias();
-
- // the format of hdfs folders in the implementation has a "/" at the
very end, we need to remove it.
- originFullName = originFullName.endsWith("/") ?
StringUtils.chop(originFullName) : originFullName;
- name = name.endsWith("/") ? StringUtils.chop(name) : name;
- // updated fullName
- String fullName = String.format(FORMAT_SS,
- originFullName.substring(0,
originFullName.lastIndexOf(FOLDER_SEPARATOR) + 1), name);
- if (!originResourceName.equals(name)) {
- try {
- if (checkResourceExists(fullName)) {
- log.error("resource {} already exists, can't recreate",
fullName);
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- } catch (Exception e) {
- throw new ServiceException(String.format("error occurs while
querying resource: %s", fullName));
- }
+ @Autowired
+ private RenameFileRequestTransformer renameFileRequestTransformer;
- }
+ @Autowired
+ private RenameFileDtoValidator renameFileDtoValidator;
- result = verifyFile(name, type, file);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- return result;
- }
+ @Autowired
+ private FileFromContentRequestTransformer
createFileFromContentRequestTransformer;
- Date now = new Date();
+ @Autowired
+ private CreateFileFromContentDtoValidator
createFileFromContentDtoValidator;
- resource.setAlias(name);
- resource.setFileName(name);
- resource.setFullName(fullName);
- resource.setUpdateTime(now);
- if (file != null) {
- resource.setSize(file.getSize());
- }
+ @Autowired
+ private FetchFileContentDtoValidator fetchFileContentDtoValidator;
- // if name unchanged, return directly without moving on HDFS
- if (originResourceName.equals(name) && file == null) {
- return result;
- }
+ @Autowired
+ private UpdateFileFromContentRequestTransformer
updateFileFromContentRequestTransformer;
- if (file != null) {
- // fail upload
- if (!upload(loginUser, fullName, file, type)) {
- log.error("Storage operation error, resourceName:{},
originFileName:{}.", name,
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.HDFS_OPERATION_ERROR);
- throw new ServiceException(
- String.format("upload resource: %s file: %s failed.",
name, file.getOriginalFilename()));
- }
- if (!fullName.equals(originFullName)) {
- try {
- storageOperate.delete(originFullName, false);
- } catch (IOException e) {
- log.error("Resource delete error, resourceFullName:{}.",
originFullName, e);
- throw new ServiceException(String.format("delete resource:
%s failed.", originFullName));
- }
- }
+ @Autowired
+ private UpdateFileFromContentDtoValidator
updateFileFromContentDtoValidator;
- ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
- return result;
- }
+ @Autowired
+ private FileRequestTransformer createFileRequestTransformer;
- // get the path of dest file in hdfs
- String destHdfsFileName = fullName;
- try {
- log.info("start copy {} -> {}", originFullName, destHdfsFileName);
- storageOperate.copy(originFullName, destHdfsFileName, true, true);
- putMsg(result, Status.SUCCESS);
- } catch (Exception e) {
- log.error(MessageFormat.format(" copy {0} -> {1} fail",
originFullName, destHdfsFileName), e);
- putMsg(result, Status.HDFS_COPY_FAIL);
- throw new ServiceException(
- MessageFormat.format(Status.HDFS_COPY_FAIL.getMsg(),
originFullName, destHdfsFileName));
- }
+ @Autowired
+ private CreateFileDtoValidator createFileDtoValidator;
- return result;
- }
+ @Autowired
+ private UpdateFileRequestTransformer updateFileRequestTransformer;
- private Result<Object> verifyFile(String name, ResourceType type,
MultipartFile file) {
- Result<Object> result = new Result<>();
- putMsg(result, Status.SUCCESS);
+ @Autowired
+ private UpdateFileDtoValidator updateFileDtoValidator;
- if (FileUtils.directoryTraversal(name)) {
- log.warn("Parameter file alias name verify failed,
fileAliasName:{}.", RegexUtils.escapeNRT(name));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
+ @Autowired
+ private DeleteResourceDtoValidator deleteResourceDtoValidator;
- if (file != null &&
FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename())))
{
- log.warn("File original name verify failed, fileOriginalName:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
- return result;
- }
+ @Autowired
+ private DownloadFileDtoValidator downloadFileDtoValidator;
- if (file != null) {
- // file is empty
- if (file.isEmpty()) {
- log.warn("Parameter file is empty, fileOriginalName:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.RESOURCE_FILE_IS_EMPTY);
- return result;
- }
-
- // file suffix
- String fileSuffix =
Files.getFileExtension(file.getOriginalFilename());
- String nameSuffix = Files.getFileExtension(name);
-
- // determine file suffix
- if (!fileSuffix.equalsIgnoreCase(nameSuffix)) {
- // rename file suffix and original suffix must be consistent
- log.warn("Rename file suffix and original suffix must be
consistent, fileOriginalName:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()));
- putMsg(result, Status.RESOURCE_SUFFIX_FORBID_CHANGE);
- return result;
- }
-
- // If resource type is UDF, only jar packages are allowed to be
uploaded, and the suffix must be .jar
- if (Constants.UDF.equals(type.name()) &&
!JAR.equalsIgnoreCase(fileSuffix)) {
- log.warn(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg());
- putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR);
- return result;
- }
- if (file.getSize() > Constants.MAX_FILE_SIZE) {
- log.warn(
- "Resource file size is larger than max file size,
fileOriginalName:{}, fileSize:{}, maxFileSize:{}.",
- RegexUtils.escapeNRT(file.getOriginalFilename()),
file.getSize(), Constants.MAX_FILE_SIZE);
- putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT);
- return result;
- }
- }
- return result;
- }
+ @Autowired
+ private PagingResourceItemRequestTransformer
pagingResourceItemRequestTransformer;
- /**
- * query resources list paging
- *
- * @param loginUser login user
- * @param fullName resource full name
- * @param resTenantCode tenantCode in the request field "resTenantCode"
for tenant code owning the resource,
- * can be different from the login user in the case
of logging in as admin users.
- * @param type resource type
- * @param searchVal search value
- * @param pageNo page number
- * @param pageSize page size
- * @return resource list page
- */
@Override
- public Result<PageInfo<StorageEntity>> queryResourceListPaging(User
loginUser, String fullName,
- String
resTenantCode, ResourceType type,
- String
searchVal, Integer pageNo, Integer pageSize) {
- Result<PageInfo<StorageEntity>> result = new Result<>();
- PageInfo<StorageEntity> pageInfo = new PageInfo<>(pageNo, pageSize);
- if (storageOperate == null) {
- log.warn("The resource storage is not opened.");
- return Result.success(pageInfo);
- }
-
- User user = userMapper.selectById(loginUser.getId());
- if (user == null) {
- log.error("user {} not exists", loginUser.getId());
- putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
- return result;
- }
-
- String tenantCode = getTenantCode(user);
- checkFullName(tenantCode, fullName);
+ public void createDirectory(CreateDirectoryRequest createDirectoryRequest)
{
+ CreateDirectoryDto createDirectoryDto =
createDirectoryRequestTransformer.transform(createDirectoryRequest);
+ createDirectoryDtoValidator.validate(createDirectoryDto);
- if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode))
{
- log.error("current user does not have permission");
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
-
- List<StorageEntity> resourcesList;
- try {
- resourcesList = queryStorageEntityList(loginUser, fullName, type,
tenantCode, false);
- } catch (ServiceException e) {
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
-
- // remove leading and trailing spaces in searchVal
- String trimmedSearchVal = searchVal != null ? searchVal.trim() : "";
- // filter based on trimmed searchVal
- List<StorageEntity> filteredResourceList = resourcesList.stream()
- .filter(x ->
x.getFileName().contains(trimmedSearchVal)).collect(Collectors.toList());
- // inefficient pagination
- List<StorageEntity> slicedResourcesList =
filteredResourceList.stream().skip((long) (pageNo - 1) * pageSize)
- .limit(pageSize).collect(Collectors.toList());
-
- pageInfo.setTotal(filteredResourceList.size());
- pageInfo.setTotalList(slicedResourcesList);
- result.setData(pageInfo);
- putMsg(result, Status.SUCCESS);
- return result;
- }
-
- private List<StorageEntity> queryStorageEntityList(User loginUser, String
fullName, ResourceType type,
- String tenantCode,
boolean recursive) {
- String defaultPath = "";
- List<StorageEntity> resourcesList = new ArrayList<>();
- String resourceStorageType =
- PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE,
ResUploadType.LOCAL.name());
- if (isAdmin(loginUser) && StringUtils.isBlank(fullName)) {
- // list all tenants' resources to admin users in the root directory
- List<User> userList = userMapper.selectList(null);
- Set<String> visitedTenantEntityCode = new HashSet<>();
- for (User userEntity : userList) {
- String tenantEntityCode = getTenantCode(userEntity);
- if (!visitedTenantEntityCode.contains(tenantEntityCode)) {
- defaultPath = storageOperate.getResDir(tenantEntityCode);
- if (type.equals(ResourceType.UDF)) {
- defaultPath =
storageOperate.getUdfDir(tenantEntityCode);
- }
- try {
- resourcesList.addAll(recursive
- ?
storageOperate.listFilesStatusRecursively(defaultPath, defaultPath,
tenantEntityCode,
- type)
- : storageOperate.listFilesStatus(defaultPath,
defaultPath, tenantEntityCode, type));
-
- visitedTenantEntityCode.add(tenantEntityCode);
- } catch (Exception e) {
- log.error(e.getMessage() + " Resource path: {}",
defaultPath, e);
- throw new ServiceException(
- String.format(e.getMessage() + " make sure
resource path: %s exists in %s", defaultPath,
- resourceStorageType));
- }
- }
- }
- } else {
- defaultPath = storageOperate.getResDir(tenantCode);
- if (type.equals(ResourceType.UDF)) {
- defaultPath = storageOperate.getUdfDir(tenantCode);
- }
-
- try {
- if (StringUtils.isBlank(fullName)) {
- fullName = defaultPath;
- }
- resourcesList =
- recursive ?
storageOperate.listFilesStatusRecursively(fullName, defaultPath, tenantCode,
type)
- : storageOperate.listFilesStatus(fullName,
defaultPath, tenantCode, type);
- } catch (Exception e) {
- log.error(e.getMessage() + " Resource path: {}", fullName, e);
- throw new ServiceException(String.format(e.getMessage() + "
make sure resource path: %s exists in %s",
- defaultPath, resourceStorageType));
- }
- }
-
- return resourcesList;
+
storageOperator.createStorageDir(createDirectoryDto.getDirectoryAbsolutePath());
+ log.info("Success create directory: {}",
createDirectoryRequest.getParentAbsoluteDirectory());
}
- /**
- * create directory
- * xxx The steps to verify resources are cumbersome and can be optimized
- *
- * @param loginUser login user
- * @param fullName full name
- * @param type resource type
- * @param result Result
- */
- private void createDirectory(User loginUser, String fullName, ResourceType
type, Result<Object> result) {
- String tenantCode =
tenantMapper.queryById(loginUser.getTenantId()).getTenantCode();
- // String directoryName = storageOperate.getFileName(type, tenantCode,
fullName);
- String resourceRootPath = storageOperate.getDir(type, tenantCode);
+ @Override
+ public void createFile(CreateFileRequest createFileRequest) {
+ CreateFileDto createFileDto =
createFileRequestTransformer.transform(createFileRequest);
+ createFileDtoValidator.validate(createFileDto);
+
+ // todo: use storage proxy
+ MultipartFile file = createFileDto.getFile();
+ String fileAbsolutePath = createFileDto.getFileAbsolutePath();
+ String srcLocalTmpFileAbsolutePath = copyFileToLocal(file);
try {
- if (!storageOperate.exists(resourceRootPath)) {
- storageOperate.createTenantDirIfNotExists(tenantCode);
- }
-
- if (!storageOperate.mkdir(tenantCode, fullName)) {
- throw new ServiceException(String.format("Create resource
directory: %s failed.", fullName));
- }
- putMsg(result, Status.SUCCESS);
- } catch (Exception e) {
- throw new ServiceException(String.format("create resource
directory: %s failed.", fullName));
+ storageOperator.upload(srcLocalTmpFileAbsolutePath,
fileAbsolutePath, true, false);
+ ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
+ log.info("Success upload resource file: {} complete.",
fileAbsolutePath);
+ } catch (Exception ex) {
+ // If exception, clear the tmp path
+ FileUtils.deleteFile(srcLocalTmpFileAbsolutePath);
+ throw ex;
Review Comment:
It seems that the exception stack information cannot be seen here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
