GaOrtiga commented on code in PR #13935: URL: https://github.com/apache/cloudstack/pull/13935#discussion_r3845610780
########## api/src/main/java/org/apache/cloudstack/api/command/user/backup/DownloadBackupFileCmd.java: ########## @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.backup; + +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.user.Account; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ExtractResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; + +import javax.inject.Inject; + +@APICommand(name = "downloadBackupFile", + description = "Download a file from a backup", + responseObject = ExtractResponse.class, since = "4.24.0.0") Review Comment: ```suggestion responseObject = ExtractResponse.class, since = "24.0.0") ``` ########## api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupFilesCmd.java: ########## @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.backup; + +import com.cloud.user.Account; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; + +import javax.inject.Inject; +import java.util.List; + +@APICommand(name = "listBackupFiles", + description = "List a backup inner files", + responseObject = DataStoreObjectResponse.class, since = "4.24.0.0") Review Comment: ```suggestion responseObject = DataStoreObjectResponse.class, since = "24.0.0") ``` ########## api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupFilesystemsCmd.java: ########## @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.backup; + +import com.cloud.user.Account; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; + +import javax.inject.Inject; +import java.util.List; + +@APICommand(name = "listBackupFilesystems", + description = "List a backup inner file systems", + responseObject = DataStoreObjectResponse.class, since = "4.24.0.0") Review Comment: ```suggestion responseObject = DataStoreObjectResponse.class, since = "24.0.0") ``` ########## server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java: ########## @@ -1900,6 +1907,107 @@ private void checkForPendingBackupJobs(final BackupVO backup) { } } + @Override + public List<DataStoreObjectResponse> listBackupFilesystems(long backupId, Long volumeId) { + BackupVO backupVO = backupDao.findById(backupId); + Backup.VolumeInfo backupVolumeInfo = validateBackupAndGetVolumeInfo(backupId, volumeId, backupVO); + + BackupOfferingVO backupOfferingVO = backupOfferingDao.findById(backupVO.getBackupOfferingId()); + BackupProvider backupProvider = getBackupProvider(backupOfferingVO.getProvider()); + if (!KBOSS_BACKUP_PROVIDER.equals(backupProvider.getName())) { + throw new CloudRuntimeException(String.format("This feature is only supported for the %s provider currently", KBOSS_BACKUP_PROVIDER)); + } + + List<FilesystemInfoTO> filesystems = backupProvider.listBackupFilesystems(backupVO, backupVolumeInfo); + + List<DataStoreObjectResponse> dataStoreObjectResponseList = new ArrayList<>(); + for (FilesystemInfoTO fsInfoTO : filesystems) { + DataStoreObjectResponse dataStoreObjectResponse = new DataStoreObjectResponse(fsInfoTO.getName(), false, fsInfoTO.getSize(), null); + dataStoreObjectResponse.setIsFilesystem(true); + dataStoreObjectResponse.setFormat(fsInfoTO.getFilesystem()); + + VolumeVO volume = volumeDao.findByIdIncludingRemoved(fsInfoTO.getVolumeId()); + dataStoreObjectResponse.setVolumeId(volume.getUuid()); + dataStoreObjectResponse.setVolumeName(volume.getName()); + dataStoreObjectResponseList.add(dataStoreObjectResponse); + } + + return dataStoreObjectResponseList; + } + + @Override + public List<DataStoreObjectResponse> listBackupFiles(long backupId, Long volumeId, String filesystem, String directory, Boolean isSymlink) { + BackupVO backupVO = backupDao.findById(backupId); + Backup.VolumeInfo backupVolumeInfo = validateBackupAndGetVolumeInfo(backupId, volumeId, backupVO); + + BackupOfferingVO backupOfferingVO = backupOfferingDao.findById(backupVO.getBackupOfferingId()); + BackupProvider backupProvider = getBackupProvider(backupOfferingVO.getProvider()); + if (!KBOSS_BACKUP_PROVIDER.equals(backupProvider.getName())) { + throw new CloudRuntimeException(String.format("This feature is only supported for the %s provider currently", KBOSS_BACKUP_PROVIDER)); + } + + return backupProvider.listBackupFiles(backupVO, backupVolumeInfo, filesystem, directory, isSymlink); + } + + @Override + public ExtractResponse downloadBackupFile(long backupId, Long volumeId, String filesystem, String file) { + BackupVO backupVO = backupDao.findById(backupId); + Backup.VolumeInfo backupVolumeInfo = validateBackupAndGetVolumeInfo(backupId, volumeId, backupVO); + + BackupOfferingVO backupOfferingVO = backupOfferingDao.findById(backupVO.getBackupOfferingId()); + BackupProvider backupProvider = getBackupProvider(backupOfferingVO.getProvider()); + if (!KBOSS_BACKUP_PROVIDER.equals(backupProvider.getName())) { + throw new CloudRuntimeException(String.format("This feature is only supported for the %s provider currently", KBOSS_BACKUP_PROVIDER)); + } + + String url = backupProvider.downloadBackupFile(backupVO, backupVolumeInfo, filesystem, file); + + ExtractResponse response = new ExtractResponse(); + if (url == null) { + response.setState(Upload.Status.DOWNLOAD_URL_NOT_CREATED.toString()); + return response; + } + + response.setUrl(url); + response.setName(file.substring(file.lastIndexOf("/") + 1)); + response.setState(Upload.Status.DOWNLOAD_URL_CREATED.toString()); + return response; + } + + private Backup.VolumeInfo validateBackupAndGetVolumeInfo(long backupId, Long volumeId, BackupVO backupVO) { + if (backupVO == null) { + logger.warn("Unable to find backup with ID [{}].", backupId); + throw new InvalidParameterValueException("Unable to find backup with given ID."); + } + + if (backupVO.getStatus() != Backup.Status.BackedUp) { + throw new InvalidParameterValueException(String.format("Backup [%s] is not in the right state to list its filesystems. It should be in [%s] state, but it is in [%s] " + + "state.", backupVO.getUuid(), Backup.Status.BackedUp.name(), backupVO.getStatus().name())); + } + + Long vmId = backupVO.getVmId(); + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); + if (vm == null) { + throw new CloudRuntimeException(String.format("Unable to find VM for backup [%s].", backupVO.getUuid())); Review Comment: ```suggestion throw new CloudRuntimeException(String.format("Unable to find VM from backup [%s].", backupVO.getUuid())); ``` ########## api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupFilesCmd.java: ########## @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.backup; + +import com.cloud.user.Account; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; + +import javax.inject.Inject; +import java.util.List; + +@APICommand(name = "listBackupFiles", + description = "List a backup inner files", + responseObject = DataStoreObjectResponse.class, since = "4.24.0.0") +public class ListBackupFilesCmd extends BaseListCmd { + + @Inject + private BackupManager backupManager; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL + @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, required = true, + description = "ID of the backup to list the files.") + private Long backupId; + + @ACL + @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.UUID, entityType = VolumeResponse.class, required = true, + description = "ID of the volume.") + private Long volumeId; + + @Parameter(name = ApiConstants.FILESYSTEM, type = CommandType.STRING, required = true, + description = "Filesystem to list the files in.", validations = {ApiArgValidator.LimitedSpecialCharacters}) + private String filesystem; + + @Parameter(name = ApiConstants.PATH, type = CommandType.STRING, required = true, + description = "Path to list files in the backed-up volume.", validations = {ApiArgValidator.LimitedSpecialCharacters}) + private String path; + + @Parameter(name = ApiConstants.IS_SYMLINK, type = CommandType.BOOLEAN, + description = "Path to list files in the backed-up volume.") Review Comment: ```suggestion description = "Indicates whether the path references a symlink.") ``` ########## plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/GuestfishClient.java: ########## @@ -0,0 +1,420 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils.qemu; + +import com.cloud.agent.api.to.FilesystemInfoTO; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * This class was built to use guestfish commands on qcow2 volumes. + * It should always be instanced with try-with-resources + * Otherwise, be sure to call close(). + * */ + +public class GuestfishClient implements AutoCloseable { + private Logger logger = LogManager.getLogger(getClass()); + + private static final int MODE_MASK = 0_170_000; + private static final int SYMLINK = 0_120_000; + private static final int DIRECTORY = 0_040_000; + private static final int FILE = 0_100_000; + + private static final String GUESTFISH = "guestfish"; + private static final String GUESTFISH_PID_STRING = "GUESTFISH_PID="; + private static final String MODE = "st_mode"; + private static final String SIZE = "st_size"; + private static final String ATIME_SEC = "st_atime_sec"; + + // Guestfish commands + protected static final String MOUNT_RO = "mount-ro"; + protected static final String UMOUNT_ALL = "umount-all"; + protected static final String TAR_OUT = "tar-out"; + protected static final String COMPRESS_OUT = "compress-out"; + protected static final String LIST_FILESYSTEMS = "list-filesystems"; + protected static final String LSTATNS = "lstatns"; + protected static final String LSTATNSLIST = "lstatnslist"; + protected static final String IS_SYMLINK = "is-symlink"; + protected static final String READLINK = "readlink"; + protected static final String BLOCKDEV_GETSIZE_64 = "blockdev-getsize64"; + protected static final String LS = "ls"; + protected static final String EXISTS = "exists"; + + private static final int MILI = 1000; + + private final String guestfishPid; + private final long volumeId; + + /** + * No-arg constructor for test purposes + */ + protected GuestfishClient() { + guestfishPid = "0"; + volumeId = 0; + } + + public GuestfishClient(String qcow2Path, long volumeId) { + this.volumeId = volumeId; + + Script script = new Script(GUESTFISH); + script.add("--listen"); + script.add("--ro"); + + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String scriptResult = script.execute(parser); + if (scriptResult != null) { + throw new CloudRuntimeException("Could not start guestfish. Is it installed?"); + } + + String output = parser.getLine(); + + this.guestfishPid = parseGuestfishPid(output); + + if (guestfishPid == null) { + throw new CloudRuntimeException("Could not parse guestfish PID."); + } + + runGuestfishRemoteCommand("add-drive-ro", qcow2Path); + runGuestfishRemoteCommand("run"); + } + + /** + * List filesystems. Will not return swap or unknown filesystems. + */ + public List<FilesystemInfoTO> listFilesystems() { + String output = runGuestfishRemoteCommand(LIST_FILESYSTEMS); + List<FilesystemInfoTO> result = new ArrayList<>(); + + logger.debug("Got the following output from list-filesystems: [{}]", output); + output.lines().forEach( line ->{ + line = line.trim(); + if (line.isEmpty()) { + return; + } + + String[] parts = line.split(":"); + if (parts.length != 2) { + return; + } + + String device = parts[0].trim(); + String type = parts[1].trim(); + if (type.equalsIgnoreCase("swap") || type.equalsIgnoreCase("unknown")) { + logger.debug("Ignoring filesystem [{}] with [{}] type.", device, type); + return; + } + + String size = runGuestfishRemoteCommand(BLOCKDEV_GETSIZE_64, device); + + result.add(new FilesystemInfoTO(device, type, parseLong(size), volumeId)); + }); + + return result; + } + + /** + * List files in the given directory. If the directory is a symlink, will try to resolve it, if we are unable to resolve it in one hop, we will return an error. + * */ + public ListDataStoreObjectsAnswer listFiles(String filesystem, String directory, Boolean isSymlink) { + try { + mount(filesystem); + + if (isSymlink == null) { + String isSymlinkString = runGuestfishRemoteCommand(IS_SYMLINK, directory); + isSymlink = Boolean.parseBoolean(isSymlinkString); + } + + if (isSymlink) { + logger.debug("Directory [{}] is a symlink, will try resolve it.", directory); + directory = (directory.charAt(directory.length() - 1) == '/' ? directory.substring(0, directory.length() - 1) : directory); + String canonicalPath = getCanonicalPath(directory); + if (canonicalPath == null) { + return new ListDataStoreObjectsAnswer(); + } + + String isSymlinkString = runGuestfishRemoteCommand(IS_SYMLINK, canonicalPath); + isSymlink = Boolean.parseBoolean(isSymlinkString); + if (isSymlink) { + logger.warn("Directory [{}] is a symlink chain. Unable to list its files. Please try to list the real directory.", directory); + return new ListDataStoreObjectsAnswer(); + } + logger.debug("Directory [{}] was resolved to [{}].", directory, canonicalPath); + directory = canonicalPath; + } + + String lsOutput = runGuestfishRemoteCommand(LS, directory); + + List<String> fileList = lsOutput.lines().filter(StringUtils::isNotBlank).collect(Collectors.toList()); + + List<String> names = new ArrayList<>(); + List<String> paths = new ArrayList<>(); + List<String> canonicalPaths = new ArrayList<>(); + List<Boolean> isDirs = new ArrayList<>(); + List<Boolean> isSymlinks = new ArrayList<>(); + List<Long> sizes = new ArrayList<>(); + List<Long> modifiedList = new ArrayList<>(); + + if (fileList.isEmpty()) { + return new ListDataStoreObjectsAnswer(true, 0, names, paths, canonicalPaths, isDirs, sizes, modifiedList); + } + + String details = getDetails(directory, fileList); + logger.trace("Got the following details for these files [{}]: [{}]", fileList, details); + + addFilesToLists(directory, details, fileList, names, paths, canonicalPaths, isDirs, isSymlinks, sizes, modifiedList); + + return new ListDataStoreObjectsAnswer(true, names.size(), names, paths, canonicalPaths, isDirs, isSymlinks, sizes, modifiedList); + } finally { + runGuestfishRemoteCommand(UMOUNT_ALL); + } + } + + /** + * Extracts a file from the given filesystem and path to the destination. If followSymlink is true will try to resolve it, if we cannot do it in one hop we throw an error. + * */ + public boolean extractFile(String filesystem, String filePath, String destination, boolean followSymlink) { + mount(filesystem); + + String details = runGuestfishRemoteCommand(LSTATNS, filePath); + Map<String, String> stat = parseKeyValueOutput(details); + long mode = parseLong(stat.get(MODE)); + long fileType = mode & MODE_MASK; + boolean isSymlink = fileType == SYMLINK; + + if (isSymlink && followSymlink) { + logger.debug("File is a symlink, will try to resolve it in order to download the actual file."); + String canonicalPath = getCanonicalPath(filePath); + if (canonicalPath == null) { + runGuestfishRemoteCommand(UMOUNT_ALL); + throw new CloudRuntimeException(String.format("File [%s] is a symlink to a non-existent file. Unable to download.", fileType)); + } + + return extractFile(filesystem, canonicalPath, destination, false); + } else if (isSymlink) { + runGuestfishRemoteCommand(UMOUNT_ALL); + throw new CloudRuntimeException(String.format("Unable to extract file at [%s]. It seems like a symlink chain. Try to download the actual file at the end of the chain.", + filePath)); + } + + boolean isDirectory = fileType == DIRECTORY; + boolean isFile = fileType == FILE; + try { + if (isDirectory) { + logger.debug("Extracting directory at [{}] to [{}].", filePath, destination); + runGuestfishRemoteCommand(TAR_OUT, filePath, destination, "compress:gzip"); + } else if (isFile) { + logger.debug("Extracting file at [{}] to [{}].", filePath, destination); + runGuestfishRemoteCommand(COMPRESS_OUT, "gzip", filePath, destination); + } else { + throw new CloudRuntimeException(String.format("Unable to extract file at [%s]. It is neither a file nor a directory.", filePath)); + } + } catch (Exception e) { + logger.error("Caught exception while extracting file, will try to delete any leftovers and rethrow the exception.", e); + try { + Files.deleteIfExists(Path.of(destination)); + } catch (IOException ignored) { + } + throw e; + } finally { + runGuestfishRemoteCommand(UMOUNT_ALL); + } + return isDirectory; + } + + protected String getDetails(String directory, List<String> fileList) { + String fileListString = formatFileListForLstatnslist(fileList); + + return runGuestfishRemoteCommand(LSTATNSLIST, directory, fileListString); + } + + + protected String formatFileListForLstatnslist(List<String> fileList) { + StringBuilder fileListStringBuilder = new StringBuilder(); + for (String string : fileList) { + fileListStringBuilder.append("'"); + fileListStringBuilder.append(string); + fileListStringBuilder.append("'"); + fileListStringBuilder.append(" "); + } + fileListStringBuilder.deleteCharAt(fileListStringBuilder.length() - 1); + return fileListStringBuilder.toString(); + } + + /** + * Given the directory, file list and details of those files, will populate the names, paths, canonicalPaths, isDirs, isSymlinks, sizes and modifiedList lists. Review Comment: ```suggestion * Given the directory, file list, and details of those files, will populate the names, paths, canonicalPaths, isDirs, isSymlinks, sizes and modifiedList lists. ``` An oxford comma here makes it more readable IMO. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
