Mmuzaf commented on a change in pull request #7984: URL: https://github.com/apache/ignite/pull/7984#discussion_r525123270
########## File path: modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/defragmentation/DefragmentationFileUtils.java ########## @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.defragmentation; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; +import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; + +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.nio.file.StandardOpenOption.CREATE_NEW; +import static java.nio.file.StandardOpenOption.WRITE; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.FILE_SUFFIX; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_PREFIX; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.TMP_SUFFIX; + +/** + * Everything related to file management during defragmentation process. + */ +public class DefragmentationFileUtils { + /** Prefix for link mapping files. */ + private static final String DFRG_LINK_MAPPING_FILE_PREFIX = PART_FILE_PREFIX + "map-"; + + /** Link mapping file template. */ + private static final String DFRG_LINK_MAPPING_FILE_TEMPLATE = DFRG_LINK_MAPPING_FILE_PREFIX + "%d" + FILE_SUFFIX; + + /** Defragmentation complation marker file name. */ + private static final String DFRG_COMPLETION_MARKER_FILE_NAME = "dfrg-completion-marker"; + + /** Name of defragmentated index partition file. */ + private static final String DFRG_INDEX_FILE_NAME = INDEX_FILE_PREFIX + "-dfrg" + FILE_SUFFIX; + + /** Name of defragmentated index partition temporary file. */ + private static final String DFRG_INDEX_TMP_FILE_NAME = DFRG_INDEX_FILE_NAME + TMP_SUFFIX; + + /** Prefix for defragmented partition files. */ + private static final String DFRG_PARTITION_FILE_PREFIX = PART_FILE_PREFIX + "dfrg-"; + + /** Defragmented partition file template. */ + private static final String DFRG_PARTITION_FILE_TEMPLATE = DFRG_PARTITION_FILE_PREFIX + "%d" + FILE_SUFFIX; + + /** Defragmented partition temp file template. */ + private static final String DFRG_PARTITION_TMP_FILE_TEMPLATE = DFRG_PARTITION_FILE_TEMPLATE + TMP_SUFFIX; + + /** + * Performs cleanup of work dir before initializing file page stores. + * Will finish batch renaming if defragmentation was completed or delete garbage if it wasn't. + * + * @param workDir Cache group working directory. + * @param log Logger to write messages. + * @throws IgniteCheckedException If {@link IOException} occurred. + */ + public static void beforeInitPageStores(File workDir, IgniteLogger log) throws IgniteCheckedException { + batchRenameDefragmentedCacheGroupPartitions(workDir, log); + + U.delete(defragmentationCompletionMarkerFile(workDir)); + + for (File file : workDir.listFiles()) { + String fileName = file.getName(); + + if ( + fileName.startsWith(DFRG_PARTITION_FILE_PREFIX) + || fileName.startsWith(DFRG_INDEX_FILE_NAME) + || fileName.startsWith(DFRG_LINK_MAPPING_FILE_PREFIX) + ) + U.delete(file); + } + } + + /** + * Checks whether cache group defragmentation completed or not. Completes it if all that's left is renaming. + * + * @param workDir Cache group working directory. + * @param grpId Cache group Id of cache group belonging to the given working directory. + * @param log Logger to write messages. + * @return {@code true} if given cache group is already defragmented. + * @throws IgniteCheckedException If {@link IOException} occurred. + * + * @see DefragmentationFileUtils#defragmentationCompletionMarkerFile(File) + */ + public static boolean skipAlreadyDefragmentedCacheGroup(File workDir, int grpId, IgniteLogger log) throws IgniteCheckedException { + File completionMarkerFile = defragmentationCompletionMarkerFile(workDir); + + if (completionMarkerFile.exists()) { + if (log.isInfoEnabled()) { + log.info(S.toString( + "Skipping already defragmented page group", + "grpId", grpId, false, + "markerFileName", completionMarkerFile.getName(), false, + "workDir", workDir.getAbsolutePath(), false + )); + } + + batchRenameDefragmentedCacheGroupPartitions(workDir, log); + + return true; + } + + return false; + } + + /** + * Checks whether partition has already been defragmented or not. Cleans corrupted data if previous failed + * defragmentation attempt was found. + * + * @param workDir Cache group working directory. + * @param grpId Cache group Id of cache group belonging to the given working directory. + * @param partId Partionion index to check. + * @param log Logger to write messages. + * @return {@code true} if given partition is already defragmented. + * @throws IgniteCheckedException If {@link IOException} occurred. + * + * @see DefragmentationFileUtils#defragmentedPartTmpFile(File, int) + * @see DefragmentationFileUtils#defragmentedPartFile(File, int) + * @see DefragmentationFileUtils#defragmentedPartMappingFile(File, int) + */ + public static boolean skipAlreadyDefragmentedPartition(File workDir, int grpId, int partId, IgniteLogger log) throws IgniteCheckedException { + File defragmentedPartFile = defragmentedPartFile(workDir, partId); + File defragmentedPartMappingFile = defragmentedPartMappingFile(workDir, partId); + + if (defragmentedPartFile.exists() && defragmentedPartMappingFile.exists()) { + if (log.isInfoEnabled()) { + log.info(S.toString( + "Skipping already defragmented partition", + "grpId", grpId, false, + "partId", partId, false, + "partFileName", defragmentedPartFile.getName(), false, + "mappingFileName", defragmentedPartMappingFile.getName(), false, + "workDir", workDir.getAbsolutePath(), false + )); + } + + return true; + } + + File defragmentedPartTmpFile = defragmentedPartTmpFile(workDir, partId); + + try { + Files.deleteIfExists(defragmentedPartTmpFile.toPath()); + + Files.deleteIfExists(defragmentedPartFile.toPath()); + + Files.deleteIfExists(defragmentedPartMappingFile.toPath()); + } + catch (IOException e) { + handleIoException(e); + } + + return false; + } + + /** + * Failure-tolerant batch rename of defragmented partition files. + * + * Deletes all link mapping files old partition and index files, renaming defragmentated files in the process. Can + * be run on the same folder multiple times if failed for some reason. + * + * Does something only if completion marker is present in the folder. This marker won't be deleted in the end. + * Deletion of the marker must be done outside of defragmentation mode to prevent cache groups to be defragmentated + * several times in case of failures. + * + * @param workDir Cache group working directory. + * @param log Logger to write messages. + * @throws IgniteCheckedException If {@link IOException} occurred. + * + * @see DefragmentationFileUtils#writeDefragmentationCompletionMarker(FileIOFactory, File, IgniteLogger) + */ + public static void batchRenameDefragmentedCacheGroupPartitions(File workDir, IgniteLogger log) throws IgniteCheckedException { + File completionMarkerFile = defragmentationCompletionMarkerFile(workDir); + + if (!completionMarkerFile.exists()) + return; + + try { + for (File mappingFile : workDir.listFiles((dir, name) -> name.startsWith(DFRG_LINK_MAPPING_FILE_PREFIX))) + Files.delete(mappingFile.toPath()); + + for (File partFile : workDir.listFiles((dir, name) -> name.startsWith(DFRG_PARTITION_FILE_PREFIX))) { + int partId = extractPartId(partFile.getName()); + + File oldPartFile = new File(workDir, String.format(PART_FILE_TEMPLATE, partId)); + + Files.move(partFile.toPath(), oldPartFile.toPath(), ATOMIC_MOVE, REPLACE_EXISTING); + } + + File idxFile = new File(workDir, DFRG_INDEX_FILE_NAME); + + if (idxFile.exists()) { + File oldIdxFile = new File(workDir, INDEX_FILE_NAME); + + Files.move(idxFile.toPath(), oldIdxFile.toPath(), ATOMIC_MOVE, REPLACE_EXISTING); + } + } + catch (IOException e) { + handleIoException(e); + } + } + + /** + * Extracts partition number from file names like {@code part-dfrg-%d.bin}. + * + * @param dfrgPartFileName Defragmented partition file name. + * @return Partition index. + * + * @see DefragmentationFileUtils#defragmentedPartFile(File, int) + */ + private static int extractPartId(String dfrgPartFileName) { Review comment: Maybe I'm missing something, but from my understanding the logic is: - if the fragment of code is used only once there is no reason to do something - if the fragment of code is used may times this is the reason to create a private method inside the class - if the fragment of code is used may times between classes this is the reason to move it to util class If you have a method with 1000 lines and try to reduce it by 'naming' some pieces of code (creating methods for some fragments of code) than you should think about of completely reworking the method body. So, my opinion is - such methods floods the source code and must be inlined. As an advantage of this - you will simplify the whole PR with reducing the changes. But it's up to you :-) ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
