This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new d8a1a93 [NO ISSUE][REP] Ensure files delete order from replicas d8a1a93 is described below commit d8a1a93f99ce7c789dcc78afa1df3398d833cb15 Author: Murtadha Hubail <murtadha.hub...@couchbase.com> AuthorDate: Tue Sep 14 21:21:17 2021 +0300 [NO ISSUE][REP] Ensure files delete order from replicas - user model changes: no - storage format changes: no - interface changes: no Details: - When deleting files from a replica, delete the resource files last to ensure no orphaned component files if the replica fails after deleting the resource file. - When replicating a resource file, ensure the index directory is empty. Change-Id: I7dbb9960cb644c063de49dd0920a1709964de2f8 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13224 Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Murtadha Hubail <mhub...@apache.org> Reviewed-by: Ali Alsuliman <ali.al.solai...@gmail.com> --- .../java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java | 1 - .../org/apache/asterix/replication/messaging/ReplicateFileTask.java | 5 +++++ .../apache/asterix/replication/sync/ReplicaFilesSynchronizer.java | 5 +++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java index a926c66..dd7a663 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java @@ -46,7 +46,6 @@ public class LocalStorageCleanupTask implements INCLifecycleTask { deleteInvalidMetadataIndexes(localResourceRepository); final Set<Integer> nodePartitions = appContext.getReplicaManager().getPartitions(); localResourceRepository.deleteCorruptedResources(); - //TODO optimize this to cleanup all active partitions at once for (Integer partition : nodePartitions) { localResourceRepository.cleanup(partition); } diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java index 7f26b96..bf1613c 100644 --- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java +++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java @@ -39,6 +39,7 @@ import org.apache.asterix.common.utils.StorageConstants; import org.apache.asterix.replication.api.IReplicaTask; import org.apache.asterix.replication.api.IReplicationWorker; import org.apache.asterix.replication.management.NetworkingUtil; +import org.apache.commons.io.FileUtils; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; @@ -72,6 +73,10 @@ public class ReplicateFileTask implements IReplicaTask { // resolve path final FileReference localPath = ioManager.resolve(file); final Path resourceDir = Files.createDirectories(localPath.getFile().getParentFile().toPath()); + if (indexMetadata) { + // ensure clean index directory + FileUtils.cleanDirectory(resourceDir.toFile()); + } // create mask final Path maskPath = Paths.get(resourceDir.toString(), StorageConstants.MASK_FILE_PREFIX + localPath.getFile().getName()); diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java index 735318d..477559c 100644 --- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java +++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java @@ -21,6 +21,7 @@ package org.apache.asterix.replication.sync; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -127,6 +128,10 @@ public class ReplicaFilesSynchronizer { private void deleteInvalidFiles(List<String> files) { final FileSynchronizer sync = new FileSynchronizer(appCtx, replica); + // sort files to ensure index metadata files starting with "." are deleted last + files.sort(String::compareTo); + Collections.reverse(files); + LOGGER.info("deleting {}", files); files.forEach(sync::delete); }