This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 91e18a7aba1c3cbd10b7d414271af3e587b51a3e Author: Murtadha Hubail <[email protected]> AuthorDate: Mon Oct 28 16:38:08 2019 +0300 [NO ISSUE][STO] Force Checkpoints/Index Metadata Files to Disk - user model changes: no - storage format changes: no - interface changes: no Details: - Ensure the system checkpoint file is forced to disk. - Ensure index metadata files and checkpoints are forced to disk. Change-Id: I27ab8ed39e23760b514640d77532a1dc24a97992 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/3943 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Reviewed-by: Michael Blow <[email protected]> --- .../java/org/apache/asterix/app/nc/IndexCheckpointManager.java | 5 ++--- .../management/resource/PersistentLocalResourceRepository.java | 4 ++-- .../management/service/recovery/AbstractCheckpointManager.java | 9 +++++++-- .../src/main/java/org/apache/hyracks/util/file/FileUtil.java | 9 +++++++++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java index 420585a..54647f8 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java @@ -36,6 +36,7 @@ import org.apache.asterix.common.storage.IndexCheckpoint; import org.apache.asterix.common.utils.StorageConstants; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.util.annotations.ThreadSafe; +import org.apache.hyracks.util.file.FileUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -191,9 +192,7 @@ public class IndexCheckpointManager implements IIndexCheckpointManager { if (checkpointPath.toFile().exists()) { Files.delete(checkpointPath); } - try (BufferedWriter writer = Files.newBufferedWriter(checkpointPath)) { - writer.write(checkpoint.asJson()); - } + FileUtil.writeAndForce(checkpointPath, checkpoint.asJson().getBytes()); // ensure it was written correctly by reading it read(checkpointPath); return; diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java index aef7bbd..9f5300e 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java @@ -71,6 +71,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentId; import org.apache.hyracks.storage.common.ILocalResourceRepository; import org.apache.hyracks.storage.common.LocalResource; import org.apache.hyracks.util.ExitUtil; +import org.apache.hyracks.util.file.FileUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -194,8 +195,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito try { createResourceFileMask(resourceFile); byte[] bytes = OBJECT_MAPPER.writeValueAsBytes(resource.toJson(persistedResourceRegistry)); - final Path path = Paths.get(resourceFile.getAbsolutePath()); - Files.write(path, bytes); + FileUtil.writeAndForce(Paths.get(resourceFile.getAbsolutePath()), bytes); indexCheckpointManagerProvider.get(DatasetResourceReference.of(resource)).init(Long.MIN_VALUE, 0, LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID.getMaxId()); deleteResourceFileMask(resourceFile); diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java index 81002be..717af81 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java @@ -22,11 +22,11 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.io.OutputStream; +import java.io.RandomAccessFile; import java.nio.channels.ClosedByInterruptException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -42,6 +42,8 @@ import org.apache.asterix.common.transactions.ITransactionSubsystem; import org.apache.asterix.common.utils.StorageConstants; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.IPersistedResourceRegistry; +import org.apache.hyracks.api.util.IoUtil; +import org.apache.hyracks.util.file.FileUtil; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -172,8 +174,11 @@ public abstract class AbstractCheckpointManager implements ICheckpointManager { } // Write checkpoint file to disk try { + if (path.toFile().exists()) { + IoUtil.delete(path); + } byte[] bytes = OBJECT_MAPPER.writeValueAsBytes(checkpoint.toJson(persistedResourceRegistry)); - Files.write(path, bytes, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + FileUtil.writeAndForce(path, bytes); readCheckpoint(path); } catch (IOException e) { LOGGER.log(Level.ERROR, "Failed to write checkpoint to disk", e); diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java index 443640a..fe60653 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/file/FileUtil.java @@ -20,6 +20,8 @@ package org.apache.hyracks.util.file; import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.file.Path; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; @@ -86,4 +88,11 @@ public class FileUtil { } } } + + public static void writeAndForce(Path path, byte[] data) throws IOException { + try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "rw");) { + raf.write(data); + raf.getChannel().force(true); + } + } }
