This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new dda1e45d513 Fix LOAD snapshot races and stabilize cluster sync tests
(#19031)
dda1e45d513 is described below
commit dda1e45d513f2c1656fe8ba6a24984f7f957a3dc
Author: Caideyipi <[email protected]>
AuthorDate: Tue Nov 3 15:51:25 2026 +0800
Fix LOAD snapshot races and stabilize cluster sync tests (#19031)
* Fix LOAD snapshot races during concurrent piece writes
* Keep LOAD snapshots independent of writable staged files
* Stabilize LOAD cluster restart integration tests
* Fix LOAD snapshot recovery with missing progress logs
* Fix LOAD snapshot copy during concurrent writes
* Stabilize concurrent LOAD snapshot restore test
---
.../iotdb/db/it/IoTDBLoadTsFileClusterIT.java | 113 +++++-
.../db/storageengine/load/LoadTsFileProgress.java | 73 ++--
.../db/storageengine/load/LoadTsFileSnapshot.java | 59 ++-
.../storageengine/load/LoadTsFileProgressTest.java | 149 ++++++++
.../storageengine/load/LoadTsFileSnapshotTest.java | 420 ++++++++++++++++++---
5 files changed, 706 insertions(+), 108 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java
index dadd5a7b4ba..dd1c1eef59d 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileClusterIT.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import
org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.isession.SessionConfig;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.env.cluster.ClusterConstant;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
@@ -65,6 +66,7 @@ import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.file.Files;
+import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -85,6 +87,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
@@ -296,18 +299,22 @@ public class IoTDBLoadTsFileClusterIT {
statement.execute("load \"" + tmpDir.getAbsolutePath() + "\"");
}
+ awaitAllReplicasHaveData();
+
EnvFactory.getEnv().getDataNodeWrapper(0).stop();
Thread.sleep(1_000L);
- final long secondPointCount = generateTsFile(tmpDir, "load-second.tsfile",
3_000, 100_000L);
+ final File secondDir = new File(tmpDir, "load-second");
+ Assert.assertTrue(secondDir.mkdirs());
+ final long secondPointCount = generateTsFile(secondDir,
"load-second.tsfile", 3_000, 100_000L);
try (final Connection connection =
DriverManager.getConnection(
Config.IOTDB_URL_PREFIX
+
EnvFactory.getEnv().getDataNodeWrapper(1).getIpAndPortString(),
- "root",
- "root");
+ SessionConfig.DEFAULT_USER,
+ SessionConfig.DEFAULT_PASSWORD);
final Statement statement = connection.createStatement()) {
- statement.execute("load \"" + tmpDir.getAbsolutePath() + "\"");
+ statement.execute("load \"" + secondDir.getAbsolutePath() + "\"");
final long expectedPointCount = firstPointCount + secondPointCount;
long actualPointCount = queryPointCount(statement);
@@ -334,23 +341,26 @@ public class IoTDBLoadTsFileClusterIT {
statement.execute("load \"" + tmpDir.getAbsolutePath() + "\"");
awaitPointCount(statement, firstPointCount);
}
+ awaitAllReplicasHaveData();
- // LOAD the second file while DataNode 0 is down. With three replicas it
holds a copy of every
- // region, so it is the DataNode that has to catch up with the LOAD when
it comes back.
+ // LOAD only the second file while DataNode 0 is down. With three replicas
it holds a copy of
+ // every region, so it is the DataNode that has to catch up with the LOAD
when it comes back.
final DataNodeWrapper restartedDataNode =
EnvFactory.getEnv().getDataNodeWrapper(0);
restartedDataNode.stop();
Assert.assertFalse(restartedDataNode.isAlive());
- final long secondPointCount = generateTsFile(tmpDir, "sync-second.tsfile",
3_000, 100_000L);
+ final File secondDir = new File(tmpDir, "sync-second");
+ Assert.assertTrue(secondDir.mkdirs());
+ final long secondPointCount = generateTsFile(secondDir,
"sync-second.tsfile", 3_000, 100_000L);
final long expectedPointCount = firstPointCount + secondPointCount;
try (final Connection connection =
DriverManager.getConnection(
Config.IOTDB_URL_PREFIX
+
EnvFactory.getEnv().getDataNodeWrapper(1).getIpAndPortString(),
- "root",
- "root");
+ SessionConfig.DEFAULT_USER,
+ SessionConfig.DEFAULT_PASSWORD);
final Statement statement = connection.createStatement()) {
- statement.execute("load \"" + tmpDir.getAbsolutePath() + "\"");
+ statement.execute("load \"" + secondDir.getAbsolutePath() + "\"");
awaitPointCount(statement, expectedPointCount);
}
@@ -556,6 +566,85 @@ public class IoTDBLoadTsFileClusterIT {
+ pointCounts);
}
+ /** Whether a local region contains at least one TsFile. */
+ private static boolean hasData(final Long size) {
+ return size != null && size > 0;
+ }
+
+ /** Waits until every DataNode has materialized each DataRegion replica
locally. */
+ private static void awaitAllReplicasHaveData() {
+ Awaitility.await()
+ .atMost(2, TimeUnit.MINUTES)
+ .pollInterval(1, TimeUnit.SECONDS)
+ .ignoreExceptions()
+ .untilAsserted(
+ () -> {
+ final Map<Integer, Map<Integer, Long>> regionSizes =
dataRegionSizes();
+ Assert.assertFalse(
+ "the loaded database must have data regions",
regionSizes.isEmpty());
+ for (final Map<Integer, Long> replicaSizes :
regionSizes.values()) {
+ Assert.assertEquals(DATA_NODE_NUM, replicaSizes.size());
+ Assert.assertTrue(
+
replicaSizes.values().stream().allMatch(IoTDBLoadTsFileClusterIT::hasData));
+ }
+ });
+ }
+
+ /** The local TsFile sizes of this database, mapped to each DataNode's
region replica. */
+ private static Map<Integer, Map<Integer, Long>> dataRegionSizes() throws
Exception {
+ final Map<Integer, Map<Integer, Long>> regionSizes = new HashMap<>();
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ for (int dataNodeIndex = 0; dataNodeIndex < DATA_NODE_NUM;
dataNodeIndex++) {
+ final DataNodeWrapper dataNode =
EnvFactory.getEnv().getDataNodeWrapper(dataNodeIndex);
+ final int dataNodeId = dataNodeId(statement, dataNode);
+ for (final String order : Arrays.asList("sequence", "unsequence")) {
+ final java.nio.file.Path databaseDir =
+ Paths.get(dataNode.getDataNodeDir(), "data", order, DATABASE);
+ if (!Files.isDirectory(databaseDir)) {
+ continue;
+ }
+ try (Stream<java.nio.file.Path> regionDirectories =
Files.list(databaseDir)) {
+ regionDirectories
+ .filter(Files::isDirectory)
+ .forEach(
+ regionDirectory -> {
+ try {
+ final int regionId =
+
Integer.parseInt(regionDirectory.getFileName().toString());
+ final long size = tsFileSize(regionDirectory);
+ regionSizes
+ .computeIfAbsent(regionId, ignored -> new
HashMap<>())
+ .merge(dataNodeId, size, Long::sum);
+ } catch (final IOException | NumberFormatException e) {
+ throw new AssertionError(
+ "cannot inspect " + regionDirectory + " on " +
dataNode, e);
+ }
+ });
+ }
+ }
+ }
+ }
+ return regionSizes;
+ }
+
+ private static long tsFileSize(final java.nio.file.Path regionDirectory)
throws IOException {
+ try (Stream<java.nio.file.Path> files = Files.walk(regionDirectory)) {
+ return files
+ .filter(Files::isRegularFile)
+ .filter(path -> path.getFileName().toString().endsWith(".tsfile"))
+ .mapToLong(
+ path -> {
+ try {
+ return Files.size(path);
+ } catch (final IOException e) {
+ throw new AssertionError("cannot read " + path, e);
+ }
+ })
+ .sum();
+ }
+ }
+
/** The cluster id of the given DataNode, matched by its client RPC
endpoint. */
private static int dataNodeId(final Statement statement, final
DataNodeWrapper dataNode)
throws SQLException {
@@ -612,8 +701,8 @@ public class IoTDBLoadTsFileClusterIT {
DriverManager.getConnection(
Config.IOTDB_URL_PREFIX
+
EnvFactory.getEnv().getDataNodeWrapper(0).getIpAndPortString(),
- "root",
- "root");
+ SessionConfig.DEFAULT_USER,
+ SessionConfig.DEFAULT_PASSWORD);
final Statement statement = connection.createStatement()) {
statement.setQueryTimeout(DEGRADED_LOAD_TIMEOUT_SECONDS);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileProgress.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileProgress.java
index cf876348894..abfdcd91611 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileProgress.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileProgress.java
@@ -130,7 +130,7 @@ public class LoadTsFileProgress {
// -------------------------------------------------------------------------
private void ensureActivated() throws IOException {
- if (!progressFile.exists()) {
+ if (!progressFile.exists() || progressFile.length() == 0L) {
final File parent = progressFile.getParentFile();
if (parent != null && !parent.exists()) {
parent.mkdirs();
@@ -394,7 +394,7 @@ public class LoadTsFileProgress {
}
public List<ChunkRangeRecord> readAllRecords() throws IOException {
- if (!progressFile.isFile()) {
+ if (!progressFile.isFile() || progressFile.length() == 0L) {
return Collections.emptyList();
}
@@ -460,7 +460,7 @@ public class LoadTsFileProgress {
* destructive.
*/
public static TerminalRecord readTerminal(final File progressFile) throws
IOException {
- if (progressFile == null || !progressFile.isFile()) {
+ if (progressFile == null || !progressFile.isFile() ||
progressFile.length() == 0L) {
return null;
}
TerminalRecord terminal = null;
@@ -516,49 +516,56 @@ public class LoadTsFileProgress {
return null;
}
- /** Locates the offset of an incomplete trailing append entry, or -1 when
the file is intact. */
+ /** Locates an incomplete header or trailing entry, or -1 when the file is
intact. */
private long findTornTailOffset() throws IOException {
try (final CountingInputStream countingIn =
new CountingInputStream(new BufferedInputStream(new
FileInputStream(progressFile)));
final DataInputStream in = new DataInputStream(countingIn)) {
- final String magic = ReadWriteIOUtils.readString(in);
- if (!MAGIC.equals(magic)) {
+ final long fileLength = progressFile.length();
+ if (fileLength < Integer.BYTES) {
+ return 0L;
+ }
+ final int magicLength = in.readInt();
+ if (magicLength != MAGIC.length()) {
return -1L;
}
- skipUuid(in);
-
- // The length prefix is read into the same four bytes for every entry.
- final byte[] lengthBuf = new byte[Integer.BYTES];
- final ByteBuffer lengthView = ByteBuffer.wrap(lengthBuf);
+ if (fileLength - countingIn.getBytesRead() < magicLength) {
+ return 0L;
+ }
+ final byte[] magic = new byte[magicLength];
+ in.readFully(magic);
+ if (!MAGIC.equals(new String(magic, StandardCharsets.UTF_8))) {
+ return -1L;
+ }
+ if (fileLength - countingIn.getBytesRead() < Integer.BYTES) {
+ return 0L;
+ }
+ final int uuidLength = in.readInt();
+ if (uuidLength < 0) {
+ return -1L;
+ }
+ if (fileLength - countingIn.getBytesRead() < uuidLength) {
+ return 0L;
+ }
+ in.skipNBytes(uuidLength);
- while (true) {
+ while (countingIn.getBytesRead() < fileLength) {
final long entryStart = countingIn.getBytesRead();
- // The mark keeps skip() inside the buffer. Without it, a skip that
runs past the end of the
- // file is delegated to the file stream, which reports it as a full
skip and hides the tear.
- in.mark(Integer.BYTES);
- final int read = in.read(lengthBuf);
- if (read < 0) {
- return -1L; // Clean EOF
- }
- if (read < Integer.BYTES) {
- return entryStart; // Partially written length header
+ if (fileLength - entryStart < Integer.BYTES) {
+ return entryStart;
}
-
- lengthView.clear();
- final int entryLength = lengthView.getInt();
+ final int entryLength = in.readInt();
if (entryLength <= 0) {
- return -1L; // Corrupted marker, not a simple append truncation
+ return -1L;
}
-
- long skipped = 0;
- while (skipped < entryLength) {
- final long step = in.skip(entryLength - skipped);
- if (step <= 0) {
- return entryStart; // Incomplete entry payload
- }
- skipped += step;
+ // FileInputStream.skip() can report success beyond EOF. Check the
captured file length
+ // before skipping, including for entries larger than a
BufferedInputStream's buffer.
+ if (fileLength - countingIn.getBytesRead() < entryLength) {
+ return entryStart;
}
+ in.skipNBytes(entryLength);
}
+ return -1L;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
index 0d375b8a1ff..64e3e44a524 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshot.java
@@ -24,8 +24,11 @@ import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
-import org.apache.iotdb.db.storageengine.load.util.LoadUtil;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.external.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -157,6 +160,31 @@ public final class LoadTsFileSnapshot {
// 3. Catch up staged files if progress logs advanced during file copy
catchUpStagedFiles(taskDir, targetDir);
+
+ // A piece is applied only after its chunk records are appended. Without
any records, the
+ // restoring manager cannot resume this staged file; leave replay free to
create it again.
+ discardUnrecordedStagedFiles(targetDir);
+ }
+
+ private static void discardUnrecordedStagedFiles(final File snapshotTaskDir)
throws IOException {
+ for (final File stagedFile : listFiles(snapshotTaskDir)) {
+ if (!stagedFile.isFile() ||
!stagedFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
+ continue;
+ }
+
+ final File progressFile = LoadTsFileProgress.progressFileFor(stagedFile);
+ if (progressFile.isFile()
+ && !new
LoadTsFileProgress(stagedFile).readAllRecordsRepairingTornTail().isEmpty()) {
+ continue;
+ }
+
+ Files.deleteIfExists(stagedFile.toPath());
+ Files.deleteIfExists(
+ new File(stagedFile.getPath() +
TsFileResource.RESOURCE_SUFFIX).toPath());
+
Files.deleteIfExists(ModificationFile.getExclusiveMods(stagedFile).toPath());
+ Files.deleteIfExists(
+ new File(stagedFile.getPath() +
ModificationFileV1.FILE_SUFFIX).toPath());
+ }
}
/**
@@ -176,9 +204,14 @@ public final class LoadTsFileSnapshot {
final File snapshotStagedFile = new File(snapshotTaskDir, baseName);
final File sourceStagedFile = new File(sourceTaskDir, baseName);
- if (!snapshotStagedFile.isFile() || !sourceStagedFile.isFile()) {
+ if (!sourceStagedFile.isFile()) {
continue;
}
+ // A new staged file may appear between the data-file and progress-log
directory scans.
+ // Its log was copied by the second pass, so the snapshot must also
contain its data.
+ if (!snapshotStagedFile.isFile()) {
+ publishAtomicCopy(sourceStagedFile, snapshotStagedFile);
+ }
long expectedLength = recordedLengthOf(snapshotStagedFile);
for (int i = 0;
@@ -337,7 +370,9 @@ public final class LoadTsFileSnapshot {
final File tempTarget =
new File(parent, target.getName() + TEMP_FILE_SUFFIX +
UUID.randomUUID());
try {
- LoadUtil.transferFile(source, tempTarget, true);
+ // Staged files and their progress logs remain writable, including after
restore. A hard
+ // link would let appends or torn-tail repairs change both the source
and its snapshot.
+ copyPrefixBytes(source, tempTarget, source.length());
atomicMoveOrFallback(tempTarget, target);
} catch (final IOException e) {
Files.deleteIfExists(tempTarget.toPath());
@@ -352,10 +387,22 @@ public final class LoadTsFileSnapshot {
final File tempTarget =
new File(parent, target.getName() + TEMP_FILE_SUFFIX +
UUID.randomUUID());
+ try {
+ copyPrefixBytes(source, tempTarget, length);
+ atomicMoveOrFallback(tempTarget, target);
+ } catch (final IOException e) {
+ Files.deleteIfExists(tempTarget.toPath());
+ throw e;
+ }
+ }
+
+ /** Copies through a shared-read channel so a live writer can keep appending
on Windows. */
+ private static void copyPrefixBytes(final File source, final File target,
final long length)
+ throws IOException {
try (final FileChannel srcChannel = FileChannel.open(source.toPath(),
StandardOpenOption.READ);
final FileChannel dstChannel =
FileChannel.open(
- tempTarget.toPath(),
+ target.toPath(),
StandardOpenOption.CREATE,
StandardOpenOption.WRITE,
StandardOpenOption.TRUNCATE_EXISTING)) {
@@ -368,10 +415,6 @@ public final class LoadTsFileSnapshot {
}
position += transferred;
}
- atomicMoveOrFallback(tempTarget, target);
- } catch (final IOException e) {
- Files.deleteIfExists(tempTarget.toPath());
- throw e;
}
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileProgressTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileProgressTest.java
index ebf07540e0b..78a182654c7 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileProgressTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileProgressTest.java
@@ -39,6 +39,7 @@ import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.util.Arrays;
public class LoadTsFileProgressTest {
@@ -180,6 +181,154 @@ public class LoadTsFileProgressTest {
Assert.assertEquals(chunk.getChunkStatistic().getSerializedSize(),
statisticsBody.remaining());
}
+ @Test
+ public void testEveryIncompleteHeaderIsRepairedBeforeAppending() throws
Exception {
+ final File tsFile = Files.createTempFile("load-progress-header",
".tsfile").toFile();
+ final File progressFile = LoadTsFileProgress.progressFileFor(tsFile);
+ try {
+ recordChunk(tsFile, 1L);
+ final byte[] complete = Files.readAllBytes(progressFile.toPath());
+ final int headerLength =
+ Integer.BYTES
+ + "LTPROG1".length()
+ + Integer.BYTES
+ +
tsFile.getParentFile().getName().getBytes(StandardCharsets.UTF_8).length;
+ for (int length = 0; length < headerLength; length++) {
+ Files.write(progressFile.toPath(), Arrays.copyOf(complete, length));
+
+ Assert.assertTrue(
+ "incomplete header at byte " + length,
+ new
LoadTsFileProgress(tsFile).readAllRecordsRepairingTornTail().isEmpty());
+ Assert.assertEquals(0L, progressFile.length());
+ Assert.assertTrue(new
LoadTsFileProgress(tsFile).readAllRecords().isEmpty());
+ Assert.assertNull(LoadTsFileProgress.readTerminal(progressFile));
+
+ // A restored task must be able to append again after its empty header
was repaired.
+ recordChunk(tsFile, 2L);
+ Assert.assertEquals(1, new
LoadTsFileProgress(tsFile).readAllRecords().size());
+ }
+ } finally {
+ Files.deleteIfExists(progressFile.toPath());
+ Files.deleteIfExists(tsFile.toPath());
+ }
+ }
+
+ @Test
+ public void testEveryTornEntryKeepsTheCompletePrefix() throws Exception {
+ final File tsFile = Files.createTempFile("load-progress-torn-entry",
".tsfile").toFile();
+ final File progressFile = LoadTsFileProgress.progressFileFor(tsFile);
+ try {
+ recordChunk(tsFile, 1L);
+ final long firstEnd = progressFile.length();
+ recordChunk(tsFile, 2L);
+ final long secondEnd = progressFile.length();
+ new
LoadTsFileProgress(tsFile).recordTerminal(LoadTsFileConsensusOp.COMMIT, 3L);
+ final byte[] complete = Files.readAllBytes(progressFile.toPath());
+
+ final int headerLength =
+ Integer.BYTES
+ + "LTPROG1".length()
+ + Integer.BYTES
+ +
tsFile.getParentFile().getName().getBytes(StandardCharsets.UTF_8).length;
+ for (int length = headerLength; length < complete.length; length++) {
+ Files.write(progressFile.toPath(), Arrays.copyOf(complete, length));
+ final int expectedRecords = length < firstEnd ? 0 : length < secondEnd
? 1 : 2;
+ final long expectedLength =
+ length < firstEnd ? headerLength : length < secondEnd ? firstEnd :
secondEnd;
+ Assert.assertEquals(
+ "torn entry at byte " + length,
+ expectedRecords,
+ new
LoadTsFileProgress(tsFile).readAllRecordsRepairingTornTail().size());
+ Assert.assertEquals(expectedLength, progressFile.length());
+ Assert.assertNull(LoadTsFileProgress.readTerminal(progressFile));
+ }
+ } finally {
+ Files.deleteIfExists(progressFile.toPath());
+ Files.deleteIfExists(tsFile.toPath());
+ }
+ }
+
+ @Test
+ public void testTornEntryLargerThanTheReadBufferIsRepaired() throws
Exception {
+ final File tsFile = Files.createTempFile("load-progress-large-torn",
".tsfile").toFile();
+ final File progressFile = LoadTsFileProgress.progressFileFor(tsFile);
+ try {
+ recordChunk(tsFile, 1L);
+ final long completeLength = progressFile.length();
+ try (final DataOutputStream out =
+ new DataOutputStream(new FileOutputStream(progressFile, true))) {
+ out.writeInt(32769);
+ out.writeByte(99);
+ out.write(new byte[24576]);
+ }
+
+ Assert.assertEquals(
+ 1, new
LoadTsFileProgress(tsFile).readAllRecordsRepairingTornTail().size());
+ Assert.assertEquals(completeLength, progressFile.length());
+ } finally {
+ Files.deleteIfExists(progressFile.toPath());
+ Files.deleteIfExists(tsFile.toPath());
+ }
+ }
+
+ @Test
+ public void testCompleteEntryLargerThanTheReadBufferIsPreserved() throws
Exception {
+ final File tsFile = Files.createTempFile("load-progress-large-complete",
".tsfile").toFile();
+ final File progressFile = LoadTsFileProgress.progressFileFor(tsFile);
+ try {
+ recordChunk(tsFile, 1L);
+ try (final DataOutputStream out =
+ new DataOutputStream(new FileOutputStream(progressFile, true))) {
+ out.writeInt(32769);
+ out.writeByte(99);
+ out.write(new byte[32768]);
+ }
+ new
LoadTsFileProgress(tsFile).recordTerminal(LoadTsFileConsensusOp.COMMIT, 3L);
+ final long completeLength = progressFile.length();
+
+ Assert.assertEquals(
+ 1, new
LoadTsFileProgress(tsFile).readAllRecordsRepairingTornTail().size());
+ Assert.assertEquals(completeLength, progressFile.length());
+ Assert.assertEquals(3L,
LoadTsFileProgress.readTerminal(progressFile).searchIndex());
+ } finally {
+ Files.deleteIfExists(progressFile.toPath());
+ Files.deleteIfExists(tsFile.toPath());
+ }
+ }
+
+ @Test
+ public void testInvalidMagicIsNotRepairedAsATornHeader() throws Exception {
+ final File tsFile = Files.createTempFile("load-progress-invalid-magic",
".tsfile").toFile();
+ final File progressFile = LoadTsFileProgress.progressFileFor(tsFile);
+ try {
+ recordChunk(tsFile, 1L);
+ final byte[] corrupt = Files.readAllBytes(progressFile.toPath());
+ corrupt[Integer.BYTES] = 'X';
+ Files.write(progressFile.toPath(), corrupt);
+
+ Assert.assertThrows(
+ java.io.IOException.class,
+ () -> new
LoadTsFileProgress(tsFile).readAllRecordsRepairingTornTail());
+ Assert.assertArrayEquals(corrupt,
Files.readAllBytes(progressFile.toPath()));
+ } finally {
+ Files.deleteIfExists(progressFile.toPath());
+ Files.deleteIfExists(tsFile.toPath());
+ }
+ }
+
+ private static void recordChunk(final File tsFile, final long searchIndex)
throws Exception {
+ new LoadTsFileProgress(tsFile)
+ .recordChunk(
+ new StringArrayDeviceID("root", "sg", "d1"),
+ false,
+ 7L,
+ 30L,
+ true,
+ createChunk(),
+ 40L,
+ searchIndex);
+ }
+
private static Chunk createChunk() {
final ChunkHeader chunkHeader =
new ChunkHeader(
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshotTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshotTest.java
index d56ef4c22db..f78eb8ff244 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshotTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/LoadTsFileSnapshotTest.java
@@ -43,14 +43,20 @@ import org.junit.Test;
import org.mockito.Mockito;
import java.io.File;
+import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -72,9 +78,6 @@ public class LoadTsFileSnapshotTest {
private static final String REGION_ID = "0";
private static final String REGION_DIR_NAME = DATABASE_NAME + "-" +
REGION_ID;
- /** Bounds the loop that snapshots a task while its pieces are being
applied. */
- private static final int MAX_SNAPSHOTS = 500;
-
private File tempDir;
private String[] originalLoadBaseDirs;
private IoTDBConfig config;
@@ -147,6 +150,92 @@ public class LoadTsFileSnapshotTest {
collected.contains("load/roots"));
}
+ @Test
+ public void testSnapshotDoesNotChangeWhenMorePiecesAreApplied() throws
Exception {
+ final String loadId = "snapshot-before-append";
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+
Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager));
+ final NonAlignedChunkData first = laidOutChunk();
+ final NonAlignedChunkData second =
+ createNonAlignedChunkData(
+ new StringArrayDeviceID("root", "snapshot_test", "d1"), "s1", 100,
10);
+ final ChunkOffsetCalculator calculator = new ChunkOffsetCalculator();
+ calculator.assign(first);
+ calculator.assign(second);
+ final LoadTsFileConsensusNode firstPiece = stagedPiece(loadId, first);
+ manager.writePiece(firstPiece);
+ final File staged = new File(tempDir,
firstPiece.getPieceRefs().get(0).getRelativePath());
+ final File snapshotDir = new File(tempDir, "snapshot-before-append");
+ assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir));
+ final File copiedStaged =
+ new File(
+ new File(new File(snapshotDir,
LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME), loadId),
+ staged.getName());
+ final File copiedProgress =
LoadTsFileProgress.progressFileFor(copiedStaged);
+ final byte[] copiedData = Files.readAllBytes(copiedStaged.toPath());
+ final byte[] copiedLog = Files.readAllBytes(copiedProgress.toPath());
+
+ manager.writePiece(stagedPiece(loadId, second));
+
+ assertTrue(staged.length() > copiedData.length);
+ assertEquals(2, new LoadTsFileProgress(staged).readAllRecords().size());
+ assertArrayEquals(copiedData, Files.readAllBytes(copiedStaged.toPath()));
+ assertArrayEquals(copiedLog, Files.readAllBytes(copiedProgress.toPath()));
+ assertEquals(1, new
LoadTsFileProgress(copiedStaged).readAllRecords().size());
+ }
+
+ @Test
+ public void testSnapshotRepairsIncompleteHeaderWithoutChangingSource()
throws Exception {
+ assertSnapshotRepairsProgressWithoutChangingSource(true);
+ }
+
+ @Test
+ public void testSnapshotRepairsTornEntryWithoutChangingSource() throws
Exception {
+ assertSnapshotRepairsProgressWithoutChangingSource(false);
+ }
+
+ private void assertSnapshotRepairsProgressWithoutChangingSource(final
boolean incompleteHeader)
+ throws Exception {
+ final String loadId = "snapshot-torn-progress";
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+
Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager));
+ final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk());
+ manager.writePiece(piece);
+ final File staged = new File(tempDir,
piece.getPieceRefs().get(0).getRelativePath());
+ final File progressFile = LoadTsFileProgress.progressFileFor(staged);
+ final byte[] completeLog = Files.readAllBytes(progressFile.toPath());
+ final byte[] tornLog;
+ if (incompleteHeader) {
+ tornLog = Arrays.copyOf(completeLog, Integer.BYTES - 1);
+ } else {
+ tornLog = Arrays.copyOf(completeLog, completeLog.length + Integer.BYTES
+ 1);
+ ByteBuffer.wrap(tornLog, completeLog.length, Integer.BYTES).putInt(32);
+ }
+ Files.write(progressFile.toPath(), tornLog);
+ final File snapshotDir = new File(tempDir, "snapshot-torn-progress");
+
+ assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir));
+
+ // Repair only the copied log. The live log may still be in the middle of
an append.
+ assertArrayEquals(tornLog, Files.readAllBytes(progressFile.toPath()));
+ final File copiedStaged =
+ new File(
+ new File(new File(snapshotDir,
LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME), loadId),
+ staged.getName());
+ if (incompleteHeader) {
+ assertFalse(copiedStaged.exists());
+ }
+ assertEquals(
+ incompleteHeader ? 0L : completeLog.length,
+ LoadTsFileProgress.progressFileFor(copiedStaged).length());
+ assertEquals(
+ incompleteHeader ? 0 : 1, new
LoadTsFileProgress(copiedStaged).readAllRecords().size());
+ if (incompleteHeader) {
+ assertPieceCanBeReplayedAfterRestore(
+ snapshotDir, loadId, Files.readAllBytes(staged.toPath()));
+ }
+ }
+
@Test
public void testSnapshotOfARegionWithoutStagingTasksIsEmpty() throws
Exception {
final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
@@ -331,6 +420,99 @@ public class LoadTsFileSnapshotTest {
assertFalse(new File(new File(tempDir, REGION_DIR_NAME),
"finished-task").exists());
}
+ @Test
+ public void testSnapshotCopiesAStagedFileThatAppearsBetweenDirectoryScans()
throws Exception {
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ final LoadTsFileConsensusNode piece = stagedPiece("new-staged-file",
laidOutChunk());
+ manager.writePiece(piece);
+ final File staged = new File(tempDir,
piece.getPieceRefs().get(0).getRelativePath());
+ // The task is initially empty. Its staged file and progress log appear
before the second
+ // directory scan, so the data-file pass misses the file that the log now
describes.
+ final File taskDir =
+ new File(staged.getParent()) {
+ private boolean firstScan = true;
+
+ @Override
+ public File[] listFiles() {
+ if (firstScan) {
+ firstScan = false;
+ return new File[0];
+ }
+ return super.listFiles();
+ }
+ };
+ final LoadTsFileManager snapshotManager =
Mockito.mock(LoadTsFileManager.class);
+ Mockito.when(snapshotManager.getActiveTaskDirs())
+ .thenReturn(Collections.singletonList(taskDir));
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent())
+ .thenReturn(Optional.of(snapshotManager));
+ final File snapshotDir = new File(tempDir, "snapshot-new-file");
+
+ assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir));
+
+ assertEveryCopiedLogFitsItsCopiedFile(snapshotDir);
+ assertEquals(staged.length(), lengthOfTheStagedFile(snapshotDir));
+ }
+
+ @Test
+ public void testSnapshotDropsAStagedFileWithoutAProgressLog() throws
Exception {
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ final String loadId = "staged-without-progress";
+ final LoadTsFileConsensusNode piece = stagedPiece(loadId, laidOutChunk());
+ manager.writePiece(piece);
+ final File staged = new File(tempDir,
piece.getPieceRefs().get(0).getRelativePath());
+ final byte[] stagedBytes = Files.readAllBytes(staged.toPath());
+ final byte[] progressBytes =
+
Files.readAllBytes(LoadTsFileProgress.progressFileFor(staged).toPath());
+ // The first piece has created its data file, but no progress log is
visible to either scan yet.
+ final File taskDir =
+ new File(staged.getParent()) {
+ @Override
+ public File[] listFiles() {
+ return new File[] {staged};
+ }
+ };
+ final LoadTsFileManager snapshotManager =
Mockito.mock(LoadTsFileManager.class);
+ Mockito.when(snapshotManager.getActiveTaskDirs())
+ .thenReturn(Collections.singletonList(taskDir));
+ Mockito.when(dataRegion.getLoadTsFileManagerIfPresent())
+ .thenReturn(Optional.of(snapshotManager));
+ final File snapshotDir = new File(tempDir, "snapshot-without-progress");
+
+ assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir));
+
+ final File copiedStaged =
+ new File(
+ new File(
+ new File(snapshotDir, LoadTsFileSnapshot.SNAPSHOT_SUBDIR_NAME),
+ staged.getParentFile().getName()),
+ staged.getName());
+ assertFalse(copiedStaged.exists());
+ assertArrayEquals(stagedBytes, Files.readAllBytes(staged.toPath()));
+ assertArrayEquals(
+ progressBytes,
Files.readAllBytes(LoadTsFileProgress.progressFileFor(staged).toPath()));
+ assertPieceCanBeReplayedAfterRestore(snapshotDir, loadId, stagedBytes);
+ }
+
+ private void assertPieceCanBeReplayedAfterRestore(
+ final File snapshotDir, final String loadId, final byte[]
expectedStagedBytes)
+ throws Exception {
+ final File restoredRoot = new File(tempDir, "restored-load-root");
+ setLoadRoots(restoredRoot);
+ LoadTsFileSnapshot.restore(DATABASE_NAME, REGION_ID, snapshotDir);
+ final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
+ try {
+ final LoadTsFileConsensusNode replayed = stagedPiece(loadId,
laidOutChunk());
+ recovered.writePiece(replayed);
+ final File restoredStaged =
+ new File(restoredRoot,
replayed.getPieceRefs().get(0).getRelativePath());
+ assertArrayEquals(expectedStagedBytes,
Files.readAllBytes(restoredStaged.toPath()));
+ assertEquals(1, new
LoadTsFileProgress(restoredStaged).readAllRecords().size());
+ } finally {
+ recovered.stop();
+ }
+ }
+
/**
* A snapshot taken while pieces are still being applied has to be
restorable: every range a
* copied progress log describes has to be inside the bytes that were copied
with it.
@@ -343,58 +525,122 @@ public class LoadTsFileSnapshotTest {
*/
@Test
public void testSnapshotTakenWhilePiecesAreAppliedStaysRestorable() throws
Exception {
- final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
-
Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager));
- final String loadId = "concurrent-snapshot";
-
- // Ten chunks of one partition, laid out one after the other, so every
piece appends to the same
- // staged file and its log.
- final ChunkOffsetCalculator calculator = new ChunkOffsetCalculator();
- final List<NonAlignedChunkData> chunks = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- final NonAlignedChunkData chunk =
- createNonAlignedChunkData(
- new StringArrayDeviceID("root", "snapshot_test", "d" + i), "s" +
i, i * 100, 10);
- calculator.assign(chunk);
- chunks.add(chunk);
- }
+ final int originalWriteBufferSize = config.getLoadTsFileWriteBufferSize();
+ config.setLoadTsFileWriteBufferSize(1024);
+ try {
+ final LoadTsFileManager manager = new LoadTsFileManager(dataRegion);
+ try {
+
Mockito.when(dataRegion.getLoadTsFileManagerIfPresent()).thenReturn(Optional.of(manager));
+ final String loadId = "concurrent-snapshot";
+
+ // Keep a live writer open on one staged file while later pieces
append to it.
+ final ChunkOffsetCalculator calculator = new ChunkOffsetCalculator();
+ final NonAlignedChunkData firstChunk =
+ createNonAlignedChunkData(
+ new StringArrayDeviceID("root", "snapshot_test", "d0"), "s0",
0, 1000);
+ calculator.assign(firstChunk);
+ manager.writePiece(loadId, Collections.singletonList(firstChunk));
+ final File stagedFile =
+ new File(tempDir,
firstChunk.getChunkPayloadRefs().get(0).getFilePath());
+ final long firstPieceLength = stagedFile.length();
+
+ final List<NonAlignedChunkData> chunks = new ArrayList<>();
+ for (int i = 1; i <= 50; i++) {
+ final NonAlignedChunkData chunk =
+ i == 1
+ ? blockingChunk(
+ new StringArrayDeviceID("root", "snapshot_test", "d" +
i),
+ "s" + i,
+ i * 1000,
+ 1000)
+ : createNonAlignedChunkData(
+ new StringArrayDeviceID("root", "snapshot_test", "d" +
i),
+ "s" + i,
+ i * 1000,
+ 1000);
+ calculator.assign(chunk);
+ chunks.add(chunk);
+ }
- final Set<Long> stagedLengths = new HashSet<>();
- final AtomicBoolean writing = new AtomicBoolean(true);
- final Thread writer =
- new Thread(
- () -> {
- try {
- for (final NonAlignedChunkData chunk : chunks) {
- manager.writePiece(loadId, Collections.singletonList(chunk));
- Thread.sleep(5L);
- }
- } catch (final Exception e) {
- // Whatever was staged is what the snapshots below are checked
against.
- } finally {
- writing.set(false);
+ final Set<Long> stagedLengths = new HashSet<>();
+ final ExecutorService writer = Executors.newSingleThreadExecutor();
+ final BlockingNonAlignedChunkData blockingChunk =
+ (BlockingNonAlignedChunkData) chunks.get(0);
+ File overlappedSnapshotDir = null;
+ try {
+ for (int i = 0; i < chunks.size(); i++) {
+ final NonAlignedChunkData chunk = chunks.get(i);
+ final Future<?> applied =
+ writer.submit(
+ () -> {
+ manager.writePiece(loadId,
Collections.singletonList(chunk));
+ return null;
+ });
+ final File snapshotDir = new File(tempDir, "snapshot-concurrent-"
+ i);
+ if (i == 0) {
+ assertTrue(
+ "the writer has to have appended bytes before the snapshot
starts",
+ blockingChunk.awaitBytesWritten());
+ assertTrue(
+ "the in-flight piece has to be visible in the staged file",
+ stagedFile.length() > firstPieceLength);
+ overlappedSnapshotDir = snapshotDir;
+ }
+ try {
+ assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir));
+ assertEveryCopiedLogFitsItsCopiedFile(snapshotDir);
+ } finally {
+ if (i == 0) {
+ blockingChunk.releaseWrite();
}
- });
- writer.start();
- try {
- // Snapshot the task while the pieces of the writer thread are still
being applied.
- int snapshots = 0;
- while (writing.get() && snapshots < MAX_SNAPSHOTS) {
- final File snapshotDir = new File(tempDir, "snapshot-" + snapshots);
- assertTrue(LoadTsFileSnapshot.snapshot(dataRegion, snapshotDir));
- assertEveryCopiedLogFitsItsCopiedFile(snapshotDir);
- stagedLengths.add(lengthOfTheStagedFile(snapshotDir));
- snapshots++;
- Thread.sleep(1L);
+ }
+
+ // Propagate writer failures and capture each completed piece
before allowing the next
+ // one.
+ applied.get(30, TimeUnit.SECONDS);
+ final File completedSnapshotDir = new File(tempDir,
"snapshot-completed-" + i);
+ assertTrue(LoadTsFileSnapshot.snapshot(dataRegion,
completedSnapshotDir));
+ assertEveryCopiedLogFitsItsCopiedFile(completedSnapshotDir);
+ stagedLengths.add(lengthOfTheStagedFile(completedSnapshotDir));
+ }
+ } finally {
+ blockingChunk.releaseWrite();
+ writer.shutdownNow();
+ assertTrue(
+ "the writer has to have finished", writer.awaitTermination(30,
TimeUnit.SECONDS));
+ }
+ assertTrue(
+ "the snapshots have to have caught the staged file while it was
still growing",
+ stagedLengths.size() > 1);
+
+ manager.stop();
+ setLoadRoots(tempDir);
+ LoadTsFileSnapshot.clear(DATABASE_NAME, REGION_ID);
+ LoadTsFileSnapshot.restore(DATABASE_NAME, REGION_ID,
overlappedSnapshotDir);
+
+ final LoadTsFileManager recovered = new LoadTsFileManager(dataRegion);
+ try {
+ // The overlapped snapshot has the first completed piece, but not
the paused piece's
+ // record.
+ // Recreate the paused piece with its original layout. Reusing the
original object would
+ // retain source file references populated by the first writer.
+ final NonAlignedChunkData resumedChunk =
+ createNonAlignedChunkData(
+ new StringArrayDeviceID("root", "snapshot_test", "d1"),
"s1", 1000, 1000);
+ resumedChunk.setChunkLayout(chunks.get(0).getChunkLayout());
+ assertTrue(stagedFile.isFile());
+ assertEquals(1, new
LoadTsFileProgress(stagedFile).readAllRecords().size());
+ recovered.writePiece(loadId,
Collections.singletonList(resumedChunk));
+ assertEquals(2, new
LoadTsFileProgress(stagedFile).readAllRecords().size());
+ } finally {
+ recovered.stop();
+ }
+ } finally {
+ manager.stop();
}
- assertTrue("the snapshots have to have been taken while the task was
staged", snapshots > 0);
} finally {
- writer.join();
+ config.setLoadTsFileWriteBufferSize(originalWriteBufferSize);
}
- assertFalse("the writer has to have finished by now", writing.get());
- assertTrue(
- "the snapshots have to have caught the staged file while it was still
growing",
- stagedLengths.size() > 1);
}
/** The length of the staged file a snapshot holds, or -1 when it holds none
yet. */
@@ -446,13 +692,19 @@ public class LoadTsFileSnapshotTest {
file.getName()
.substring(
0, file.getName().length() -
LoadTsFileProgress.PROGRESS_SUFFIX.length()));
- assertTrue(
- "a copied log has no staged file next to it: " + file.getName(),
staged.isFile());
final LoadTsFileProgress progress = new LoadTsFileProgress(staged);
// Reading a copied log repairs the fragment of an entry that was
caught half appended,
// exactly as the node that restores this snapshot does.
- for (final LoadTsFileProgress.ChunkRangeRecord record :
- progress.readAllRecordsRepairingTornTail()) {
+ final List<LoadTsFileProgress.ChunkRangeRecord> records =
+ progress.readAllRecordsRepairingTornTail();
+ if (records.isEmpty()) {
+ assertFalse(
+ "an unrecorded staged file blocks replay: " + staged.getName(),
staged.exists());
+ continue;
+ }
+ assertTrue(
+ "a copied log has no staged file next to it: " + file.getName(),
staged.isFile());
+ for (final LoadTsFileProgress.ChunkRangeRecord record : records) {
assertTrue(
"a copied log describes bytes the copied file does not hold: "
+ record.physicalEnd()
@@ -527,6 +779,64 @@ public class LoadTsFileSnapshotTest {
return chunkData;
}
+ private static BlockingNonAlignedChunkData blockingChunk(
+ final IDeviceID device, final String measurement, final int valueBase,
final int pointCount) {
+ final BlockingNonAlignedChunkData chunkData =
+ new BlockingNonAlignedChunkData(
+ device,
+ new ChunkHeader(
+ measurement,
+ 0,
+ TSDataType.INT32,
+ CompressionType.UNCOMPRESSED,
+ TSEncoding.PLAIN,
+ 0),
+ new TTimePartitionSlot(0L));
+ final long[] times = new long[pointCount];
+ final Object[] values = new Object[pointCount];
+ for (int i = 0; i < pointCount; i++) {
+ times[i] = i + 1L;
+ values[i] = valueBase + i;
+ }
+ chunkData.writeDecodePage(times, values, pointCount);
+ chunkData.endChunk();
+ return chunkData;
+ }
+
+ private static final class BlockingNonAlignedChunkData extends
NonAlignedChunkData {
+ private final CountDownLatch bytesWritten = new CountDownLatch(1);
+ private final CountDownLatch release = new CountDownLatch(1);
+
+ private BlockingNonAlignedChunkData(
+ final IDeviceID device,
+ final ChunkHeader chunkHeader,
+ final TTimePartitionSlot timePartitionSlot) {
+ super(device, chunkHeader, timePartitionSlot);
+ }
+
+ @Override
+ public void setChunkPayloadRefs(final List<ChunkPayloadRef>
chunkPayloadRefs) {
+ super.setChunkPayloadRefs(chunkPayloadRefs);
+ bytesWritten.countDown();
+ try {
+ if (!release.await(30, TimeUnit.SECONDS)) {
+ throw new AssertionError("the snapshot test did not release the
writer");
+ }
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError("the snapshot test writer was interrupted",
e);
+ }
+ }
+
+ private boolean awaitBytesWritten() throws InterruptedException {
+ return bytesWritten.await(30, TimeUnit.SECONDS);
+ }
+
+ private void releaseWrite() {
+ release.countDown();
+ }
+ }
+
private static void deleteRecursively(final File file) {
if (file == null || !file.exists()) {
return;