This is an automated email from the ASF dual-hosted git repository. tanxinyu pushed a commit to branch wal_compression in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 06d3dba2a6613f5a9efb8a5363ad8e10dc153461 Author: OneSizeFitQuorum <[email protected]> AuthorDate: Wed Jun 19 13:33:22 2024 +0800 enhance Signed-off-by: OneSizeFitQuorum <[email protected]> --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +- .../storageengine/dataregion/wal/io/LogWriter.java | 35 ++++++------ .../dataregion/wal/io/WALInputStream.java | 64 +++++++++++++--------- .../dataregion/wal/io/WALMetaData.java | 12 ++-- .../storageengine/dataregion/wal/io/WALWriter.java | 14 +++-- .../dataregion/wal/recover/WALRecoverWriter.java | 12 ++-- .../wal/compression/WALCompressionTest.java | 12 ++-- .../wal/recover/WALRecoverWriterTest.java | 4 +- .../resources/conf/iotdb-system.properties | 8 ++- 10 files changed, 93 insertions(+), 72 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 3e3fa348ed6..df82bedc465 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1138,7 +1138,7 @@ public class IoTDBConfig { */ private String RateLimiterType = "FixedIntervalRateLimiter"; - private CompressionType WALCompressionAlgorithm = CompressionType.UNCOMPRESSED; + private CompressionType WALCompressionAlgorithm = CompressionType.LZ4; IoTDBConfig() {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index dcf8792a145..f2865c8865d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -418,7 +418,7 @@ public class IoTDBDescriptor { "io_task_queue_size_for_flushing", Integer.toString(conf.getIoTaskQueueSizeForFlushing())))); boolean enableWALCompression = - Boolean.parseBoolean(properties.getProperty("enable_wal_compression", "false")); + Boolean.parseBoolean(properties.getProperty("enable_wal_compression", "true")); conf.setWALCompressionAlgorithm( enableWALCompression ? CompressionType.LZ4 : CompressionType.UNCOMPRESSED); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 29335efde5c..3aa3e7deb73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint; import org.apache.tsfile.compress.ICompressor; import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.bouncycastle.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,13 +50,16 @@ public abstract class LogWriter implements ILogWriter { protected final FileChannel logChannel; protected long size = 0; protected long originalSize = 0; - private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES * 2 + 1); + private final int COMPRESSED_HEADER_SIZE = Bytes.BYTES + Integer.BYTES * 2; + private final int UN_COMPRESSED_HEADER_SIZE = +Bytes.BYTES + Integer.BYTES; + private final ByteBuffer headerBuffer = ByteBuffer.allocate(UN_COMPRESSED_HEADER_SIZE); private ICompressor compressor = ICompressor.getCompressor( IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm()); private ByteBuffer compressedByteBuffer; - // Minimum size to compress, default is 32 KB - private static long minCompressionSize = 32 * 1024L; + + /** Minimum size to compress, default is 32 KB */ + private static final long MIN_COMPRESSION_SIZE = 32 * 1024L; protected LogWriter(File logFile) throws IOException { this.logFile = logFile; @@ -63,23 +67,16 @@ public abstract class LogWriter implements ILogWriter { this.logChannel = this.logStream.getChannel(); if (!logFile.exists() || logFile.length() == 0) { this.logChannel.write( - ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes(StandardCharsets.UTF_8))); + ByteBuffer.wrap(WALWriter.MAGIC_STRING_V2.getBytes(StandardCharsets.UTF_8))); size += logChannel.position(); } - if (IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm() - != CompressionType.UNCOMPRESSED) { - // TODO: Use a dynamic strategy to enlarge the buffer size - compressedByteBuffer = - ByteBuffer.allocate( - compressor.getMaxBytesForCompression( - IoTDBDescriptor.getInstance().getConfig().getWalBufferSize())); - } else { - compressedByteBuffer = null; - } } @Override public double write(ByteBuffer buffer) throws IOException { + // To support hot loading, we can't define it as a variable, + // because we need to dynamically check whether wal compression is enabled + // each time the buffer is serialized CompressionType compressionType = IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); int bufferSize = buffer.position(); @@ -92,12 +89,13 @@ public abstract class LogWriter implements ILogWriter { int uncompressedSize = bufferSize; if (compressionType != CompressionType.UNCOMPRESSED /* Do not compress buffer that is less than min size */ - && bufferSize > minCompressionSize) { + && bufferSize > MIN_COMPRESSION_SIZE) { if (Objects.isNull(compressedByteBuffer)) { + // TODO: Use a dynamic strategy to enlarge the buffer size compressedByteBuffer = ByteBuffer.allocate( compressor.getMaxBytesForCompression( - IoTDBDescriptor.getInstance().getConfig().getWalBufferSize())); + IoTDBDescriptor.getInstance().getConfig().getWalBufferSize() / 2)); } compressedByteBuffer.clear(); if (compressor.getType() != compressionType) { @@ -108,6 +106,11 @@ public abstract class LogWriter implements ILogWriter { bufferSize = buffer.position(); buffer.flip(); compressed = true; + originalSize += COMPRESSED_HEADER_SIZE; + size += COMPRESSED_HEADER_SIZE; + } else { + originalSize += UN_COMPRESSED_HEADER_SIZE; + size += UN_COMPRESSED_HEADER_SIZE; } size += bufferSize; /* diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 844c06436b7..a1a3b5ea3d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -38,8 +38,11 @@ public class WALInputStream extends InputStream implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(WALInputStream.class); private final FileChannel channel; - private final ByteBuffer segmentHeaderBuffer = ByteBuffer.allocate(Integer.BYTES + Byte.BYTES); - private final ByteBuffer compressedHeader = ByteBuffer.allocate(Integer.BYTES); + + private final int DEFAULT_BATCH_READ_SIZE = 128 * 1024; + private final ByteBuffer segmentHeaderWithoutConpressedSizeBuffer = + ByteBuffer.allocate(Integer.BYTES + Byte.BYTES); + private final ByteBuffer compressedSizeHeaderBuffer = ByteBuffer.allocate(Integer.BYTES); private ByteBuffer dataBuffer = null; private ByteBuffer compressedBuffer = null; private long fileSize; @@ -69,7 +72,7 @@ public class WALInputStream extends InputStream implements AutoCloseable { } private void getEndOffset() throws IOException { - if (channel.size() < WALWriter.MAGIC_STRING_BYTES + Integer.BYTES) { + if (channel.size() < WALWriter.MAGIC_STRING_V2_BYTES + Integer.BYTES) { // An broken file endOffset = channel.size(); return; @@ -79,17 +82,17 @@ public class WALInputStream extends InputStream implements AutoCloseable { try { if (version == FileVersion.V2) { // New Version - ByteBuffer magicStringBuffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); - channel.read(magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_BYTES); + ByteBuffer magicStringBuffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_V2_BYTES); + channel.read(magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_V2_BYTES); magicStringBuffer.flip(); if (!new String(magicStringBuffer.array(), StandardCharsets.UTF_8) - .equals(WALWriter.MAGIC_STRING)) { + .equals(WALWriter.MAGIC_STRING_V2)) { // This is a broken wal file endOffset = channel.size(); return; } else { // This is a normal wal file - position = channel.size() - WALWriter.MAGIC_STRING_BYTES - Integer.BYTES; + position = channel.size() - WALWriter.MAGIC_STRING_V2_BYTES - Integer.BYTES; } } else { // Old version @@ -115,29 +118,35 @@ public class WALInputStream extends InputStream implements AutoCloseable { channel.read(metadataSizeBuf, position); metadataSizeBuf.flip(); int metadataSize = metadataSizeBuf.getInt(); - endOffset = channel.size() - WALWriter.MAGIC_STRING_BYTES - Integer.BYTES - metadataSize - 1; + // -1 is for the endmarker + endOffset = + channel.size() - WALWriter.MAGIC_STRING_V2_BYTES - Integer.BYTES - metadataSize - 1; } finally { - channel.position(WALWriter.MAGIC_STRING_BYTES); + if (version == FileVersion.V2) { + channel.position(WALWriter.MAGIC_STRING_V2_BYTES); + } else { + channel.position(WALWriter.MAGIC_STRING_V1_BYTES); + } } } private void analyzeFileVersion() throws IOException { - if (channel.size() < WALWriter.MAGIC_STRING_BYTES) { + if (channel.size() < WALWriter.MAGIC_STRING_V2_BYTES) { version = FileVersion.UNKNOWN; return; } - if (isCurrentVersion()) { + if (isV2Version()) { this.version = FileVersion.V2; return; } this.version = FileVersion.V1; } - private boolean isCurrentVersion() throws IOException { + private boolean isV2Version() throws IOException { channel.position(0); - ByteBuffer buffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); + ByteBuffer buffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_V2_BYTES); channel.read(buffer); - return new String(buffer.array(), StandardCharsets.UTF_8).equals(WALWriter.MAGIC_STRING); + return new String(buffer.array(), StandardCharsets.UTF_8).equals(WALWriter.MAGIC_STRING_V2); } @Override @@ -206,7 +215,7 @@ public class WALInputStream extends InputStream implements AutoCloseable { } if (Objects.isNull(dataBuffer)) { // read 128 KB - dataBuffer = ByteBuffer.allocate(128 * 1024); + dataBuffer = ByteBuffer.allocate(DEFAULT_BATCH_READ_SIZE); } dataBuffer.clear(); channel.read(dataBuffer); @@ -292,9 +301,9 @@ public class WALInputStream extends InputStream implements AutoCloseable { */ public void skipToGivenLogicalPosition(long pos) throws IOException { if (version == FileVersion.V2) { - channel.position(WALWriter.MAGIC_STRING_BYTES); + channel.position(WALWriter.MAGIC_STRING_V2_BYTES); long posRemain = pos; - SegmentInfo segmentInfo = null; + SegmentInfo segmentInfo; do { segmentInfo = getNextSegmentInfo(); if (posRemain >= segmentInfo.uncompressedSize) { @@ -340,24 +349,25 @@ public class WALInputStream extends InputStream implements AutoCloseable { } private SegmentInfo getNextSegmentInfo() throws IOException { - segmentHeaderBuffer.clear(); - channel.read(segmentHeaderBuffer); - segmentHeaderBuffer.flip(); + segmentHeaderWithoutConpressedSizeBuffer.clear(); + channel.read(segmentHeaderWithoutConpressedSizeBuffer); + segmentHeaderWithoutConpressedSizeBuffer.flip(); SegmentInfo info = new SegmentInfo(); - info.compressionType = CompressionType.deserialize(segmentHeaderBuffer.get()); - info.dataInDiskSize = segmentHeaderBuffer.getInt(); + info.compressionType = + CompressionType.deserialize(segmentHeaderWithoutConpressedSizeBuffer.get()); + info.dataInDiskSize = segmentHeaderWithoutConpressedSizeBuffer.getInt(); if (info.compressionType != CompressionType.UNCOMPRESSED) { - compressedHeader.clear(); - channel.read(compressedHeader); - compressedHeader.flip(); - info.uncompressedSize = compressedHeader.getInt(); + compressedSizeHeaderBuffer.clear(); + channel.read(compressedSizeHeaderBuffer); + compressedSizeHeaderBuffer.flip(); + info.uncompressedSize = compressedSizeHeaderBuffer.getInt(); } else { info.uncompressedSize = info.dataInDiskSize; } return info; } - private class SegmentInfo { + private static class SegmentInfo { public CompressionType compressionType; public int dataInDiskSize; public int uncompressedSize; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java index 7fa634ffbdc..9f23fa7f34e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java @@ -85,7 +85,7 @@ public class WALMetaData implements SerializedSize { + (memTablesId.isEmpty() ? 0 : Integer.BYTES + memTablesId.size() * Long.BYTES); } - public void serialize(File file, ByteBuffer buffer) { + public void serialize(ByteBuffer buffer) { buffer.putLong(firstSearchIndex); buffer.putInt(buffersSize.size()); for (int size : buffersSize) { @@ -129,12 +129,12 @@ public class WALMetaData implements SerializedSize { } public static WALMetaData readFromWALFile(File logFile, FileChannel channel) throws IOException { - if (channel.size() < WALWriter.MAGIC_STRING_BYTES || !isValidMagicString(channel)) { + if (channel.size() < WALWriter.MAGIC_STRING_V2_BYTES || !isValidMagicString(channel)) { throw new IOException(String.format("Broken wal file %s", logFile)); } // load metadata size ByteBuffer metadataSizeBuf = ByteBuffer.allocate(Integer.BYTES); - long position = channel.size() - WALWriter.MAGIC_STRING_BYTES - Integer.BYTES; + long position = channel.size() - WALWriter.MAGIC_STRING_V2_BYTES - Integer.BYTES; channel.read(metadataSizeBuf, position); metadataSizeBuf.flip(); // load metadata @@ -159,11 +159,11 @@ public class WALMetaData implements SerializedSize { } private static boolean isValidMagicString(FileChannel channel) throws IOException { - ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); - channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_BYTES); + ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_V2_BYTES); + channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_V2_BYTES); magicStringBytes.flip(); String magicString = new String(magicStringBytes.array(), StandardCharsets.UTF_8); - return magicString.equals(WALWriter.MAGIC_STRING) + return magicString.equals(WALWriter.MAGIC_STRING_V2) || magicString.contains(WALWriter.MAGIC_STRING_V1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java index 7017b4be6cb..9283c2b897f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java @@ -36,8 +36,12 @@ import java.nio.charset.StandardCharsets; public class WALWriter extends LogWriter { private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); public static final String MAGIC_STRING_V1 = "WAL"; - public static final String MAGIC_STRING = "V2-WAL"; - public static final int MAGIC_STRING_BYTES = MAGIC_STRING.getBytes(StandardCharsets.UTF_8).length; + public static final String MAGIC_STRING_V2 = "V2-WAL"; + + public static final int MAGIC_STRING_V1_BYTES = + MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length; + public static final int MAGIC_STRING_V2_BYTES = + MAGIC_STRING_V2.getBytes(StandardCharsets.UTF_8).length; private WALFileStatus walFileStatus = WALFileStatus.CONTAINS_NONE_SEARCH_INDEX; // wal files' metadata @@ -68,14 +72,14 @@ public class WALWriter extends LogWriter { int metaDataSize = metaData.serializedSize(); ByteBuffer buffer = ByteBuffer.allocate( - endMarker.serializedSize() + metaDataSize + Integer.BYTES + MAGIC_STRING_BYTES); + endMarker.serializedSize() + metaDataSize + Integer.BYTES + MAGIC_STRING_V2_BYTES); // mark info part ends endMarker.serialize(buffer); // flush meta data - metaData.serialize(logFile, buffer); + metaData.serialize(buffer); buffer.putInt(metaDataSize); // add magic string - buffer.put(MAGIC_STRING.getBytes(StandardCharsets.UTF_8)); + buffer.put(MAGIC_STRING_V2.getBytes(StandardCharsets.UTF_8)); size += buffer.position(); writeMetadata(buffer); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java index 5db0c17c987..b0d8f38d4d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java @@ -28,8 +28,8 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.StandardOpenOption; -import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING; -import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING_BYTES; +import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING_V2; +import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING_V2_BYTES; /** Check whether the wal file is broken and recover it. */ public class WALRecoverWriter { @@ -42,10 +42,10 @@ public class WALRecoverWriter { public void recover(WALMetaData metaData) throws IOException { // locate broken data long truncateSize; - if (logFile.length() < MAGIC_STRING_BYTES) { // file without magic string + if (logFile.length() < MAGIC_STRING_V2_BYTES) { // file without magic string truncateSize = 0; } else { - if (readTailMagic().equals(MAGIC_STRING)) { // complete file + if (readTailMagic().equals(MAGIC_STRING_V2)) { // complete file return; } else { // file with broken magic string truncateSize = metaData.getTruncateOffSet(); @@ -63,8 +63,8 @@ public class WALRecoverWriter { private String readTailMagic() throws IOException { try (FileChannel channel = FileChannel.open(logFile.toPath(), StandardOpenOption.READ)) { - ByteBuffer magicStringBytes = ByteBuffer.allocate(MAGIC_STRING_BYTES); - channel.read(magicStringBytes, channel.size() - MAGIC_STRING_BYTES); + ByteBuffer magicStringBytes = ByteBuffer.allocate(MAGIC_STRING_V2_BYTES); + channel.read(magicStringBytes, channel.size() - MAGIC_STRING_V2_BYTES); magicStringBytes.flip(); return new String(magicStringBytes.array()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java index d187f6107b6..8e49b4b7e89 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java @@ -192,10 +192,10 @@ public class WALCompressionTest { try (DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(Files.newInputStream(walFile.toPath())))) { - byte[] magicStringBytes = new byte[WALWriter.MAGIC_STRING_BYTES]; + byte[] magicStringBytes = new byte[WALWriter.MAGIC_STRING_V2_BYTES]; // head magic string dataInputStream.readFully(magicStringBytes); - Assert.assertEquals(WALWriter.MAGIC_STRING, new String(magicStringBytes)); + Assert.assertEquals(WALWriter.MAGIC_STRING_V2, new String(magicStringBytes)); Assert.assertEquals( CompressionType.UNCOMPRESSED, CompressionType.deserialize(dataInputStream.readByte())); Assert.assertEquals(buf.array().length, dataInputStream.readInt()); @@ -209,7 +209,7 @@ public class WALCompressionTest { dataInputStream.readFully(metadataBuf.array()); // Tail magic string dataInputStream.readFully(magicStringBytes); - Assert.assertEquals(WALWriter.MAGIC_STRING, new String(magicStringBytes)); + Assert.assertEquals(WALWriter.MAGIC_STRING_V2, new String(magicStringBytes)); } } @@ -243,10 +243,10 @@ public class WALCompressionTest { try (DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(Files.newInputStream(walFile.toPath())))) { - byte[] magicStringBytes = new byte[WALWriter.MAGIC_STRING_BYTES]; + byte[] magicStringBytes = new byte[WALWriter.MAGIC_STRING_V2_BYTES]; // head magic string dataInputStream.readFully(magicStringBytes); - Assert.assertEquals(WALWriter.MAGIC_STRING, new String(magicStringBytes)); + Assert.assertEquals(WALWriter.MAGIC_STRING_V2, new String(magicStringBytes)); Assert.assertEquals( CompressionType.LZ4, CompressionType.deserialize(dataInputStream.readByte())); Assert.assertEquals(compressed.length, dataInputStream.readInt()); @@ -263,7 +263,7 @@ public class WALCompressionTest { dataInputStream.readFully(metadataBuf.array()); // Tail magic string dataInputStream.readFully(magicStringBytes); - Assert.assertEquals(WALWriter.MAGIC_STRING, new String(magicStringBytes)); + Assert.assertEquals(WALWriter.MAGIC_STRING_V2, new String(magicStringBytes)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java index 97b00f379f2..7d49c448354 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java @@ -77,7 +77,7 @@ public class WALRecoverWriterTest { Byte.BYTES + (Long.BYTES + Integer.BYTES) + Integer.BYTES - + WALWriter.MAGIC_STRING_BYTES * 2, + + WALWriter.MAGIC_STRING_V2_BYTES * 2, logFile.length()); try (WALByteBufReader reader = new WALByteBufReader(logFile)) { Assert.assertFalse(reader.hasNext()); @@ -102,7 +102,7 @@ public class WALRecoverWriterTest { Byte.BYTES + (Long.BYTES + Integer.BYTES) + Integer.BYTES - + WALWriter.MAGIC_STRING_BYTES * 2, + + WALWriter.MAGIC_STRING_V2_BYTES * 2, logFile.length()); try (WALByteBufReader reader = new WALByteBufReader(logFile)) { Assert.assertFalse(reader.hasNext()); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties index 7dbf5f5b716..17bb6c70aca 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties @@ -1432,8 +1432,12 @@ data_replication_factor=1 # iot_consensus_cache_window_time_in_ms=-1 # Enable Write Ahead Log compression. -# Option: true, false -# enable_wal_compression=false +# With this parameter enabled, IoTDB can save a lot of IO resources at the cost of a small amount +# of additional CPU resources, which is generally suitable for the scenario +# where CPU is not the bottleneck but IO is the bottleneck. +# effectiveMode: hot_reload +# Datatype: boolean +# enable_wal_compression=true #################### ### IoTConsensus Configuration
