This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch wal-compress-formal-branch in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0e0236650d4703baa26418adb4f03fed206f9a63 Author: Liu Xuxin <[email protected]> AuthorDate: Mon Jun 17 14:45:58 2024 +0800 Indicate the encoding of String --- .../dataregion/wal/buffer/WALBuffer.java | 1 + .../db/storageengine/dataregion/wal/io/LogWriter.java | 4 +++- .../dataregion/wal/io/WALByteBufReader.java | 1 + .../dataregion/wal/io/WALInputStream.java | 19 +++++++++++++------ .../storageengine/dataregion/wal/io/WALMetaData.java | 3 ++- .../db/storageengine/dataregion/wal/io/WALWriter.java | 5 +++-- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 627771bde38..20970aea34b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -540,6 +540,7 @@ public class WALBuffer extends AbstractWALBuffer { boolean forceSuccess = false; // try to roll log writer if (info.rollWALFileWriterListener != null + // TODO: Control the wal file by the number of WALEntry || (forceFlag && currentWALFileWriter.originalSize() >= config.getWalFileSizeThresholdInByte())) { try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 98a42b75280..29335efde5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.util.Objects; /** @@ -61,7 +62,8 @@ public abstract class LogWriter implements ILogWriter { this.logStream = new FileOutputStream(logFile, true); this.logChannel = this.logStream.getChannel(); if (!logFile.exists() || logFile.length() == 0) { - this.logChannel.write(ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes())); + this.logChannel.write( + ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes(StandardCharsets.UTF_8))); size += logChannel.position(); } if (IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java index 3e0909d6c52..882b5ea468c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java @@ -66,6 +66,7 @@ public class WALByteBufReader implements Closeable { */ public ByteBuffer next() throws IOException { int size = sizeIterator.next(); + // TODO: Reuse this buffer ByteBuffer buffer = ByteBuffer.allocate(size); /* Notice, we don't need to flip the buffer after calling diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 780be52671c..7cdcb63095f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.util.Objects; public class WALInputStream extends InputStream implements AutoCloseable { @@ -78,7 +79,8 @@ public class WALInputStream extends InputStream implements AutoCloseable { ByteBuffer magicStringBuffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_BYTES); magicStringBuffer.flip(); - if (!new String(magicStringBuffer.array()).equals(WALWriter.MAGIC_STRING)) { + if (!new String(magicStringBuffer.array(), StandardCharsets.UTF_8) + .equals(WALWriter.MAGIC_STRING)) { // This is a broken wal file endOffset = channel.size(); return; @@ -89,16 +91,21 @@ public class WALInputStream extends InputStream implements AutoCloseable { } else { // Old version ByteBuffer magicStringBuffer = - ByteBuffer.allocate(WALWriter.MAGIC_STRING_V1.getBytes().length); + ByteBuffer.allocate(WALWriter.MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length); channel.read( - magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length); + magicStringBuffer, + channel.size() - WALWriter.MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length); magicStringBuffer.flip(); - if (!new String(magicStringBuffer.array()).equals(WALWriter.MAGIC_STRING_V1)) { + if (!new String(magicStringBuffer.array(), StandardCharsets.UTF_8) + .equals(WALWriter.MAGIC_STRING_V1)) { // this is a broken wal file endOffset = channel.size(); return; } else { - position = channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length - Integer.BYTES; + position = + channel.size() + - WALWriter.MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length + - Integer.BYTES; } } // Read the meta data size @@ -127,7 +134,7 @@ public class WALInputStream extends InputStream implements AutoCloseable { channel.position(0); ByteBuffer buffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(buffer); - return new String(buffer.array()).equals(WALWriter.MAGIC_STRING); + return new String(buffer.array(), StandardCharsets.UTF_8).equals(WALWriter.MAGIC_STRING); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java index 8824bff3a26..7fa634ffbdc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java @@ -29,6 +29,7 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -161,7 +162,7 @@ public class WALMetaData implements SerializedSize { ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_BYTES); magicStringBytes.flip(); - String magicString = new String(magicStringBytes.array()); + String magicString = new String(magicStringBytes.array(), StandardCharsets.UTF_8); return magicString.equals(WALWriter.MAGIC_STRING) || magicString.contains(WALWriter.MAGIC_STRING_V1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java index c5be73892e7..7017b4be6cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java @@ -30,13 +30,14 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; /** WALWriter writes the binary {@link WALEntry} into .wal file. */ public class WALWriter extends LogWriter { private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); public static final String MAGIC_STRING_V1 = "WAL"; public static final String MAGIC_STRING = "V2-WAL"; - public static final int MAGIC_STRING_BYTES = MAGIC_STRING.getBytes().length; + public static final int MAGIC_STRING_BYTES = MAGIC_STRING.getBytes(StandardCharsets.UTF_8).length; private WALFileStatus walFileStatus = WALFileStatus.CONTAINS_NONE_SEARCH_INDEX; // wal files' metadata @@ -74,7 +75,7 @@ public class WALWriter extends LogWriter { metaData.serialize(logFile, buffer); buffer.putInt(metaDataSize); // add magic string - buffer.put(MAGIC_STRING.getBytes()); + buffer.put(MAGIC_STRING.getBytes(StandardCharsets.UTF_8)); size += buffer.position(); writeMetadata(buffer); }
