(tsfile) branch develop updated: Fixed the bug that ReadWriteIOUtils does not guarantee to use UTF-8 in deserialization

[jackietien]

This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git


The following commit(s) were added to refs/heads/develop by this push:
     new d36093df Fixed the bug that ReadWriteIOUtils does not guarantee to use 
UTF-8 in deserialization
d36093df is described below

commit d36093df847d085fa9f66083425cf5572b5b9518
Author: Caideyipi <87789683+caidey...@users.noreply.github.com>
AuthorDate: Thu Jan 9 10:24:26 2025 +0800

    Fixed the bug that ReadWriteIOUtils does not guarantee to use UTF-8 in 
deserialization
---
 NOTICE                                             |  2 +-
 .../org/apache/tsfile/utils/ReadWriteIOUtils.java  | 10 +++----
 .../apache/tsfile/utils/ReadWriteIOUtilsTest.java  | 35 ++++++++++++++++++++--
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/NOTICE b/NOTICE
index 6e2085f6..c50186ee 100644
--- a/NOTICE
+++ b/NOTICE
@@ -6,7 +6,7 @@ The Apache Software Foundation (http://www.apache.org/).
 
 ============================================================================
 
-TsFile project uses 4 Chinese Patents:
+TsFile project uses 2 Chinese Patents:
 * 201711384490X
 * 201711319331.1
 
diff --git 
a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java 
b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
index 50aed117..88f1388d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
@@ -617,7 +617,7 @@ public class ReadWriteIOUtils {
     if (readLen != strLength) {
       throw new IOException(String.format(RETURN_ERROR, strLength, readLen));
     }
-    return new String(bytes, 0, strLength);
+    return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET);
   }
 
   /** String length's type is varInt */
@@ -633,7 +633,7 @@ public class ReadWriteIOUtils {
     if (readLen != strLength) {
       throw new IOException(String.format(RETURN_ERROR, strLength, readLen));
     }
-    return new String(bytes, 0, strLength);
+    return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET);
   }
 
   /** Read string from byteBuffer. */
@@ -646,7 +646,7 @@ public class ReadWriteIOUtils {
     }
     byte[] bytes = new byte[strLength];
     buffer.get(bytes, 0, strLength);
-    return new String(bytes, 0, strLength);
+    return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET);
   }
 
   /** String length's type is varInt */
@@ -659,7 +659,7 @@ public class ReadWriteIOUtils {
     }
     byte[] bytes = new byte[strLength];
     buffer.get(bytes, 0, strLength);
-    return new String(bytes, 0, strLength);
+    return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET);
   }
 
   /** Read string from byteBuffer with user define length. */
@@ -671,7 +671,7 @@ public class ReadWriteIOUtils {
     }
     byte[] bytes = new byte[length];
     buffer.get(bytes, 0, length);
-    return new String(bytes, 0, length);
+    return new String(bytes, 0, length, TSFileConfig.STRING_CHARSET);
   }
 
   public static ByteBuffer getByteBuffer(String s) {
diff --git 
a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java 
b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java
index 17400fe1..a0cb9a0a 100644
--- 
a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java
+++ 
b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java
@@ -54,7 +54,21 @@ public class ReadWriteIOUtilsTest {
     Assert.assertNotNull(result);
     Assert.assertEquals(str, result);
 
-    // 2. null value
+    // 2. Chinese value
+    str = "中文";
+    byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
+    stream = new DataOutputStream(byteArrayOutputStream);
+    try {
+      ReadWriteIOUtils.write(str, stream);
+    } catch (IOException e) {
+      fail(e.toString());
+    }
+
+    result = 
ReadWriteIOUtils.readString(ByteBuffer.wrap(byteArrayOutputStream.toByteArray()));
+    Assert.assertNotNull(result);
+    Assert.assertEquals(str, result);
+
+    // 3. null value
     str = null;
     byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
     stream = new DataOutputStream(byteArrayOutputStream);
@@ -153,7 +167,24 @@ public class ReadWriteIOUtilsTest {
     Assert.assertNotNull(result);
     Assert.assertTrue(result.isEmpty());
 
-    // 6. null
+    // 6. key: chinese; value: chinese
+    key = "中文";
+    value = "中文";
+    map = new HashMap<>();
+    map.put(key, value);
+    byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
+    stream = new DataOutputStream(byteArrayOutputStream);
+    try {
+      ReadWriteIOUtils.write(map, stream);
+    } catch (IOException e) {
+      fail(e.toString());
+    }
+
+    result = 
ReadWriteIOUtils.readMap(ByteBuffer.wrap(byteArrayOutputStream.toByteArray()));
+    Assert.assertNotNull(result);
+    Assert.assertEquals(map, result);
+
+    // 7. null
     map = null;
     byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
     stream = new DataOutputStream(byteArrayOutputStream);