761417898 opened a new pull request, #123: URL: https://github.com/apache/tsfile/pull/123
issue: https://github.com/apache/tsfile/issues/113 It seems that BitpackCodec corresponds to the IntRleCodec in the Java version, but `BitpackCodec` miscalculates the length of the `length_` field during encoding and the length of the `bit_width_` field during decoding. This also causes DictionaryCodec to be inconsistent with the Java version. The following are the relevant unit tests. ``` TEST_F(BitPackEncoderTest, Encode) { BitPackEncoder encoder; common::ByteStream stream(1024, common::MOD_ENCODER_OBJ); int test_data[] = {5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 9, 9}; for (int value : test_data) { encoder.encode(value, stream); } encoder.encode_flush(stream); EXPECT_EQ(stream.total_size(), 12); uint32_t want_len = 12, read_len; uint8_t real_buf[12] = {}; stream.read_buf(real_buf, want_len, read_len); EXPECT_EQ(want_len, read_len); // Generated using Java Edition uint8_t expected_buf[12] = {11, 4, 5, 7, 85, 86, 102, 103, 120, 153, 153, 144}; for (int i = 0; i < 12; i++) { EXPECT_EQ(real_buf[i], expected_buf[i]); } } TEST_F(BitPackDecoderTest, ReadInt) { BitPackDecoder decoder; common::ByteStream stream(1024, common::MOD_ENCODER_OBJ); BitPackEncoder encoder; for (int i = 0; i < 1024; i++) { encoder.encode(int32_t(i), stream); } encoder.encode_flush(stream); decoder.init(); for (int i = 0; i < 1024; i++) { EXPECT_EQ(int32_t(i), decoder.read_int(stream)); } } TEST_F(DictionaryTest, DictionaryEncoder) { DictionaryEncoder encoder; common::ByteStream stream(1024, common::MOD_DICENCODE_OBJ); encoder.init(); encoder.encode("apple", stream); encoder.encode("banana", stream); encoder.encode("cherry", stream); encoder.encode("apple", stream); encoder.flush(stream); uint8_t buf[1024] = {0}; uint32_t want_len, read_len; want_len = stream.total_size(); stream.read_buf(buf, want_len, read_len); // Generated using Java Edition uint8_t expected_buf[] = {6, 10, 97, 112, 112, 108, 101, 12, 98, 97, 110, 97, 110, 97, 12, 99, 104, 101, 114, 114, 121, 5, 2, 3, 4, 24, 0}; EXPECT_EQ(read_len, sizeof(expected_buf)); for (int i = 0; i < sizeof(expected_buf); i++) { EXPECT_EQ(expected_buf[i], buf[i]); } } TEST_F(DictionaryTest, DictionaryEncoderAndDecoder) { DictionaryEncoder encoder; common::ByteStream stream(1024, common::MOD_DICENCODE_OBJ); encoder.init(); encoder.encode("apple", stream); encoder.encode("banana", stream); encoder.encode("cherry", stream); encoder.encode("apple", stream); encoder.flush(stream); DictionaryDecoder decoder; decoder.init(); ASSERT_TRUE(decoder.has_next(stream)); ASSERT_EQ(decoder.read_string(stream), "apple"); ASSERT_TRUE(decoder.has_next(stream)); ASSERT_EQ(decoder.read_string(stream), "banana"); ASSERT_TRUE(decoder.has_next(stream)); ASSERT_EQ(decoder.read_string(stream), "cherry"); ASSERT_TRUE(decoder.has_next(stream)); ASSERT_EQ(decoder.read_string(stream), "apple"); } ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
