This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new af2a7ee4 Fix tag empty error and disorder timestamp. (#489)
af2a7ee4 is described below
commit af2a7ee472dfcdc3e807f5c9fcfe19cff4e513b2
Author: Colin Lee <[email protected]>
AuthorDate: Thu Jun 19 09:47:48 2025 +0800
Fix tag empty error and disorder timestamp. (#489)
* add timestamp check.
* fix read error when tag is empty.
* fix tag empty error.
* add no_str_to_read.
* fix fmt.
* fix memory leak.
* use std::string tag in path.
* fix memory leak.
* fix memory leak.
* fix err in win.
* fix.
---
cpp/src/common/allocator/byte_stream.h | 49 ++++
cpp/src/common/allocator/my_string.h | 8 +-
cpp/src/common/constant/tsfile_constant.h | 6 +
cpp/src/common/device_id.h | 164 ++++++++-----
cpp/src/common/row_record.h | 5 +-
cpp/src/common/statistic.h | 5 +
cpp/src/common/tablet.cc | 22 +-
cpp/src/common/tsblock/tsblock.cc | 8 +-
cpp/src/common/tsblock/tsblock.h | 9 +
.../common/tsblock/vector/variable_length_vector.h | 1 +
cpp/src/common/tsfile_common.h | 31 ++-
cpp/src/file/tsfile_io_writer.cc | 1 -
cpp/src/reader/aligned_chunk_reader.cc | 3 +-
.../reader/block/single_device_tsblock_reader.cc | 22 +-
cpp/src/reader/chunk_reader.cc | 2 +-
cpp/src/reader/device_meta_iterator.cc | 1 -
cpp/src/reader/qds_with_timegenerator.cc | 4 +-
cpp/src/reader/qds_without_timegenerator.cc | 4 +-
cpp/src/reader/table_result_set.cc | 8 +-
cpp/src/utils/db_utils.h | 80 -------
cpp/src/utils/storage_utils.h | 4 +-
cpp/src/writer/time_page_writer.h | 4 +
cpp/src/writer/tsfile_writer.cc | 4 +-
cpp/test/common/device_id_test.cc | 59 +++++
cpp/test/common/row_record_test.cc | 6 +-
.../reader/table_view/tsfile_reader_table_test.cc | 12 +-
cpp/test/reader/tsfile_reader_test.cc | 4 +-
cpp/test/utils/db_utils_test.cc | 50 ----
.../writer/table_view/tsfile_writer_table_test.cc | 265 ++++++++++++++++++++-
cpp/test/writer/time_chunk_writer_test.cc | 5 -
cpp/test/writer/tsfile_writer_test.cc | 12 +-
31 files changed, 587 insertions(+), 271 deletions(-)
diff --git a/cpp/src/common/allocator/byte_stream.h
b/cpp/src/common/allocator/byte_stream.h
index 78366345..47c5148f 100644
--- a/cpp/src/common/allocator/byte_stream.h
+++ b/cpp/src/common/allocator/byte_stream.h
@@ -20,6 +20,7 @@
#ifndef COMMON_ALLOCATOR_BYTE_STREAM_H
#define COMMON_ALLOCATOR_BYTE_STREAM_H
+#include <common/constant/tsfile_constant.h>
#include <stdio.h>
#include <stdlib.h>
@@ -1047,6 +1048,54 @@ class SerializationUtil {
}
return ret;
}
+
+ // If the str is nullptr, NO_STR_TO_READ will be added instead.
+ FORCE_INLINE static int write_var_char_ptr(const std::string *str,
+ ByteStream &out) {
+ int ret = common::E_OK;
+ if (str == nullptr) {
+ write_var_int(storage::NO_STR_TO_READ, out);
+ return ret;
+ }
+ size_t str_len = str->length();
+ if (RET_FAIL(write_var_int(str_len, out))) {
+ return ret;
+ } else if (RET_FAIL(out.write_buf(str->c_str(), str_len))) {
+ return ret;
+ }
+ return ret;
+ }
+
+ // If `str` is not a nullptr after calling `read_var_char_ptr`, it
+ // indicates that memory has been allocated and must be freed.
+ FORCE_INLINE static int read_var_char_ptr(std::string *&str,
+ ByteStream &in) {
+ int ret = common::E_OK;
+ int32_t len = 0;
+ int32_t read_len = 0;
+ if (RET_FAIL(read_var_int(len, in))) {
+ return ret;
+ } else {
+ if (len == storage::NO_STR_TO_READ) {
+ str = nullptr;
+ return ret;
+ } else {
+ char *tmp_buf = static_cast<char *>(malloc(len));
+ if (RET_FAIL(in.read_buf(tmp_buf, len, read_len))) {
+ free(tmp_buf);
+ return ret;
+ } else if (len != read_len) {
+ free(tmp_buf);
+ ret = E_BUF_NOT_ENOUGH;
+ } else {
+ str = new std::string(tmp_buf, len);
+ free(tmp_buf);
+ }
+ }
+ }
+ return ret;
+ }
+
FORCE_INLINE static int read_var_str(std::string &str, ByteStream &in) {
int ret = common::E_OK;
int32_t len = 0;
diff --git a/cpp/src/common/allocator/my_string.h
b/cpp/src/common/allocator/my_string.h
index 9f5d8a5a..1f7bf00a 100644
--- a/cpp/src/common/allocator/my_string.h
+++ b/cpp/src/common/allocator/my_string.h
@@ -35,11 +35,12 @@ struct String {
String() : buf_(nullptr), len_(0) {}
String(char *buf, uint32_t len) : buf_(buf), len_(len) {}
- String(const std::string& str, common::PageArena& pa) : buf_(nullptr),
len_(0) {
+ String(const std::string &str, common::PageArena &pa)
+ : buf_(nullptr), len_(0) {
dup_from(str, pa);
}
- String(const std::string& str) {
- buf_ = (char*)str.c_str();
+ String(const std::string &str) {
+ buf_ = (char *)str.c_str();
len_ = str.size();
}
FORCE_INLINE bool is_null() const { return buf_ == nullptr && len_ == 0; }
@@ -67,6 +68,7 @@ struct String {
FORCE_INLINE int dup_from(const String &str, common::PageArena &pa) {
len_ = str.len_;
if (UNLIKELY(len_ == 0)) {
+ buf_ = nullptr;
return common::E_OK;
}
buf_ = pa.alloc(len_);
diff --git a/cpp/src/common/constant/tsfile_constant.h
b/cpp/src/common/constant/tsfile_constant.h
index af5d70f7..e1f4fd34 100644
--- a/cpp/src/common/constant/tsfile_constant.h
+++ b/cpp/src/common/constant/tsfile_constant.h
@@ -16,6 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
+
+#ifndef COMMON_CONSTANT_TSFILE_CONSTANT_H_
+#define COMMON_CONSTANT_TSFILE_CONSTANT_H_
#include <string>
#include <regex>
@@ -38,8 +41,11 @@ namespace storage
static const unsigned char VALUE_COLUMN_MASK = 0x40;
static const std::string TIME_COLUMN_ID = "";
+ static const int NO_STR_TO_READ = -1;
static const std::regex
IDENTIFIER_PATTERN("([a-zA-Z0-9_\\u2E80-\\u9FFF]+)");
static const std::regex
NODE_NAME_PATTERN("(\\*{0,2}[a-zA-Z0-9_\\u2E80-\\u9FFF]+\\*{0,2})");
static const int DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3;
} // namespace storage
+
+#endif
diff --git a/cpp/src/common/device_id.h b/cpp/src/common/device_id.h
index 021cb6aa..b3a173db 100644
--- a/cpp/src/common/device_id.h
+++ b/cpp/src/common/device_id.h
@@ -35,25 +35,25 @@
namespace storage {
class IDeviceID {
-public:
+ public:
virtual ~IDeviceID() = default;
virtual int serialize(common::ByteStream& write_stream) { return 0; }
virtual int deserialize(common::ByteStream& read_stream) { return 0; }
virtual std::string get_table_name() { return ""; }
virtual int segment_num() { return 0; }
- virtual const std::vector<std::string>& get_segments() const {
+ virtual const std::vector<std::string*>& get_segments() const {
return empty_segments_;
}
virtual std::string get_device_name() const { return ""; };
- virtual bool operator<(const IDeviceID& other) { return 0; }
+ virtual bool operator<(const IDeviceID& other) { return false; }
virtual bool operator==(const IDeviceID& other) { return false; }
virtual bool operator!=(const IDeviceID& other) { return false; }
-protected:
+ protected:
IDeviceID() : empty_segments_() {}
-private:
- const std::vector<std::string> empty_segments_;
+ private:
+ const std::vector<std::string*> empty_segments_;
};
struct IDeviceIDComparator {
@@ -64,23 +64,51 @@ struct IDeviceIDComparator {
};
class StringArrayDeviceID : public IDeviceID {
-public:
+ public:
explicit StringArrayDeviceID(const std::vector<std::string>& segments)
: segments_(formalize(segments)) {}
- explicit StringArrayDeviceID(const std::string& device_id_string)
- : segments_(split_device_id_string(device_id_string)) {}
+ explicit StringArrayDeviceID(const std::string& device_id_string) {
+ auto segments = split_device_id_string(device_id_string);
+ segments_.reserve(segments.size());
+ for (const auto& segment : segments) {
+ segments_.push_back(new std::string(segment));
+ }
+ }
+
+ explicit StringArrayDeviceID(const std::vector<std::string*>& segments) {
+ segments_.reserve(segments.size());
+ for (const auto& segment : segments) {
+ segments_.push_back(segment == nullptr ? nullptr
+ : new
std::string(*segment));
+ }
+ }
explicit StringArrayDeviceID() : segments_() {}
- ~StringArrayDeviceID() override = default;
+ ~StringArrayDeviceID() override {
+ for (const auto& segment : segments_) {
+ delete segment;
+ }
+ }
std::string get_device_name() const override {
- return segments_.empty() ? "" :
std::accumulate(std::next(segments_.begin()), segments_.end(),
- segments_.front(),
- [](std::string a, const std::string& b) {
- return std::move(a) + "." + b;
- });
+ if (segments_.empty()) {
+ return "";
+ }
+
+ std::string result(*segments_.front());
+ for (auto it = std::next(segments_.begin()); it != segments_.end();
+ ++it) {
+ result += '.';
+ if (*it != nullptr) {
+ result += **it;
+ } else {
+ result += "null";
+ }
+ }
+
+ return result;
};
int serialize(common::ByteStream& write_stream) override {
@@ -88,12 +116,12 @@ public:
if (RET_FAIL(common::SerializationUtil::write_var_uint(segment_num(),
write_stream)))
{
return ret;
- }
+ }
for (const auto& segment : segments_) {
- if (RET_FAIL(common::SerializationUtil::write_var_str(segment,
- write_stream))) {
+ if (RET_FAIL(common::SerializationUtil::write_var_char_ptr(
+ segment, write_stream))) {
return ret;
- }
+ }
}
return ret;
}
@@ -101,13 +129,23 @@ public:
int deserialize(common::ByteStream& read_stream) override {
int ret = common::E_OK;
uint32_t num_segments;
- if (RET_FAIL(common::SerializationUtil::read_var_uint(num_segments,
read_stream))) {
+ if (RET_FAIL(common::SerializationUtil::read_var_uint(num_segments,
+ read_stream))) {
return ret;
}
+
+ for (auto& segment : segments_) {
+ if (segment != nullptr) {
+ delete segment;
+ }
+ }
+
segments_.clear();
for (uint32_t i = 0; i < num_segments; ++i) {
- std::string segment;
- if (RET_FAIL(common::SerializationUtil::read_var_str(segment,
read_stream))) {
+ std::string* segment;
+ if (RET_FAIL(common::SerializationUtil::read_var_char_ptr(
+ segment, read_stream))) {
+ delete segment;
return ret;
}
segments_.push_back(segment);
@@ -116,52 +154,69 @@ public:
}
std::string get_table_name() override {
- return segments_.empty() ? "" : segments_[0];
+ return segments_.empty() ? "" : *segments_[0];
}
int segment_num() override { return static_cast<int>(segments_.size()); }
- const std::vector<std::string>& get_segments() const override {
+ const std::vector<std::string*>& get_segments() const override {
return segments_;
}
- virtual bool operator<(const IDeviceID& other) override {
+ bool operator<(const IDeviceID& other) override {
auto other_segments = other.get_segments();
- return std::lexicographical_compare(segments_.begin(), segments_.end(),
- other_segments.begin(),
- other_segments.end());
+ return std::lexicographical_compare(
+ segments_.begin(), segments_.end(), other_segments.begin(),
+ other_segments.end(),
+ [](const std::string* a, const std::string* b) {
+ if (a == nullptr && b == nullptr) return false; // equal
+ if (a == nullptr) return true; // nullptr < any string
+ if (b == nullptr) return false; // any string > nullptr
+ return *a < *b;
+ });
}
bool operator==(const IDeviceID& other) override {
auto other_segments = other.get_segments();
return (segments_.size() == other_segments.size()) &&
std::equal(segments_.begin(), segments_.end(),
- other_segments.begin());
+ other_segments.begin(),
+ [](const std::string* a, const std::string* b) {
+ if (a == nullptr && b == nullptr) return true;
+ if (a == nullptr || b == nullptr) return false;
+ return *a == *b;
+ });
}
bool operator!=(const IDeviceID& other) override {
return !(*this == other);
}
-private:
- std::vector<std::string> segments_;
+ private:
+ std::vector<std::string*> segments_;
- std::vector<std::string> formalize(
+ static std::vector<std::string*> formalize(
const std::vector<std::string>& segments) {
auto it =
std::find_if(segments.rbegin(), segments.rend(),
[](const std::string& seg) { return !seg.empty(); });
- return std::vector<std::string>(segments.begin(), it.base());
+ std::vector<std::string> validate_segments(segments.begin(),
it.base());
+ std::vector<std::string*> result;
+ result.reserve(validate_segments.size());
+ for (const auto& segment : validate_segments) {
+ result.emplace_back(new std::string(segment));
+ }
+ return result;
}
- std::vector<std::string> split_device_id_string(
+ static std::vector<std::string> split_device_id_string(
const std::string& device_id_string) {
auto splits =
storage::PathNodesGenerator::invokeParser(device_id_string);
return split_device_id_string(splits);
}
- std::vector<std::string> split_device_id_string(
+ static std::vector<std::string> split_device_id_string(
const std::vector<std::string>& splits) {
size_t segment_cnt = splits.size();
std::vector<std::string> final_segments;
@@ -173,8 +228,9 @@ private:
if (segment_cnt == 1) {
// "root" -> {"root"}
final_segments.push_back(splits[0]);
- } else if (segment_cnt < static_cast<size_t>(
- storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME + 1)) {
+ } else if (segment_cnt <
+ static_cast<size_t>(
+ storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME + 1)) {
// "root.a" -> {"root", "a"}
// "root.a.b" -> {"root.a", "b"}
std::string table_name = std::accumulate(
@@ -184,26 +240,26 @@ private:
});
final_segments.push_back(table_name);
final_segments.push_back(splits.back());
- } else {
- // "root.a.b.c" -> {"root.a.b", "c"}
- // "root.a.b.c.d" -> {"root.a.b", "c", "d"}
- std::string table_name = std::accumulate(
- splits.begin(),
- splits.begin() +
storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME,
- std::string(), [](const std::string& a, const std::string&
b) {
- return a.empty() ? b : a + storage::PATH_SEPARATOR + b;
- });
-
- final_segments.emplace_back(std::move(table_name));
- final_segments.insert(
- final_segments.end(),
- splits.begin() +
storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME,
- splits.end());
- }
+ } else {
+ // "root.a.b.c" -> {"root.a.b", "c"}
+ // "root.a.b.c.d" -> {"root.a.b", "c", "d"}
+ std::string table_name = std::accumulate(
+ splits.begin(),
+ splits.begin() + storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME,
+ std::string(), [](const std::string& a, const std::string& b) {
+ return a.empty() ? b : a + storage::PATH_SEPARATOR + b;
+ });
+
+ final_segments.emplace_back(std::move(table_name));
+ final_segments.insert(
+ final_segments.end(),
+ splits.begin() + storage::DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME,
+ splits.end());
+ }
return final_segments;
}
};
-}
+} // namespace storage
#endif
\ No newline at end of file
diff --git a/cpp/src/common/row_record.h b/cpp/src/common/row_record.h
index 2e449a5f..902913eb 100644
--- a/cpp/src/common/row_record.h
+++ b/cpp/src/common/row_record.h
@@ -55,8 +55,7 @@ struct Field {
is_type(common::NULL_TYPE);
}
- template <class T>
- FORCE_INLINE void set_value(common::TSDataType type, T val,
+ void set_value(common::TSDataType type, void* val, size_t len,
common::PageArena &pa) {
if (val == nullptr) {
type_ = common::NULL_TYPE;
@@ -86,7 +85,7 @@ struct Field {
}
case common::STRING: {
value_.strval_ = new common::String();
- value_.strval_->dup_from(*(common::String *)val, pa);
+ value_.strval_->dup_from(std::string(static_cast<char*>(val),
len), pa);
break;
}
// case common::TEXT: {
diff --git a/cpp/src/common/statistic.h b/cpp/src/common/statistic.h
index 8eb866e2..7590a6e0 100644
--- a/cpp/src/common/statistic.h
+++ b/cpp/src/common/statistic.h
@@ -164,6 +164,11 @@ class Statistic {
ASSERT(false);
return 0;
}
+
+ int get_count() const { return count_; }
+
+ int64_t get_end_time() const { return end_time_; }
+
virtual int deserialize_from(common::ByteStream &in) {
int ret = common::E_OK;
if (RET_FAIL(common::SerializationUtil::read_var_uint(
diff --git a/cpp/src/common/tablet.cc b/cpp/src/common/tablet.cc
index 8bc801cf..08671793 100644
--- a/cpp/src/common/tablet.cc
+++ b/cpp/src/common/tablet.cc
@@ -334,22 +334,36 @@ void Tablet::set_column_categories(
}
std::shared_ptr<IDeviceID> Tablet::get_device_id(int i) const {
- std::vector<std::string> id_array;
- id_array.push_back(insert_target_name_);
+ std::vector<std::string *> id_array;
+ id_array.push_back(new std::string(insert_target_name_));
for (auto id_column_idx : id_column_indexes_) {
common::TSDataType data_type = INVALID_DATATYPE;
void *value_ptr = get_value(i, id_column_idx, data_type);
+ if (value_ptr == nullptr) {
+ id_array.push_back(nullptr);
+ continue;
+ }
common::String str;
switch (data_type) {
case STRING:
str = *static_cast<common::String *>(value_ptr);
- id_array.push_back(str.to_std_string());
+ if (str.buf_ == nullptr || str.len_ == 0) {
+ id_array.push_back(new std::string());
+ } else {
+ id_array.push_back(new std::string(str.buf_, str.len_));
+ }
break;
default:
break;
}
}
- return std::make_shared<StringArrayDeviceID>(id_array);
+ auto res = std::make_shared<StringArrayDeviceID>(id_array);
+ for (auto &id : id_array) {
+ if (id != nullptr) {
+ delete id;
+ }
+ }
+ return res;
}
} // end namespace storage
\ No newline at end of file
diff --git a/cpp/src/common/tsblock/tsblock.cc
b/cpp/src/common/tsblock/tsblock.cc
index f290e3f6..8b93a501 100644
--- a/cpp/src/common/tsblock/tsblock.cc
+++ b/cpp/src/common/tsblock/tsblock.cc
@@ -72,24 +72,24 @@ void TsBlock::write_data(ByteStream *__restrict byte_stream,
switch (type) {
case common::INT64: {
int64_t ival = *reinterpret_cast<int64_t *>(val);
- strval = to_string(ival);
+ strval = std::to_string(ival);
break;
}
case common::INT32: {
int32_t ival = *reinterpret_cast<int32_t *>(val);
- strval = to_string(ival);
+ strval = std::to_string(ival);
break;
}
case common::FLOAT: {
float ival = *reinterpret_cast<float *>(
val); // cppcheck-suppress invalidPointerCast
- strval = to_string(ival);
+ strval = std::to_string(ival);
break;
}
case common::DOUBLE: {
double ival = *reinterpret_cast<double *>(
val); // cppcheck-suppress invalidPointerCast
- strval = to_string(ival);
+ strval = std::to_string(ival);
break;
}
case common::BOOLEAN: {
diff --git a/cpp/src/common/tsblock/tsblock.h b/cpp/src/common/tsblock/tsblock.h
index 47f110cc..4316e8f6 100644
--- a/cpp/src/common/tsblock/tsblock.h
+++ b/cpp/src/common/tsblock/tsblock.h
@@ -202,6 +202,15 @@ class ColAppender {
FORCE_INLINE uint32_t get_col_row_count() { return column_row_count_; }
FORCE_INLINE uint32_t get_column_index() { return column_index_; }
+ FORCE_INLINE int fill_null(uint32_t end_index) {
+ while (column_row_count_ < end_index) {
+ if (!add_row()) {
+ return E_INVALID_ARG;
+ }
+ append_null();
+ }
+ return E_OK;
+ }
FORCE_INLINE int fill(const char *value, uint32_t len,
uint32_t end_index) {
while (column_row_count_ < end_index) {
diff --git a/cpp/src/common/tsblock/vector/variable_length_vector.h
b/cpp/src/common/tsblock/vector/variable_length_vector.h
index bec599ac..23ebb75d 100644
--- a/cpp/src/common/tsblock/vector/variable_length_vector.h
+++ b/cpp/src/common/tsblock/vector/variable_length_vector.h
@@ -61,6 +61,7 @@ class VariableLengthVector : public Vector {
*null = nulls_.test(rowid);
} else {
*null = false;
+ *len = 0;
}
if (LIKELY(!(*null))) {
char *result = values_.read(offset_, len);
diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h
index 878fc4d2..f20632ee 100644
--- a/cpp/src/common/tsfile_common.h
+++ b/cpp/src/common/tsfile_common.h
@@ -621,10 +621,10 @@ class TSMIterator {
common::String &ret_measurement_name,
TimeseriesIndex &ret_ts_index);
-private:
- common::SimpleList<ChunkGroupMeta *> &chunk_group_meta_list_;
- common::SimpleList<ChunkGroupMeta *>::Iterator chunk_group_meta_iter_;
- common::SimpleList<ChunkMeta *>::Iterator chunk_meta_iter_;
+ private:
+ common::SimpleList<ChunkGroupMeta *> &chunk_group_meta_list_;
+ common::SimpleList<ChunkGroupMeta *>::Iterator chunk_group_meta_iter_;
+ common::SimpleList<ChunkMeta *>::Iterator chunk_meta_iter_;
// timeseries measurenemnt chunk meta info
// map <device_name, <measurement_name, vector<chunk_meta>>>
@@ -651,10 +651,10 @@ struct IComparable {
virtual int compare(const IComparable &other) {
if (this->operator<(other)) {
return -1;
- } else if (this->operator>(other)) {
- return 1;
- } else {
+ } else if (this->operator==(other)) {
return 0;
+ } else {
+ return 1;
}
}
virtual std::string to_string() const = 0;
@@ -670,24 +670,21 @@ struct DeviceIDComparable : IComparable {
const auto *other_device =
dynamic_cast<const DeviceIDComparable *>(&other);
if (!other_device) throw std::runtime_error("Incompatible comparison");
- return device_id_->get_device_name() <
- other_device->device_id_->get_device_name();
+ return *device_id_ < *other_device->device_id_;
}
bool operator>(const IComparable &other) const override {
const auto *other_device =
dynamic_cast<const DeviceIDComparable *>(&other);
if (!other_device) throw std::runtime_error("Incompatible comparison");
- return device_id_->get_device_name() >
- other_device->device_id_->get_device_name();
+ return *device_id_ != *other_device->device_id_ && !(*device_id_ <
*other_device->device_id_);
}
bool operator==(const IComparable &other) const override {
const auto *other_device =
dynamic_cast<const DeviceIDComparable *>(&other);
if (!other_device) throw std::runtime_error("Incompatible comparison");
- return device_id_->get_device_name() ==
- other_device->device_id_->get_device_name();
+ return *device_id_ == *other_device->device_id_;
}
std::string to_string() const override {
@@ -908,10 +905,10 @@ struct MetaIndexNode {
}
}
- int binary_search_children(std::shared_ptr<IComparable> key,
- bool exact_search,
- std::shared_ptr<IMetaIndexEntry>
&ret_index_entry,
- int64_t &ret_end_offset);
+ int binary_search_children(
+ std::shared_ptr<IComparable> key, bool exact_search,
+ std::shared_ptr<IMetaIndexEntry> &ret_index_entry,
+ int64_t &ret_end_offset);
int serialize_to(common::ByteStream &out) {
int ret = common::E_OK;
diff --git a/cpp/src/file/tsfile_io_writer.cc b/cpp/src/file/tsfile_io_writer.cc
index af4c29dc..4d19b12e 100644
--- a/cpp/src/file/tsfile_io_writer.cc
+++ b/cpp/src/file/tsfile_io_writer.cc
@@ -409,7 +409,6 @@ int TsFileIOWriter::write_file_index() {
// Time column also need add to bloom filter.
ret = filter.add_path_entry(tmp_device_name, measurement_name);
-
if (RET_FAIL(ts_index.serialize_to(write_stream_))) {
} else {
#if DEBUG_SE
diff --git a/cpp/src/reader/aligned_chunk_reader.cc
b/cpp/src/reader/aligned_chunk_reader.cc
index 06af8699..230661f3 100644
--- a/cpp/src/reader/aligned_chunk_reader.cc
+++ b/cpp/src/reader/aligned_chunk_reader.cc
@@ -209,7 +209,6 @@ int AlignedChunkReader::get_next_page(TsBlock *ret_tsblock,
int ret = E_OK;
Filter *filter =
(oneshoot_filter != nullptr ? oneshoot_filter : time_filter_);
-
if (prev_time_page_not_finish() && prev_value_page_not_finish()) {
ret = decode_time_value_buf_into_tsblock(ret_tsblock, oneshoot_filter);
return ret;
@@ -550,7 +549,7 @@ int AlignedChunkReader::decode_time_value_buf_into_tsblock(
continue;
\
} else {
\
/*std::cout << "decoder: time=" << time << ", value=" << value
\
- * << std::endl;*/
\
+ * << std::endl;*/ \
row_appender.append(0, (char *)&time, sizeof(time));
\
row_appender.append(1, (char *)&value, sizeof(value));
\
}
\
diff --git a/cpp/src/reader/block/single_device_tsblock_reader.cc
b/cpp/src/reader/block/single_device_tsblock_reader.cc
index ec8059f9..90a642b2 100644
--- a/cpp/src/reader/block/single_device_tsblock_reader.cc
+++ b/cpp/src/reader/block/single_device_tsblock_reader.cc
@@ -67,9 +67,6 @@ int SingleDeviceTsBlockReader::init(DeviceQueryTask*
device_query_task,
device_query_task->get_device_id(),
device_query_task->get_column_mapping()->get_measurement_columns(),
time_series_indexs, pa_);
- for (auto measurement_column :
- device_query_task->get_column_mapping()->get_measurement_columns()) {
- }
for (const auto& time_series_index : time_series_indexs) {
construct_column_context(time_series_index, time_filter);
}
@@ -104,9 +101,11 @@ int SingleDeviceTsBlockReader::has_next(bool& has_next) {
has_next = false;
return common::E_OK;
}
+
for (auto col_appender : col_appenders_) {
col_appender->reset();
}
+
current_block_->reset();
bool next_time_set = false;
@@ -199,11 +198,20 @@ int SingleDeviceTsBlockReader::fill_ids() {
for (const auto& entry : id_column_contexts_) {
const auto& id_column_context = entry.second;
for (int32_t pos : id_column_context.pos_in_result_) {
- common::String device_id(
- device_query_task_->get_device_id()->get_segments().at(
- id_column_context.pos_in_device_id_));
+ std::string* device_tag = nullptr;
+ device_tag =
device_query_task_->get_device_id()->get_segments().at(
+ id_column_context.pos_in_device_id_);
+ if (device_tag == nullptr) {
+ ret = col_appenders_[pos + 1]->fill_null(
+ current_block_->get_row_count());
+ if (ret != common::E_OK) {
+ return ret;
+ }
+ continue;
+ }
+
if (RET_FAIL(col_appenders_[pos + 1]->fill(
- (char*)&device_id, sizeof(device_id),
+ device_tag->c_str(), device_tag->length(),
current_block_->get_row_count()))) {
return ret;
}
diff --git a/cpp/src/reader/chunk_reader.cc b/cpp/src/reader/chunk_reader.cc
index f1e395da..afe7740a 100644
--- a/cpp/src/reader/chunk_reader.cc
+++ b/cpp/src/reader/chunk_reader.cc
@@ -437,7 +437,7 @@ int
ChunkReader::STRING_DECODE_TYPED_TV_INTO_TSBLOCK(ByteStream &time_in,
continue;
} else {
row_appender.append(0, (char *)&time, sizeof(time));
- row_appender.append(1, (char *)&value, sizeof(value));
+ row_appender.append(1, value.buf_, value.len_);
}
}
return ret;
diff --git a/cpp/src/reader/device_meta_iterator.cc
b/cpp/src/reader/device_meta_iterator.cc
index d4512263..e3c61dc8 100644
--- a/cpp/src/reader/device_meta_iterator.cc
+++ b/cpp/src/reader/device_meta_iterator.cc
@@ -28,7 +28,6 @@ bool DeviceMetaIterator::has_next() {
if (load_results() != common::E_OK) {
return false;
}
-
return !result_cache_.empty();
}
diff --git a/cpp/src/reader/qds_with_timegenerator.cc
b/cpp/src/reader/qds_with_timegenerator.cc
index fd0e1292..c93eb5c0 100644
--- a/cpp/src/reader/qds_with_timegenerator.cc
+++ b/cpp/src/reader/qds_with_timegenerator.cc
@@ -357,7 +357,7 @@ int QDSWithTimeGenerator::next(bool &has_next) {
return E_OK;
}
row_record_->set_timestamp(timestamp);
- row_record_->get_field(0)->set_value(TSDataType::INT64, ×tamp, pa_);
+ row_record_->get_field(0)->set_value(TSDataType::INT64, ×tamp,
sizeof(timestamp), pa_);
#if DEBUG_SE
std::cout << "QDSWithTimeGenerator::get_next: timestamp=" << timestamp
<< ", will generate row at this timestamp." << std::endl;
@@ -366,7 +366,7 @@ int QDSWithTimeGenerator::next(bool &has_next) {
for (size_t i = 0; i < value_at_vec_.size(); i++) {
ValueAt &va = value_at_vec_[i];
void *val_obj_ptr = va.at(timestamp);
- row_record_->get_field(i + 1)->set_value(va.data_type_, val_obj_ptr,
+ row_record_->get_field(i + 1)->set_value(va.data_type_, val_obj_ptr,
get_len(va.data_type_),
pa_);
}
diff --git a/cpp/src/reader/qds_without_timegenerator.cc
b/cpp/src/reader/qds_without_timegenerator.cc
index 805c9c4d..b2323da8 100644
--- a/cpp/src/reader/qds_without_timegenerator.cc
+++ b/cpp/src/reader/qds_without_timegenerator.cc
@@ -117,7 +117,7 @@ int QDSWithoutTimeGenerator::next(bool &has_next) {
}
int64_t time = heap_time_.begin()->first;
row_record_->set_timestamp(time);
- row_record_->get_field(0)->set_value(INT64, &time, pa_);
+ row_record_->get_field(0)->set_value(INT64, &time, get_len(INT64), pa_);
uint32_t count = heap_time_.count(time);
std::multimap<int64_t, uint32_t>::iterator iter = heap_time_.find(time);
@@ -125,7 +125,7 @@ int QDSWithoutTimeGenerator::next(bool &has_next) {
uint32_t len = 0;
row_record_->get_field(iter->second + 1)
->set_value(value_iters_[iter->second]->get_data_type(),
- value_iters_[iter->second]->read(&len), pa_);
+ value_iters_[iter->second]->read(&len), len, pa_);
value_iters_[iter->second]->next();
if (!time_iters_[iter->second]->end()) {
int64_t timev = *(int64_t
*)(time_iters_[iter->second]->read(&len));
diff --git a/cpp/src/reader/table_result_set.cc
b/cpp/src/reader/table_result_set.cc
index bbadc44a..396913e9 100644
--- a/cpp/src/reader/table_result_set.cc
+++ b/cpp/src/reader/table_result_set.cc
@@ -70,9 +70,11 @@ int TableResultSet::next(bool& has_next) {
bool null = false;
row_record_->reset();
for (uint32_t i = 0; i < row_iterator_->get_column_count(); ++i) {
- row_record_->get_field(i)->set_value(
- row_iterator_->get_data_type(i),
- row_iterator_->read(i, &len, &null), pa_);
+ const auto value = row_iterator_->read(i, &len, &null);
+ if (!null) {
+
row_record_->get_field(i)->set_value(row_iterator_->get_data_type(i),
+ value, len, pa_);
+ }
}
row_iterator_->next();
}
diff --git a/cpp/src/utils/db_utils.h b/cpp/src/utils/db_utils.h
index 4b5aca9b..592bb0ed 100644
--- a/cpp/src/utils/db_utils.h
+++ b/cpp/src/utils/db_utils.h
@@ -150,86 +150,6 @@ struct TsID {
}
};
-struct DeviceID {
- NodeID db_nid_;
- NodeID device_nid_;
-
- DeviceID() : db_nid_(0), device_nid_(0) {}
- DeviceID(const NodeID db_nid, const NodeID device_nid)
- : db_nid_(db_nid), device_nid_(device_nid) {}
- explicit DeviceID(const TsID &ts_id)
- : db_nid_(ts_id.db_nid_), device_nid_(ts_id.device_nid_) {}
-
- FORCE_INLINE bool operator==(const DeviceID &other) const {
- return db_nid_ == other.db_nid_ && device_nid_ == other.device_nid_;
- }
- FORCE_INLINE bool operator!=(const DeviceID &other) const {
- return db_nid_ != other.db_nid_ || device_nid_ != other.device_nid_;
- }
- FORCE_INLINE void from(const TsID &ts_id) {
- db_nid_ = ts_id.db_nid_;
- device_nid_ = ts_id.device_nid_;
- }
- FORCE_INLINE bool operator<(const DeviceID &that) const {
- int32_t this_i32 = (((int32_t)db_nid_) << 16) | (device_nid_);
- int32_t that_i32 = (((int32_t)that.db_nid_) << 16) |
(that.device_nid_);
- return this_i32 < that_i32;
- }
-};
-
-#define INVALID_TTL (-1)
-
-// describe single database
-struct DatabaseDesc {
- int64_t ttl_;
- std::string db_name_;
- TsID ts_id_;
-
- DatabaseDesc() : ttl_(INVALID_TTL), db_name_(""), ts_id_() {}
- DatabaseDesc(uint64_t ttl, const std::string &name, const TsID &ts_id)
- : ttl_(ttl), db_name_(name), ts_id_(ts_id) {}
-};
-
-enum WALFlushPolicy {
- WAL_DISABLED = 0,
- WAL_ASYNC = 1,
- WAL_FLUSH = 2,
- WAL_SYNC = 3,
-};
-
-template <typename T>
-std::string to_string(const T &val) {
- // todo: There may be a better way to avoid the memory problem of
- // ostringstream
- std::ostringstream oss;
- oss << val;
- return oss.str();
-}
-
-// TODO rename to DatabaseIdTTL
-struct DatabaseIdTTL {
- NodeID db_nid_;
- int64_t ttl_;
- DatabaseIdTTL() {}
- DatabaseIdTTL(NodeID db_nid, int64_t ttl) : db_nid_(db_nid), ttl_(ttl) {}
- DatabaseIdTTL(const DatabaseIdTTL &other)
- : db_nid_(other.db_nid_), ttl_(other.ttl_) {}
- DatabaseIdTTL &operator=(const DatabaseIdTTL &other) {
- this->db_nid_ = other.db_nid_;
- this->ttl_ = other.ttl_;
- return *this;
- }
- bool operator==(const DatabaseIdTTL &other) {
- if (db_nid_ != other.db_nid_ || ttl_ != other.ttl_) {
- return false;
- }
- return true;
- }
- friend std::ostream &operator<<(std::ostream &out, DatabaseIdTTL &di) {
- out << "(" << di.db_nid_ << ", " << di.ttl_ << ") ";
- return out;
- }
-};
/**
* @brief Represents the schema information for a single measurement.
diff --git a/cpp/src/utils/storage_utils.h b/cpp/src/utils/storage_utils.h
index 698a7303..a152a57e 100644
--- a/cpp/src/utils/storage_utils.h
+++ b/cpp/src/utils/storage_utils.h
@@ -75,12 +75,12 @@ FORCE_INLINE std::string get_file_path_from_file_id(
return oss.str();
}
-static void to_lowercase_inplace(std::string &str) {
+FORCE_INLINE static void to_lowercase_inplace(std::string &str) {
std::transform(
str.begin(), str.end(), str.begin(),
[](unsigned char c) -> unsigned char { return std::tolower(c); });
}
-static std::string to_lower(const std::string &str) {
+FORCE_INLINE static std::string to_lower(const std::string &str) {
std::string result;
std::transform(
str.begin(), str.end(), std::back_inserter(result),
diff --git a/cpp/src/writer/time_page_writer.h
b/cpp/src/writer/time_page_writer.h
index bc5b8212..bbf70165 100644
--- a/cpp/src/writer/time_page_writer.h
+++ b/cpp/src/writer/time_page_writer.h
@@ -73,6 +73,10 @@ class TimePageWriter {
FORCE_INLINE int write(int64_t timestamp) {
int ret = common::E_OK;
+ if (statistic_->count_ != 0 && is_inited_ &&
+ timestamp <= statistic_->end_time_) {
+ return common::E_OUT_OF_ORDER;
+ }
if (RET_FAIL(time_encoder_->encode(timestamp, time_out_stream_))) {
} else {
statistic_->update(timestamp);
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index 0f47756d..dcf6608b 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -751,7 +751,9 @@ int TsFileWriter::write_table(Tablet &tablet) {
return ret;
}
for (int i = start_idx; i < end_idx; i++) {
- time_chunk_writer->write(tablet.timestamps_[i]);
+ if (RET_FAIL(time_chunk_writer->write(tablet.timestamps_[i])))
{
+ return ret;
+ }
}
uint32_t field_col_count = 0;
for (uint32_t i = 0; i < tablet.get_column_count(); ++i) {
diff --git a/cpp/test/common/device_id_test.cc
b/cpp/test/common/device_id_test.cc
new file mode 100644
index 00000000..64a294c4
--- /dev/null
+++ b/cpp/test/common/device_id_test.cc
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License a
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "common/device_id.h"
+
+#include <gtest/gtest.h>
+#include "common/tablet.h"
+
+namespace storage {
+using namespace ::common;
+TEST(DeviceIdTest, NormalTest) {
+ std::string device_id_string = "root.db.tb.device1";
+ StringArrayDeviceID device_id = StringArrayDeviceID(device_id_string);
+ ASSERT_EQ("root.db.tb.device1", device_id.get_device_name());
+}
+
+TEST(DeviceIdTest, TabletDeviceId) {
+ std::vector<TSDataType> measurement_types{
+ TSDataType::STRING, TSDataType::STRING, TSDataType::STRING,
TSDataType::INT32};
+ std::vector<ColumnCategory> column_categories{ColumnCategory::TAG,
ColumnCategory::TAG, ColumnCategory::TAG, ColumnCategory::FIELD};
+ std::vector<std::string> measurement_names{"tag1", "tag2",
+ "tag3", "value"};
+
+ Tablet tablet("test_device0", measurement_names, measurement_types,
column_categories);
+ tablet.add_timestamp(0, 1);
+ tablet.add_value(0, 0, "t1");
+ tablet.add_value(0, 1, "t2");
+ tablet.add_value(0, 2, "t3");
+ tablet.add_value(1, 0, "");
+ tablet.add_value(1, 1, "t2");
+ tablet.add_value(1, 2, "t3");
+ tablet.add_value(2, 1, "t2");
+ tablet.add_value(2, 2, "t3");
+ auto device_id = std::make_shared<StringArrayDeviceID>(
+ std::vector<std::string>({"test_device0", "t1", "t2", "t3"}));
+ auto device_id2 = tablet.get_device_id(0);
+ ASSERT_TRUE(*device_id2 == *device_id);
+
+ ASSERT_EQ("test_device0..t2.t3",
tablet.get_device_id(1)->get_device_name());
+ ASSERT_EQ("test_device0.null.t2.t3",
tablet.get_device_id(2)->get_device_name());
+
+}
+}
diff --git a/cpp/test/common/row_record_test.cc
b/cpp/test/common/row_record_test.cc
index fd6e41f0..b753dbf5 100644
--- a/cpp/test/common/row_record_test.cc
+++ b/cpp/test/common/row_record_test.cc
@@ -22,6 +22,8 @@
#include <vector>
+#include "common/tsblock/tuple_desc.h"
+
namespace storage {
TEST(FieldTest, DefaultConstructor) {
@@ -62,12 +64,12 @@ TEST(FieldTest, SetValue) {
Field field;
common::PageArena pa; // dosen't matter
int32_t i32_val = 123;
- field.set_value(common::INT32, &i32_val, pa);
+ field.set_value(common::INT32, &i32_val, common::get_len(common::INT32),
pa);
EXPECT_EQ(field.type_, common::INT32);
EXPECT_EQ(field.value_.ival_, 123);
double d_val = 3.14;
- field.set_value(common::DOUBLE, &d_val, pa);
+ field.set_value(common::DOUBLE, &d_val, common::get_len(common::DOUBLE),
pa);
EXPECT_EQ(field.type_, common::DOUBLE);
EXPECT_DOUBLE_EQ(field.value_.dval_, 3.14);
}
diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc
b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
index d2f4a8a3..2bdc74b2 100644
--- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc
+++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc
@@ -80,17 +80,17 @@ class TsFileTableReaderTest : public ::testing::Test {
int measurement_schema_num = 5;
for (int i = 0; i < id_schema_num; i++) {
measurement_schemas.emplace_back(new MeasurementSchema(
- "id" + to_string(i), TSDataType::STRING, TSEncoding::PLAIN,
+ "id" + std::to_string(i), TSDataType::STRING,
TSEncoding::PLAIN,
CompressionType::UNCOMPRESSED));
column_categories.emplace_back(ColumnCategory::TAG);
}
for (int i = 0; i < measurement_schema_num; i++) {
measurement_schemas.emplace_back(new MeasurementSchema(
- "s" + to_string(i), TSDataType::INT64, TSEncoding::PLAIN,
+ "s" + std::to_string(i), TSDataType::INT64, TSEncoding::PLAIN,
CompressionType::UNCOMPRESSED));
column_categories.emplace_back(ColumnCategory::FIELD);
}
- return new TableSchema("testTable" + to_string(table_num),
+ return new TableSchema("testTable" + std::to_string(table_num),
measurement_schemas, column_categories);
}
@@ -254,12 +254,12 @@ TEST_F(TsFileTableReaderTest, TableModelResultMetadata) {
ASSERT_EQ(result_set_metadata->get_column_type(1), INT64);
for (int i = 2; i <= 6; i++) {
ASSERT_EQ(result_set_metadata->get_column_name(i),
- "id" + to_string(i - 2));
+ "id" + std::to_string(i - 2));
ASSERT_EQ(result_set_metadata->get_column_type(i), TSDataType::STRING);
}
for (int i = 7; i <= 11; i++) {
ASSERT_EQ(result_set_metadata->get_column_name(i),
- "s" + to_string(i - 7));
+ "s" + std::to_string(i - 7));
ASSERT_EQ(result_set_metadata->get_column_type(i), TSDataType::INT64);
}
reader.destroy_query_data_set(table_result_set);
@@ -296,7 +296,7 @@ TEST_F(TsFileTableReaderTest, TableModelGetSchema) {
ASSERT_EQ(table_schemas.size(), 10);
for (int i = 0; i < 10; i++) {
ASSERT_EQ(table_schemas[i]->get_table_name(),
- "testtable" + to_string(i));
+ "testtable" + std::to_string(i));
for (int j = 0; j < 5; j++) {
ASSERT_EQ(table_schemas[i]->get_data_types()[j],
TSDataType::STRING);
diff --git a/cpp/test/reader/tsfile_reader_test.cc
b/cpp/test/reader/tsfile_reader_test.cc
index a52eec7a..0b360eb4 100644
--- a/cpp/test/reader/tsfile_reader_test.cc
+++ b/cpp/test/reader/tsfile_reader_test.cc
@@ -160,13 +160,13 @@ TEST_F(TsFileReaderTest, GetAllDevice) {
for (size_t i = 0; i < 1024; i++) {
tsfile_writer_->register_timeseries(
- "device.ln" + to_string(i),
+ "device.ln" + std::to_string(i),
storage::MeasurementSchema(measurement_name, data_type, encoding,
compression_type));
}
for (size_t i = 0; i < 1024; i++) {
- TsRecord record(1622505600000, "device.ln" + to_string(i));
+ TsRecord record(1622505600000, "device.ln" + std::to_string(i));
record.add_point(measurement_name, (int32_t)0);
ASSERT_EQ(tsfile_writer_->write_record(record), E_OK);
}
diff --git a/cpp/test/utils/db_utils_test.cc b/cpp/test/utils/db_utils_test.cc
index ae0e8376..fb455eea 100644
--- a/cpp/test/utils/db_utils_test.cc
+++ b/cpp/test/utils/db_utils_test.cc
@@ -117,56 +117,6 @@ TEST(TsIDTest, OperatorLess) {
EXPECT_FALSE(ts_id2 < ts_id1);
}
-TEST(DeviceIDTest, Constructor) {
- DeviceID device_id;
- EXPECT_EQ(device_id.db_nid_, 0);
- EXPECT_EQ(device_id.device_nid_, 0);
-}
-
-TEST(DeviceIDTest, ParameterizedConstructor) {
- DeviceID device_id(1, 2);
- EXPECT_EQ(device_id.db_nid_, 1);
- EXPECT_EQ(device_id.device_nid_, 2);
-}
-
-TEST(DeviceIDTest, TsIDConstructor) {
- TsID ts_id(1, 2, 3);
- DeviceID device_id(ts_id);
- EXPECT_EQ(device_id.db_nid_, 1);
- EXPECT_EQ(device_id.device_nid_, 2);
-}
-
-TEST(DeviceIDTest, OperatorEqual) {
- DeviceID device_id1(1, 2);
- DeviceID device_id2(1, 2);
- EXPECT_TRUE(device_id1 == device_id2);
- device_id2.db_nid_ = 3;
- EXPECT_FALSE(device_id1 == device_id2);
-}
-
-TEST(DeviceIDTest, OperatorNotEqual) {
- DeviceID device_id1(1, 2);
- DeviceID device_id2(1, 2);
- EXPECT_FALSE(device_id1 != device_id2);
- device_id2.db_nid_ = 3;
- EXPECT_TRUE(device_id1 != device_id2);
-}
-
-TEST(DatabaseDescTest, Constructor) {
- DatabaseDesc db_desc;
- EXPECT_EQ(db_desc.ttl_, INVALID_TTL);
- EXPECT_EQ(db_desc.db_name_, "");
- EXPECT_EQ(db_desc.ts_id_.db_nid_, 0);
-}
-
-TEST(DatabaseDescTest, ParameterizedConstructor) {
- TsID ts_id(1, 2, 3);
- DatabaseDesc db_desc(1000, "test_db", ts_id);
- EXPECT_EQ(db_desc.ttl_, 1000);
- EXPECT_EQ(db_desc.db_name_, "test_db");
- EXPECT_EQ(db_desc.ts_id_, ts_id);
-}
-
TEST(ColumnSchemaTest, Constructor) {
ColumnSchema col_schema;
EXPECT_EQ(col_schema.data_type_, INVALID_DATATYPE);
diff --git a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
index a13c20be..44636078 100644
--- a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
+++ b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
@@ -37,7 +37,7 @@ class TsFileWriterTableTest : public ::testing::Test {
libtsfile_init();
file_name_ = std::string("tsfile_writer_table_test_") +
generate_random_string(10) + std::string(".tsfile");
- remove(file_name_.c_str());
+ int ret = remove(file_name_.c_str());
int flags = O_WRONLY | O_CREAT | O_TRUNC;
#ifdef _WIN32
flags |= O_BINARY;
@@ -45,7 +45,10 @@ class TsFileWriterTableTest : public ::testing::Test {
mode_t mode = 0666;
write_file_.create(file_name_, flags, mode);
}
- void TearDown() override { remove(file_name_.c_str()); }
+ void TearDown() override {
+ int ret = remove(file_name_.c_str());
+ ASSERT_EQ(ret, 0);
+ }
std::string file_name_;
WriteFile write_file_;
@@ -76,17 +79,17 @@ class TsFileWriterTableTest : public ::testing::Test {
int measurement_schema_num = 5;
for (int i = 0; i < id_schema_num; i++) {
measurement_schemas.emplace_back(new MeasurementSchema(
- "id" + to_string(i), TSDataType::STRING, TSEncoding::PLAIN,
+ "id" + std::to_string(i), TSDataType::STRING,
TSEncoding::PLAIN,
CompressionType::UNCOMPRESSED));
column_categories.emplace_back(ColumnCategory::TAG);
}
for (int i = 0; i < measurement_schema_num; i++) {
measurement_schemas.emplace_back(new MeasurementSchema(
- "s" + to_string(i), TSDataType::INT64, TSEncoding::PLAIN,
+ "s" + std::to_string(i), TSDataType::INT64, TSEncoding::PLAIN,
CompressionType::UNCOMPRESSED));
column_categories.emplace_back(ColumnCategory::FIELD);
}
- return new TableSchema("testTable" + to_string(table_num),
+ return new TableSchema("testTable" + std::to_string(table_num),
measurement_schemas, column_categories);
}
@@ -187,6 +190,56 @@ TEST_F(TsFileWriterTableTest, WithoutTagAndMultiPage) {
delete table_schema;
}
+TEST_F(TsFileWriterTableTest, WriteDisorderTest) {
+ auto table_schema = gen_table_schema(0);
+ auto tsfile_table_writer_ =
+ std::make_shared<TsFileTableWriter>(&write_file_, table_schema);
+
+ int device_num = 1;
+ int num_timestamp_per_device = 10;
+ int offset = 0;
+ storage::Tablet tablet(table_schema->get_measurement_names(),
+ table_schema->get_data_types(),
+ device_num * num_timestamp_per_device);
+
+ char* literal = new char[std::strlen("device_id") + 1];
+ std::strcpy(literal, "device_id");
+ String literal_str(literal, std::strlen("device_id"));
+ for (int i = 0; i < device_num; i++) {
+ for (int l = 0; l < num_timestamp_per_device; l++) {
+ int row_index = i * num_timestamp_per_device + l;
+ // disordered timestamp.
+ tablet.add_timestamp(row_index, l > num_timestamp_per_device / 2
+ ? l - num_timestamp_per_device
+ : offset + l);
+ auto column_schemas = table_schema->get_measurement_schemas();
+ for (const auto& column_schema : column_schemas) {
+ switch (column_schema->data_type_) {
+ case TSDataType::INT64:
+ tablet.add_value(row_index,
+ column_schema->measurement_name_,
+ static_cast<int64_t>(i));
+ break;
+ case TSDataType::STRING:
+ tablet.add_value(row_index,
+ column_schema->measurement_name_,
+ literal_str);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ }
+ delete[] literal;
+
+ ASSERT_EQ(tsfile_table_writer_->write_table(tablet),
+ common::E_OUT_OF_ORDER);
+ ASSERT_EQ(tsfile_table_writer_->flush(), common::E_OK);
+ ASSERT_EQ(tsfile_table_writer_->close(), common::E_OK);
+ delete table_schema;
+}
+
TEST_F(TsFileWriterTableTest, WriteTableTestMultiFlush) {
auto table_schema = gen_table_schema(0);
auto tsfile_table_writer_ = std::make_shared<TsFileTableWriter>(
@@ -250,6 +303,62 @@ TEST_F(TsFileWriterTableTest, WriterWithMemoryThreshold) {
delete table_schema;
}
+TEST_F(TsFileWriterTableTest, EmptyTagWrite) {
+ std::vector<MeasurementSchema*> measurement_schemas;
+ std::vector<ColumnCategory> column_categories;
+ measurement_schemas.resize(3);
+ measurement_schemas[0] = new MeasurementSchema("device1", STRING);
+ measurement_schemas[1] = new MeasurementSchema("device2", STRING);
+ measurement_schemas[2] = new MeasurementSchema("value", DOUBLE);
+ column_categories.emplace_back(ColumnCategory::TAG);
+ column_categories.emplace_back(ColumnCategory::TAG);
+ column_categories.emplace_back(ColumnCategory::FIELD);
+ TableSchema* table_schema =
+ new TableSchema("test_table", measurement_schemas, column_categories);
+ auto tsfile_table_writer =
+ std::make_shared<TsFileTableWriter>(&write_file_, table_schema);
+ Tablet tablet = Tablet(table_schema->get_measurement_names(),
+ table_schema->get_data_types());
+ tablet.set_table_name("test_table");
+ for (int i = 0; i < 100; i++) {
+ tablet.add_timestamp(i, static_cast<int64_t>(i));
+ tablet.add_value(i, "device1",
+ std::string("device" + std::to_string(i)).c_str());
+ tablet.add_value(i, "device2", "");
+ tablet.add_value(i, "value", i * 1.1);
+ }
+ tsfile_table_writer->write_table(tablet);
+ tsfile_table_writer->flush();
+ tsfile_table_writer->close();
+
+ TsFileReader reader = TsFileReader();
+ reader.open(write_file_.get_file_path());
+ ResultSet* ret = nullptr;
+ int ret_value =
+ reader.query("test_table", {"device1", "device2", "value"}, 0, 50,
ret);
+ ASSERT_EQ(common::E_OK, ret_value);
+
+ ASSERT_EQ(ret_value, 0);
+ auto* table_result_set = (TableResultSet*)ret;
+ bool has_next = false;
+ int cur_line = 0;
+ while (IS_SUCC(table_result_set->next(has_next)) && has_next) {
+ cur_line++;
+ int64_t timestamp = table_result_set->get_value<int64_t>("time");
+ ASSERT_EQ(table_result_set->get_value<common::String*>("device1")
+ ->to_std_string(),
+ "device" + std::to_string(timestamp));
+ ASSERT_EQ(table_result_set->get_value<double>("value"),
+ timestamp * 1.1);
+ }
+ ASSERT_EQ(cur_line, 51);
+ table_result_set->close();
+ reader.destroy_query_data_set(table_result_set);
+
+ reader.close();
+ delete table_schema;
+}
+
TEST_F(TsFileWriterTableTest, WritehDataTypeMisMatch) {
auto table_schema = gen_table_schema(0);
auto tsfile_table_writer_ = std::make_shared<TsFileTableWriter>(
@@ -263,9 +372,6 @@ TEST_F(TsFileWriterTableTest, WritehDataTypeMisMatch) {
storage::Tablet tablet(table_schema->get_measurement_names(), datatypes,
device_num * num_timestamp_per_device);
- char* literal = new char[std::strlen("device_id") + 1];
- std::strcpy(literal, "device_id");
- String literal_str(literal, std::strlen("device_id"));
for (int i = 0; i < device_num; i++) {
for (int l = 0; l < num_timestamp_per_device; l++) {
int row_index = i * num_timestamp_per_device + l;
@@ -284,9 +390,9 @@ TEST_F(TsFileWriterTableTest, WritehDataTypeMisMatch) {
static_cast<int32_t>(i));
break;
case TSDataType::STRING:
- tablet.add_value(row_index,
-
column_schemas[idx]->measurement_name_,
- literal_str);
+ tablet.add_value(
+ row_index, column_schemas[idx]->measurement_name_,
+ std::string("device" + std::to_string(i)).c_str());
break;
default:
break;
@@ -294,7 +400,6 @@ TEST_F(TsFileWriterTableTest, WritehDataTypeMisMatch) {
}
}
}
- delete[] literal;
delete table_schema;
ASSERT_EQ(E_TYPE_NOT_MATCH, tsfile_table_writer_->write_table(tablet));
@@ -341,7 +446,7 @@ TEST_F(TsFileWriterTableTest, WriteAndReadSimple) {
int64_t timestamp = table_result_set->get_value<int64_t>("time");
ASSERT_EQ(table_result_set->get_value<common::String*>("device")
->to_std_string(),
- "device" + to_string(timestamp));
+ "device" + std::to_string(timestamp));
ASSERT_EQ(table_result_set->get_value<double>("value"),
timestamp * 1.1);
}
@@ -388,5 +493,139 @@ TEST_F(TsFileWriterTableTest, DuplicateColumnName) {
ASSERT_EQ(E_INVALID_ARG, tsfile_table_writer->write_table(tablet));
ASSERT_EQ(E_INVALID_ARG, tsfile_table_writer->register_table(
std::make_shared<TableSchema>(*table_schema)));
+ tsfile_table_writer->close();
+ delete table_schema;
+}
+
+TEST_F(TsFileWriterTableTest, WriteWithNullAndEmptyTag) {
+ std::vector<MeasurementSchema*> measurement_schemas;
+ std::vector<ColumnCategory> column_categories;
+ for (int i = 0; i < 3; i++) {
+ measurement_schemas.emplace_back(new MeasurementSchema(
+ "id" + std::to_string(i), TSDataType::STRING));
+ column_categories.emplace_back(ColumnCategory::TAG);
+ }
+ measurement_schemas.emplace_back(new MeasurementSchema("value", DOUBLE));
+ column_categories.emplace_back(ColumnCategory::FIELD);
+ auto table_schema =
+ new TableSchema("testTable", measurement_schemas, column_categories);
+ auto tsfile_table_writer =
+ std::make_shared<TsFileTableWriter>(&write_file_, table_schema);
+ int time = 0;
+ Tablet tablet = Tablet(table_schema->get_measurement_names(),
+ table_schema->get_data_types(), 10);
+
+ for (int i = 0; i < 10; i++) {
+ tablet.add_timestamp(i, static_cast<int64_t>(time++));
+ tablet.add_value(i, 0, "tag1");
+ tablet.add_value(i, 1, "tag2");
+ tablet.add_value(i, 2, "tag3");
+ tablet.add_value(i, 3, 100.0f);
+ }
+
+ tsfile_table_writer->write_table(tablet);
+ Tablet tablet2 = Tablet(table_schema->get_measurement_names(),
+ table_schema->get_data_types(), 10);
+
+ for (int i = 0; i < 10; i++) {
+ tablet2.add_timestamp(i, static_cast<int64_t>(time++));
+ tablet2.add_value(i, 0, i % 2 == 0 ? "" : "tag4");
+ tablet2.add_value(i, 1, i % 2 == 1 ? "" : "tag5");
+ tablet2.add_value(i, 2, i % 3 == 0 ? "" : "tag6");
+ tablet2.add_value(i, 3, 101.0f);
+ }
+ tsfile_table_writer->write_table(tablet2);
+
+ Tablet tablet3 = Tablet(table_schema->get_measurement_names(),
+ table_schema->get_data_types(), 10);
+ for (int i = 0; i < 10; i++) {
+ tablet3.add_timestamp(i, static_cast<int64_t>(time++));
+ tablet3.add_value(i, 0, "tag7");
+ if (i % 2 == 0) {
+ tablet3.add_value(i, 1, "tag8\0ta");
+ } else {
+ tablet3.add_value(i, 2, "tag9");
+ }
+ tablet3.add_value(i, 3, 102.0f);
+ }
+
+ tsfile_table_writer->write_table(tablet3);
+ tsfile_table_writer->flush();
+ tsfile_table_writer->close();
+
delete table_schema;
+
+ auto reader = TsFileReader();
+ reader.open(write_file_.get_file_path());
+ ResultSet* ret = nullptr;
+ int ret_value =
+ reader.query("testTable", {"id0", "id1", "id2", "value"}, 0, 50, ret);
+ ASSERT_EQ(common::E_OK, ret_value);
+
+ auto table_result_set = (TableResultSet*)ret;
+ bool has_next = false;
+ int cur_line = 0;
+ auto schema = table_result_set->get_metadata();
+ while (IS_SUCC(table_result_set->next(has_next)) && has_next) {
+ int64_t timestamp = table_result_set->get_value<int64_t>(1);
+ switch (timestamp) {
+ case 0: {
+ // All tag fields have valid values.
+ ASSERT_EQ(common::String(std::string("tag1")),
+ *table_result_set->get_value<common::String*>(2));
+ ASSERT_EQ(common::String(std::string("tag2")),
+ *table_result_set->get_value<common::String*>(3));
+ ASSERT_EQ(common::String(std::string("tag3")),
+ *table_result_set->get_value<common::String*>(4));
+ ASSERT_EQ(100.0f, table_result_set->get_value<double>(5));
+ break;
+ }
+ case 10: {
+ // The first and last tag fields are empty strings.
+ ASSERT_EQ(common::String(std::string("")),
+ *table_result_set->get_value<common::String*>(2));
+ ASSERT_EQ(common::String(std::string("tag5")),
+ *table_result_set->get_value<common::String*>(3));
+ ASSERT_EQ(common::String(std::string("")),
+ *table_result_set->get_value<common::String*>(4));
+ ASSERT_EQ(101.0f, table_result_set->get_value<double>(5));
+ break;
+ }
+ case 11: {
+ // The middle tag field is an empty string.
+ ASSERT_EQ(common::String(std::string("tag4")),
+ *table_result_set->get_value<common::String*>(2));
+ ASSERT_EQ(common::String(std::string("")),
+ *table_result_set->get_value<common::String*>(3));
+ ASSERT_EQ(common::String(std::string("tag6")),
+ *table_result_set->get_value<common::String*>(4));
+ ASSERT_EQ(101.0f, table_result_set->get_value<double>(5));
+ break;
+ }
+ case 20: {
+ // The last tag field is null.
+ ASSERT_EQ(common::String(std::string("tag7")),
+ *table_result_set->get_value<common::String*>(2));
+ ASSERT_EQ(common::String(std::string("tag8\0ta")),
+ *table_result_set->get_value<common::String*>(3));
+ ASSERT_TRUE(table_result_set->is_null(4));
+ ASSERT_EQ(102.0f, table_result_set->get_value<double>(5));
+ break;
+ }
+ case 21: {
+ // The middle tag field is null.
+ ASSERT_EQ(common::String(std::string("tag7")),
+ *table_result_set->get_value<common::String*>(2));
+ ASSERT_EQ(common::String(std::string("tag9")),
+ *table_result_set->get_value<common::String*>(4));
+ ASSERT_TRUE(table_result_set->is_null(3));
+ ASSERT_EQ(102.0f, table_result_set->get_value<double>(5));
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ reader.destroy_query_data_set(table_result_set);
+ ASSERT_EQ(reader.close(), common::E_OK);
}
\ No newline at end of file
diff --git a/cpp/test/writer/time_chunk_writer_test.cc
b/cpp/test/writer/time_chunk_writer_test.cc
index 1f1910c4..df79ab28 100644
--- a/cpp/test/writer/time_chunk_writer_test.cc
+++ b/cpp/test/writer/time_chunk_writer_test.cc
@@ -52,11 +52,6 @@ TEST_F(TimeChunkWriterTest, InitWithParameters) {
writer.destroy();
}
-TEST_F(TimeChunkWriterTest, WriteBoolean) {
- EXPECT_EQ(time_chunk_writer.write(true), E_OK);
- EXPECT_EQ(time_chunk_writer.write(false), E_OK);
-}
-
TEST_F(TimeChunkWriterTest, WriteLargeDataSet) {
for (int i = 0; i < 10000; ++i) {
time_chunk_writer.write(i);
diff --git a/cpp/test/writer/tsfile_writer_test.cc
b/cpp/test/writer/tsfile_writer_test.cc
index 6f58f0cb..d8c0d414 100644
--- a/cpp/test/writer/tsfile_writer_test.cc
+++ b/cpp/test/writer/tsfile_writer_test.cc
@@ -706,7 +706,7 @@ TEST_F(TsFileWriterTest, WriteAlignedTimeseries) {
std::string device_name = "device";
std::vector<std::string> measurement_names;
for (int i = 0; i < measurement_num; i++) {
- measurement_names.emplace_back("temperature" + to_string(i));
+ measurement_names.emplace_back("temperature" + std::to_string(i));
}
common::TSDataType data_type = common::TSDataType::INT32;
@@ -735,7 +735,7 @@ TEST_F(TsFileWriterTest, WriteAlignedTimeseries) {
std::vector<storage::Path> select_list;
for (int i = 0; i < measurement_num; ++i) {
- std::string measurement_name = "temperature" + to_string(i);
+ std::string measurement_name = "temperature" + std::to_string(i);
storage::Path path(device_name, measurement_name);
select_list.push_back(path);
}
@@ -776,7 +776,7 @@ TEST_F(TsFileWriterTest, WriteAlignedMultiFlush) {
std::string device_name = "device";
std::vector<std::string> measurement_names;
for (int i = 0; i < measurement_num; i++) {
- measurement_names.emplace_back("temperature" + to_string(i));
+ measurement_names.emplace_back("temperature" + std::to_string(i));
}
common::TSDataType data_type = common::TSDataType::INT32;
@@ -805,7 +805,7 @@ TEST_F(TsFileWriterTest, WriteAlignedMultiFlush) {
std::vector<storage::Path> select_list;
for (int i = 0; i < measurement_num; ++i) {
- std::string measurement_name = "temperature" + to_string(i);
+ std::string measurement_name = "temperature" + std::to_string(i);
storage::Path path(device_name, measurement_name);
select_list.push_back(path);
}
@@ -846,7 +846,7 @@ TEST_F(TsFileWriterTest, WriteAlignedPartialData) {
std::string device_name = "device";
std::vector<std::string> measurement_names;
for (int i = 0; i < measurement_num; i++) {
- measurement_names.emplace_back("temperature" + to_string(i));
+ measurement_names.emplace_back("temperature" + std::to_string(i));
}
common::TSDataType data_type = common::TSDataType::INT32;
@@ -874,7 +874,7 @@ TEST_F(TsFileWriterTest, WriteAlignedPartialData) {
std::vector<storage::Path> select_list;
for (int i = 0; i < measurement_num; ++i) {
- std::string measurement_name = "temperature" + to_string(i);
+ std::string measurement_name = "temperature" + std::to_string(i);
storage::Path path(device_name, measurement_name);
select_list.push_back(path);
}
