This is an automated email from the ASF dual-hosted git repository.
hulk pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 38c36e1e Unify the way of parsing metadata (#1964)
38c36e1e is described below
commit 38c36e1e9082b6bc2f8af12a0080e507a2684786
Author: hulk <[email protected]>
AuthorDate: Sun Dec 24 12:59:00 2023 +0800
Unify the way of parsing metadata (#1964)
---
src/storage/redis_db.cc | 23 +++++++--------
src/storage/redis_db.h | 4 +--
src/types/redis_bitmap.cc | 21 ++------------
src/types/redis_json.cc | 73 ++++++++++++++++-------------------------------
src/types/redis_json.h | 3 +-
5 files changed, 40 insertions(+), 84 deletions(-)
diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc
index 29dc277f..e58c1f6f 100644
--- a/src/storage/redis_db.cc
+++ b/src/storage/redis_db.cc
@@ -38,7 +38,9 @@ Database::Database(engine::Storage *storage, std::string ns)
: storage_(storage)
metadata_cf_handle_ = storage->GetCFHandle("metadata");
}
-rocksdb::Status Database::ParseMetadata(RedisType type, Slice *bytes, Metadata
*metadata) {
+// Some data types may support reading multiple types of metadata.
+// For example, bitmap supports reading string metadata and bitmap metadata.
+rocksdb::Status Database::ParseMetadata(const std::vector<RedisType> &types,
Slice *bytes, Metadata *metadata) {
std::string old_metadata;
metadata->Encode(&old_metadata);
@@ -50,7 +52,10 @@ rocksdb::Status Database::ParseMetadata(RedisType type,
Slice *bytes, Metadata *
auto _ [[maybe_unused]] = metadata->Decode(old_metadata);
return rocksdb::Status::NotFound(kErrMsgKeyExpired);
}
- if (metadata->Type() != type && (metadata->size > 0 ||
metadata->IsEmptyableType())) {
+
+ bool is_type_matched = std::find(types.begin(), types.end(),
metadata->Type()) != types.end();
+ // if type is not matched, we still need to check if the metadata is valid.
+ if (!is_type_matched && (metadata->size > 0 || metadata->IsEmptyableType()))
{
// error discarded here since it already failed
auto _ [[maybe_unused]] = metadata->Decode(old_metadata);
return rocksdb::Status::InvalidArgument(kErrMsgWrongType);
@@ -64,11 +69,9 @@ rocksdb::Status Database::ParseMetadata(RedisType type,
Slice *bytes, Metadata *
}
rocksdb::Status Database::GetMetadata(RedisType type, const Slice &ns_key,
Metadata *metadata) {
- std::string bytes;
- auto s = GetRawMetadata(ns_key, &bytes);
- if (!s.ok()) return s;
- Slice bytes_slice(bytes);
- return ParseMetadata(type, &bytes_slice, metadata);
+ std::string raw_value;
+ Slice rest;
+ return GetMetadata(type, ns_key, &raw_value, metadata, &rest);
}
rocksdb::Status Database::GetMetadata(RedisType type, const Slice &ns_key,
std::string *raw_value, Metadata *metadata,
@@ -76,7 +79,7 @@ rocksdb::Status Database::GetMetadata(RedisType type, const
Slice &ns_key, std::
auto s = GetRawMetadata(ns_key, raw_value);
*rest = *raw_value;
if (!s.ok()) return s;
- return ParseMetadata(type, rest, metadata);
+ return ParseMetadata({type}, rest, metadata);
}
rocksdb::Status Database::GetRawMetadata(const Slice &ns_key, std::string
*bytes) {
@@ -86,10 +89,6 @@ rocksdb::Status Database::GetRawMetadata(const Slice
&ns_key, std::string *bytes
return storage_->Get(read_options, metadata_cf_handle_, ns_key, bytes);
}
-rocksdb::Status Database::GetRawMetadataByUserKey(const Slice &user_key,
std::string *bytes) {
- return GetRawMetadata(AppendNamespacePrefix(user_key), bytes);
-}
-
rocksdb::Status Database::Expire(const Slice &user_key, uint64_t timestamp) {
std::string ns_key = AppendNamespacePrefix(user_key);
diff --git a/src/storage/redis_db.h b/src/storage/redis_db.h
index c77135df..252e5d30 100644
--- a/src/storage/redis_db.h
+++ b/src/storage/redis_db.h
@@ -34,12 +34,12 @@ class Database {
static constexpr uint64_t RANDOM_KEY_SCAN_LIMIT = 60;
explicit Database(engine::Storage *storage, std::string ns = "");
- [[nodiscard]] static rocksdb::Status ParseMetadata(RedisType type, Slice
*bytes, Metadata *metadata);
+ [[nodiscard]] static rocksdb::Status ParseMetadata(const
std::vector<RedisType> &types, Slice *bytes,
+ Metadata *metadata);
[[nodiscard]] rocksdb::Status GetMetadata(RedisType type, const Slice
&ns_key, Metadata *metadata);
[[nodiscard]] rocksdb::Status GetMetadata(RedisType type, const Slice
&ns_key, std::string *raw_value,
Metadata *metadata, Slice *rest);
[[nodiscard]] rocksdb::Status GetRawMetadata(const Slice &ns_key,
std::string *bytes);
- [[nodiscard]] rocksdb::Status GetRawMetadataByUserKey(const Slice &user_key,
std::string *bytes);
[[nodiscard]] rocksdb::Status Expire(const Slice &user_key, uint64_t
timestamp);
[[nodiscard]] rocksdb::Status Del(const Slice &user_key);
[[nodiscard]] rocksdb::Status MDel(const std::vector<Slice> &keys, uint64_t
*deleted_cnt);
diff --git a/src/types/redis_bitmap.cc b/src/types/redis_bitmap.cc
index b0fc65f6..37fcc0e3 100644
--- a/src/types/redis_bitmap.cc
+++ b/src/types/redis_bitmap.cc
@@ -58,28 +58,11 @@ void ExpandBitmapSegment(std::string *segment, size_t
min_bytes) {
}
rocksdb::Status Bitmap::GetMetadata(const Slice &ns_key, BitmapMetadata
*metadata, std::string *raw_value) {
- std::string old_metadata;
- metadata->Encode(&old_metadata);
auto s = GetRawMetadata(ns_key, raw_value);
if (!s.ok()) return s;
- s = metadata->Decode(*raw_value);
- if (!s.ok()) return s;
- if (metadata->Expired()) {
- // error discarded here since it already failed
- auto _ [[maybe_unused]] = metadata->Decode(old_metadata);
- return rocksdb::Status::NotFound(kErrMsgKeyExpired);
- }
- if (metadata->Type() == kRedisString) return s;
- if (metadata->Type() != kRedisBitmap && metadata->size > 0) {
- auto _ [[maybe_unused]] = metadata->Decode(old_metadata);
- return rocksdb::Status::InvalidArgument(kErrMsgWrongType);
- }
- if (metadata->size == 0) {
- auto _ [[maybe_unused]] = metadata->Decode(old_metadata);
- return rocksdb::Status::NotFound("no elements");
- }
- return s;
+ Slice slice = *raw_value;
+ return ParseMetadata({kRedisBitmap, kRedisString}, &slice, metadata);
}
rocksdb::Status Bitmap::GetBit(const Slice &user_key, uint32_t offset, bool
*bit) {
diff --git a/src/types/redis_json.cc b/src/types/redis_json.cc
index b1755e3a..b7c0d4c4 100644
--- a/src/types/redis_json.cc
+++ b/src/types/redis_json.cc
@@ -54,6 +54,22 @@ rocksdb::Status Json::write(Slice ns_key, JsonMetadata
*metadata, const JsonValu
return storage_->Write(storage_->DefaultWriteOptions(),
batch->GetWriteBatch());
}
+rocksdb::Status Json::parse(const JsonMetadata &metadata, const Slice
&json_bytes, JsonValue *value) {
+ if (metadata.format == JsonStorageFormat::JSON) {
+ auto res = JsonValue::FromString(json_bytes.ToStringView());
+ if (!res) return rocksdb::Status::Corruption(res.Msg());
+ *value = *std::move(res);
+ } else if (metadata.format == JsonStorageFormat::CBOR) {
+ auto res = JsonValue::FromCBOR(json_bytes.ToStringView());
+ if (!res) return rocksdb::Status::Corruption(res.Msg());
+ *value = *std::move(res);
+ } else {
+ return rocksdb::Status::NotSupported("JSON storage format not supported");
+ }
+
+ return rocksdb::Status::OK();
+}
+
rocksdb::Status Json::read(const Slice &ns_key, JsonMetadata *metadata,
JsonValue *value) {
std::string bytes;
Slice rest;
@@ -61,19 +77,7 @@ rocksdb::Status Json::read(const Slice &ns_key, JsonMetadata
*metadata, JsonValu
auto s = GetMetadata(kRedisJson, ns_key, &bytes, metadata, &rest);
if (!s.ok()) return s;
- if (metadata->format == JsonStorageFormat::JSON) {
- auto origin_res = JsonValue::FromString(rest.ToStringView());
- if (!origin_res) return rocksdb::Status::Corruption(origin_res.Msg());
- *value = *std::move(origin_res);
- } else if (metadata->format == JsonStorageFormat::CBOR) {
- auto origin_res = JsonValue::FromCBOR(rest.ToStringView());
- if (!origin_res) return rocksdb::Status::Corruption(origin_res.Msg());
- *value = *std::move(origin_res);
- } else {
- return rocksdb::Status::NotSupported("JSON storage format not supported");
- }
-
- return rocksdb::Status::OK();
+ return parse(*metadata, rest, value);
}
rocksdb::Status Json::create(const std::string &ns_key, JsonMetadata
&metadata, const std::string &value) {
@@ -545,52 +549,23 @@ std::vector<rocksdb::Status> Json::MGet(const
std::vector<std::string> &user_key
}
std::vector<rocksdb::Status> Json::readMulti(const std::vector<Slice>
&ns_keys, std::vector<JsonValue> &values) {
- std::vector<std::string> raw_values;
- std::vector<JsonMetadata> meta_data;
- raw_values.resize(ns_keys.size());
- meta_data.resize(ns_keys.size());
-
- auto statuses = getRawMetaData(ns_keys, meta_data, &raw_values);
- for (size_t i = 0; i < ns_keys.size(); i++) {
- if (!statuses[i].ok()) continue;
- if (meta_data[i].format == JsonStorageFormat::JSON) {
- auto res = JsonValue::FromString(raw_values[i]);
- if (!res) {
- statuses[i] = rocksdb::Status::Corruption(res.Msg());
- continue;
- }
- values[i] = *std::move(res);
- } else if (meta_data[i].format == JsonStorageFormat::CBOR) {
- auto res = JsonValue::FromCBOR(raw_values[i]);
- if (!res) {
- statuses[i] = rocksdb::Status::Corruption(res.Msg());
- continue;
- }
- values[i] = *std::move(res);
- } else {
- statuses[i] = rocksdb::Status::NotSupported("JSON storage format not
supported");
- }
- }
- return statuses;
-}
-
-std::vector<rocksdb::Status> Json::getRawMetaData(const std::vector<Slice>
&ns_keys,
- std::vector<JsonMetadata>
&metadatas,
- std::vector<std::string>
*raw_values) {
rocksdb::ReadOptions read_options = storage_->DefaultMultiGetOptions();
LatestSnapShot ss(storage_);
read_options.snapshot = ss.GetSnapShot();
- raw_values->resize(ns_keys.size());
+
std::vector<rocksdb::Status> statuses(ns_keys.size());
std::vector<rocksdb::PinnableSlice> pin_values(ns_keys.size());
storage_->MultiGet(read_options, metadata_cf_handle_, ns_keys.size(),
ns_keys.data(), pin_values.data(),
statuses.data());
for (size_t i = 0; i < ns_keys.size(); i++) {
if (!statuses[i].ok()) continue;
- Slice slice(pin_values[i].data(), pin_values[i].size());
- statuses[i] = ParseMetadata(kRedisJson, &slice, &metadatas[i]);
+ Slice rest(pin_values[i].data(), pin_values[i].size());
+ JsonMetadata metadata;
+ statuses[i] = ParseMetadata({kRedisJson}, &rest, &metadata);
+ if (!statuses[i].ok()) continue;
+
+ statuses[i] = parse(metadata, rest, &values[i]);
if (!statuses[i].ok()) continue;
- (*raw_values)[i].assign(slice.data(), slice.size());
}
return statuses;
}
diff --git a/src/types/redis_json.h b/src/types/redis_json.h
index a3993786..a2135b75 100644
--- a/src/types/redis_json.h
+++ b/src/types/redis_json.h
@@ -70,13 +70,12 @@ class Json : public Database {
private:
rocksdb::Status write(Slice ns_key, JsonMetadata *metadata, const JsonValue
&json_val);
rocksdb::Status read(const Slice &ns_key, JsonMetadata *metadata, JsonValue
*value);
+ static rocksdb::Status parse(const JsonMetadata &metadata, const Slice
&json_byt, JsonValue *value);
rocksdb::Status create(const std::string &ns_key, JsonMetadata &metadata,
const std::string &value);
rocksdb::Status del(const Slice &ns_key);
rocksdb::Status numop(JsonValue::NumOpEnum op, const std::string &user_key,
const std::string &path,
const std::string &value, JsonValue *result);
std::vector<rocksdb::Status> readMulti(const std::vector<Slice> &ns_keys,
std::vector<JsonValue> &values);
- std::vector<rocksdb::Status> getRawMetaData(const std::vector<Slice>
&ns_keys, std::vector<JsonMetadata> &metadatas,
- std::vector<std::string>
*raw_values);
};
} // namespace redis
