acelyc111 commented on code in PR #1706: URL: https://github.com/apache/incubator-pegasus/pull/1706#discussion_r1435979125
########## src/replica/kms_key_provider.h: ########## @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <string> +#include <utility> +#include <vector> + +#include "runtime/security/kms_client.h" +#include "utils/errors.h" + +namespace dsn { +namespace replication { +class replica_kms_info; +} // namespace replication + +namespace security { +// This class is to generating EEK IV KV from KMS (a.k.a Key Manager Service) and get DEK from KMS. +class KMSKeyProvider +{ +public: + ~KMSKeyProvider() {} + + KMSKeyProvider(const std::vector<std::string> &kms_url, std::string cluster_key_name) + : client_(kms_url, std::move(cluster_key_name)) + { + } + + // Use KMS client which decrypted the encryption key from KMS and decrypted key is a hex string + // which could be used derectly. + dsn::error_s DecryptEncryptionKey(const dsn::replication::replica_kms_info &kms_info, + std::string *decrypted_key); + + // Use KMS client which generated an encryption key from KMS (the generated key is encrypted). Review Comment: ```suggestion // Generate an encryption key from KMS. ``` ########## src/replica/test/replica_http_service_test.cpp: ########## @@ -49,6 +52,9 @@ class replica_http_service_test : public replica_test_base // Disable unnecessary works before starting stub. FLAGS_fd_disabled = true; FLAGS_duplication_enabled = false; + // Set FLAGS_enable_acl true to make group validator encrypt_data_at_rest_pre_check succeed + // when encrypt_data_at_rest is true. + dsn::security::FLAGS_enable_acl = true; Review Comment: Set them together at the same place to reduce confusion. ########## src/replica/replica_stub.cpp: ########## @@ -291,13 +300,55 @@ DSN_DEFINE_int32( 10, "if tcmalloc reserved but not-used memory exceed this percentage of application allocated " "memory, replica server will release the exceeding memory back to operating system"); +DSN_DEFINE_string(pegasus.server, + encryption_cluster_key_name, + "pegasus", + "The cluster name of encrypted server which use to get server key from kms."); + +DSN_DEFINE_string( + pegasus.server, + hadoop_kms_url, + "", + "Where the server key of file system can get from. " + "Url should be comma-separated list, such as 'hostname1:1234/kms,hostname2:1234/kms'"); DSN_DECLARE_bool(duplication_enabled); DSN_DECLARE_int32(fd_beacon_interval_seconds); DSN_DECLARE_int32(fd_check_interval_seconds); DSN_DECLARE_int32(fd_grace_seconds); DSN_DECLARE_int32(fd_lease_seconds); DSN_DECLARE_int32(gc_interval_ms); +DSN_DECLARE_string(data_dirs); +DSN_DEFINE_group_validator(encrypt_data_at_rest_pre_check, [](std::string &message) -> bool { + if (!dsn::security::FLAGS_enable_acl && FLAGS_encrypt_data_at_rest) { + message = fmt::format("[pegasus.server] encrypt_data_at_rest should be enabled only if " + "[security] enable_acl is enabled."); + return false; + } + return true; +}); + +DSN_DEFINE_group_validator(encrypt_data_not_support_close, [](std::string &message) -> bool { + std::vector<std::string> dirs; + std::string data_dirs; + // In some unit test FLAGS_data_dirs may not set. + if (!dsn::utils::is_empty(FLAGS_data_dirs)) { + data_dirs = FLAGS_data_dirs; + } else { + return true; + } + utils::split_args(data_dirs.c_str(), dirs, ','); + std::string kms_path = utils::filesystem::path_combine(dirs[0], ".kms_info"); + if (!FLAGS_encrypt_data_at_rest && utils::filesystem::path_exists(kms_path)) { + message = fmt::format( + "[pegasus.server] encrypt_data_at_rest = ({}), but kms_info file path = ({}) is exist." + "Pegasus dont support close encrypte after enable encrypte.", Review Comment: ```suggestion "Encryption in Pegasus is irreversible after its initial activation.", ``` ########## src/replica/replica_stub.cpp: ########## @@ -389,9 +440,39 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f } } + std::string server_key; + dsn::replication::replica_kms_info kms_info; + if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) { + key_provider.reset(new dsn::security::KMSKeyProvider( + ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()), + FLAGS_encryption_cluster_key_name)); + auto err = kms_info.load(_options.data_dirs[0]); Review Comment: Add a check before this line: ``` CHECK_TRUE(!_options.data_dirs.empty()); ``` ########## src/runtime/security/kms_client.cpp: ########## @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <initializer_list> +#include <map> +#include <stdexcept> +#include <string> +#include <vector> + +#include "absl/strings/escaping.h" +#include "absl/strings/substitute.h" +#include "http/http_client.h" +#include "http/http_method.h" +#include "nlohmann/json.hpp" +#include "nlohmann/json_fwd.hpp" +#include "runtime/security/kms_client.h" +#include "runtime/security/replica_kms_info.h" +#include "utils/error_code.h" +#include "utils/fmt_logging.h" + +namespace dsn { +namespace security { + +dsn::error_s KMSClient::DecryptEncryptionKey(const dsn::replication::replica_kms_info &kms_info, + std::string *decrypted_key) +{ + nlohmann::json post; + post["name"] = cluster_key_name_; + std::string iv_plain = ::absl::HexStringToBytes(kms_info.iv); + std::string iv_b64; + ::absl::WebSafeBase64Escape(iv_plain, &iv_b64); + post["iv"] = iv_b64; + std::string eek_plain = ::absl::HexStringToBytes(kms_info.eek); + std::string eek_b64; + ::absl::WebSafeBase64Escape(eek_plain, &eek_b64); + post["material"] = eek_b64; + + http_client client; + auto err = client.init(); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "Start http client failed"); + } + + err = client.set_auth(http_auth_type::SPNEGO); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set auth type failed"); + } + + std::vector<std::string> urls; + urls.reserve(kms_urls_.size()); + for (const auto &url : kms_urls_) { + urls.emplace_back( + ::absl::Substitute("$0/v1/keyversion/$1/_eek?eek_op=decrypt", url, kms_info.kv)); + } + + client.clear_header_fields(); + client.set_content_type("application/json"); + client.set_accept("*/*"); + err = client.with_post_method(post.dump()); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set method failed"); + } + + nlohmann::json j; + for (const auto &url : urls) { + err = client.set_url(url); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http clientt set url failed"); + } + std::string resp; + err = client.exec_method(&resp); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client exec post method failed"); + } + long http_status; + client.get_http_status(http_status); + if (http_status == 200) { + j = nlohmann::json::parse(resp); Review Comment: Just break when parse OK? ########## src/runtime/security/kms_client.cpp: ########## @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <initializer_list> +#include <map> +#include <stdexcept> +#include <string> +#include <vector> + +#include "absl/strings/escaping.h" +#include "absl/strings/substitute.h" +#include "http/http_client.h" +#include "http/http_method.h" +#include "nlohmann/json.hpp" +#include "nlohmann/json_fwd.hpp" +#include "runtime/security/kms_client.h" +#include "runtime/security/replica_kms_info.h" +#include "utils/error_code.h" +#include "utils/fmt_logging.h" + +namespace dsn { +namespace security { + +dsn::error_s KMSClient::DecryptEncryptionKey(const dsn::replication::replica_kms_info &kms_info, + std::string *decrypted_key) +{ + nlohmann::json post; + post["name"] = cluster_key_name_; + std::string iv_plain = ::absl::HexStringToBytes(kms_info.iv); + std::string iv_b64; + ::absl::WebSafeBase64Escape(iv_plain, &iv_b64); + post["iv"] = iv_b64; + std::string eek_plain = ::absl::HexStringToBytes(kms_info.eek); + std::string eek_b64; + ::absl::WebSafeBase64Escape(eek_plain, &eek_b64); + post["material"] = eek_b64; + + http_client client; + auto err = client.init(); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "Start http client failed"); + } + + err = client.set_auth(http_auth_type::SPNEGO); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set auth type failed"); + } + + std::vector<std::string> urls; + urls.reserve(kms_urls_.size()); + for (const auto &url : kms_urls_) { + urls.emplace_back( + ::absl::Substitute("$0/v1/keyversion/$1/_eek?eek_op=decrypt", url, kms_info.kv)); + } + + client.clear_header_fields(); + client.set_content_type("application/json"); + client.set_accept("*/*"); + err = client.with_post_method(post.dump()); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set method failed"); + } + + nlohmann::json j; + for (const auto &url : urls) { + err = client.set_url(url); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http clientt set url failed"); + } + std::string resp; + err = client.exec_method(&resp); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client exec post method failed"); + } + long http_status; + client.get_http_status(http_status); + if (http_status == 200) { + j = nlohmann::json::parse(resp); + } else { + LOG_WARNING("The http status is ({}), and url is ({})", http_status, url); + } + } + + std::string dek_b64; + if (j.contains("material")) { + dek_b64 = j.at("material"); + } else { + return dsn::error_s::make(ERR_INVALID_DATA, "Null material received"); + } + std::string dek_plain; + if (!::absl::WebSafeBase64Unescape(dek_b64, &dek_plain)) { + return dsn::error_s::make(ERR_INVALID_DATA, "Invalid IV received"); + } + *decrypted_key = ::absl::BytesToHexString(dek_plain); + return dsn::error_s::ok(); +} + +dsn::error_s KMSClient::GenerateEncryptionKeyFromKMS(const std::string &key_name, + dsn::replication::replica_kms_info *kms_info) +{ + http_client client; + auto err = client.init(); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "Start http client failed"); + } + err = client.set_auth(http_auth_type::SPNEGO); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set auth type failed"); + } + std::vector<std::string> urls; + urls.reserve(kms_urls_.size()); + for (const auto &url : kms_urls_) { + urls.emplace_back( + ::absl::Substitute("$0/v1/key/$1/_eek?eek_op=generate&num_keys=1", url, key_name)); + } + + nlohmann::json j = nlohmann::json::object(); + for (const auto &url : urls) { + err = client.set_url(url); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set url failed"); + } + + err = client.with_get_method(); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set get method failed"); + } + + std::string resp; + err = client.exec_method(&resp); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client exec get method failed"); + } + + long http_status; + client.get_http_status(http_status); + if (http_status == 200) { + j = nlohmann::json::parse(resp); + nlohmann::json jsonObject = j.at(0); + std::string res = jsonObject.dump(); + j = nlohmann::json::parse(res); Review Comment: Break when parse OK. ########## src/runtime/security/replica_kms_info.h: ########## @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include <string> + +#include "common/json_helper.h" +#include "utils/error_code.h" +#include "utils/ports.h" + +namespace dsn { +namespace replication { + +// This class is to store and load EEK, IV, KV from KMS as json file. +// Get decrypted key should POST EEK, IV, KV both to KMS. +class replica_kms_info +{ +public: + std::string eek; // a.k.a encrypted encryption key + std::string iv; // a.k.a initialization vector + std::string kv; // a.k.a key version + DEFINE_JSON_SERIALIZATION(eek, iv, kv) + static const std::string kKmsInfo; // json file name + +public: + replica_kms_info(const std::string &e_key = "", + const std::string &i = "", + const std::string &k_version = "") + : eek(e_key), iv(i), kv(k_version) + { + } + // load replica_kms_info object from json file Review Comment: ```suggestion // Load replica_kms_info object from a json file. ``` ########## src/runtime/security/kms_client.h: ########## @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <algorithm> +#include <string> +#include <utility> +#include <vector> + +#include "utils/errors.h" + +namespace dsn { +namespace replication { +class replica_kms_info; +} // namespace replication + +namespace security { +// A class to generate encryption_key from KMS for writing file which implemented based on http +// client. +// This class is not thread-safe. Thus maintain one instance for each thread. +class KMSClient +{ +public: + KMSClient(const std::vector<std::string> &kms_url, std::string cluster_key_name) + : kms_urls_(kms_url), cluster_key_name_(std::move(cluster_key_name)) + { + } + + // Get the Decrypted Encryption Key(dek) back from KMS. The EEK, IV, KV need generated from KMS + // by GenerateEncryptionKey function first. Review Comment: Retrieve the Decrypted Encryption Key (DEK) from KMS after generating the EEK, IV, and KV. ########## src/replica/replica_stub.cpp: ########## @@ -291,13 +300,55 @@ DSN_DEFINE_int32( 10, "if tcmalloc reserved but not-used memory exceed this percentage of application allocated " "memory, replica server will release the exceeding memory back to operating system"); +DSN_DEFINE_string(pegasus.server, + encryption_cluster_key_name, + "pegasus", + "The cluster name of encrypted server which use to get server key from kms."); + +DSN_DEFINE_string( + pegasus.server, + hadoop_kms_url, + "", + "Where the server key of file system can get from. " + "Url should be comma-separated list, such as 'hostname1:1234/kms,hostname2:1234/kms'"); DSN_DECLARE_bool(duplication_enabled); DSN_DECLARE_int32(fd_beacon_interval_seconds); DSN_DECLARE_int32(fd_check_interval_seconds); DSN_DECLARE_int32(fd_grace_seconds); DSN_DECLARE_int32(fd_lease_seconds); DSN_DECLARE_int32(gc_interval_ms); +DSN_DECLARE_string(data_dirs); +DSN_DEFINE_group_validator(encrypt_data_at_rest_pre_check, [](std::string &message) -> bool { + if (!dsn::security::FLAGS_enable_acl && FLAGS_encrypt_data_at_rest) { + message = fmt::format("[pegasus.server] encrypt_data_at_rest should be enabled only if " + "[security] enable_acl is enabled."); + return false; + } + return true; +}); + +DSN_DEFINE_group_validator(encrypt_data_not_support_close, [](std::string &message) -> bool { + std::vector<std::string> dirs; + std::string data_dirs; + // In some unit test FLAGS_data_dirs may not set. + if (!dsn::utils::is_empty(FLAGS_data_dirs)) { + data_dirs = FLAGS_data_dirs; + } else { + return true; + } + utils::split_args(data_dirs.c_str(), dirs, ','); + std::string kms_path = utils::filesystem::path_combine(dirs[0], ".kms_info"); Review Comment: Use replica_kms_info::kKmsInfo instead another raw string. ########## src/runtime/security/replica_kms_info.cpp: ########## @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one Review Comment: Unify to use `//` style for licenses. ########## src/replica/replica_stub.cpp: ########## @@ -389,9 +440,39 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f } } + std::string server_key; + dsn::replication::replica_kms_info kms_info; + if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) { Review Comment: Deal the case of "FLAGS_encrypt_data_at_rest && utils::is_empty(FLAGS_hadoop_kms_url)" ########## src/replica/replica_stub.cpp: ########## @@ -291,13 +300,55 @@ DSN_DEFINE_int32( 10, "if tcmalloc reserved but not-used memory exceed this percentage of application allocated " "memory, replica server will release the exceeding memory back to operating system"); +DSN_DEFINE_string(pegasus.server, + encryption_cluster_key_name, + "pegasus", + "The cluster name of encrypted server which use to get server key from kms."); + +DSN_DEFINE_string( + pegasus.server, + hadoop_kms_url, + "", + "Where the server key of file system can get from. " + "Url should be comma-separated list, such as 'hostname1:1234/kms,hostname2:1234/kms'"); DSN_DECLARE_bool(duplication_enabled); DSN_DECLARE_int32(fd_beacon_interval_seconds); DSN_DECLARE_int32(fd_check_interval_seconds); DSN_DECLARE_int32(fd_grace_seconds); DSN_DECLARE_int32(fd_lease_seconds); DSN_DECLARE_int32(gc_interval_ms); +DSN_DECLARE_string(data_dirs); +DSN_DEFINE_group_validator(encrypt_data_at_rest_pre_check, [](std::string &message) -> bool { + if (!dsn::security::FLAGS_enable_acl && FLAGS_encrypt_data_at_rest) { + message = fmt::format("[pegasus.server] encrypt_data_at_rest should be enabled only if " + "[security] enable_acl is enabled."); + return false; + } + return true; +}); + +DSN_DEFINE_group_validator(encrypt_data_not_support_close, [](std::string &message) -> bool { + std::vector<std::string> dirs; + std::string data_dirs; + // In some unit test FLAGS_data_dirs may not set. + if (!dsn::utils::is_empty(FLAGS_data_dirs)) { + data_dirs = FLAGS_data_dirs; + } else { + return true; + } + utils::split_args(data_dirs.c_str(), dirs, ','); Review Comment: Unify to use ::absl::StrSplit ########## src/replica/replica_stub.cpp: ########## @@ -291,13 +300,55 @@ DSN_DEFINE_int32( 10, "if tcmalloc reserved but not-used memory exceed this percentage of application allocated " "memory, replica server will release the exceeding memory back to operating system"); +DSN_DEFINE_string(pegasus.server, + encryption_cluster_key_name, + "pegasus", + "The cluster name of encrypted server which use to get server key from kms."); + +DSN_DEFINE_string( + pegasus.server, + hadoop_kms_url, + "", + "Where the server key of file system can get from. " + "Url should be comma-separated list, such as 'hostname1:1234/kms,hostname2:1234/kms'"); Review Comment: Provide the comma-separated list of URLs from which to retrieve the file system's server key. Example format: 'hostname1:1234/kms,hostname2:1234/kms'. ########## src/runtime/security/replica_kms_info.h: ########## @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include <string> + +#include "common/json_helper.h" +#include "utils/error_code.h" +#include "utils/ports.h" + +namespace dsn { +namespace replication { + +// This class is to store and load EEK, IV, KV from KMS as json file. +// Get decrypted key should POST EEK, IV, KV both to KMS. +class replica_kms_info +{ +public: + std::string eek; // a.k.a encrypted encryption key + std::string iv; // a.k.a initialization vector + std::string kv; // a.k.a key version + DEFINE_JSON_SERIALIZATION(eek, iv, kv) + static const std::string kKmsInfo; // json file name + +public: + replica_kms_info(const std::string &e_key = "", + const std::string &i = "", + const std::string &k_version = "") + : eek(e_key), iv(i), kv(k_version) + { + } + // load replica_kms_info object from json file + error_code load(const std::string &dir) WARN_UNUSED_RESULT; + // store replica_kms_info object to json file Review Comment: ```suggestion // Store replica_kms_info object to a json file. ``` ########## src/replica/replica_stub.cpp: ########## @@ -389,9 +440,39 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f } } + std::string server_key; + dsn::replication::replica_kms_info kms_info; + if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) { + key_provider.reset(new dsn::security::KMSKeyProvider( + ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()), + FLAGS_encryption_cluster_key_name)); + auto err = kms_info.load(_options.data_dirs[0]); + if (err != dsn::ERR_OK) { + LOG_WARNING("Can't open kms-info file to read, this is normal when first launch " + "process. err = {}", + err); + } + // The encryption key should empty when process upon the first launch. And the process will + // get EEK, IV, KV from KMS. + // After first launch, the encryption key should not empty and get from kms-info file. The Review Comment: ```suggestion // After the first launch, the encryption key should not empty and get from kms-info file. The ``` ########## src/replica/kms_key_provider.h: ########## @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include <string> +#include <utility> +#include <vector> + +#include "runtime/security/kms_client.h" +#include "utils/errors.h" + +namespace dsn { +namespace replication { +class replica_kms_info; +} // namespace replication + +namespace security { +// This class is to generating EEK IV KV from KMS (a.k.a Key Manager Service) and get DEK from KMS. +class KMSKeyProvider +{ +public: + ~KMSKeyProvider() {} + + KMSKeyProvider(const std::vector<std::string> &kms_url, std::string cluster_key_name) + : client_(kms_url, std::move(cluster_key_name)) + { + } + + // Use KMS client which decrypted the encryption key from KMS and decrypted key is a hex string + // which could be used derectly. Review Comment: ```suggestion // Decrypt the encryption key in 'kms_info' via KMS. 'decrypted_key' is a hex string. ``` ########## src/replica/replica_stub.cpp: ########## @@ -291,13 +300,55 @@ DSN_DEFINE_int32( 10, "if tcmalloc reserved but not-used memory exceed this percentage of application allocated " "memory, replica server will release the exceeding memory back to operating system"); +DSN_DEFINE_string(pegasus.server, + encryption_cluster_key_name, + "pegasus", + "The cluster name of encrypted server which use to get server key from kms."); Review Comment: ```suggestion "The cluster name of the server is used to retrieve its encryption key from KMS."); ``` ########## src/replica/replica_stub.cpp: ########## @@ -389,9 +440,39 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f } } + std::string server_key; + dsn::replication::replica_kms_info kms_info; + if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) { + key_provider.reset(new dsn::security::KMSKeyProvider( + ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()), + FLAGS_encryption_cluster_key_name)); + auto err = kms_info.load(_options.data_dirs[0]); + if (err != dsn::ERR_OK) { + LOG_WARNING("Can't open kms-info file to read, this is normal when first launch " + "process. err = {}", + err); + } + // The encryption key should empty when process upon the first launch. And the process will Review Comment: ```suggestion // The encryption key should be empty when process upon the first launch. And the process will ``` ########## src/replica/replica_stub.cpp: ########## @@ -389,9 +440,39 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f } } + std::string server_key; + dsn::replication::replica_kms_info kms_info; + if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) { + key_provider.reset(new dsn::security::KMSKeyProvider( + ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()), + FLAGS_encryption_cluster_key_name)); + auto err = kms_info.load(_options.data_dirs[0]); + if (err != dsn::ERR_OK) { + LOG_WARNING("Can't open kms-info file to read, this is normal when first launch " + "process. err = {}", + err); + } + // The encryption key should empty when process upon the first launch. And the process will + // get EEK, IV, KV from KMS. + // After first launch, the encryption key should not empty and get from kms-info file. The + // process get DEK from KMS. + if (kms_info.eek.empty()) { + auto err = key_provider->GenerateEncryptionKey(&kms_info); + CHECK(err, "get encryption key failed, err = {}", err); + } + CHECK(key_provider->DecryptEncryptionKey(kms_info, &server_key), + "get decryption key failed"); + FLAGS_server_key = server_key.c_str(); + } + // Initialize the file system manager. _fs_manager.initialize(_options.data_dirs, _options.data_dir_tags); + if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) { Review Comment: Just check if key_provider is valid is enough. ########## src/replica/replica_stub.cpp: ########## @@ -389,9 +440,39 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f } } + std::string server_key; + dsn::replication::replica_kms_info kms_info; + if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) { + key_provider.reset(new dsn::security::KMSKeyProvider( + ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()), + FLAGS_encryption_cluster_key_name)); + auto err = kms_info.load(_options.data_dirs[0]); + if (err != dsn::ERR_OK) { + LOG_WARNING("Can't open kms-info file to read, this is normal when first launch " + "process. err = {}", + err); + } + // The encryption key should empty when process upon the first launch. And the process will + // get EEK, IV, KV from KMS. + // After first launch, the encryption key should not empty and get from kms-info file. The + // process get DEK from KMS. + if (kms_info.eek.empty()) { Review Comment: Generate the key only if the file doesn't exist (I guess there is a specify code return from kms_info.load), otherwise, it must encounter some intolerable error, just interrupt the procedure. ########## src/replica/replica_stub.cpp: ########## @@ -389,9 +440,39 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f } } + std::string server_key; + dsn::replication::replica_kms_info kms_info; + if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) { + key_provider.reset(new dsn::security::KMSKeyProvider( + ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()), + FLAGS_encryption_cluster_key_name)); + auto err = kms_info.load(_options.data_dirs[0]); + if (err != dsn::ERR_OK) { + LOG_WARNING("Can't open kms-info file to read, this is normal when first launch " Review Comment: It's normal to encounter a temporary inability to open the kms-info file during the first process launch. ########## src/replica/replica_stub.cpp: ########## @@ -389,9 +440,39 @@ void replica_stub::initialize(const replication_options &opts, bool clear /* = f } } + std::string server_key; + dsn::replication::replica_kms_info kms_info; + if (FLAGS_encrypt_data_at_rest && !utils::is_empty(FLAGS_hadoop_kms_url)) { + key_provider.reset(new dsn::security::KMSKeyProvider( + ::absl::StrSplit(FLAGS_hadoop_kms_url, ",", ::absl::SkipEmpty()), + FLAGS_encryption_cluster_key_name)); + auto err = kms_info.load(_options.data_dirs[0]); + if (err != dsn::ERR_OK) { + LOG_WARNING("Can't open kms-info file to read, this is normal when first launch " + "process. err = {}", + err); + } + // The encryption key should empty when process upon the first launch. And the process will + // get EEK, IV, KV from KMS. + // After first launch, the encryption key should not empty and get from kms-info file. The + // process get DEK from KMS. + if (kms_info.eek.empty()) { + auto err = key_provider->GenerateEncryptionKey(&kms_info); + CHECK(err, "get encryption key failed, err = {}", err); + } + CHECK(key_provider->DecryptEncryptionKey(kms_info, &server_key), Review Comment: Also log the err message. ########## src/runtime/security/kms_client.cpp: ########## @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <initializer_list> +#include <map> +#include <stdexcept> +#include <string> +#include <vector> + +#include "absl/strings/escaping.h" +#include "absl/strings/substitute.h" +#include "http/http_client.h" +#include "http/http_method.h" +#include "nlohmann/json.hpp" +#include "nlohmann/json_fwd.hpp" +#include "runtime/security/kms_client.h" +#include "runtime/security/replica_kms_info.h" +#include "utils/error_code.h" +#include "utils/fmt_logging.h" + +namespace dsn { +namespace security { + +dsn::error_s KMSClient::DecryptEncryptionKey(const dsn::replication::replica_kms_info &kms_info, + std::string *decrypted_key) +{ + nlohmann::json post; Review Comment: It would be better to use `payload`, "post" is a verb. ########## src/runtime/security/kms_client.cpp: ########## @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <initializer_list> +#include <map> +#include <stdexcept> +#include <string> +#include <vector> + +#include "absl/strings/escaping.h" +#include "absl/strings/substitute.h" +#include "http/http_client.h" +#include "http/http_method.h" +#include "nlohmann/json.hpp" +#include "nlohmann/json_fwd.hpp" +#include "runtime/security/kms_client.h" +#include "runtime/security/replica_kms_info.h" +#include "utils/error_code.h" +#include "utils/fmt_logging.h" + +namespace dsn { +namespace security { + +dsn::error_s KMSClient::DecryptEncryptionKey(const dsn::replication::replica_kms_info &kms_info, + std::string *decrypted_key) +{ + nlohmann::json post; + post["name"] = cluster_key_name_; + std::string iv_plain = ::absl::HexStringToBytes(kms_info.iv); + std::string iv_b64; + ::absl::WebSafeBase64Escape(iv_plain, &iv_b64); + post["iv"] = iv_b64; + std::string eek_plain = ::absl::HexStringToBytes(kms_info.eek); + std::string eek_b64; + ::absl::WebSafeBase64Escape(eek_plain, &eek_b64); + post["material"] = eek_b64; + + http_client client; + auto err = client.init(); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "Start http client failed"); + } + + err = client.set_auth(http_auth_type::SPNEGO); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set auth type failed"); + } + + std::vector<std::string> urls; + urls.reserve(kms_urls_.size()); + for (const auto &url : kms_urls_) { + urls.emplace_back( + ::absl::Substitute("$0/v1/keyversion/$1/_eek?eek_op=decrypt", url, kms_info.kv)); Review Comment: Use fmt::format ########## src/runtime/security/kms_client.cpp: ########## @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <initializer_list> +#include <map> +#include <stdexcept> +#include <string> +#include <vector> + +#include "absl/strings/escaping.h" +#include "absl/strings/substitute.h" +#include "http/http_client.h" +#include "http/http_method.h" +#include "nlohmann/json.hpp" +#include "nlohmann/json_fwd.hpp" +#include "runtime/security/kms_client.h" +#include "runtime/security/replica_kms_info.h" +#include "utils/error_code.h" +#include "utils/fmt_logging.h" + +namespace dsn { +namespace security { + +dsn::error_s KMSClient::DecryptEncryptionKey(const dsn::replication::replica_kms_info &kms_info, + std::string *decrypted_key) +{ + nlohmann::json post; + post["name"] = cluster_key_name_; + std::string iv_plain = ::absl::HexStringToBytes(kms_info.iv); + std::string iv_b64; + ::absl::WebSafeBase64Escape(iv_plain, &iv_b64); + post["iv"] = iv_b64; + std::string eek_plain = ::absl::HexStringToBytes(kms_info.eek); + std::string eek_b64; + ::absl::WebSafeBase64Escape(eek_plain, &eek_b64); + post["material"] = eek_b64; + + http_client client; + auto err = client.init(); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "Start http client failed"); + } + + err = client.set_auth(http_auth_type::SPNEGO); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set auth type failed"); + } + + std::vector<std::string> urls; + urls.reserve(kms_urls_.size()); + for (const auto &url : kms_urls_) { + urls.emplace_back( + ::absl::Substitute("$0/v1/keyversion/$1/_eek?eek_op=decrypt", url, kms_info.kv)); + } + + client.clear_header_fields(); + client.set_content_type("application/json"); + client.set_accept("*/*"); + err = client.with_post_method(post.dump()); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set method failed"); + } + + nlohmann::json j; + for (const auto &url : urls) { + err = client.set_url(url); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http clientt set url failed"); + } + std::string resp; + err = client.exec_method(&resp); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client exec post method failed"); + } + long http_status; + client.get_http_status(http_status); + if (http_status == 200) { + j = nlohmann::json::parse(resp); + } else { + LOG_WARNING("The http status is ({}), and url is ({})", http_status, url); + } + } + + std::string dek_b64; + if (j.contains("material")) { + dek_b64 = j.at("material"); + } else { + return dsn::error_s::make(ERR_INVALID_DATA, "Null material received"); + } + std::string dek_plain; + if (!::absl::WebSafeBase64Unescape(dek_b64, &dek_plain)) { + return dsn::error_s::make(ERR_INVALID_DATA, "Invalid IV received"); + } + *decrypted_key = ::absl::BytesToHexString(dek_plain); + return dsn::error_s::ok(); +} + +dsn::error_s KMSClient::GenerateEncryptionKeyFromKMS(const std::string &key_name, + dsn::replication::replica_kms_info *kms_info) +{ + http_client client; + auto err = client.init(); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "Start http client failed"); + } + err = client.set_auth(http_auth_type::SPNEGO); + if (!err.is_ok()) { + return dsn::error_s::make(ERR_CURL_FAILED, "http client set auth type failed"); + } + std::vector<std::string> urls; + urls.reserve(kms_urls_.size()); + for (const auto &url : kms_urls_) { + urls.emplace_back( + ::absl::Substitute("$0/v1/key/$1/_eek?eek_op=generate&num_keys=1", url, key_name)); Review Comment: Use fmt::format -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
