This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch aegis in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 34c5b852c86cd43109fd4a3df87ec27b3a7c6f54 Author: Robert Newson <[email protected]> AuthorDate: Wed Apr 8 16:40:26 2020 +0100 Add encryption for database values --- configure | 19 +++ rebar.config.script | 1 + rel/reltool.config | 1 + src/aegis/src/aegis.app.src | 31 +++++ src/aegis/src/aegis.erl | 132 +++++++++++++++++++++ src/aegis/src/aegis.hrl | 57 +++++++++ .../src/aegis_keywrap.erl} | 34 +++--- src/aegis/test/aegis_basic_test.erl | 17 +++ src/couch/rebar.config.script | 11 +- src/couch_views/src/couch_views_fdb.erl | 8 +- src/fabric/include/fabric2.hrl | 1 + src/fabric/src/fabric2_fdb.erl | 31 +++-- 12 files changed, 305 insertions(+), 38 deletions(-) diff --git a/configure b/configure index 38e62e3..5bd40d3 100755 --- a/configure +++ b/configure @@ -96,6 +96,24 @@ parse_opts() { continue ;; + --key-manager) + if [ -n "$2" ]; then + eval AEGIS_KEY_MANAGER=$2 + shift 2 + continue + else + printf 'ERROR: "--key-manager" requires a non-empty argument.\n' >&2 + exit 1 + fi + ;; + --key-manager=?*) + eval AEGIS_KEY_MANAGER=${1#*=} + ;; + --key-manager=) + printf 'ERROR: "--key-manager" requires a non-empty argument.\n' >&2 + exit 1 + ;; + --dev) WITH_DOCS=0 WITH_FAUXTON=0 @@ -241,6 +259,7 @@ cat > $rootdir/config.erl << EOF {with_curl, $WITH_CURL}. {with_proper, $WITH_PROPER}. {erlang_md5, $ERLANG_MD5}. +{aegis_key_manager, "$AEGIS_KEY_MANAGER"}, {spidermonkey_version, "$SM_VSN"}. EOF diff --git a/rebar.config.script b/rebar.config.script index 6f9f65c..118a99e 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -114,6 +114,7 @@ os:putenv("COUCHDB_APPS_CONFIG_DIR", filename:join([COUCHDB_ROOT, "rel/apps"])). SubDirs = [ %% must be compiled first as it has a custom behavior "src/couch_epi", + "src/aegis", "src/couch_log", "src/chttpd", "src/couch", diff --git a/rel/reltool.config b/rel/reltool.config index 9fbf285..1e64a80 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -90,6 +90,7 @@ {app, xmerl, [{incl_cond, include}]}, %% couchdb + {app, aegis, [{incl_cond, include}]}, {app, b64url, [{incl_cond, include}]}, {app, bear, [{incl_cond, include}]}, {app, chttpd, [{incl_cond, include}]}, diff --git a/src/aegis/src/aegis.app.src b/src/aegis/src/aegis.app.src new file mode 100644 index 0000000..51b608d --- /dev/null +++ b/src/aegis/src/aegis.app.src @@ -0,0 +1,31 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +{application, aegis, + [ + {description, "If it's good enough for Zeus, it's good enough for CouchDB"}, + {vsn, git}, + {applications, + [kernel, + stdlib, + crypto, + couch_log, + base64, + erlfdb + ]}, + {env,[]}, + {modules, []}, + {maintainers, []}, + {licenses, []}, + {links, []} + ] +}. diff --git a/src/aegis/src/aegis.erl b/src/aegis/src/aegis.erl new file mode 100644 index 0000000..dc8271f --- /dev/null +++ b/src/aegis/src/aegis.erl @@ -0,0 +1,132 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis). +-include("aegis.hrl"). +-include_lib("fabric/include/fabric2.hrl"). + +%% TODO - get from key manager +-define(ROOT_KEY, <<1:256>>). + +-define(WRAPPED_KEY, {?DB_AEGIS, 1}). + + +-export([ + create/2, + open/2, + + decrypt/2, + decrypt/3, + encrypt/3, + wrap_fold_fun/2 +]). + +create(#{} = Db, Options) -> + #{ + tx := Tx, + db_prefix := DbPrefix + } = Db, + + % Generate new key + DbKey = crypto:strong_rand_bytes(32), + + % protect it with root key + WrappedKey = aegis_keywrap:key_wrap(?ROOT_KEY, DbKey), + + % And store it + FDBKey = erlfdb_tuple:pack(?WRAPPED_KEY, DbPrefix), + ok = erlfdb:set(Tx, FDBKey, WrappedKey), + + Db#{ + aegis => DbKey + }. + + +open(#{} = Db, Options) -> + #{ + tx := Tx, + db_prefix := DbPrefix + } = Db, + + % Fetch wrapped key + FDBKey = erlfdb_tuple:pack(?WRAPPED_KEY, DbPrefix), + WrappedKey = erlfdb:wait(erlfdb:get(Tx, FDBKey)), + + % Unwrap it + DbKey = aegis_keywrap:key_unwrap(?ROOT_KEY, WrappedKey), + + Db#{ + aegis => DbKey + }. + + +encrypt(#{} = _Db, _Key, <<>>) -> + <<>>; + +encrypt(#{} = Db, Key, Value) when is_binary(Key), is_binary(Value) -> + #{ + uuid := UUID, + aegis := DbKey + } = Db, + + EncryptionKey = crypto:strong_rand_bytes(32), + <<WrappedKey:320>> = aegis_keywrap:key_wrap(DbKey, EncryptionKey), + + {CipherText, <<CipherTag:128>>} = + ?aes_gcm_encrypt( + EncryptionKey, + <<0:96>>, + <<UUID/binary, 0:8, Key/binary>>, + Value), + <<1:8, WrappedKey:320, CipherTag:128, CipherText/binary>>. + + +decrypt(#{} = Db, Rows) when is_list(Rows) -> + lists:map(fun({Key, Value}) -> + {Key, decrypt(Db, Key, Value)} + end, Rows). + +decrypt(#{} = _Db, _Key, <<>>) -> + <<>>; + +decrypt(#{} = Db, Key, Value) when is_binary(Key), is_binary(Value) -> + #{ + uuid := UUID, + aegis := DbKey + } = Db, + + case Value of + <<1:8, WrappedKey:320, CipherTag:128, CipherText/binary>> -> + case aegis_keywrap:key_unwrap(DbKey, <<WrappedKey:320>>) of + fail -> + erlang:error(decryption_failed); + DecryptionKey -> + Decrypted = + ?aes_gcm_decrypt( + DecryptionKey, + <<0:96>>, + <<UUID/binary, 0:8, Key/binary>>, + CipherText, + <<CipherTag:128>>), + if Decrypted /= error -> Decrypted; true -> + erlang:error(decryption_failed) + end + end; + _ -> + erlang:error(not_ciphertext) + end. + + +wrap_fold_fun(Db, Fun) when is_function(Fun, 2) -> + fun({Key, Value}, Acc) -> + Fun({Key, decrypt(Db, Key, Value)}, Acc) + end. diff --git a/src/aegis/src/aegis.hrl b/src/aegis/src/aegis.hrl new file mode 100644 index 0000000..2a2a2dc --- /dev/null +++ b/src/aegis/src/aegis.hrl @@ -0,0 +1,57 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +%% Assume old crypto api + +-define(sha256_hmac(Key, PlainText), crypto:hmac(sha256, Key, PlainText)). + +-define(aes_gcm_encrypt(Key, IV, AAD, Data), + crypto:block_encrypt(aes_gcm, Key, IV, {AAD, Data, 16})). + +-define(aes_gcm_decrypt(Key, IV, AAD, CipherText, CipherTag), + crypto:block_decrypt(aes_gcm, Key, IV, {AAD, CipherText, CipherTag})). + +-define(aes_ecb_encrypt(Key, Data), + crypto:block_encrypt(aes_ecb, Key, Data)). + +-define(aes_ecb_decrypt(Key, Data), + crypto:block_decrypt(aes_ecb, Key, Data)). + +%% Replace macros if new crypto api is available +-ifdef(OTP_RELEASE). +-if(?OTP_RELEASE >= 22). + +-undef(sha256_hmac). +-define(sha256_hmac(Key, PlainText), crypto:mac(hmac, sha256, Key, PlainText)). + +-undef(aes_gcm_encrypt). +-define(aes_gcm_encrypt(Key, IV, AAD, Data), + crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, Data, AAD, 16, true)). + +-undef(aes_gcm_decrypt). +-define(aes_gcm_decrypt(Key, IV, AAD, CipherText, CipherTag), + crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, CipherText, + AAD, CipherTag, false)). + +-define(key_alg(Key), case bit_size(Key) of + 128 -> aes_128_ecb; 192 -> aes_192_ecb; 256 -> aes_256_ecb end). + +-undef(aes_ecb_encrypt). +-define(aes_ecb_encrypt(Key, Data), + crypto:crypto_one_time(?key_alg(Key), Key, Data, true)). + +-undef(aes_ecb_decrypt). +-define(aes_ecb_decrypt(Key, Data), + crypto:crypto_one_time(?key_alg(Key), Key, Data, false)). + +-endif. +-endif. \ No newline at end of file diff --git a/src/couch/src/couch_keywrap.erl b/src/aegis/src/aegis_keywrap.erl similarity index 84% rename from src/couch/src/couch_keywrap.erl rename to src/aegis/src/aegis_keywrap.erl index 0d1e3f5..58c7668 100644 --- a/src/couch/src/couch_keywrap.erl +++ b/src/aegis/src/aegis_keywrap.erl @@ -1,4 +1,17 @@ --module(couch_keywrap). +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_keywrap). +-include("aegis.hrl"). %% Implementation of NIST Special Publication 800-38F %% For wrapping and unwrapping keys with AES. @@ -7,25 +20,6 @@ -define(ICV1, 16#A6A6A6A6A6A6A6A6). -%% Assume old crypto api --define(aes_ecb_encrypt(Key, Data), - crypto:block_encrypt(aes_ecb, Key, Data)). --define(aes_ecb_decrypt(Key, Data), - crypto:block_decrypt(aes_ecb, Key, Data)). - -%% Replace macros if new crypto api is available --ifdef(OTP_RELEASE). --if(?OTP_RELEASE >= 22). --define(key_alg(Key), case bit_size(Key) of 128 -> aes_128_ecb; 192 -> aes_192_ecb; 256 -> aes_256_ecb end). --undef(aes_ecb_encrypt). --define(aes_ecb_encrypt(Key, Data), - crypto:crypto_one_time(?key_alg(Key), Key, Data, true)). --undef(aes_ecb_decrypt). --define(aes_ecb_decrypt(Key, Data), - crypto:crypto_one_time(?key_alg(Key), Key, Data, false)). --endif. --endif. - -spec key_wrap(WrappingKey :: binary(), KeyToWrap :: binary()) -> binary(). key_wrap(WrappingKey, KeyToWrap) when is_binary(WrappingKey), bit_size(KeyToWrap) rem 64 == 0 -> diff --git a/src/aegis/test/aegis_basic_test.erl b/src/aegis/test/aegis_basic_test.erl new file mode 100644 index 0000000..61d9737 --- /dev/null +++ b/src/aegis/test/aegis_basic_test.erl @@ -0,0 +1,17 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_basic_test). + +-include_lib("eunit/include/eunit.hrl"). + +-define(DB, #{uuid => <<"foo">>}). diff --git a/src/couch/rebar.config.script b/src/couch/rebar.config.script index 91e24d9..e281eab 100644 --- a/src/couch/rebar.config.script +++ b/src/couch/rebar.config.script @@ -92,6 +92,15 @@ MD5Config = case lists:keyfind(erlang_md5, 1, CouchConfig) of [] end, +AegisConfig = case lists:keyfind(crypto_module, 1, CouchConfig) of + {aegis_key_manager, ""} -> + []; + {aegis_key_manager, Module} -> + [{d, 'AEGIS_KEY_MANAGER', list_to_existing_atom(Module)}]; + _ -> + [] +end, + ProperConfig = case code:lib_dir(proper) of {error, bad_name} -> []; _ -> [{d, 'WITH_PROPER'}] @@ -223,7 +232,7 @@ AddConfig = [ {d, 'COUCHDB_VERSION', Version}, {d, 'COUCHDB_GIT_SHA', GitSha}, {i, "../"} - ] ++ MD5Config ++ ProperConfig}, + ] ++ MD5Config ++ AegisConfig ++ ProperConfig}, {port_env, PortEnvOverrides}, {eunit_compile_opts, PlatformDefines} ]. diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl index 3b008d4..dacfdf9 100644 --- a/src/couch_views/src/couch_views_fdb.erl +++ b/src/couch_views/src/couch_views_fdb.erl @@ -158,7 +158,7 @@ fold_map_idx(TxDb, Sig, ViewId, Options, Callback, Acc0) -> callback => Callback, acc => Acc0 }, - Fun = fun fold_fwd/2, + Fun = aegis:wrap_fold_fun(TxDb, fun fold_fwd/2), #{ acc := Acc1 @@ -283,7 +283,7 @@ update_id_idx(TxDb, Sig, ViewId, DocId, NewRows, KVSize) -> Key = id_idx_key(DbPrefix, Sig, DocId, ViewId), Val = couch_views_encoding:encode([length(NewRows), KVSize, Unique]), - ok = erlfdb:set(Tx, Key, Val). + ok = erlfdb:set(Tx, Key, aegis:encrypt(TxDb, Key, Val)). update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows) -> @@ -303,7 +303,7 @@ update_map_idx(TxDb, Sig, ViewId, DocId, ExistingKeys, NewRows) -> lists:foreach(fun({DupeId, Key1, Key2, EV}) -> KK = map_idx_key(MapIdxPrefix, {Key1, DocId}, DupeId), Val = erlfdb_tuple:pack({Key2, EV}), - ok = erlfdb:set(Tx, KK, Val) + ok = erlfdb:set(Tx, KK, aegis:encrypt(TxDb, KK, Val)) end, KVsToAdd). @@ -318,7 +318,7 @@ get_view_keys(TxDb, Sig, DocId) -> erlfdb_tuple:unpack(K, DbPrefix), [TotalKeys, TotalSize, UniqueKeys] = couch_views_encoding:decode(V), {ViewId, TotalKeys, TotalSize, UniqueKeys} - end, erlfdb:get_range(Tx, Start, End, [])). + end, aegis:decrypt(TxDb, erlfdb:get_range(Tx, Start, End, []))). update_row_count(TxDb, Sig, ViewId, Increment) -> diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl index 0c07575..b4fe4f7 100644 --- a/src/fabric/include/fabric2.hrl +++ b/src/fabric/include/fabric2.hrl @@ -40,6 +40,7 @@ -define(DB_LOCAL_DOC_BODIES, 25). -define(DB_ATT_NAMES, 26). -define(DB_SEARCH, 27). +-define(DB_AEGIS, 28). % Versions diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl index 2295a56..96f60e6 100644 --- a/src/fabric/src/fabric2_fdb.erl +++ b/src/fabric/src/fabric2_fdb.erl @@ -177,7 +177,7 @@ create(#{} = Db0, Options) -> name := DbName, tx := Tx, layer_prefix := LayerPrefix - } = Db = ensure_current(Db0, false), + } = Db1 = ensure_current(Db0, false), DbKey = erlfdb_tuple:pack({?ALL_DBS, DbName}, LayerPrefix), HCA = erlfdb_hca:create(erlfdb_tuple:pack({?DB_HCA}, LayerPrefix)), @@ -220,7 +220,7 @@ create(#{} = Db0, Options) -> UserCtx = fabric2_util:get_value(user_ctx, Options, #user_ctx{}), Options1 = lists:keydelete(user_ctx, 1, Options), - Db#{ + Db2 = Db1#{ uuid => UUID, db_prefix => DbPrefix, db_version => DbVersion, @@ -235,7 +235,8 @@ create(#{} = Db0, Options) -> % All other db things as we add features, db_options => Options1 - }. + }, + aegis:create(Db2, Options). open(#{} = Db0, Options) -> @@ -280,14 +281,15 @@ open(#{} = Db0, Options) -> }, Db3 = load_config(Db2), + Db4 = aegis:open(Db3, Options), - case {UUID, Db3} of + case {UUID, Db4} of {undefined, _} -> ok; {<<_/binary>>, #{uuid := UUID}} -> ok; {<<_/binary>>, #{uuid := _}} -> erlang:error(database_does_not_exist) end, - load_validate_doc_funs(Db3). + load_validate_doc_funs(Db4). % Match on `name` in the function head since some non-fabric2 db @@ -630,9 +632,10 @@ get_doc_body_wait(#{} = Db0, DocId, RevInfo, Future) -> rev_path := RevPath } = RevInfo, - RevBodyRows = erlfdb:fold_range_wait(Tx, Future, fun({_K, V}, Acc) -> + FoldFun = aegis:wrap_fold_fun(Db, fun({_K, V}, Acc) -> [V | Acc] - end, []), + end), + RevBodyRows = erlfdb:fold_range_wait(Tx, Future, FoldFun, []), BodyRows = lists:reverse(RevBodyRows), fdb_to_doc(Db, DocId, RevPos, [Rev | RevPath], BodyRows). @@ -649,7 +652,7 @@ get_local_doc(#{} = Db0, <<?LOCAL_DOC_PREFIX, _/binary>> = DocId) -> Prefix = erlfdb_tuple:pack({?DB_LOCAL_DOC_BODIES, DocId}, DbPrefix), Future = erlfdb:get_range_startswith(Tx, Prefix), - Chunks = lists:map(fun({_K, V}) -> V end, erlfdb:wait(Future)), + {_, Chunks} = lists:unzip(aegis:decrypt(Db, erlfdb:wait(Future))), fdb_to_local_doc(Db, DocId, Rev, Chunks). @@ -878,7 +881,9 @@ write_local_doc(#{} = Db0, Doc) -> % Make sure to clear the whole range, in case there was a larger % document body there before. erlfdb:clear_range_startswith(Tx, BPrefix), - lists:foreach(fun({K, V}) -> erlfdb:set(Tx, K, V) end, Rows) + lists:foreach(fun({K, V}) -> + erlfdb:set(Tx, K, aegis:encrypt(Db, K, V)) + end, Rows) end, case {WasDeleted, Doc#doc.deleted} of @@ -906,8 +911,8 @@ read_attachment(#{} = Db, DocId, AttId) -> not_found -> throw({not_found, missing}); KVs -> - Vs = [V || {_K, V} <- KVs], - iolist_to_binary(Vs) + {_, Chunks} = lists:unzip(aegis:decrypt(Db, KVs)), + iolist_to_binary(Chunks) end. @@ -925,7 +930,7 @@ write_attachment(#{} = Db, DocId, Data) when is_binary(Data) -> lists:foldl(fun(Chunk, ChunkId) -> AttKey = erlfdb_tuple:pack({?DB_ATTS, DocId, AttId, ChunkId}, DbPrefix), - ok = erlfdb:set(Tx, AttKey, Chunk), + ok = erlfdb:set(Tx, AttKey, aegis:encrypt(Db, AttKey, Chunk)), ChunkId + 1 end, 0, Chunks), {ok, AttId}. @@ -1193,7 +1198,7 @@ write_doc_body(#{} = Db0, #doc{} = Doc) -> Rows = doc_to_fdb(Db, Doc), lists:foreach(fun({Key, Value}) -> - ok = erlfdb:set(Tx, Key, Value) + ok = erlfdb:set(Tx, Key, aegis:encrypt(Db, Key, Value)) end, Rows).
