This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch database_encryption in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 14fb6776e4a2b9ebdc1d1896cf71c4be78f18df6 Author: Robert Newson <[email protected]> AuthorDate: Thu May 5 14:39:37 2022 +0100 Database Encryption Support CouchDB can optionally encrypt databases and views. We use AES in Counter Mode, which ensures we can encrypt and decrypt any section of the file without padding or alignment. The ciphertext is the same length as the plaintext. This mode provides confidentiality but not authentication. Key management is configurable, a system administrator can write a module implementing the aegis_key_manager behaviour with any implementation. CouchDB ships with an implementation that stores keys in the config file as an example, but this is not suitable for production. --- rebar.config.script | 1 + rel/overlay/etc/default.ini | 7 + rel/reltool.config | 2 + src/aegis/src/aegis.app.src | 28 ++++ src/aegis/src/aegis.erl | 60 ++++++++ src/aegis/src/aegis_cmac.erl | 83 +++++++++++ src/aegis/src/aegis_key_manager.erl | 41 ++++++ src/aegis/src/aegis_key_manager_config.erl | 62 ++++++++ src/aegis/src/aegis_s2v.erl | 53 +++++++ src/aegis/src/aegis_siv.erl | 140 ++++++++++++++++++ src/aegis/src/aegis_util.erl | 88 +++++++++++ src/couch/src/couch_file.erl | 225 ++++++++++++++++++++++++----- src/couch/src/couch_util.erl | 42 +++++- 13 files changed, 793 insertions(+), 39 deletions(-) diff --git a/rebar.config.script b/rebar.config.script index 44c4d61b1..0e6af11d3 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -113,6 +113,7 @@ os:putenv("COUCHDB_APPS_CONFIG_DIR", filename:join([COUCHDB_ROOT, "rel/apps"])). SubDirs = [ %% must be compiled first as it has a custom behavior "src/couch_epi", + "src/aegis", "src/couch_log", "src/chttpd", "src/couch", diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 162ccb926..19b02ed30 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -747,3 +747,10 @@ port = {{prometheus_port}} ; to disable this setting could be if the views need an upgrade but located on ; read-only file system. ;commit_on_header_upgrade = true + +[encryption] +wrapping_key_id = bec46c7438e685d6 + +[encryption_keys] +254abda45788029b = 12b856a100016bbf5e3bab3f7db22cb3c6a7a9442322f16aa998649198df3445 +bec46c7438e685d6 = f9dae865d5396cf146745df47b68f6bb8f3a3118b39758d4cb2f44cf746d3dcb diff --git a/rel/reltool.config b/rel/reltool.config index ab26fb2ed..a7ab87c5f 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -26,6 +26,7 @@ syntax_tools, xmerl, %% couchdb + aegis, b64url, bear, chttpd, @@ -90,6 +91,7 @@ {app, xmerl, [{incl_cond, include}]}, %% couchdb + {app, aegis, [{incl_cond, include}]}, {app, b64url, [{incl_cond, include}]}, {app, bear, [{incl_cond, include}]}, {app, chttpd, [{incl_cond, include}]}, diff --git a/src/aegis/src/aegis.app.src b/src/aegis/src/aegis.app.src new file mode 100644 index 000000000..9088ec46c --- /dev/null +++ b/src/aegis/src/aegis.app.src @@ -0,0 +1,28 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +{application, aegis, + [{description, "An OTP application"}, + {vsn, git}, + {registered, []}, + {applications, + [kernel, + stdlib, + crypto + ]}, + {env,[]}, + {modules, []}, + + {maintainers, []}, + {licenses, []}, + {links, []} + ]}. diff --git a/src/aegis/src/aegis.erl b/src/aegis/src/aegis.erl new file mode 100644 index 000000000..ac7b49b6d --- /dev/null +++ b/src/aegis/src/aegis.erl @@ -0,0 +1,60 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis). + +-export([wrap_key/3, unwrap_key/3]). + +wrap_key(KEK, AAD, DEK) when is_binary(KEK), is_list(AAD), is_binary(DEK) -> + ExpandedKey = aegis_util:expand(KEK), + {CipherText, CipherTag} = + aegis_siv:block_encrypt( + ExpandedKey, + AAD, + DEK + ), + <<CipherTag/binary, CipherText/binary>>. + +unwrap_key(KEK, AAD, <<CipherTag:16/binary, CipherText/binary>>) when + is_binary(KEK), is_list(AAD) +-> + ExpandedKey = aegis_util:expand(KEK), + aegis_siv:block_decrypt( + ExpandedKey, + AAD, + {CipherText, CipherTag} + ). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +aegis_test_() -> + [ + ?_assertEqual( + <<91, 78, 2, 43, 95, 157, 34, 252, 93, 35, 150, 141, 155, 139, 247, 136, 154, 203, 16, + 143, 196, 78, 93, 9, 189, 119, 22, 27, 60, 47, 186, 114, 70, 231, 113, 189, 36, 236, + 139, 153, 85, 58, 207, 165, 169, 70, 67, 61>>, + wrap_key(<<0:256>>, [], <<1:256>>) + ), + ?_assertEqual( + <<1:256>>, + unwrap_key( + <<0:256>>, + [], + <<91, 78, 2, 43, 95, 157, 34, 252, 93, 35, 150, 141, 155, 139, 247, 136, 154, 203, + 16, 143, 196, 78, 93, 9, 189, 119, 22, 27, 60, 47, 186, 114, 70, 231, 113, 189, + 36, 236, 139, 153, 85, 58, 207, 165, 169, 70, 67, 61>> + ) + ) + ]. + +-endif. diff --git a/src/aegis/src/aegis_cmac.erl b/src/aegis/src/aegis_cmac.erl new file mode 100644 index 000000000..9f1f05fcd --- /dev/null +++ b/src/aegis/src/aegis_cmac.erl @@ -0,0 +1,83 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_cmac). + +-export([cmac/2]). + +cmac(Key, Message) -> + cmac(Key, <<0:128>>, Message). + +cmac(Key, X, <<Last:16/binary>>) -> + {K1, _K2} = generate_subkeys(Key), + crypto:crypto_one_time(cmac_cipher(Key), Key, crypto:exor(X, crypto:exor(Last, K1)), true); +cmac(Key, X, <<Block:16/binary, Rest/binary>>) -> + cmac(Key, crypto:crypto_one_time(cmac_cipher(Key), Key, crypto:exor(X, Block), true), Rest); +cmac(Key, X, Last) -> + {_K1, K2} = generate_subkeys(Key), + crypto:crypto_one_time( + cmac_cipher(Key), + Key, + crypto:exor(X, crypto:exor(aegis_util:pad(Last), K2)), + true + ). + +generate_subkeys(Key) -> + L = crypto:crypto_one_time(cmac_cipher(Key), Key, <<0:128>>, true), + K1 = aegis_util:double(L), + K2 = aegis_util:double(K1), + {K1, K2}. + +cmac_cipher(Key) when bit_size(Key) == 128 -> + aes_128_ecb; +cmac_cipher(Key) when bit_size(Key) == 256 -> + aes_256_ecb. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +cmac_test_() -> + [ + ?_assertEqual( + <<16#bb1d6929e95937287fa37d129b756746:128>>, + cmac( + <<16#2b7e151628aed2a6abf7158809cf4f3c:128>>, + <<>> + ) + ), + + ?_assertEqual( + <<16#070a16b46b4d4144f79bdd9dd04a287c:128>>, + cmac( + <<16#2b7e151628aed2a6abf7158809cf4f3c:128>>, + <<16#6bc1bee22e409f96e93d7e117393172a:128>> + ) + ), + + ?_assertEqual( + <<16#028962f61b7bf89efc6b551f4667d983:128>>, + cmac( + <<16#603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4:256>>, + <<>> + ) + ), + + ?_assertEqual( + <<16#28a7023f452e8f82bd4bf28d8c37c35c:128>>, + cmac( + <<16#603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4:256>>, + <<16#6bc1bee22e409f96e93d7e117393172a:128>> + ) + ) + ]. + +-endif. diff --git a/src/aegis/src/aegis_key_manager.erl b/src/aegis/src/aegis_key_manager.erl new file mode 100644 index 000000000..62793438a --- /dev/null +++ b/src/aegis/src/aegis_key_manager.erl @@ -0,0 +1,41 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_key_manager). + +-export([ + wrap_key/1, + unwrap_key/1 +]). + +-type dek() :: binary(). +-type wek() :: binary(). + +-callback wrap_key(DataEncryptionKey :: dek()) -> + {ok, WrappedKey :: wek()} + | {error, Reason :: term()}. + +-callback unwrap_key(WrappedKey :: wek()) -> + {ok, DataEncryptionKey :: dek()} + | {error, Reason :: term()}. + +wrap_key(DataEncryptionKey) -> + Module = key_manager_module(), + Module:wrap_key(DataEncryptionKey). + +unwrap_key(WrappedKey) -> + Module = key_manager_module(), + Module:unwrap_key(WrappedKey). + +key_manager_module() -> + Module = config:get("aegis", "key_manager_module", "aegis_key_manager_config"), + list_to_atom(Module). diff --git a/src/aegis/src/aegis_key_manager_config.erl b/src/aegis/src/aegis_key_manager_config.erl new file mode 100644 index 000000000..a52de335a --- /dev/null +++ b/src/aegis/src/aegis_key_manager_config.erl @@ -0,0 +1,62 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_key_manager_config). +-behaviour(aegis_key_manager). + +-export([ + wrap_key/1, + unwrap_key/1 +]). + +wrap_key(DataEncryptionKey) when is_binary(DataEncryptionKey) -> + {ok, WrappingKeyId, WrappingKey} = current_wrapping_key(), + WrappedKey = aegis:wrap_key(WrappingKey, [WrappingKeyId], DataEncryptionKey), + {ok, <<(byte_size(WrappingKeyId)):8, WrappingKeyId/binary, WrappedKey/binary>>}. + +unwrap_key(<<WrappingKeyIdLen:8, WrappingKeyId:WrappingKeyIdLen/binary, WrappedKey/binary>>) -> + case wrapping_key(WrappingKeyId) of + {ok, WrappingKeyId, WrappingKey} -> + case aegis:unwrap_key(WrappingKey, [WrappingKeyId], WrappedKey) of + fail -> + {error, unwrap_failed}; + Key when is_binary(Key) -> + {ok, Key} + end; + {error, Reason} -> + {error, Reason} + end; +unwrap_key(_) -> + {error, invalid_key}. + +current_wrapping_key() -> + wrapping_key(config:get("encryption", "wrapping_key_id")). + +wrapping_key(KeyId) when is_binary(KeyId) -> + wrapping_key(binary_to_list(KeyId)); +wrapping_key(KeyId) when is_list(KeyId), length(KeyId) == 16 -> + case get_config_binary("encryption_keys", KeyId, undefined) of + Hex when is_binary(Hex), byte_size(Hex) == 64 -> + {ok, list_to_binary(KeyId), couch_util:from_hex(Hex)}; + undefined -> + {error, no_key}; + _ -> + {error, invalid_key} + end. + +get_config_binary(Section, Key, Default) -> + case config:get(Section, Key) of + undefined -> + Default; + Value -> + list_to_binary(Value) + end. diff --git a/src/aegis/src/aegis_s2v.erl b/src/aegis/src/aegis_s2v.erl new file mode 100644 index 000000000..0bc6f49cd --- /dev/null +++ b/src/aegis/src/aegis_s2v.erl @@ -0,0 +1,53 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_s2v). + +-export([s2v/3]). + +s2v(Key, [], <<>>) -> + aegis_cmac:cmac(Key, <<1:128>>); +s2v(Key, AAD, PlainText) when length(AAD) < 127 -> + s2v(Key, AAD, PlainText, aegis_cmac:cmac(Key, <<0:128>>)). + +s2v(Key, [], PlainText, Acc) when bit_size(PlainText) >= 128 -> + aegis_cmac:cmac(Key, aegis_util:xorend(PlainText, Acc)); +s2v(Key, [], PlainText, Acc) -> + aegis_cmac:cmac( + Key, + crypto:exor(aegis_util:double(Acc), aegis_util:pad(PlainText)) + ); +s2v(Key, [H | T], PlainText, Acc0) -> + Acc1 = crypto:exor(aegis_util:double(Acc0), aegis_cmac:cmac(Key, H)), + s2v(Key, T, PlainText, Acc1). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +s2v_0_test() -> + ?assertEqual( + <<16#85632d07c6e8f37f950acd320a2ecc93:128>>, + s2v( + <<16#fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0:128>>, + [<<16#101112131415161718191a1b1c1d1e1f2021222324252627:192>>], + <<16#112233445566778899aabbccddee:112>> + ) + ). + +%% for test coverage only. this value does not come from a test vector. +s2v_1_test() -> + ?assertEqual( + <<106, 56, 130, 35, 180, 192, 121, 7, 97, 30, 181, 248, 111, 114, 85, 151>>, + s2v(<<0:128>>, [], <<>>) + ). + +-endif. diff --git a/src/aegis/src/aegis_siv.erl b/src/aegis/src/aegis_siv.erl new file mode 100644 index 000000000..efe3a1809 --- /dev/null +++ b/src/aegis/src/aegis_siv.erl @@ -0,0 +1,140 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_siv). + +-export([block_encrypt/3, block_decrypt/3]). + +-spec block_encrypt(binary(), list(), binary()) -> {binary(), binary()}. +block_encrypt(Key, AAD, PlainText) when + bit_size(Key) == 256; bit_size(Key) == 512 +-> + {K1, K2} = split(Key), + <<V:128>> = aegis_s2v:s2v(K1, AAD, PlainText), + Q = V band 16#ffffffffffffffff7fffffff7fffffff, + CipherText = aes_ctr(K2, <<Q:128>>, PlainText), + {CipherText, <<V:128>>}. + +block_decrypt(Key, AAD, {CipherText, <<V:128>>}) when + bit_size(Key) == 256; bit_size(Key) == 512 +-> + {K1, K2} = split(Key), + Q = V band 16#ffffffffffffffff7fffffff7fffffff, + PlainText = aes_ctr(K2, <<Q:128>>, CipherText), + <<T:128>> = aegis_s2v:s2v(K1, AAD, PlainText), + case V == T of + true -> + PlainText; + false -> + fail + end. + +split(Key) -> + Half = byte_size(Key) div 2, + <<K1:Half/binary, K2:Half/binary>> = Key, + {K1, K2}. + +aes_ctr(Key, IV, Data) -> + Cipher = ctr_cipher(Key), + crypto:crypto_one_time(Cipher, Key, IV, Data, true). + +ctr_cipher(Key) when bit_size(Key) == 128 -> + aes_128_ctr; +ctr_cipher(Key) when bit_size(Key) == 256 -> + aes_256_ctr. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +encrypt_test_() -> + [ + ?_assertEqual( + {<<16#40c02b9690c4dc04daef7f6afe5c:112>>, <<16#85632d07c6e8f37f950acd320a2ecc93:128>>}, + block_encrypt( + <<16#fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:256>>, + [<<16#101112131415161718191a1b1c1d1e1f2021222324252627:192>>], + <<16#112233445566778899aabbccddee:112>> + ) + ), + + ?_assertEqual( + { + << + 16#cb900f2fddbe404326601965c889bf17dba77ceb094fa663b7a3f748ba8af829ea64ad544a272e9c485b62a3fd5c0d:376 + >>, + <<16#7bdb6e3b432667eb06f4d14bff2fbd0f:128>> + }, + block_encrypt( + <<16#7f7e7d7c7b7a79787776757473727170404142434445464748494a4b4c4d4e4f:256>>, + [ + << + 16#00112233445566778899aabbccddeeffdeaddadadeaddadaffeeddccbbaa99887766554433221100:320 + >>, + <<16#102030405060708090a0:80>>, + <<16#09f911029d74e35bd84156c5635688c0:128>> + ], + << + 16#7468697320697320736f6d6520706c61696e7465787420746f20656e6372797074207573696e67205349562d414553:376 + >> + ) + ) + ]. + +decrypt_test_() -> + [ + ?_assertEqual( + <<16#112233445566778899aabbccddee:112>>, + block_decrypt( + <<16#fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:256>>, + [<<16#101112131415161718191a1b1c1d1e1f2021222324252627:192>>], + {<<16#40c02b9690c4dc04daef7f6afe5c:112>>, << + 16#85632d07c6e8f37f950acd320a2ecc93:128 + >>} + ) + ), + + ?_assertEqual( + fail, + block_decrypt( + <<16#fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff:256>>, + [<<16#101112131415161718191a1b1c1d1e1f2021222324252627:192>>], + {<<16#40c02b9690c4dc04daef7f6afe5c:112>>, << + 16#85632d07c6e8f37f950acd320a2ecc94:128 + >>} + ) + ), + + ?_assertEqual( + << + 16#7468697320697320736f6d6520706c61696e7465787420746f20656e6372797074207573696e67205349562d414553:376 + >>, + + block_decrypt( + <<16#7f7e7d7c7b7a79787776757473727170404142434445464748494a4b4c4d4e4f:256>>, + [ + << + 16#00112233445566778899aabbccddeeffdeaddadadeaddadaffeeddccbbaa99887766554433221100:320 + >>, + <<16#102030405060708090a0:80>>, + <<16#09f911029d74e35bd84156c5635688c0:128>> + ], + { + << + 16#cb900f2fddbe404326601965c889bf17dba77ceb094fa663b7a3f748ba8af829ea64ad544a272e9c485b62a3fd5c0d:376 + >>, + <<16#7bdb6e3b432667eb06f4d14bff2fbd0f:128>> + } + ) + ) + ]. + +-endif. diff --git a/src/aegis/src/aegis_util.erl b/src/aegis/src/aegis_util.erl new file mode 100644 index 000000000..ec60338c3 --- /dev/null +++ b/src/aegis/src/aegis_util.erl @@ -0,0 +1,88 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(aegis_util). + +-export([ + double/1, + expand/1, + pad/1, + xorend/2 +]). + +%% @doc double +%% is the multiplication of S and 0...010 in the finite field +%% represented using the primitive polynomial +%% x<sup>128</sup> + x<sup>7</sup> + x<sup>2</sup> + x + 1. +%% @end +-spec double(Val :: binary()) -> binary(). +double(<<0:1, Lo:127>>) -> + <<(Lo bsl 1):128>>; +double(<<1:1, Lo:127>>) -> + crypto:exor(<<(Lo bsl 1):128>>, <<16#87:128>>). + +%% because SIV only uses half the bits of the input key +%% to encrypt and the other half for the authentication/IV +%% we expand our keys to 512 to ensure an overall security +%% threshold of 256. +expand(Key) when bit_size(Key) == 256 -> + %% expansion technique from Bjoern Tackmann - IBM Zurich + K0 = crypto:crypto_one_time(aes_256_ecb, Key, <<0:128>>, true), + K1 = crypto:crypto_one_time(aes_256_ecb, Key, <<1:128>>, true), + K2 = crypto:crypto_one_time(aes_256_ecb, Key, <<2:128>>, true), + K3 = crypto:crypto_one_time(aes_256_ecb, Key, <<3:128>>, true), + <<K0/binary, K1/binary, K2/binary, K3/binary>>. + +%% @doc pad +%% indicates padding of string X, len(X) < 128, out to 128 bits by +%% the concatenation of a single bit of 1 followed by as many 0 bits +%% as are necessary. +%% @end +-spec pad(binary()) -> binary(). +pad(Val) when bit_size(Val) =< 128 -> + Pad = 128 - bit_size(Val) - 1, + <<Val/binary, 1:1, 0:Pad>>. + +%% @doc xorend +%% where len(A) >= len(B), means xoring a string B onto the end of +%% string A -- i.e., leftmost(A, len(A)-len(B)) || (rightmost(A, +%% len(B)) xor B). +%% @end +-spec xorend(binary(), binary()) -> binary(). +xorend(A, B) when byte_size(A) >= byte_size(B) -> + Diff = byte_size(A) - byte_size(B), + <<Left:Diff/binary, Right/binary>> = A, + Xor = crypto:exor(Right, B), + <<Left/binary, Xor/binary>>. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +double_0_test() -> + ?assertEqual( + <<16#1c09bf5f83df7e080280b050b37e0e74:128>>, + double(<<16#0e04dfafc1efbf040140582859bf073a:128>>) + ). + +double_1_test() -> + ?assertEqual( + <<16#dbe13bd0ed8c85dc9af179c99ddbf819:128>>, + double(<<16#edf09de876c642ee4d78bce4ceedfc4f:128>>) + ). + +pad_test() -> + ?assertEqual( + <<16#112233445566778899aabbccddee8000:128>>, + pad(<<16#112233445566778899aabbccddee:112>>) + ). + +-endif. diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl index ba8d9c42f..801b98b3a 100644 --- a/src/couch/src/couch_file.erl +++ b/src/couch/src/couch_file.erl @@ -23,6 +23,19 @@ -define(IS_OLD_STATE(S), is_pid(S#file.db_monitor)). -define(PREFIX_SIZE, 5). -define(DEFAULT_READ_COUNT, 1024). +-define(ENCRYPTED_HEADER, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15). + +%% Database encryption design details + +%% If, at couch_file creation time, encryption is enabled, couch_file +%% generates a random 256-bit AES key and a random 128 bit +%% initialisation vector. The currently configured aegis key manager +%% module is asked to wrap the key and return it as a +%% binary. couch_file stores the IV and the wrapped key in a special +%% header block, always block 0, in the first 4 KiB of the file. All +%% data, including all headers except the encryption header at block +%% 0, are encrypted using AES in Counter Mode, where the counter is +%% calculated as IV plus the file offset, in AES block increments. -type block_id() :: non_neg_integer(). -type location() :: non_neg_integer(). @@ -33,7 +46,10 @@ is_sys, eof = 0, db_monitor, - pread_limit = 0 + pread_limit = 0, + iv, + enc, + dec }). % public API @@ -439,7 +455,7 @@ init({Filepath, Options, ReturnPid, Ref}) -> ok = file:sync(Fd), maybe_track_open_os_files(Options), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}}; + init_crypto(#file{fd = Fd, is_sys = IsSys, pread_limit = Limit}); false -> ok = file:close(Fd), init_status_error(ReturnPid, Ref, {error, eexist}) @@ -447,7 +463,7 @@ init({Filepath, Options, ReturnPid, Ref}) -> false -> maybe_track_open_os_files(Options), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}} + init_crypto(#file{fd = Fd, is_sys = IsSys, pread_limit = Limit}) end; Error -> init_status_error(ReturnPid, Ref, Error) @@ -464,7 +480,9 @@ init({Filepath, Options, ReturnPid, Ref}) -> maybe_track_open_os_files(Options), {ok, Eof} = file:position(Fd, eof), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd = Fd, eof = Eof, is_sys = IsSys, pread_limit = Limit}}; + init_crypto(#file{ + fd = Fd, eof = Eof, is_sys = IsSys, pread_limit = Limit + }); Error -> init_status_error(ReturnPid, Ref, Error) end; @@ -567,20 +585,25 @@ handle_call({truncate, Pos}, _From, #file{fd = Fd} = File) -> {ok, Pos} = file:position(Fd, Pos), case file:truncate(Fd) of ok -> - {reply, ok, File#file{eof = Pos}}; + case init_crypto(File#file{eof = Pos}) of + {ok, File1} -> + {reply, ok, File1}; + {error, Reason} -> + {error, Reason} + end; Error -> {reply, Error, File} end; -handle_call({append_bin, Bin}, _From, #file{fd = Fd, eof = Pos} = File) -> +handle_call({append_bin, Bin}, _From, #file{eof = Pos} = File) -> Blocks = make_blocks(Pos rem ?SIZE_BLOCK, Bin), Size = iolist_size(Blocks), - case file:write(Fd, Blocks) of + case encrypted_write(File, Blocks) of ok -> {reply, {ok, Pos, Size}, File#file{eof = Pos + Size}}; Error -> {reply, Error, reset_eof(File)} end; -handle_call({append_bins, Bins}, _From, #file{fd = Fd, eof = Pos} = File) -> +handle_call({append_bins, Bins}, _From, #file{eof = Pos} = File) -> {BlockResps, FinalPos} = lists:mapfoldl( fun(Bin, PosAcc) -> Blocks = make_blocks(PosAcc rem ?SIZE_BLOCK, Bin), @@ -591,13 +614,13 @@ handle_call({append_bins, Bins}, _From, #file{fd = Fd, eof = Pos} = File) -> Bins ), {AllBlocks, Resps} = lists:unzip(BlockResps), - case file:write(Fd, AllBlocks) of + case encrypted_write(File, AllBlocks) of ok -> {reply, {ok, Resps}, File#file{eof = FinalPos}}; Error -> {reply, Error, reset_eof(File)} end; -handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) -> +handle_call({write_header, Bin}, _From, #file{eof = Pos} = File) -> BinSize = byte_size(Bin), case Pos rem ?SIZE_BLOCK of 0 -> @@ -606,14 +629,14 @@ handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) -> Padding = <<0:(8 * (?SIZE_BLOCK - BlockOffset))>> end, FinalBin = [Padding, <<1, BinSize:32/integer>> | make_blocks(5, [Bin])], - case file:write(Fd, FinalBin) of + case encrypted_write(File, FinalBin) of ok -> {reply, ok, File#file{eof = Pos + iolist_size(FinalBin)}}; Error -> {reply, Error, reset_eof(File)} end; -handle_call(find_header, _From, #file{fd = Fd, eof = Pos} = File) -> - {reply, find_header(Fd, Pos div ?SIZE_BLOCK), File}. +handle_call(find_header, _From, #file{eof = Pos} = File) -> + {reply, find_header(File, Pos div ?SIZE_BLOCK), File}. handle_cast(close, Fd) -> {stop, normal, Fd}. @@ -641,26 +664,26 @@ format_status(_Opt, [PDict, #file{} = File]) -> {_Fd, FilePath} = couch_util:get_value(couch_file_fd, PDict), [{data, [{"State", File}, {"InitialFilePath", FilePath}]}]. -find_header(Fd, Block) -> - case (catch load_header(Fd, Block)) of +find_header(#file{} = File, Block) -> + case (catch load_header(File, Block)) of {ok, Bin} -> {ok, Bin}; _Error -> ReadCount = config:get_integer( "couchdb", "find_header_read_count", ?DEFAULT_READ_COUNT ), - find_header(Fd, Block - 1, ReadCount) + find_header(File, Block - 1, ReadCount) end. -load_header(Fd, Block) -> +load_header(#file{} = File, Block) -> {ok, <<1, HeaderLen:32/integer, RestBlock/binary>>} = - file:pread(Fd, Block * ?SIZE_BLOCK, ?SIZE_BLOCK), - load_header(Fd, Block * ?SIZE_BLOCK, HeaderLen, RestBlock). + encrypted_pread(File, Block * ?SIZE_BLOCK, ?SIZE_BLOCK), + load_header(File, Block * ?SIZE_BLOCK, HeaderLen, RestBlock). -load_header(Fd, Pos, HeaderLen) -> - load_header(Fd, Pos, HeaderLen, <<>>). +load_header(#file{} = File, Pos, HeaderLen) -> + load_header(File, Pos, HeaderLen, <<>>). -load_header(Fd, Pos, HeaderLen, RestBlock) -> +load_header(#file{} = File, Pos, HeaderLen, RestBlock) -> TotalBytes = calculate_total_read_len(?PREFIX_SIZE, HeaderLen), RawBin = case TotalBytes =< byte_size(RestBlock) of @@ -670,7 +693,7 @@ load_header(Fd, Pos, HeaderLen, RestBlock) -> false -> ReadStart = Pos + ?PREFIX_SIZE + byte_size(RestBlock), ReadLen = TotalBytes - byte_size(RestBlock), - {ok, Missing} = file:pread(Fd, ReadStart, ReadLen), + {ok, Missing} = encrypted_pread(File, ReadStart, ReadLen), <<RestBlock/binary, Missing/binary>> end, <<Md5Sig:16/binary, HeaderBin/binary>> = @@ -681,12 +704,12 @@ load_header(Fd, Pos, HeaderLen, RestBlock) -> %% Read multiple block locations using a single file:pread/2. -spec find_header(file:fd(), block_id(), non_neg_integer()) -> {ok, binary()} | no_valid_header. -find_header(_Fd, Block, _ReadCount) when Block < 0 -> +find_header(_File, Block, _ReadCount) when Block < 0 -> no_valid_header; -find_header(Fd, Block, ReadCount) -> +find_header(#file{} = File, Block, ReadCount) -> FirstBlock = max(0, Block - ReadCount + 1), BlockLocations = [?SIZE_BLOCK * B || B <- lists:seq(FirstBlock, Block)], - {ok, DataL} = file:pread(Fd, [{L, ?PREFIX_SIZE} || L <- BlockLocations]), + {ok, DataL} = encrypted_pread(File, [{L, ?PREFIX_SIZE} || L <- BlockLocations]), %% Since BlockLocations are ordered from oldest to newest, we rely %% on lists:foldl/3 to reverse the order, making HeaderLocations %% correctly ordered from newest to oldest. @@ -700,27 +723,27 @@ find_header(Fd, Block, ReadCount) -> [], lists:zip(BlockLocations, DataL) ), - case find_newest_header(Fd, HeaderLocations) of + case find_newest_header(File, HeaderLocations) of {ok, _Location, HeaderBin} -> {ok, HeaderBin}; _ -> ok = file:advise( - Fd, hd(BlockLocations), ReadCount * ?SIZE_BLOCK, dont_need + File#file.fd, hd(BlockLocations), ReadCount * ?SIZE_BLOCK, dont_need ), NextBlock = hd(BlockLocations) div ?SIZE_BLOCK - 1, - find_header(Fd, NextBlock, ReadCount) + find_header(File, NextBlock, ReadCount) end. -spec find_newest_header(file:fd(), [{location(), header_size()}]) -> {ok, location(), binary()} | not_found. -find_newest_header(_Fd, []) -> +find_newest_header(_File, []) -> not_found; -find_newest_header(Fd, [{Location, Size} | LocationSizes]) -> - case (catch load_header(Fd, Location, Size)) of +find_newest_header(#file{} = File, [{Location, Size} | LocationSizes]) -> + case (catch load_header(File, Location, Size)) of {ok, HeaderBin} -> {ok, Location, HeaderBin}; _Error -> - find_newest_header(Fd, LocationSizes) + find_newest_header(File, LocationSizes) end. -spec read_raw_iolist_int(#file{}, Pos :: non_neg_integer(), Len :: non_neg_integer()) -> @@ -728,9 +751,9 @@ find_newest_header(Fd, [{Location, Size} | LocationSizes]) -> % 0110 UPGRADE CODE read_raw_iolist_int(Fd, {Pos, _Size}, Len) -> read_raw_iolist_int(Fd, Pos, Len); -read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) -> +read_raw_iolist_int(#file{} = File, Pos, Len) -> {Pos, TotalBytes} = get_pread_locnum(File, Pos, Len), - case catch file:pread(Fd, Pos, TotalBytes) of + case catch encrypted_pread(File, Pos, TotalBytes) of {ok, <<RawBin:TotalBytes/binary>>} -> {remove_block_prefixes(Pos rem ?SIZE_BLOCK, RawBin), Pos + TotalBytes}; Else -> @@ -744,15 +767,15 @@ read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) -> throw({file_truncate_error, Else, Filepath}) end. -% TODO: check if this is really unused -read_multi_raw_iolists_int(#file{fd = Fd} = File, PosLens) -> +% used in couch_bt_engine_compactor.erl via pread_terms/2 +read_multi_raw_iolists_int(#file{} = File, PosLens) -> LocNums = lists:map( fun({Pos, Len}) -> get_pread_locnum(File, Pos, Len) end, PosLens ), - {ok, Bins} = file:pread(Fd, LocNums), + {ok, Bins} = encrypted_pread(File, LocNums), lists:zipwith( fun({Pos, TotalBytes}, Bin) -> <<RawBin:TotalBytes/binary>> = Bin, @@ -905,6 +928,132 @@ reset_eof(#file{} = File) -> {ok, Eof} = file:position(File#file.fd, eof), File#file{eof = Eof}. +%% new file or we've wiped all the data, including the wrapped key, so we need a new one. +init_crypto(#file{eof = 0} = File0) -> + DataEncryptionKey = crypto:strong_rand_bytes(32), + IV = crypto:strong_rand_bytes(16), + case aegis_key_manager:wrap_key(DataEncryptionKey) of + {ok, WrappedKey} -> + case write_encryption_header(File0, WrappedKey, IV) of + {ok, File1} -> + ok = file:sync(File1#file.fd), + {ok, init_ciphers(File1, DataEncryptionKey, IV)}; + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end; +%% we're opening an existing file and need to unwrap the key if file is encrypted. +init_crypto(#file{eof = Pos, enc = undefined, dec = undefined} = File) when Pos > 0 -> + case read_encryption_header(File) of + {ok, WrappedKey, IV} -> + case aegis_key_manager:unwrap_key(WrappedKey) of + {ok, DataEncryptionKey} -> + {ok, init_ciphers(File, DataEncryptionKey, IV)}; + {error, Reason} -> + {error, Reason} + end; + not_encrypted -> + {ok, File}; + {error, Reason} -> + {error, Reason} + end. + +init_ciphers(#file{} = File, DataEncryptionKey, IV) when + is_binary(DataEncryptionKey), is_binary(IV) +-> + EncState = crypto:crypto_dyn_iv_init(aes_256_ctr, DataEncryptionKey, true), + DecState = crypto:crypto_dyn_iv_init(aes_256_ctr, DataEncryptionKey, false), + File#file{iv = crypto:bytes_to_integer(IV), enc = EncState, dec = DecState}. + +write_encryption_header(#file{eof = 0} = File, WrappedKey, IV) when + byte_size(WrappedKey) < 1024, bit_size(IV) == 128 +-> + Header = [<<?ENCRYPTED_HEADER>>, IV, <<(byte_size(WrappedKey)):16>>, WrappedKey], + PaddedHeader = [Header, <<0:((?SIZE_BLOCK - iolist_size(Header) - 32) * 8)>>], + DigestHeader = [PaddedHeader, crypto:hash(sha256, PaddedHeader)], + ?SIZE_BLOCK = iolist_size(DigestHeader), + case file:write(File#file.fd, DigestHeader) of + ok -> + {ok, File#file{eof = ?SIZE_BLOCK}}; + {error, Reason} -> + {error, Reason} + end. + +read_encryption_header(#file{} = File) -> + case file:pread(File#file.fd, 0, ?SIZE_BLOCK) of + {ok, + <<?ENCRYPTED_HEADER, IV:16/binary, WrappedKeyLen:16, WrappedKey:(WrappedKeyLen)/binary, + _/binary>> = DigestHeader} -> + Header = binary:part(DigestHeader, 0, ?SIZE_BLOCK - 32), + Digest = binary:part(DigestHeader, ?SIZE_BLOCK - 32, 32), + case Digest == crypto:hash(sha256, Header) of + true -> + {ok, WrappedKey, IV}; + false -> + {error, corrupted_encryption_header} + end; + {ok, _} -> + not_encrypted; + {error, Reason} -> + {error, Reason} + end. + +%% We can encrypt any section of the file but we must make +%% sure we align with the key stream. +encrypted_write(#file{enc = undefined} = File, Data) -> + file:write(File#file.fd, Data); +encrypted_write(#file{} = File, Data) -> + CipherText = encrypt(File#file.enc, File#file.iv, File#file.eof, pad(File#file.eof, Data)), + file:write(File#file.fd, unpad(File#file.eof, CipherText)). + +encrypted_pread(#file{dec = undefined} = File, LocNums) -> + file:pread(File#file.fd, LocNums); +encrypted_pread(#file{} = File, LocNums) -> + case file:pread(File#file.fd, LocNums) of + {ok, DataL} -> + {ok, + lists:zipwith( + fun({Pos, _Len}, CipherText) -> + PlainText = decrypt(File#file.dec, File#file.iv, Pos, pad(Pos, CipherText)), + unpad(Pos, PlainText) + end, + LocNums, + DataL + )}; + Else -> + Else + end. + +encrypted_pread(#file{dec = undefined} = File, Pos, Len) -> + file:pread(File#file.fd, Pos, Len); +encrypted_pread(#file{} = File, Pos, Len) -> + case file:pread(File#file.fd, Pos, Len) of + {ok, CipherText} -> + PlainText = decrypt(File#file.dec, File#file.iv, Pos, pad(Pos, CipherText)), + {ok, unpad(Pos, PlainText)}; + Else -> + Else + end. + +encrypt(Enc, IV, Pos, Data) -> + crypto:crypto_dyn_iv_update(Enc, Data, aes_ctr(IV, Pos)). + +decrypt(Dec, IV, Pos, Data) -> + crypto:crypto_dyn_iv_update(Dec, Data, aes_ctr(IV, Pos)). + +aes_ctr(IV, Pos) -> + <<(IV + (Pos div 16)):128>>. + +pad(Pos, IOData) -> + [<<0:(Pos rem 16 * 8)>>, IOData]. + +unpad(Pos, Bin) when is_binary(Bin) -> + Size = Pos rem 16 * 8, + <<_:Size, Result/binary>> = Bin, + Result. + -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl index 912c6dd8a..fd1c30072 100644 --- a/src/couch/src/couch_util.erl +++ b/src/couch/src/couch_util.erl @@ -17,7 +17,7 @@ -export([rand32/0, implode/2]). -export([abs_pathname/1, abs_pathname/2, trim/1, drop_dot_couch_ext/1]). -export([encodeBase64Url/1, decodeBase64Url/1]). --export([validate_utf8/1, to_hex/1, parse_term/1, dict_find/3]). +-export([validate_utf8/1, to_hex/1, from_hex/1, parse_term/1, dict_find/3]). -export([get_nested_json_value/2, json_user_ctx/1]). -export([proplist_apply_field/2, json_apply_field/2]). -export([to_binary/1, to_integer/1, to_list/1, url_encode/1]). @@ -236,6 +236,46 @@ nibble_to_hex(13) -> $d; nibble_to_hex(14) -> $e; nibble_to_hex(15) -> $f. +from_hex(<<Hi:8, Lo:8, Rest/binary>>) -> + iolist_to_binary([<<(hex_to_nibble(Hi)):4, (hex_to_nibble(Lo)):4>> | from_hex(Rest)]); +from_hex(<<>>) -> + []; +from_hex(List) when is_list(List) -> + from_hex(list_to_binary(List)). + +hex_to_nibble($0) -> + 0; +hex_to_nibble($1) -> + 1; +hex_to_nibble($2) -> + 2; +hex_to_nibble($3) -> + 3; +hex_to_nibble($4) -> + 4; +hex_to_nibble($5) -> + 5; +hex_to_nibble($6) -> + 6; +hex_to_nibble($7) -> + 7; +hex_to_nibble($8) -> + 8; +hex_to_nibble($9) -> + 9; +hex_to_nibble($a) -> + 10; +hex_to_nibble($b) -> + 11; +hex_to_nibble($c) -> + 12; +hex_to_nibble($d) -> + 13; +hex_to_nibble($e) -> + 14; +hex_to_nibble($f) -> + 15. + parse_term(Bin) when is_binary(Bin) -> parse_term(binary_to_list(Bin)); parse_term(List) ->
