On Wed, 5 Feb 2020 at 22:28, Sehrope Sarkuni <sehr...@jackdb.com> wrote:
>
> On Sat, Feb 1, 2020 at 7:02 PM Masahiko Sawada
> <masahiko.saw...@2ndquadrant.com> wrote:
> > On Sun, 2 Feb 2020 at 00:37, Sehrope Sarkuni <sehr...@jackdb.com> wrote:
> > >
> > > On Fri, Jan 31, 2020 at 1:21 AM Masahiko Sawada
> > > <masahiko.saw...@2ndquadrant.com> wrote:
> > > > On Thu, 30 Jan 2020 at 20:36, Sehrope Sarkuni <sehr...@jackdb.com>
> > > > wrote:
> > > > > That
> > > > > would allow the internal usage to have a fixed output length of
> > > > > LEN(IV) + LEN(HMAC) + LEN(DATA) = 16 + 32 + 64 = 112 bytes.
> > > >
> > > > Probably you meant LEN(DATA) is 32? DATA will be an encryption key for
> > > > AES256 (master key) internally generated.
> > >
> > > No it should be 64-bytes. That way we can have separate 32-byte
> > > encryption key (for AES256) and 32-byte MAC key (for HMAC-SHA256).
> > >
> > > While it's common to reuse the same 32-byte key for both AES256 and an
> > > HMAC-SHA256 and there aren't any known issues with doing so, when
> > > designing something from scratch it's more secure to use entirely
> > > separate keys.
> >
> > The HMAC key you mentioned above is not the same as the HMAC key
> > derived from the user provided passphrase, right? That is, individual
> > key needs to have its IV and HMAC key. Given that the HMAC key used
> > for HMAC(IV || ENCRYPT(KEY, IV, DATA)) is the latter key (derived from
> > passphrase), what will be the former key used for?
>
> It's not derived from the passphrase, it's unlocked by the passphrase (along
> with the master encryption key). The server will have 64-bytes of random
> data, saved encrypted in pg_control, which can be treated as two separate
> 32-byte keys, let's call them master_encryption_key and master_mac_key. The
> 64-bytes is unlocked by decrypting it with the user passphrase at startup
> (which itself would be split into a pair of encryption and MAC keys to do the
> unlocking).
>
> The wrap and unwrap operations would use both keys:
>
> wrap(plain_text, encryption_key, mac_key) {
> // Generate random IV:
> iv = pg_strong_random(16);
> // Encrypt:
> cipher_text = encrypt_aes256_cbc(encryption_key, iv, plain_text);
> // Compute MAC on all inputs:
> mac = hmac_sha256(mac_key, encryption_key || iv || cipher_text);
> // Concat user facing pieces together
> wrapped = mac || iv || cipher_text;
> return wrapped;
> }
>
> unwrap(wrapped, encryption_key, mac_key) {
> // Split wrapped into its pieces:
> actual_mac = wrapped.slice(0, 32);
> iv = wrapped.slice(0 + 32, 16);
> cipher_text = wrapped.slice(0 + 32 + 16);
> // Compute MAC on all inputs:
> expected_mac = hmac_sha256(mac_key, encryption_key || iv || cipher_text);
> // Compare MAC vs value in wrapped:
> if (expected_mac != actual_mac) { return Error("MAC does not match"); }
> // MAC matches so decrypt:
> plain_text = decrypt_aes256_cbc(encryption_key, iv, cipher_text);
> return plain_text;
> }
>
> Every input to the encryption operation, including the encryption key, must
> be included into the HMAC calculation. If you use the same key for both
> encryption and MAC that's not required as it's already part of the MAC
> process as the key. Using separate keys requires explicitly adding in the
> encryption key into the MAC input to ensure that it the correct key prior to
> decryption in the unwrap operation. Any additional parts of the wrapped
> output (ex: a "version" byte for the algos or padding choices) should also be
> included.
>
> The wrap / unwrap above would be used with the encryption and mac keys
> derived from the user passphrase to unlock the master_encryption_key and
> master_mac_key from pg_control. Then those would be used by the higher level
> functions:
>
> pg_kmgr_wrap(plain_text) {
> return wrap(plain_text, master_encryption_key, master_mac_key);
> }
>
> pg_kmgr_unwrap(wrapped) {
> return unwrap(wrapped, master_encryption_key, master_mac_key);
> }
Thank you for explaining the details. I had missed something.
Attached updated patch incorporated all comments I got so far. The changes are:
* Renamed data_encryption_cipher to key_management_cipher
* Renamed pg_kmgr_wrap and pg_kmgr_unwrap to pg_wrap_key and pg_unwrap_key
* Changed wrap and unwrap procedure based on the comments
* Removed the restriction of requiring the input key being a multiple
of 16 bytes.
* Created a context dedicated to wrap and unwrap data
Documentation and regression tests are still missing.
Regarding key rotation, currently we allow online key rotation by
doing pg_rotate_encryption_key after changing
cluster_passphrase_command and loading. But if the server crashed
during key rotation it might require the old passphrase in spite of
the passphrase command in postgresql.conf having been changed. We need
to deal with it but I'm not sure the best approach. Possibly having a
new frontend tool that changes the key offline would be a safe
approach.
Regards,
--
Masahiko Sawada http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services
diff --git a/configure b/configure
index 702adba839..2daa5bf0c2 100755
--- a/configure
+++ b/configure
@@ -12113,7 +12113,7 @@ done
# defines OPENSSL_VERSION_NUMBER to claim version 2.0.0, even though it
# doesn't have these OpenSSL 1.1.0 functions. So check for individual
# functions.
- for ac_func in OPENSSL_init_ssl BIO_get_data BIO_meth_new ASN1_STRING_get0_data
+ for ac_func in OPENSSL_init_ssl OPENSSL_init_crypto BIO_get_data BIO_meth_new ASN1_STRING_get0_data
do :
as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
diff --git a/configure.in b/configure.in
index 8165f70039..79329d9f15 100644
--- a/configure.in
+++ b/configure.in
@@ -1194,7 +1194,7 @@ if test "$with_openssl" = yes ; then
# defines OPENSSL_VERSION_NUMBER to claim version 2.0.0, even though it
# doesn't have these OpenSSL 1.1.0 functions. So check for individual
# functions.
- AC_CHECK_FUNCS([OPENSSL_init_ssl BIO_get_data BIO_meth_new ASN1_STRING_get0_data])
+ AC_CHECK_FUNCS([OPENSSL_init_ssl OPENSSL_init_crypto BIO_get_data BIO_meth_new ASN1_STRING_get0_data])
# OpenSSL versions before 1.1.0 required setting callback functions, for
# thread-safety. In 1.1.0, it's no longer required, and CRYPTO_lock()
# function was removed.
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 9706a95848..4ace302038 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -21,7 +21,7 @@ SUBDIRS = access bootstrap catalog parser commands executor foreign lib libpq \
main nodes optimizer partitioning port postmaster \
regex replication rewrite \
statistics storage tcop tsearch utils $(top_builddir)/src/timezone \
- jit
+ jit crypto
include $(srcdir)/common.mk
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 3813eadfb4..61c06920b0 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -41,6 +41,7 @@
#include "catalog/pg_database.h"
#include "commands/tablespace.h"
#include "common/controldata_utils.h"
+#include "crypto/kmgr.h"
#include "miscadmin.h"
#include "pg_trace.h"
#include "pgstat.h"
@@ -77,6 +78,7 @@
#include "utils/timestamp.h"
extern uint32 bootstrap_data_checksum_version;
+extern uint32 bootstrap_key_management_cipher;
/* Unsupported old recovery command file names (relative to $PGDATA) */
#define RECOVERY_COMMAND_FILE "recovery.conf"
@@ -4777,6 +4779,10 @@ ReadControlFile(void)
/* Make the initdb settings visible as GUC variables, too */
SetConfigOption("data_checksums", DataChecksumsEnabled() ? "yes" : "no",
PGC_INTERNAL, PGC_S_OVERRIDE);
+
+ SetConfigOption("key_management_cipher",
+ kmgr_cipher_string(GetKeyManagementCipher()),
+ PGC_INTERNAL, PGC_S_OVERRIDE);
}
/*
@@ -4809,6 +4815,16 @@ GetMockAuthenticationNonce(void)
return ControlFile->mock_authentication_nonce;
}
+/*
+ * Returns the wrapped master keys from control file..
+ */
+uint8 *
+GetMasterEncryptionKey(void)
+{
+ Assert(ControlFile != NULL);
+ return ControlFile->masterkey;
+}
+
/*
* Are checksums enabled for data pages?
*/
@@ -4819,6 +4835,16 @@ DataChecksumsEnabled(void)
return (ControlFile->data_checksum_version > 0);
}
+/*
+ * Return the key management cipher from control file.
+ */
+int
+GetKeyManagementCipher(void)
+{
+ Assert(ControlFile != NULL);
+ return ControlFile->key_management_cipher;
+}
+
/*
* Returns a fake LSN for unlogged relations.
*
@@ -5085,6 +5111,7 @@ BootStrapXLOG(void)
XLogPageHeader page;
XLogLongPageHeader longpage;
XLogRecord *record;
+ uint8 *masterkey;
char *recptr;
bool use_existent;
uint64 sysidentifier;
@@ -5248,6 +5275,11 @@ BootStrapXLOG(void)
ControlFile->wal_log_hints = wal_log_hints;
ControlFile->track_commit_timestamp = track_commit_timestamp;
ControlFile->data_checksum_version = bootstrap_data_checksum_version;
+ ControlFile->key_management_cipher = bootstrap_key_management_cipher;
+
+ /* Bootstrap the key manager and store master keys into the control file */
+ if ((masterkey = BootStrapKmgr(bootstrap_key_management_cipher)) != NULL)
+ memcpy(&(ControlFile->masterkey), masterkey, KMGR_WRAPPED_KEY_LEN);
/* some additional ControlFile fields are set in WriteControlFile() */
diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c
index bfc629c753..34c20764de 100644
--- a/src/backend/bootstrap/bootstrap.c
+++ b/src/backend/bootstrap/bootstrap.c
@@ -28,6 +28,7 @@
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
#include "common/link-canary.h"
+#include "crypto/kmgr.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
@@ -51,6 +52,7 @@
#include "utils/relmapper.h"
uint32 bootstrap_data_checksum_version = 0; /* No checksum */
+uint32 bootstrap_key_management_cipher = KMGR_CIPHER_OFF;
#define ALLOC(t, c) \
@@ -226,7 +228,7 @@ AuxiliaryProcessMain(int argc, char *argv[])
/* If no -x argument, we are a CheckerProcess */
MyAuxProcType = CheckerProcess;
- while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:x:X:-:")) != -1)
+ while ((flag = getopt(argc, argv, "B:c:d:D:e:Fkr:x:X:-:")) != -1)
{
switch (flag)
{
@@ -249,6 +251,9 @@ AuxiliaryProcessMain(int argc, char *argv[])
pfree(debugstr);
}
break;
+ case 'e':
+ bootstrap_key_management_cipher = kmgr_cipher_value(optarg);
+ break;
case 'F':
SetConfigOption("fsync", "false", PGC_POSTMASTER, PGC_S_ARGV);
break;
diff --git a/src/backend/crypto/Makefile b/src/backend/crypto/Makefile
new file mode 100644
index 0000000000..a641860a0f
--- /dev/null
+++ b/src/backend/crypto/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile
+# Makefile for src/backend/crypto
+#
+# IDENTIFICATION
+# src/backend/crypto/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/crypto
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = kmgr.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/crypto/kmgr.c b/src/backend/crypto/kmgr.c
new file mode 100644
index 0000000000..dfdb8f8e36
--- /dev/null
+++ b/src/backend/crypto/kmgr.c
@@ -0,0 +1,282 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr.c
+ * Key manager interface routines
+ *
+ * Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/backend/storage/encryption/kmgr.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <unistd.h>
+
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+
+#include "access/xlog.h"
+#include "common/sha2.h"
+#include "common/kmgr_utils.h"
+#include "crypto/kmgr.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/shmem.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+
+/* GUC variable */
+char *cluster_passphrase_command = NULL;
+int key_management_cipher;
+
+static MemoryContext KmgrCtx = NULL;
+
+/* Raw master encryption key and HMAC key */
+static uint8 masterKeys[KMGR_KEY_AND_HMACKEY_LEN];
+
+/* Key wrap and unwrap contexts initialized with the master keys */
+static KeyWrapCtx *WrapCtx = NULL;
+static KeyWrapCtx *UnwrapCtx = NULL;
+
+static void ShutdownKmgr(int code, Datum arg);
+
+/*
+ * This function must be called ONCE on system install.
+ */
+uint8 *
+BootStrapKmgr(int bootstrap_key_management_cipher)
+{
+ KeyWrapCtx *ctx;
+ char passphrase[KMGR_MAX_PASSPHRASE_LEN];
+ uint8 kek[KMGR_KEY_LEN];
+ uint8 kekhmac[KMGR_HMACKEY_LEN];
+ uint8 masterkeys[KMGR_KEY_AND_HMACKEY_LEN];
+ uint8 *wrapped_key;
+ int wrapped_keylen;
+ int passlen;
+
+ if (bootstrap_key_management_cipher == KMGR_CIPHER_OFF)
+ return NULL;
+
+#ifndef USE_OPENSSL
+ ereport(ERROR,
+ (errcode(ERRCODE_CONFIG_FILE_ERROR),
+ (errmsg("cluster encryption is not supported because OpenSSL is not supported by this build"),
+ errhint("Compile with --with-openssl to use cluster encryption."))));
+#endif
+
+ /* Get key encryption key from passphrase command */
+ passlen = kmgr_run_cluster_passphrase_command(cluster_passphrase_command,
+ passphrase, KMGR_MAX_PASSPHRASE_LEN);
+ if (passlen < KMGR_MIN_PASSPHRASE_LEN)
+ ereport(ERROR,
+ (errmsg("passphrase must be more than %d bytes",
+ KMGR_MIN_PASSPHRASE_LEN)));
+
+ /* Get key encryption key and HMAC key from passphrase */
+ kmgr_derive_keys(passphrase, passlen, kek, kekhmac);
+ ctx = create_keywrap_ctx(kek, kekhmac, true);
+ if (!ctx)
+ ereport(ERROR,
+ (errmsg("could not initialize cipher contect")));
+
+ /* Generate the master encryption key and HMAC key */
+ if (!pg_strong_random(masterkeys, KMGR_KEY_LEN))
+ ereport(ERROR,
+ (errmsg("failed to generate cluster encryption key")));
+ if (!pg_strong_random(masterkeys + KMGR_KEY_LEN, KMGR_HMACKEY_LEN))
+ ereport(ERROR,
+ (errmsg("failed to generate cluster hmac key")));
+
+ /* Wrap the combined master keys by the key encryption keys */
+ wrapped_key = palloc0(KMGR_WRAPPED_KEY_LEN);
+ if (!kmgr_wrap_key(ctx, masterkeys, KMGR_KEY_AND_HMACKEY_LEN,
+ wrapped_key, &wrapped_keylen))
+ {
+ free_keywrap_ctx(ctx);
+ ereport(ERROR,
+ (errmsg("failed to wrap cluster key")));
+ }
+ Assert(wrapped_keylen == KMGR_WRAPPED_KEY_LEN);
+
+ free_keywrap_ctx(ctx);
+ return wrapped_key;
+}
+
+/*
+ * Get encryption key passphrase and verify it, then get the un-wrapped
+ * master encryption key and HMAC key. This function is called by postmaster
+ * at startup time.
+ */
+void
+InitializeKmgr(void)
+{
+ MemoryContext oldctx;
+ char passphrase[KMGR_MAX_PASSPHRASE_LEN];
+ uint8 kek_hmackey[KMGR_KEY_AND_HMACKEY_LEN];
+ uint8 *wrapped_key;
+ uint8 *key;
+ uint8 *hmackey;
+ int passlen;
+
+ if (!KeyManagementEnabled())
+ return;
+
+ /* Get cluster passphrase */
+ passlen = kmgr_run_cluster_passphrase_command(cluster_passphrase_command,
+ passphrase, KMGR_MAX_PASSPHRASE_LEN);
+
+ /* Get wrapped master keys: encryption key and HMAC key */
+ wrapped_key = GetMasterEncryptionKey();
+
+ /* Verify the correctness of given passphrase */
+ if (!kmgr_verify_passphrase(passphrase, passlen, wrapped_key, kek_hmackey))
+ ereport(ERROR,
+ (errmsg("cluster passphrase does not match expected passphrase")));
+
+ /* Get raw master key and hmac key */
+ key = kek_hmackey;
+ hmackey = (uint8 *) ((char *) kek_hmackey + KMGR_KEY_LEN);
+
+ KmgrCtx = AllocSetContextCreate(TopMemoryContext,
+ "Key manager context",
+ ALLOCSET_DEFAULT_SIZES);
+ oldctx = MemoryContextSwitchTo(KmgrCtx);
+
+ /* Set wrap and unwrap context with the master keys */
+ WrapCtx = create_keywrap_ctx(key, hmackey, true);
+ UnwrapCtx = create_keywrap_ctx(key, hmackey, false);
+
+ MemoryContextSwitchTo(oldctx);
+
+ /* Cache the raw master keys */
+ memcpy(masterKeys, kek_hmackey, KMGR_KEY_AND_HMACKEY_LEN);
+
+ on_shmem_exit(ShutdownKmgr, 0);
+
+}
+
+/*
+ * This must be called once during postmaster shutdown.
+ */
+static void
+ShutdownKmgr(int code, Datum arg)
+{
+ if (WrapCtx)
+ free_keywrap_ctx(WrapCtx);
+ if (UnwrapCtx)
+ free_keywrap_ctx(UnwrapCtx);
+}
+
+/*
+ * SQL function to wrap the given key by the master keys
+ */
+Datum
+pg_wrap_key(PG_FUNCTION_ARGS)
+{
+ bytea *data = PG_GETARG_BYTEA_PP(0);
+ bytea *res;
+ int datalen;
+ int reslen;
+ int len;
+
+ if (!KeyManagementEnabled())
+ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("could not wrap key because key management is not supported"),
+ errhint("Compile with --with-openssl and enable key management at initdb time")));
+
+ datalen = VARSIZE_ANY_EXHDR(data);
+ reslen = VARHDRSZ + SizeOfWrappedKey(datalen);
+ res = palloc(reslen);
+
+ if (!kmgr_wrap_key(WrapCtx, (uint8 *) VARDATA_ANY(data), datalen,
+ (uint8 *) VARDATA(res), &len))
+ ereport(ERROR,
+ (errmsg("could not wrap the given secret")));
+
+ SET_VARSIZE(res, reslen);
+ PG_RETURN_BYTEA_P(res);
+}
+
+/*
+ * SQL function to unwrap the given key by the master keys
+ */
+Datum
+pg_unwrap_key(PG_FUNCTION_ARGS)
+{
+ bytea *data = PG_GETARG_BYTEA_PP(0);
+ bytea *res;
+ int datalen;
+ int reslen;
+ int len;
+
+ if (!KeyManagementEnabled())
+ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("could not wrap key because key management is not supported"),
+ errhint("Compile with --with-openssl and enable key management at initdb time")));
+
+ datalen = VARSIZE_ANY_EXHDR(data);
+ reslen = VARHDRSZ + SizeOfUnwrappedKey(datalen);
+ res = palloc(reslen);
+
+ if (!kmgr_unwrap_key(UnwrapCtx, (uint8 *) VARDATA_ANY(data), datalen,
+ (uint8 *) VARDATA(res), &len))
+ ereport(ERROR,
+ (errmsg("could not unwrap the given secret")));
+
+ /*
+ * The size of unwrapped key can be smaller than the size estimated
+ * before unwrapping since the padding is removed during unwrapping.
+ */
+ SET_VARSIZE(res, len);
+ PG_RETURN_BYTEA_P(res);
+}
+
+/*
+ * SQL function to rotate the cluster encryption key. This function
+ * assumes that the cluster_passphrase_command is already reloaded
+ * to the new value.
+ */
+Datum
+pg_rotate_encryption_key(PG_FUNCTION_ARGS)
+{
+ KeyWrapCtx *ctx;
+ char passphrase[KMGR_MAX_PASSPHRASE_LEN];
+ uint8 new_kek[KMGR_KEY_LEN];
+ uint8 new_hmackey[KMGR_HMACKEY_LEN];
+ uint8 wrapped_keys[KMGR_KEY_AND_HMACKEY_LEN];
+ uint8 *cur_masterkey;
+ int passlen;
+ int outlen;
+
+ passlen = kmgr_run_cluster_passphrase_command(cluster_passphrase_command,
+ passphrase,
+ KMGR_MAX_PASSPHRASE_LEN);
+ if (passlen < KMGR_MIN_PASSPHRASE_LEN)
+ ereport(ERROR,
+ (errmsg("passphrase must be more than %d bytes",
+ KMGR_MIN_PASSPHRASE_LEN)));
+
+ kmgr_derive_keys(passphrase, passlen, new_kek, new_hmackey);
+
+ ctx = create_keywrap_ctx(new_kek, new_hmackey, true);
+
+ if (!kmgr_wrap_key(ctx, masterKeys, KMGR_KEY_AND_HMACKEY_LEN,
+ wrapped_keys, &outlen))
+ ereport(ERROR,
+ (errmsg("failed to wrap key")));
+
+ /* Update control file */
+ LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
+ cur_masterkey = GetMasterEncryptionKey();
+ memcpy(cur_masterkey, wrapped_keys, KMGR_WRAPPED_KEY_LEN);
+ UpdateControlFile();
+ LWLockRelease(ControlFileLock);
+
+ PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index b3986bee75..2b86047184 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -100,6 +100,7 @@
#include "common/file_perm.h"
#include "common/ip.h"
#include "common/string.h"
+#include "crypto/kmgr.h"
#include "lib/ilist.h"
#include "libpq/auth.h"
#include "libpq/libpq.h"
@@ -1335,6 +1336,11 @@ PostmasterMain(int argc, char *argv[])
*/
autovac_init();
+ /*
+ * Initialize key manager.
+ */
+ InitializeKmgr();
+
/*
* Load configuration files for client authentication.
*/
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 0a6f80963b..4ff81743fc 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -42,6 +42,7 @@
#include "catalog/pg_type.h"
#include "commands/async.h"
#include "commands/prepare.h"
+#include "crypto/kmgr.h"
#include "executor/spi.h"
#include "jit/jit.h"
#include "libpq/libpq.h"
@@ -3883,6 +3884,13 @@ PostgresMain(int argc, char *argv[],
/* Early initialization */
BaseInit();
+ /*
+ * Initialize kmgr for cluster encryption. Since kmgr needs to attach to
+ * shared memory the initialization must be called after BaseInit().
+ */
+ if (!IsUnderPostmaster)
+ InitializeKmgr();
+
/*
* Create a per-backend PGPROC struct in shared memory, except in the
* EXEC_BACKEND case where this was done in SubPostmasterMain. We must do
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index a16fe8cd5b..0eddbbaa77 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -43,6 +43,7 @@
#include "commands/vacuum.h"
#include "commands/variable.h"
#include "common/string.h"
+#include "crypto/kmgr.h"
#include "funcapi.h"
#include "jit/jit.h"
#include "libpq/auth.h"
@@ -73,6 +74,7 @@
#include "replication/walsender.h"
#include "storage/bufmgr.h"
#include "storage/dsm_impl.h"
+#include "storage/standby.h"
#include "storage/fd.h"
#include "storage/large_object.h"
#include "storage/pg_shmem.h"
@@ -465,6 +467,12 @@ const struct config_enum_entry ssl_protocol_versions_info[] = {
{NULL, 0, false}
};
+const struct config_enum_entry key_management_cipher_options[] = {
+ {"off", KMGR_CIPHER_OFF, false},
+ {"aes-256", KMGR_CIPHER_AES256, false},
+ {NULL, 0, false}
+};
+
static struct config_enum_entry shared_memory_options[] = {
#ifndef WIN32
{"sysv", SHMEM_TYPE_SYSV, false},
@@ -717,6 +725,8 @@ const char *const config_group_names[] =
gettext_noop("Statistics / Monitoring"),
/* STATS_COLLECTOR */
gettext_noop("Statistics / Query and Index Statistics Collector"),
+ /* ENCRYPTION */
+ gettext_noop("Encryption"),
/* AUTOVACUUM */
gettext_noop("Autovacuum"),
/* CLIENT_CONN */
@@ -4240,6 +4250,16 @@ static struct config_string ConfigureNamesString[] =
NULL, NULL, NULL
},
+ {
+ {"cluster_passphrase_command", PGC_SIGHUP, ENCRYPTION,
+ gettext_noop("Command to obtain passphrase for database encryption."),
+ NULL
+ },
+ &cluster_passphrase_command,
+ "",
+ NULL, NULL, NULL
+ },
+
{
{"application_name", PGC_USERSET, LOGGING_WHAT,
gettext_noop("Sets the application name to be reported in statistics and logs."),
@@ -4633,6 +4653,18 @@ static struct config_enum ConfigureNamesEnum[] =
check_ssl_max_protocol_version, NULL, NULL
},
+ {
+ {"key_management_cipher", PGC_INTERNAL, PRESET_OPTIONS,
+ gettext_noop("Specify cipher for key management to use."),
+ NULL,
+ GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE
+ },
+ &key_management_cipher,
+ KMGR_CIPHER_OFF,
+ key_management_cipher_options,
+ NULL, NULL, NULL
+ },
+
/* End-of-list marker */
{
{NULL, 0, 0, NULL, NULL}, NULL, 0, NULL, NULL, NULL, NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e1048c0047..0dd5ee946d 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -614,6 +614,11 @@
# autovacuum, -1 means use
# vacuum_cost_limit
+#------------------------------------------------------------------------------
+# ENCRYPTION
+#------------------------------------------------------------------------------
+
+#cluster_passphrase_command = ''
#------------------------------------------------------------------------------
# CLIENT CONNECTION DEFAULTS
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 7f1534aebb..4dafd129cd 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -114,6 +114,12 @@ static const char *const auth_methods_local[] = {
NULL
};
+static const char *const encryption_ciphers[] = {
+ "none",
+ "aes-256",
+ NULL
+};
+
/*
* these values are passed in by makefile defines
*/
@@ -145,6 +151,8 @@ static bool data_checksums = false;
static char *xlog_dir = NULL;
static char *str_wal_segment_size_mb = NULL;
static int wal_segment_size_mb;
+static char *enc_cipher = NULL;
+static char *cluster_passphrase = NULL;
/* internal vars */
@@ -1206,6 +1214,13 @@ setup_config(void)
"password_encryption = scram-sha-256");
}
+ if (cluster_passphrase)
+ {
+ snprintf(repltok, sizeof(repltok), "cluster_passphrase_command = '%s'",
+ escape_quotes(cluster_passphrase));
+ conflines = replace_token(conflines, "#cluster_passphrase_command = ''", repltok);
+ }
+
/*
* If group access has been enabled for the cluster then it makes sense to
* ensure that the log files also allow group access. Otherwise a backup
@@ -1416,14 +1431,15 @@ bootstrap_template1(void)
unsetenv("PGCLIENTENCODING");
snprintf(cmd, sizeof(cmd),
- "\"%s\" --boot -x1 -X %u %s %s %s",
+ "\"%s\" --boot -x1 -X %u %s %s %s %s %s",
backend_exec,
wal_segment_size_mb * (1024 * 1024),
data_checksums ? "-k" : "",
+ enc_cipher ? "-e" : "",
+ enc_cipher ? enc_cipher : "",
boot_options,
debug ? "-d 5" : "");
-
PG_CMD_OPEN;
for (line = bki_lines; *line != NULL; line++)
@@ -2312,6 +2328,9 @@ usage(const char *progname)
printf(_(" --wal-segsize=SIZE size of WAL segments, in megabytes\n"));
printf(_("\nLess commonly used options:\n"));
printf(_(" -d, --debug generate lots of debugging output\n"));
+ printf(_(" -e --enc-cipher=TYPE set encryption cipher (AES-256) for key management\n"));
+ printf(_(" -c --cluster-passphrase-command=COMMAND\n"
+ " set command to obtain passphrase for key management\n"));
printf(_(" -k, --data-checksums use data page checksums\n"));
printf(_(" -L DIRECTORY where to find the input files\n"));
printf(_(" -n, --no-clean do not clean up after errors\n"));
@@ -2377,6 +2396,42 @@ check_need_password(const char *authmethodlocal, const char *authmethodhost)
}
}
+static void
+check_encryption_cipher(const char *cipher, const char *passphrase,
+ const char *const *valid_ciphers)
+{
+ const char *const *p;
+
+ if (!cipher && !passphrase)
+ return;
+
+#ifndef USE_OPENSSL
+ pg_log_error("cluster encryption is not supported because OpenSSL is not supported by this build");
+ exit(1);
+#endif
+
+ /* Check both options must be specified at the same time */
+ if (cipher && !passphrase)
+ {
+ pg_log_error("encryption passphrase command must be specified when encryption cipher is specified");
+ exit(1);
+ }
+
+ if (!cipher && passphrase)
+ {
+ pg_log_error("encryption cipher must be specified when encryption passphrase command is specified");
+ exit(1);
+ }
+
+ for (p = valid_ciphers; *p; p++)
+ {
+ if (strcasecmp(cipher, *p) == 0)
+ return;
+ }
+
+ pg_log_error("invalid encryption cipher \"%s\"\nencryption cipher options is AES-256", cipher);
+ exit(1);
+}
void
setup_pgdata(void)
@@ -2984,6 +3039,8 @@ main(int argc, char *argv[])
{"wal-segsize", required_argument, NULL, 12},
{"data-checksums", no_argument, NULL, 'k'},
{"allow-group-access", no_argument, NULL, 'g'},
+ {"enc-cipher", required_argument, NULL, 'e'},
+ {"cluster-passphrase-command", required_argument, NULL, 'c'},
{NULL, 0, NULL, 0}
};
@@ -3025,7 +3082,7 @@ main(int argc, char *argv[])
/* process command-line options */
- while ((c = getopt_long(argc, argv, "dD:E:kL:nNU:WA:sST:X:g", long_options, &option_index)) != -1)
+ while ((c = getopt_long(argc, argv, "c:dD:E:e:kL:nNU:WA:sST:X:g", long_options, &option_index)) != -1)
{
switch (c)
{
@@ -3107,6 +3164,12 @@ main(int argc, char *argv[])
case 9:
pwfilename = pg_strdup(optarg);
break;
+ case 'e':
+ enc_cipher = pg_strdup(optarg);
+ break;
+ case 'c':
+ cluster_passphrase = pg_strdup(optarg);
+ break;
case 's':
show_setting = true;
break;
@@ -3185,6 +3248,8 @@ main(int argc, char *argv[])
check_need_password(authmethodlocal, authmethodhost);
+ check_encryption_cipher(enc_cipher, cluster_passphrase, encryption_ciphers);
+
/* set wal segment size */
if (str_wal_segment_size_mb == NULL)
wal_segment_size_mb = (DEFAULT_XLOG_SEG_SIZE) / (1024 * 1024);
@@ -3244,6 +3309,11 @@ main(int argc, char *argv[])
else
printf(_("Data page checksums are disabled.\n"));
+ if (enc_cipher)
+ printf(_("Cluster encryption key using %s is enabled.\n"), enc_cipher);
+ else
+ printf(_("Cluster encryption key is disabled.\n"));
+
if (pwprompt || pwfilename)
get_su_pwd();
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index 46ee1f1dc3..f84325fc17 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -100,6 +100,7 @@ static const char *const skip[] = {
"pg_control",
"pg_filenode.map",
"pg_internal.init",
+ "pg_kmgr",
"PG_VERSION",
#ifdef EXEC_BACKEND
"config_exec_params",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 19e21ab491..e7e06ef90b 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -25,6 +25,7 @@
#include "access/xlog_internal.h"
#include "catalog/pg_control.h"
#include "common/controldata_utils.h"
+#include "common/kmgr_utils.h"
#include "common/logging.h"
#include "getopt_long.h"
#include "pg_getopt.h"
@@ -83,7 +84,6 @@ wal_level_str(WalLevel wal_level)
return _("unrecognized wal_level");
}
-
int
main(int argc, char *argv[])
{
@@ -333,5 +333,7 @@ main(int argc, char *argv[])
ControlFile->data_checksum_version);
printf(_("Mock authentication nonce: %s\n"),
mock_auth_nonce_str);
+ printf(_("Kay manegement cipher: %s\n"),
+ kmgr_cipher_string(ControlFile->key_management_cipher));
return 0;
}
diff --git a/src/common/Makefile b/src/common/Makefile
index e757fb7399..2a8d9648f6 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -48,6 +48,7 @@ LIBS += $(PTHREAD_LIBS)
OBJS_COMMON = \
base64.o \
+ cipher.o \
config_info.o \
controldata_utils.o \
d2s.o \
@@ -58,6 +59,7 @@ OBJS_COMMON = \
ip.o \
jsonapi.o \
keywords.o \
+ kmgr_utils.o \
kwlookup.o \
link-canary.o \
md5.o \
@@ -77,6 +79,7 @@ OBJS_COMMON = \
ifeq ($(with_openssl),yes)
OBJS_COMMON += \
+ cipher_openssl.o \
protocol_openssl.o \
sha2_openssl.o
else
diff --git a/src/common/cipher.c b/src/common/cipher.c
new file mode 100644
index 0000000000..79c905b41a
--- /dev/null
+++ b/src/common/cipher.c
@@ -0,0 +1,98 @@
+/*-------------------------------------------------------------------------
+ *
+ * cipher.c
+ * Shared frontend/backend for cryptographic functions
+ *
+ * Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/cipher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/cipher.h"
+#include "common/cipher_openssl.h"
+
+void
+pg_cipher_setup(void)
+{
+#ifdef USE_OPENSSL
+ ossl_cipher_setup();
+#endif
+}
+
+pg_cipher_ctx *
+pg_cipher_ctx_create(void)
+{
+#ifdef USE_OPENSSL
+ return ossl_cipher_ctx_create();
+#endif
+ return NULL;
+}
+
+void
+pg_cipher_ctx_free(pg_cipher_ctx *ctx)
+{
+#ifdef USE_OPENSSL
+ ossl_cipher_ctx_free(ctx);
+#endif
+}
+
+bool
+pg_aes256_encrypt_init(pg_cipher_ctx *ctx, uint8 *key)
+{
+#ifdef USE_OPENSSL
+ return ossl_aes256_encrypt_init(ctx, key);
+#endif
+ return false;
+}
+
+bool
+pg_aes256_decrypt_init(pg_cipher_ctx *ctx, uint8 *key)
+{
+#ifdef USE_OPENSSL
+ return ossl_aes256_decrypt_init(ctx, key);
+#endif
+ return false;
+}
+
+bool
+pg_cipher_encrypt(pg_cipher_ctx *ctx, const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest, int *dest_size)
+{
+ bool r = false;
+#ifdef USE_OPENSSL
+ r = ossl_cipher_encrypt(ctx, input, input_size, iv, dest, dest_size);
+#endif
+ return r;
+}
+
+bool
+pg_cipher_decrypt(pg_cipher_ctx *ctx, const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest, int *dest_size)
+{
+ bool r = false;
+#ifdef USE_OPENSSL
+ r = ossl_cipher_decrypt(ctx, input, input_size, iv, dest, dest_size);
+#endif
+ return r;
+}
+
+bool
+pg_compute_HMAC(const uint8 *key, const uint8 *data,
+ int data_size, uint8 *result, int *result_size)
+{
+ bool r = true;
+#ifdef USE_OPENSSL
+ r = ossl_compute_HMAC(key, data, data_size, result,
+ result_size);
+#endif
+ return r;
+}
diff --git a/src/common/cipher_openssl.c b/src/common/cipher_openssl.c
new file mode 100644
index 0000000000..6b72b51f68
--- /dev/null
+++ b/src/common/cipher_openssl.c
@@ -0,0 +1,149 @@
+/*-------------------------------------------------------------------------
+ * cipher_openssl.c
+ * Cryptographic function using OpenSSL
+ *
+ * This contains the common low-level functions needed in both frontend and
+ * backend, for implement the database encryption.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/cipher_openssl.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/cipher_openssl.h"
+
+#include <openssl/conf.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+#include <openssl/hmac.h>
+
+bool
+ossl_cipher_setup(void)
+{
+#ifdef HAVE_OPENSSL_INIT_CRYPTO
+ /* Setup OpenSSL */
+ if (!OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL))
+ return false;
+ return true;
+#endif
+ return false;
+}
+
+pg_cipher_ctx *
+ossl_cipher_ctx_create(void)
+{
+ return EVP_CIPHER_CTX_new();
+}
+
+void
+ossl_cipher_ctx_free(pg_cipher_ctx *ctx)
+{
+ return EVP_CIPHER_CTX_free(ctx);
+}
+
+bool
+ossl_aes256_encrypt_init(pg_cipher_ctx *ctx, uint8 *key)
+{
+ if (!EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, NULL, NULL))
+ return false;
+ if (!EVP_CIPHER_CTX_set_key_length(ctx, PG_AES256_KEY_LEN))
+ return false;
+ if (!EVP_EncryptInit_ex(ctx, NULL, NULL, key, NULL))
+ return false;
+
+ /*
+ * Always enable padding. We don't need to check the return
+ * value as EVP_CIPHER_CTX_set_padding always returns 1.
+ */
+ EVP_CIPHER_CTX_set_padding(ctx, 1);
+
+ return true;
+}
+
+bool
+ossl_aes256_decrypt_init(pg_cipher_ctx *ctx, uint8 *key)
+{
+ if (!EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, NULL, NULL))
+ return false;
+ if (!EVP_CIPHER_CTX_set_key_length(ctx, PG_AES256_KEY_LEN))
+ return false;
+ if (!EVP_DecryptInit_ex(ctx, NULL, NULL, key, NULL))
+ return false;
+
+ /*
+ * Always enable padding. We don't need to check the return
+ * value as EVP_CIPHER_CTX_set_padding always returns 1.
+ */
+ EVP_CIPHER_CTX_set_padding(ctx, 1);
+
+ return true;
+}
+
+bool
+ossl_cipher_encrypt(pg_cipher_ctx *ctx,
+ const uint8 *in, int inlen,
+ const uint8 *iv, uint8 *out,
+ int *outlen)
+{
+ int len;
+ int enclen;
+
+ if (!EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv))
+ return false;
+
+ if (!EVP_EncryptUpdate(ctx, out, &len, in, inlen))
+ return false;
+
+ enclen = len;
+
+ if (!EVP_EncryptFinal_ex(ctx, (uint8 *) ((char *) out + enclen),
+ &len))
+ return false;
+
+ *outlen = enclen + len;
+
+ return true;
+}
+
+bool
+ossl_cipher_decrypt(pg_cipher_ctx *ctx,
+ const uint8 *in, int inlen,
+ const uint8 *iv, uint8 *out,
+ int *outlen)
+{
+ int declen;
+ int len;
+
+ if (!EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv))
+ return false;
+
+ if (!EVP_DecryptUpdate(ctx, out, &len, in, inlen))
+ return false;
+
+ declen = len;
+
+ if (!EVP_DecryptFinal_ex(ctx, (uint8 *) ((char *) out + declen),
+ &len))
+ return false;
+
+ *outlen = declen + len;
+
+ return true;
+}
+
+bool
+ossl_compute_HMAC(const uint8 *key, const uint8 *data,
+ int data_size, uint8 *result,
+ int *result_size)
+{
+ return HMAC(EVP_sha256(), key, PG_AES256_KEY_LEN, data,
+ (uint32) data_size, result, (uint32 *) result_size);
+}
diff --git a/src/common/kmgr_utils.c b/src/common/kmgr_utils.c
new file mode 100644
index 0000000000..cd5cdcf728
--- /dev/null
+++ b/src/common/kmgr_utils.c
@@ -0,0 +1,445 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr_utils.c
+ * Shared frontend/backend for cryptographic key management
+ *
+ * Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/common/kmgr_utils.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#ifdef FRONTEND
+#include "common/logging.h"
+#endif
+#include "common/kmgr_utils.h"
+#include "common/sha2.h"
+#include "crypto/kmgr.h"
+#include "utils/elog.h"
+#include "storage/fd.h"
+
+#define KMGR_PROMPT_MSG "Enter database encryption pass phrase:"
+
+static bool cipher_setup = false;
+
+#ifdef FRONTEND
+static FILE *open_pipe_stream(const char *command);
+static int close_pipe_stream(FILE *file);
+#endif
+
+/*
+ * Return the key wrap context initialized with the given keys. Initialize the
+ * context for key wrapping if `for_wrap` is true, otherwise for unwrapping.
+ */
+KeyWrapCtx *
+create_keywrap_ctx(uint8 key[KMGR_KEY_LEN], uint8 hmackey[KMGR_HMACKEY_LEN],
+ bool for_wrap)
+{
+ KeyWrapCtx *ctx;
+ int ret;
+
+ if (!cipher_setup)
+ {
+ pg_cipher_setup();
+ cipher_setup = true;
+ }
+
+#ifndef FRONTEND
+ ctx = (KeyWrapCtx *) palloc0(sizeof(KeyWrapCtx));
+#else
+ ctx = (KeyWrapCtx *) pg_malloc0(sizeof(KeyWrapCtx));
+#endif
+
+ /* Create a cipher context */
+ ctx->cipher = pg_cipher_ctx_create();
+ if (ctx->cipher == NULL)
+ return NULL;
+
+ /* Initialize the cipher context */
+ if (for_wrap)
+ ret = pg_aes256_encrypt_init(ctx->cipher, key);
+ else
+ ret = pg_aes256_decrypt_init(ctx->cipher, key);
+
+ if (!ret)
+ return NULL;
+
+ /* Set encryption key and HMAC key */
+ memcpy(ctx->key, key, KMGR_KEY_LEN);
+ memcpy(ctx->hmackey, hmackey, KMGR_HMACKEY_LEN);
+
+ return ctx;
+}
+
+/* Free the given cipher context */
+void
+free_keywrap_ctx(KeyWrapCtx *ctx)
+{
+ if (!ctx)
+ return;
+
+ Assert(ctx->cipher);
+
+ pg_cipher_ctx_free(ctx->cipher);
+
+#ifndef FRONTEND
+ pfree(ctx);
+#else
+ pg_free(ctx);
+#endif
+}
+
+/*
+ * Verify the correctness of the given passphrase by unwrapping the `wrapped_key`
+ * by the keys extracted from the passphrase. If the given passphrase is correct
+ * we set unwrapped keys to `raw_key` and return true. Otherwise return false.
+ */
+bool
+kmgr_verify_passphrase(char *passphrase, int passlen,
+ uint8 wrapped_key[KMGR_WRAPPED_KEY_LEN],
+ uint8 raw_key[KMGR_KEY_AND_HMACKEY_LEN])
+{
+ uint8 user_key[KMGR_KEY_LEN];
+ uint8 user_hmackey[KMGR_HMACKEY_LEN];
+ KeyWrapCtx *ctx;
+ int keylen;
+
+ /* Extract encryption key and HMAC key from the passphrase */
+ kmgr_derive_keys(passphrase, passlen, user_key, user_hmackey);
+
+ ctx = create_keywrap_ctx(user_key, user_hmackey, false);
+ if (!kmgr_unwrap_key(ctx, wrapped_key, KMGR_WRAPPED_KEY_LEN,
+ raw_key, &keylen))
+ {
+ /* The passphrase is not correct */
+ free_keywrap_ctx(ctx);
+ return false;
+ }
+
+ /* The passphrase is correct, free the cipher context */
+ free_keywrap_ctx(ctx);
+
+ return true;
+}
+
+/* Hash the given passphrase and extract it into encryption key and HMAC key */
+void
+kmgr_derive_keys(char *passphrase, Size passlen,
+ uint8 key[KMGR_KEY_LEN],
+ uint8 hmackey[KMGR_HMACKEY_LEN])
+{
+ uint8 keys[PG_SHA512_DIGEST_LENGTH];
+ pg_sha512_ctx ctx;
+
+ pg_sha512_init(&ctx);
+ pg_sha512_update(&ctx, (const uint8 *) passphrase, passlen);
+ pg_sha512_final(&ctx, keys);
+
+ /*
+ * SHA-512 results 64 bytes. We extract it into two keys for
+ * each 32 bytes.
+ */
+ if (key)
+ memcpy(key, keys, KMGR_KEY_LEN);
+ if (hmackey)
+ memcpy(hmackey, keys + KMGR_KEY_LEN, KMGR_HMACKEY_LEN);
+}
+
+/*
+ * Wrap the given key. Return true and set wrapped key to `out` if success.
+ * Otherwise return false. The caller must allocate sufficient space for
+ * wrapped key calculated by using SizeOfWrappedKey.
+ */
+bool
+kmgr_wrap_key(KeyWrapCtx *ctx, const uint8 *in, int inlen, uint8 *out, int *outlen)
+{
+ uint8 iv[AES_IV_SIZE];
+ uint8 hmac[KMGR_HMAC_LEN];
+ uint8 *keyenc;
+ int keylen;
+
+ Assert(ctx && in && out);
+
+ /* Generate IV */
+ if (!pg_strong_random(iv, AES_IV_SIZE))
+ return false;
+
+ /*
+ * To avoid allocating the memory for encrypted data, we store encrypted data
+ * directly into *out. Encrypted data places at the end.
+ */
+ keyenc = (uint8 *) ((char *) out + KMGR_HMAC_LEN + AES_IV_SIZE);
+
+ if (!pg_cipher_encrypt(ctx->cipher, in, inlen, iv, keyenc, &keylen))
+ return false;
+
+ if (!kmgr_compute_HMAC(ctx, keyenc, keylen, hmac))
+ return false;
+
+ /*
+ * Assemble the wrapped key. The order of the wrapped key is iv, hmac and
+ * encrypted data.
+ */
+ memcpy(out, hmac, KMGR_HMAC_LEN);
+ memcpy(out + KMGR_HMAC_LEN, iv, AES_IV_SIZE);
+
+ *outlen = SizeOfWrappedKey(inlen);
+
+ return true;
+}
+
+/*
+ * Unwrap the given key. Return true and set unwrapped key to `out` if success.
+ * Otherwise return false. The caller must allocate sufficient space for
+ * unwrapped key calculated by using SizeOfUnwrappedKey.
+ */
+bool
+kmgr_unwrap_key(KeyWrapCtx *ctx,const uint8 *in, int inlen, uint8 *out, int *outlen)
+{
+ uint8 hmac[KMGR_HMAC_LEN];
+ uint8 *iv;
+ uint8 *expected_hmac;
+ uint8 *keyenc;
+ int keylen;
+ char *p = (char *) in;;
+
+ Assert(ctx && in && out);
+
+ /* Disassemble the wrapped keys */
+ expected_hmac = (uint8 *) p;
+ p += KMGR_HMAC_LEN;
+ iv = (uint8 *) p;
+ p += AES_IV_SIZE;
+ keylen = inlen - (p - ((char *) in));
+ keyenc = (uint8 *) p;
+
+ /* Verify the correctness of HMAC */
+ if (!kmgr_compute_HMAC(ctx, keyenc, keylen, hmac))
+ return false;
+
+ if (memcmp(hmac, expected_hmac, KMGR_HMAC_LEN) != 0)
+ return false;
+
+ /* Decrypt encrypted data */
+ if (!pg_cipher_decrypt(ctx->cipher, keyenc, keylen, iv, out, outlen))
+ return false;
+
+ return true;
+}
+
+/*
+ * Compute HMAC of the given input. The HMAC is the fixed length,
+ * KMGR_HMAC_LEN bytes. The caller must allocate enough memory.
+ */
+bool
+kmgr_compute_HMAC(KeyWrapCtx *ctx, const uint8 *in, int inlen, uint8 *out)
+{
+ int resultsize = 0;
+
+ Assert(ctx && in && out);
+ return pg_compute_HMAC(ctx->hmackey, in, inlen, out, &resultsize);
+}
+
+/*
+ * Run cluster passphrase command.
+ *
+ * prompt will be substituted for %p.
+ *
+ * The result will be put in buffer buf, which is of size size.
+ * The return value is the length of the actual result.
+ */
+int
+kmgr_run_cluster_passphrase_command(char *passphrase_command, char *buf,
+ int size)
+{
+ char command[MAXPGPATH];
+ char *p;
+ char *dp;
+ char *endp;
+ FILE *fh;
+ int pclose_rc;
+ size_t len = 0;
+
+ Assert(size > 0);
+ buf[0] = '\0';
+
+ dp = command;
+ endp = command + MAXPGPATH - 1;
+ *endp = '\0';
+
+ for (p = passphrase_command; *p; p++)
+ {
+ if (p[0] == '%')
+ {
+ switch (p[1])
+ {
+ case 'p':
+ StrNCpy(dp, KMGR_PROMPT_MSG, strlen(KMGR_PROMPT_MSG));
+ dp += strlen(KMGR_PROMPT_MSG);
+ p++;
+ break;
+ case '%':
+ p++;
+ if (dp < endp)
+ *dp++ = *p;
+ break;
+ default:
+ if (dp < endp)
+ *dp++ = *p;
+ break;
+ }
+ }
+ else
+ {
+ if (dp < endp)
+ *dp++ = *p;
+ }
+ }
+ *dp = '\0';
+
+#ifdef FRONTEND
+ fh = open_pipe_stream(command);
+ if (fh == NULL)
+ {
+ pg_log_fatal("could not execute command \"%s\": %m",
+ command);
+ exit(EXIT_FAILURE);
+ }
+#else
+ fh = OpenPipeStream(command, "r");
+ if (fh == NULL)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not execute command \"%s\": %m",
+ command)));
+#endif
+
+ if ((len = fread(buf, sizeof(char), size, fh)) < size)
+ {
+ if (ferror(fh))
+ {
+#ifdef FRONTEND
+ pg_log_fatal("could not read from command \"%s\": %m",
+ command);
+ exit(EXIT_FAILURE);
+#else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not read from command \"%s\": %m",
+ command)));
+#endif
+ }
+ }
+
+#ifdef FRONTEND
+ pclose_rc = close_pipe_stream(fh);
+#else
+ pclose_rc = ClosePipeStream(fh);
+#endif
+
+ if (pclose_rc == -1)
+ {
+#ifdef FRONTEND
+ pg_log_fatal("could not close pipe to external command: %m");
+ exit(EXIT_FAILURE);
+#else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not close pipe to external command: %m")));
+#endif
+ }
+ else if (pclose_rc != 0)
+ {
+#ifdef FRONTEND
+ pg_log_fatal("command \"%s\" failed", command);
+ exit(EXIT_FAILURE);
+#else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("command \"%s\" failed",
+ command),
+ errdetail_internal("%s", wait_result_to_str(pclose_rc))));
+#endif
+ }
+
+ return len;
+}
+
+/* Convert cipher name string to integer value */
+int
+kmgr_cipher_value(const char *name)
+{
+ if (strcasecmp(name, "aes-256") == 0)
+ return KMGR_CIPHER_AES256;
+
+ return KMGR_CIPHER_OFF;
+}
+
+/* Convert integer value to cipher name string */
+char *
+kmgr_cipher_string(int value)
+{
+ switch (value)
+ {
+ case KMGR_CIPHER_OFF :
+ return "off";
+ case KMGR_CIPHER_AES256:
+ return "aes-256";
+ default:
+ break;
+ }
+
+ return "unknown";
+}
+
+#ifdef FRONTEND
+static FILE *
+open_pipe_stream(const char *command)
+{
+ FILE *res;
+
+#ifdef WIN32
+ size_t cmdlen = strlen(command);
+ char *buf;
+ int save_errno;
+
+ buf = malloc(cmdlen + 2 + 1);
+ if (buf == NULL)
+ {
+ errno = ENOMEM;
+ return NULL;
+ }
+ buf[0] = '"';
+ mempcy(&buf[1], command, cmdlen);
+ buf[cmdlen + 1] = '"';
+ buf[cmdlen + 2] = '\0';
+
+ res = _popen(buf, "r");
+
+ save_errno = errno;
+ free(buf);
+ errno = save_errno;
+#else
+ res = popen(command, "r");
+#endif /* WIN32 */
+ return res;
+}
+
+static int
+close_pipe_stream(FILE *file)
+{
+#ifdef WIN32
+ return _pclose(file);
+#else
+ return pclose(file);
+#endif /* WIN32 */
+}
+#endif /* FRONTEND */
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 98b033fc20..8b77a73c01 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -15,6 +15,7 @@
#include "access/xlogdefs.h"
#include "access/xloginsert.h"
#include "access/xlogreader.h"
+#include "crypto/kmgr.h"
#include "datatype/timestamp.h"
#include "lib/stringinfo.h"
#include "nodes/pg_list.h"
@@ -291,8 +292,10 @@ extern TimestampTz GetCurrentChunkReplayStartTime(void);
extern void UpdateControlFile(void);
extern uint64 GetSystemIdentifier(void);
+extern uint8 *GetMasterEncryptionKey(void);
extern char *GetMockAuthenticationNonce(void);
extern bool DataChecksumsEnabled(void);
+extern int GetKeyManagementCipher(void);
extern XLogRecPtr GetFakeLSNForUnloggedRel(void);
extern Size XLOGShmemSize(void);
extern void XLOGShmemInit(void);
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index de5670e538..9dac38a5c0 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -17,6 +17,7 @@
#include "access/transam.h"
#include "access/xlogdefs.h"
+#include "crypto/kmgr.h"
#include "pgtime.h" /* for pg_time_t */
#include "port/pg_crc32c.h"
@@ -219,6 +220,9 @@ typedef struct ControlFileData
/* Are data pages protected by checksums? Zero if no checksum version */
uint32 data_checksum_version;
+ /* Key management cipher. Off by default */
+ uint32 key_management_cipher;
+
/*
* Random nonce, used in authentication requests that need to proceed
* based on values that are cluster-unique, like a SASL exchange that
@@ -226,6 +230,9 @@ typedef struct ControlFileData
*/
char mock_authentication_nonce[MOCK_AUTH_NONCE_LEN];
+ /* Database cluster master key */
+ uint8 masterkey[KMGR_WRAPPED_KEY_LEN];
+
/* CRC of all above ... MUST BE LAST! */
pg_crc32c crc;
} ControlFileData;
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 2228256907..c7839f125d 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10768,4 +10768,17 @@
proname => 'pg_partition_root', prorettype => 'regclass',
proargtypes => 'regclass', prosrc => 'pg_partition_root' },
+# function for key managements
+{ oid => '8200', descr => 'rotate cluter encryption key',
+ proname => 'pg_rotate_encryption_key',
+ provolatile => 'v', prorettype => 'bool',
+ proargtypes => '', prosrc => 'pg_rotate_encryption_key' },
+{ oid => '8201', descr => 'wrap the given secret',
+ proname => 'pg_wrap_key',
+ provolatile => 'v', prorettype => 'bytea',
+ proargtypes => 'bytea', prosrc => 'pg_wrap_key' },
+{ oid => '8202', descr => 'unwrap the given secret',
+ proname => 'pg_unwrap_key',
+ provolatile => 'v', prorettype => 'bytea',
+ proargtypes => 'bytea', prosrc => 'pg_unwrap_key' },
]
diff --git a/src/include/common/cipher.h b/src/include/common/cipher.h
new file mode 100644
index 0000000000..fdfba2d1a6
--- /dev/null
+++ b/src/include/common/cipher.h
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * cipher.h
+ * Declarations for cryptographic functions
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/common/cipher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef CIPHER_H
+#define CIPHER_H
+
+#ifdef USE_OPENSSL
+#include <openssl/evp.h>
+#include <openssl/conf.h>
+#include <openssl/err.h>
+#endif
+
+/* Key length of AES256 */
+#define PG_AES256_KEY_LEN 32
+
+/*
+ * The encrypted data is a series of blocks of size ENCRYPTION_BLOCK.
+ * Initialization vector(IV) is the same size of cipher block.
+ */
+#define AES_BLOCK_SIZE 16
+#define AES_IV_SIZE (AES_BLOCK_SIZE)
+
+/* HMAC key and HMAC length. We use HMAC-SHA256 */
+#define PG_HMAC_SHA256_KEY_LEN 32
+#define PG_HMAC_SHA256_LEN 32
+
+#ifdef USE_OPENSSL
+typedef EVP_CIPHER_CTX pg_cipher_ctx;
+#else
+typedef void pg_cipher_ctx;
+#endif
+
+extern pg_cipher_ctx *pg_cipher_ctx_create(void);
+extern void pg_cipher_ctx_free(pg_cipher_ctx *ctx);
+extern void pg_cipher_setup(void);
+extern bool pg_aes256_encrypt_init(pg_cipher_ctx *ctx, uint8 *key);
+extern bool pg_aes256_decrypt_init(pg_cipher_ctx *ctx, uint8 *key);
+extern bool pg_cipher_encrypt(pg_cipher_ctx *ctx,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest,
+ int *dest_size);
+extern bool pg_cipher_decrypt(pg_cipher_ctx *ctx,
+ const uint8 *input, int input_size,
+ const uint8 *iv, uint8 *dest,
+ int *dest_size);
+extern bool pg_compute_HMAC(const uint8 *key, const uint8 *data,
+ int data_size, uint8 *result,
+ int *result_size);
+
+#endif /* CIPHER_H */
diff --git a/src/include/common/cipher_openssl.h b/src/include/common/cipher_openssl.h
new file mode 100644
index 0000000000..d55970b89d
--- /dev/null
+++ b/src/include/common/cipher_openssl.h
@@ -0,0 +1,39 @@
+/*-------------------------------------------------------------------------
+ *
+ * cipher_openssl.h
+ * Declarations for helper functions using OpenSSL
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/common/cipher_openssl.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef CIPHER_OPENSSL_H
+#define CIPHER_OPENSSL_H
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/cipher.h"
+
+extern pg_cipher_ctx *ossl_cipher_ctx_create(void);
+extern void ossl_cipher_ctx_free(pg_cipher_ctx *ctx);
+extern bool ossl_cipher_setup(void);
+extern bool ossl_aes256_encrypt_init(pg_cipher_ctx *ctx, uint8 *key);
+extern bool ossl_aes256_decrypt_init(pg_cipher_ctx *ctx, uint8 *key);
+extern bool ossl_cipher_encrypt(pg_cipher_ctx *ctx,
+ const uint8 *in, int inlen,
+ const uint8 *iv, uint8 *out,
+ int *outlen);
+extern bool ossl_cipher_decrypt(pg_cipher_ctx *ctx,
+ const uint8 *in, int inlen,
+ const uint8 *iv, uint8 *out,
+ int *outlen);
+extern bool ossl_compute_HMAC(const uint8 *key, const uint8 *data,
+ int data_size, uint8 *result,
+ int *result_size);
+#endif
diff --git a/src/include/common/kmgr_utils.h b/src/include/common/kmgr_utils.h
new file mode 100644
index 0000000000..42a797207c
--- /dev/null
+++ b/src/include/common/kmgr_utils.h
@@ -0,0 +1,83 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr_utils.h
+ * Declarations for utility function for key management
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/common/kmgr_utils.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef KMGR_UTILS_H
+#define KMGR_UTILS_H
+
+#include "common/cipher.h"
+
+/* As of now key length supports only AES-256 key */
+#define KMGR_KEY_LEN PG_AES256_KEY_LEN
+
+/* Key management uses HMAC-256 */
+#define KMGR_HMACKEY_LEN PG_HMAC_SHA256_KEY_LEN
+#define KMGR_HMAC_LEN PG_HMAC_SHA256_LEN
+
+/* Allowed length of cluster passphrase */
+#define KMGR_MIN_PASSPHRASE_LEN 64
+#define KMGR_MAX_PASSPHRASE_LEN 1024
+
+/*
+ * Wrapped key consists of HMAC of encrypted key, IV and encrypted key.
+ */
+#define KMGR_KEY_AND_HMACKEY_LEN (KMGR_KEY_LEN + KMGR_HMACKEY_LEN)
+#define KMGR_WRAPPED_KEY_LEN \
+ (KMGR_HMAC_LEN + AES_IV_SIZE + SizeOfKeyWithPadding(KMGR_KEY_AND_HMACKEY_LEN))
+
+/*
+ * Size of encrypted key size with padding. We use PKCS#7 padding
+ * described in RFC 5652.
+ */
+#define SizeOfKeyWithPadding(klen) \
+ ((int)(klen) + (AES_BLOCK_SIZE - ((int)(klen) % AES_BLOCK_SIZE)))
+
+/*
+ * Macro to compute the size of wrapped and unwrapped key. The wrapped
+ * key consists of HMAC of the encrypted key, IV and the encrypted data
+ * that is the same length as the input.
+ */
+#define SizeOfWrappedKey(klen) \
+ (KMGR_HMACKEY_LEN + AES_IV_SIZE + SizeOfKeyWithPadding((int)(klen)))
+#define SizeOfUnwrappedKey(klen) \
+ ((int)(klen) - (KMGR_HMACKEY_LEN + AES_IV_SIZE))
+
+/*
+ * Key wrapping cipher context.
+ */
+typedef struct KeyWrapCtx
+{
+ uint8 key[KMGR_KEY_LEN];
+ uint8 hmackey[KMGR_HMACKEY_LEN];
+ pg_cipher_ctx *cipher;
+} KeyWrapCtx;
+
+extern KeyWrapCtx *create_keywrap_ctx(uint8 key[KMGR_KEY_LEN],
+ uint8 hmackey[KMGR_HMACKEY_LEN],
+ bool for_wrap);
+extern void free_keywrap_ctx(KeyWrapCtx *ctx);
+extern void kmgr_derive_keys(char *passphrase, Size passlen,
+ uint8 key[KMGR_KEY_LEN],
+ uint8 hmackey[KMGR_HMACKEY_LEN]);
+extern bool kmgr_verify_passphrase(char *passphrase, int passlen,
+ uint8 wrapped_key[KMGR_WRAPPED_KEY_LEN],
+ uint8 raw_key[KMGR_KEY_AND_HMACKEY_LEN]);
+extern bool kmgr_wrap_key(KeyWrapCtx *ctx, const uint8 *in, int inlen,
+ uint8 *out, int *outlen);
+extern bool kmgr_unwrap_key(KeyWrapCtx *ctx,const uint8 *in, int inlen,
+ uint8 *out, int *outlen);
+extern bool kmgr_compute_HMAC(KeyWrapCtx *ctx, const uint8 *in, int inlen,
+ uint8 *out);
+extern int kmgr_run_cluster_passphrase_command(char *passphrase_command,
+ char *buf, int size);
+extern int kmgr_cipher_value(const char *name);
+extern char * kmgr_cipher_string(int value);
+
+#endif /* KMGR_UTILS_H */
diff --git a/src/include/crypto/kmgr.h b/src/include/crypto/kmgr.h
new file mode 100644
index 0000000000..e8ca106ad4
--- /dev/null
+++ b/src/include/crypto/kmgr.h
@@ -0,0 +1,37 @@
+/*-------------------------------------------------------------------------
+ *
+ * kmgr.h
+ * Key management module for transparent data encryption
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/crypto/kmgr.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef KMGR_H
+#define KMGR_H
+
+#include "common/cipher.h"
+#include "common/kmgr_utils.h"
+#include "storage/relfilenode.h"
+#include "storage/bufpage.h"
+
+#define KeyManagementEnabled() \
+ (key_management_cipher > KMGR_CIPHER_OFF)
+
+/* GUC parameter */
+extern PGDLLIMPORT int key_management_cipher;
+extern char *cluster_passphrase_command;
+
+/* Value of key_management_cipher */
+enum
+{
+ KMGR_CIPHER_OFF = 0,
+ KMGR_CIPHER_AES256
+};
+
+extern uint8 *BootStrapKmgr(int bootstrap_key_management_cipher);
+extern void InitializeKmgr(void);
+
+#endif /* KMGR_H */
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 6f485f73cd..bb35bf782c 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -415,6 +415,9 @@
/* Define to 1 if you have the `OPENSSL_init_ssl' function. */
#undef HAVE_OPENSSL_INIT_SSL
+/* Define to 1 if you have the `OPENSSL_init_crypto' function. */
+#undef HAVE_OPENSSL_INIT_CRYPTO
+
/* Define to 1 if you have the <ossp/uuid.h> header file. */
#undef HAVE_OSSP_UUID_H
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 454c2df487..c0c53b1e13 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -89,6 +89,7 @@ enum config_group
STATS,
STATS_MONITORING,
STATS_COLLECTOR,
+ ENCRYPTION,
AUTOVACUUM,
CLIENT_CONN,
CLIENT_CONN_STATEMENT,
