This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 6cd9397bb IMPALA-13705: Environment specific errors in
test_encryption_exprs
6cd9397bb is described below
commit 6cd9397bb7eb8d9ac3c6a911bc1d0a1a4cb2bf17
Author: Daniel Becker <[email protected]>
AuthorDate: Tue Jan 28 16:28:34 2025 +0100
IMPALA-13705: Environment specific errors in test_encryption_exprs
IMPALA-13039 added support for the aes_encrypt() and aes_decrypt()
functions, where the user can choose the AES operation mode. Currently
if the user chooses a mode that is not supported by the OpenSSL library
at runtime, we fall back to a supported mode. There are two issues
related to this:
1. If the mode chosen by the user requires a 16 byte encryption key but
the fallback mode needs a 32 byte one, we will read past the provided
buffer.
2. If the tests are run in an environment where some modes are not
supported, we will get incorrect results. This can for example be the
case with GCM.
The first problem is solved by disabling falling back to a supported
mode in case of aes_encrypt() and aes_decrypt(). If the mode is not
supported, an error is returned instead. Note that this does not affect
the case when the user explicitly specifies NULL as the operation mode -
a default encoding will still be chosen in that case.
The second problem is solved by dividing the test queries into separate
".test" files based on operation mode (and one for queries that are
expected to fail). Each operation mode that is not always supported is
tested first with a probing query and the corresponding tests are only
run if the probing query succeeds, i.e. the mode is supported.
Change-Id: I27146cda4fa41965de35821315738e5502d1b018
Reviewed-on: http://gerrit.cloudera.org:8080/22419
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
be/src/exprs/string-functions-ir.cc | 2 +-
be/src/exprs/string-functions.cc | 10 +-
be/src/util/openssl-util.cc | 14 +-
.../queries/QueryTest/encryption_exprs.test | 456 ---------------------
.../QueryTest/encryption_exprs_aes_128_ecb.test | 23 ++
.../QueryTest/encryption_exprs_aes_128_gcm.test | 84 ++++
.../QueryTest/encryption_exprs_aes_256_cfb.test | 79 ++++
.../QueryTest/encryption_exprs_aes_256_ctr.test | 66 +++
.../QueryTest/encryption_exprs_aes_256_ecb.test | 15 +
.../QueryTest/encryption_exprs_aes_256_gcm.test | 140 +++++++
.../queries/QueryTest/encryption_exprs_errors.test | 55 +++
tests/query_test/test_exprs.py | 63 ++-
12 files changed, 535 insertions(+), 472 deletions(-)
diff --git a/be/src/exprs/string-functions-ir.cc
b/be/src/exprs/string-functions-ir.cc
index e9803f809..b179569f2 100644
--- a/be/src/exprs/string-functions-ir.cc
+++ b/be/src/exprs/string-functions-ir.cc
@@ -259,7 +259,7 @@ StringVal StringFunctions::Rpad(FunctionContext* context,
const StringVal& str,
IntVal StringFunctions::Length(FunctionContext* context, const StringVal& str)
{
if (str.is_null) return IntVal::null();
- if (context->impl()->GetConstFnAttr(FunctionContextImpl::UTF8_MODE, 0)) {
+ if (context->impl()->GetConstFnAttr(FunctionContextImpl::UTF8_MODE)) {
return Utf8Length(context, str);
}
return IntVal(str.len);
diff --git a/be/src/exprs/string-functions.cc b/be/src/exprs/string-functions.cc
index 60835d7ea..12630087a 100644
--- a/be/src/exprs/string-functions.cc
+++ b/be/src/exprs/string-functions.cc
@@ -472,13 +472,11 @@ Status InitializeEncryptionKey(FunctionContext* ctx,
const StringVal& expr,
// GCM mode expects expression, key, AES mode and iv vector.
// CTR and CFB modes expect expression, key, AES mode and iv vector.
// ECB mode expects expression, key, AES mode.
-// If the mode passed by the user is supported by Impala (a valid member of
-// AES_CIPHER_MODE except for INVALID) but not supported by the OpenSSL
-// library used internally, then the default mode of the library is chosen.
//
-// This is different from the case where the user enters a mode that Impala
does not
-// support (e.g., a nonexistent or invalid mode). In such cases, the mode is
-// considered invalid, and an error is returned.
+// If 'mode' is NULL, a default mode will be chosen at runtime.
+//
+// If the mode passed by the user is not supported by Impala or by the OpenSSL
+// library used internally, an error is returned.
//
// Description of the modes supported:
// AES-GCM (Advanced Encryption Standard Galois/Counter Mode) is a mode of
operation
diff --git a/be/src/util/openssl-util.cc b/be/src/util/openssl-util.cc
index 7de123da1..d814d6d8c 100644
--- a/be/src/util/openssl-util.cc
+++ b/be/src/util/openssl-util.cc
@@ -32,6 +32,7 @@
#include <openssl/tls1.h>
#include "common/atomic.h"
+#include "common/status.h"
#include "gutil/port.h" // ATTRIBUTE_WEAK
#include "gutil/strings/substitute.h"
@@ -376,14 +377,13 @@ const EVP_CIPHER* EncryptionKey::GetCipher() const {
Status EncryptionKey::InitializeFields(const uint8_t* key, int key_len, const
uint8_t* iv,
int iv_len, AES_CIPHER_MODE m) {
- mode_ = m;
+ RETURN_IF_ERROR(ValidateModeAndKeyLength(m, key_len));
if (!IsModeSupported(m)) {
- mode_ = GetSupportedDefaultMode();
- LOG(WARNING) << Substitute("$0 is not supported, fall back to $1.",
- ModeToString(m), ModeToString(mode_));
+ return Status(Substitute("AES mode $0 is not supported by OpenSSL version
($1) "
+ "that Impala was built against.", ModeToString(m),
OPENSSL_VERSION_TEXT));
}
- Status status = ValidateModeAndKeyLength(m, key_len);
- RETURN_IF_ERROR(status);
+
+ mode_ = m;
key_length_ = key_len;
iv_length_ = iv_len;
@@ -477,4 +477,4 @@ AES_CIPHER_MODE EncryptionKey::StringToMode(std::
string_view str) {
}
return AES_CIPHER_MODE::INVALID;
}
-}
\ No newline at end of file
+}
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs.test
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs.test
deleted file mode 100644
index e3d004096..000000000
---
a/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs.test
+++ /dev/null
@@ -1,456 +0,0 @@
-====
----- QUERY
-# AES encryption/decryption examples:
-select
aes_decrypt(base64decode('y6Ss+zCYObpCbgfWfyNWTw=='),'1234567890123456','AES_128_ECB','');
----- RESULTS
-'ABC'
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('BQGHoM3lqYcsurCRq3PlUw=='),'1234567890123456','AES_128_ECB','');
----- RESULTS
-''
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('E1zl+pDv/GY4JLk254KAIQ=='),'12345678901234567890123456789012','AES_256_ECB','');
----- RESULTS
-'ABC'
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt('ABC',
'1234567890123456','AES_128_GCM','1234567890123456'));
----- RESULTS
-'x+am+BIqtrEK9FpC/zrvpOycjQ=='
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('x+am+BIqtrEK9FpC/zrvpOycjQ=='),'1234567890123456','AES_128_GCM','1234567890123456');
----- RESULTS
-'ABC'
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt('',
'1234567890123456','AES_128_GCM','1234567890123456'));
----- RESULTS
-'moMhTz224yot8uRtksO+pw=='
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('moMhTz224yot8uRtksO+pw=='),'1234567890123456','AES_128_GCM','1234567890123456');
----- RESULTS
-''
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM','1234567890123456'));
----- RESULTS
-'F/DLkSwEikFOlqzXVCysy1JX7Q=='
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_GCM','1234567890123456');
----- RESULTS
-'ABC'
----- TYPES
-STRING
-====
----- QUERY
-select aes_decrypt(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM','1234567890123456'),
-'12345678901234567890123456789012','AES_256_GCM','1234567890123456');
----- RESULTS
-'ABC'
----- TYPES
-STRING
-====
----- QUERY
-select
base64encode(aes_encrypt(aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),
-'12345678901234567890123456789012','AES_256_GCM','1234567890123456'),'12345678901234567890123456789012','AES_256_GCM',
-'1234567890123456'));
----- RESULTS
-'F/DLkSwEikFOlqzXVCysy1JX7Q=='
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('ofMshL41XkG6NQ+v6bawAQ=='),'1234567890123456','AES_128_ECB','');
----- RESULTS
-'Hello World!'
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt('The quick brown fox jumps over the lazy dog',
'1234567890123456','AES_128_GCM',
-'1234567890123456'));
----- RESULTS
-'0syAnPYY0qT/RNtD7s0UkV/4P7HSl8xI1ZCz9AINV3r5W0xLkUs/gAlXWD+i6TIx+WiWiuipZJMXLmM='
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('0syAnPYY0qT/RNtD7s0UkV/4P7HSl8xI1ZCz9AINV3r5W0xLkUs/gAlXWD+i6TIx+WiWiuipZJMXLmM='),
-'1234567890123456','AES_128_GCM','1234567890123456');
----- RESULTS
-'The quick brown fox jumps over the lazy dog'
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt('Impala',
'1234567890123456','AES_128_GCM','12345678901'));
----- RESULTS
-'sBDdvFMLqNtJvuHUP8s7oWaH0NWXNQ=='
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('sBDdvFMLqNtJvuHUP8s7oWaH0NWXNQ=='),'1234567890123456','AES_128_GCM','12345678901');
----- RESULTS
-'Impala'
----- TYPES
-STRING
-====
----- QUERY
-select aes_decrypt(aes_encrypt('Impala',
'1234567890123456','AES_128_GCM','12345678901'),
-'1234567890123456','AES_128_GCM','12345678901');
----- RESULTS
-'Impala'
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt('impalaaaaaaaaaaaaaaaaaaaa',
'12345678901234567890123456789012','AES_256_CFB',
-'1234567890123456'));
----- RESULTS
-'pkw4Y8WLkJnACfAL4R4BNUBzdnUeUo7vTw=='
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('pkw4Y8WLkJnACfAL4R4BNUBzdnUeUo7vTw=='),'12345678901234567890123456789012',
-'AES_256_CFB','1234567890123456');
----- RESULTS
-'impalaaaaaaaaaaaaaaaaaaaa'
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt('',
'12345678901234567890123456789012','AES_256_CFB','1234567890123456'));
----- RESULTS
-''
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode(''),'12345678901234567890123456789012','AES_256_CFB','1234567890123456');
----- RESULTS
-''
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt('impalaaaaaaaaaaaaaaaaaaaa',
'12345678901234567890123456789012','AES_256_CTR',
-'1234567890123456'));
----- RESULTS
-'pkw4Y8WLkJnACfAL4R4BNb0HYMEjpcA63A=='
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('pkw4Y8WLkJnACfAL4R4BNb0HYMEjpcA63A=='),'12345678901234567890123456789012',
-'AES_256_CTR','1234567890123456');
----- RESULTS
-'impalaaaaaaaaaaaaaaaaaaaa'
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt('',
'12345678901234567890123456789012','AES_256_CTR','1234567890123456'));
----- RESULTS
-''
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode(''),'12345678901234567890123456789012','AES_256_CTR','1234567890123456');
----- RESULTS
-''
----- TYPES
-STRING
-====
----- QUERY
-select aes_decrypt((aes_encrypt('impalaaaaaaaaaaaaaaaaaaaa',
'12345678901234567890123456789012','AES_256_CFB',
-'1234567890123456')),'12345678901234567890123456789012','AES_256_CFB','1234567890123456');
----- RESULTS
-'impalaaaaaaaaaaaaaaaaaaaa'
----- TYPES
-STRING
-====
----- QUERY
-select aes_decrypt((aes_encrypt('impalaaaaaaaaaaaaaaaaaaaa',
'12345678901234567890123456789012','AES_256_CTR',
-'1234567890123456')),'12345678901234567890123456789012','AES_256_CTR','1234567890123456');
----- RESULTS
-'impalaaaaaaaaaaaaaaaaaaaa'
----- TYPES
-STRING
-====
----- QUERY
-# Encryption/ decryption when mode is NULL, defaulting to GCM mode.
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012',NULL,'1234567890123456'));
----- RESULTS
-'F/DLkSwEikFOlqzXVCysy1JX7Q=='
----- TYPES
-STRING
-====
----- QUERY
-select aes_decrypt(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM','1234567890123456'),
-'12345678901234567890123456789012',NULL,'1234567890123456');
----- RESULTS
-'ABC'
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012',NULL,'1234567890123456');
----- RESULTS
-'ABC'
----- TYPES
-STRING
-====
----- QUERY
-# Encryption/ decryption with expr as NULL.
-select base64encode(aes_encrypt(NULL,
'12345678901234567890123456789012','AES_256_GCM','1234567890123456'));
----- RESULTS
-'NULL'
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(NULL,'12345678901234567890123456789012','AES_256_GCM','1234567890123456');
----- RESULTS
-'NULL'
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt(NULL,
'12345678901234567890123456789012','AES_256_CTR','1234567890123456'));
----- RESULTS
-'NULL'
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(NULL,'12345678901234567890123456789012','AES_256_CTR','1234567890123456');
----- RESULTS
-'NULL'
----- TYPES
-STRING
-====
----- QUERY
-select base64encode(aes_encrypt(NULL,
'12345678901234567890123456789012','AES_256_CFB','1234567890123456'));
----- RESULTS
-'NULL'
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(NULL,'12345678901234567890123456789012','AES_256_CFB','1234567890123456');
----- RESULTS
-'NULL'
----- TYPES
-STRING
-====
----- QUERY
-# Key cannot be NULL.
-select base64encode(aes_encrypt('ABC',NULL,'AES_256_CFB','1234567890123456'));
----- RESULTS
----- CATCH
-Key cannot be NULL.
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),NULL,'AES_256_CFB','1234567890123456');
----- RESULTS
----- CATCH
-UDF ERROR: Key cannot be NULL.
-====
----- QUERY
-# Key cannot be NULL, with the default mode GCM.
-select base64encode(aes_encrypt('ABC',NULL,'AES_256_GCM','1234567890123456'));
----- RESULTS
----- CATCH
-Key cannot be NULL.
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),NULL,'AES_256_GCM','1234567890123456');
----- RESULTS
----- CATCH
-UDF ERROR: Key cannot be NULL.
-====
----- QUERY
-# ECB not supported for AES encryption.
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_ECB','1234567890123456'));
----- RESULTS
----- CATCH
-ECB mode is not supported for encryption.
-====
----- QUERY
-# iv cannot be NULL for GCM mode.
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM',NULL));
----- RESULTS
----- CATCH
-IV vector required for AES_256_GCM mode
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_GCM',NULL);
----- RESULTS
----- CATCH
-UDF ERROR: IV vector required for AES_256_GCM mode
-====
----- QUERY
-# iv cannot be NULL for CFB mode.
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_CFB',NULL));
----- RESULTS
----- CATCH
-IV vector required for AES_256_CFB mode
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_CFB',NULL);
----- RESULTS
----- CATCH
-UDF ERROR: IV vector required for AES_256_CFB mode
-====
----- QUERY
-# iv cannot be NULL for CTR mode.
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_CTR',NULL));
----- RESULTS
----- CATCH
-IV vector required for AES_256_CTR mode
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_CTR',NULL);
----- RESULTS
----- CATCH
-UDF ERROR: IV vector required for AES_256_CTR mode
-====
----- QUERY
-# Error resulting due to user entered incorrect mode.
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_CTB','1234567890123456'));
----- RESULTS
----- CATCH
-Invalid AES 'mode': AES_256_CTB
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_CTB',
-'1234567890123456');
----- RESULTS
----- CATCH
-UDF ERROR: Invalid AES 'mode': AES_256_CTB
-====
----- QUERY
-# Error with incorrect key length.
-select base64encode(aes_encrypt('ABC',
'123456789012345678901234567890121','AES_256_GCM','1234567890123456'));
----- RESULTS
----- CATCH
-AES only supports 128 and 256 bit key lengths
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'123456789012345678901234567890121','AES_256_GCM',
-'1234567890123456');
----- RESULTS
----- CATCH
-UDF ERROR: AES only supports 128 and 256 bit key lengths
-====
----- QUERY
-# Error with incorrect iv length.
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM','12345678901234567'));
----- RESULTS
----- CATCH
-IV vector size is greater than 16 bytes
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_GCM',
-'12345678901234567');
----- RESULTS
----- CATCH
-UDF ERROR: IV vector size is greater than 16 bytes
-====
----- QUERY
-# Prefixes of supported modes are not accepted
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','','1234567890123456'));
----- RESULTS
----- CATCH
-Invalid AES 'mode':
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','',
-'1234567890123456');
----- RESULTS
----- CATCH
-UDF ERROR: Invalid AES 'mode':
-====
----- QUERY
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256','1234567890123456'));
----- RESULTS
----- CATCH
-Invalid AES 'mode': AES_256
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256',
-'1234567890123456');
----- RESULTS
----- CATCH
-UDF ERROR: Invalid AES 'mode': AES_256
-====
----- QUERY
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GC','1234567890123456'));
----- RESULTS
----- CATCH
-Invalid AES 'mode': AES_256_GC
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_GC',
-'1234567890123456');
----- RESULTS
----- CATCH
-UDF ERROR: Invalid AES 'mode': AES_256_GC
-====
----- QUERY
-# Modes are case-insensitive.
-select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','aes_256_gcm','1234567890123456'));
----- RESULTS
-'F/DLkSwEikFOlqzXVCysy1JX7Q=='
----- TYPES
-STRING
-====
----- QUERY
-select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','aes_256_gcm',
-'1234567890123456');
----- RESULTS
-'ABC'
----- TYPES
-STRING
-====
----- QUERY
-select count(*) from functional.alltypes where string_col =
aes_decrypt(aes_encrypt(string_col, '1234567890123456',
-'AES_128_GCM', '1234567890123456'), '1234567890123456', 'AES_128_GCM',
'1234567890123456');
----- RESULTS
-7300
----- TYPES
-BIGINT
-====
----- QUERY
-select count(*) from functional_parquet.alltypes where CAST(timestamp_col AS
STRING) =
-aes_decrypt(aes_encrypt(CAST(timestamp_col AS STRING), '1234567890123456',
'AES_128_GCM', '1234567890123456'),
-'1234567890123456', 'AES_128_GCM', '1234567890123456');
----- RESULTS
-7300
----- TYPES
-BIGINT
-====
\ No newline at end of file
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_128_ecb.test
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_128_ecb.test
new file mode 100644
index 000000000..f7a0dd887
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_128_ecb.test
@@ -0,0 +1,23 @@
+====
+---- QUERY
+# AES encryption/decryption examples:
+select
aes_decrypt(base64decode('y6Ss+zCYObpCbgfWfyNWTw=='),'1234567890123456','AES_128_ECB','');
+---- RESULTS
+'ABC'
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('BQGHoM3lqYcsurCRq3PlUw=='),'1234567890123456','AES_128_ECB','');
+---- RESULTS
+''
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('ofMshL41XkG6NQ+v6bawAQ=='),'1234567890123456','AES_128_ECB','');
+---- RESULTS
+'Hello World!'
+---- TYPES
+STRING
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_128_gcm.test
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_128_gcm.test
new file mode 100644
index 000000000..fe800a70b
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_128_gcm.test
@@ -0,0 +1,84 @@
+====
+---- QUERY
+select base64encode(aes_encrypt('ABC',
'1234567890123456','AES_128_GCM','1234567890123456'));
+---- RESULTS
+'x+am+BIqtrEK9FpC/zrvpOycjQ=='
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('x+am+BIqtrEK9FpC/zrvpOycjQ=='),'1234567890123456','AES_128_GCM','1234567890123456');
+---- RESULTS
+'ABC'
+---- TYPES
+STRING
+====
+---- QUERY
+select base64encode(aes_encrypt('',
'1234567890123456','AES_128_GCM','1234567890123456'));
+---- RESULTS
+'moMhTz224yot8uRtksO+pw=='
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('moMhTz224yot8uRtksO+pw=='),'1234567890123456','AES_128_GCM','1234567890123456');
+---- RESULTS
+''
+---- TYPES
+STRING
+====
+---- QUERY
+select count(*) from functional.alltypes where string_col =
aes_decrypt(aes_encrypt(string_col, '1234567890123456',
+'AES_128_GCM', '1234567890123456'), '1234567890123456', 'AES_128_GCM',
'1234567890123456');
+---- RESULTS
+7300
+---- TYPES
+BIGINT
+====
+---- QUERY
+select count(*) from functional_parquet.alltypes where CAST(timestamp_col AS
STRING) =
+aes_decrypt(aes_encrypt(CAST(timestamp_col AS STRING), '1234567890123456',
'AES_128_GCM', '1234567890123456'),
+'1234567890123456', 'AES_128_GCM', '1234567890123456');
+---- RESULTS
+7300
+---- TYPES
+BIGINT
+====
+---- QUERY
+select base64encode(aes_encrypt('The quick brown fox jumps over the lazy dog',
'1234567890123456','AES_128_GCM',
+'1234567890123456'));
+---- RESULTS
+'0syAnPYY0qT/RNtD7s0UkV/4P7HSl8xI1ZCz9AINV3r5W0xLkUs/gAlXWD+i6TIx+WiWiuipZJMXLmM='
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('0syAnPYY0qT/RNtD7s0UkV/4P7HSl8xI1ZCz9AINV3r5W0xLkUs/gAlXWD+i6TIx+WiWiuipZJMXLmM='),
+'1234567890123456','AES_128_GCM','1234567890123456');
+---- RESULTS
+'The quick brown fox jumps over the lazy dog'
+---- TYPES
+STRING
+====
+---- QUERY
+select base64encode(aes_encrypt('Impala',
'1234567890123456','AES_128_GCM','12345678901'));
+---- RESULTS
+'sBDdvFMLqNtJvuHUP8s7oWaH0NWXNQ=='
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('sBDdvFMLqNtJvuHUP8s7oWaH0NWXNQ=='),'1234567890123456','AES_128_GCM','12345678901');
+---- RESULTS
+'Impala'
+---- TYPES
+STRING
+====
+---- QUERY
+select aes_decrypt(aes_encrypt('Impala',
'1234567890123456','AES_128_GCM','12345678901'),
+'1234567890123456','AES_128_GCM','12345678901');
+---- RESULTS
+'Impala'
+---- TYPES
+STRING
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_cfb.test
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_cfb.test
new file mode 100644
index 000000000..e0bb171f1
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_cfb.test
@@ -0,0 +1,79 @@
+====
+---- QUERY
+select base64encode(aes_encrypt('impalaaaaaaaaaaaaaaaaaaaa',
'12345678901234567890123456789012','AES_256_CFB',
+'1234567890123456'));
+---- RESULTS
+'pkw4Y8WLkJnACfAL4R4BNUBzdnUeUo7vTw=='
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('pkw4Y8WLkJnACfAL4R4BNUBzdnUeUo7vTw=='),'12345678901234567890123456789012',
+'AES_256_CFB','1234567890123456');
+---- RESULTS
+'impalaaaaaaaaaaaaaaaaaaaa'
+---- TYPES
+STRING
+====
+---- QUERY
+select base64encode(aes_encrypt('',
'12345678901234567890123456789012','AES_256_CFB','1234567890123456'));
+---- RESULTS
+''
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode(''),'12345678901234567890123456789012','AES_256_CFB','1234567890123456');
+---- RESULTS
+''
+---- TYPES
+STRING
+====
+---- QUERY
+select aes_decrypt((aes_encrypt('impalaaaaaaaaaaaaaaaaaaaa',
'12345678901234567890123456789012','AES_256_CFB',
+'1234567890123456')),'12345678901234567890123456789012','AES_256_CFB','1234567890123456');
+---- RESULTS
+'impalaaaaaaaaaaaaaaaaaaaa'
+---- TYPES
+STRING
+====
+---- QUERY
+select base64encode(aes_encrypt(NULL,
'12345678901234567890123456789012','AES_256_CFB','1234567890123456'));
+---- RESULTS
+'NULL'
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(NULL,'12345678901234567890123456789012','AES_256_CFB','1234567890123456');
+---- RESULTS
+'NULL'
+---- TYPES
+STRING
+====
+---- QUERY
+# Key cannot be NULL.
+select base64encode(aes_encrypt('ABC',NULL,'AES_256_CFB','1234567890123456'));
+---- RESULTS
+---- CATCH
+Key cannot be NULL.
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),NULL,'AES_256_CFB','1234567890123456');
+---- RESULTS
+---- CATCH
+UDF ERROR: Key cannot be NULL.
+====
+---- QUERY
+# iv cannot be NULL for CFB mode.
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_CFB',NULL));
+---- RESULTS
+---- CATCH
+IV vector required for AES_256_CFB mode
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_CFB',NULL);
+---- RESULTS
+---- CATCH
+UDF ERROR: IV vector required for AES_256_CFB mode
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_ctr.test
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_ctr.test
new file mode 100644
index 000000000..2443b1c01
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_ctr.test
@@ -0,0 +1,66 @@
+====
+---- QUERY
+select base64encode(aes_encrypt('impalaaaaaaaaaaaaaaaaaaaa',
'12345678901234567890123456789012','AES_256_CTR',
+'1234567890123456'));
+---- RESULTS
+'pkw4Y8WLkJnACfAL4R4BNb0HYMEjpcA63A=='
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('pkw4Y8WLkJnACfAL4R4BNb0HYMEjpcA63A=='),'12345678901234567890123456789012',
+'AES_256_CTR','1234567890123456');
+---- RESULTS
+'impalaaaaaaaaaaaaaaaaaaaa'
+---- TYPES
+STRING
+====
+---- QUERY
+select base64encode(aes_encrypt('',
'12345678901234567890123456789012','AES_256_CTR','1234567890123456'));
+---- RESULTS
+''
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode(''),'12345678901234567890123456789012','AES_256_CTR','1234567890123456');
+---- RESULTS
+''
+---- TYPES
+STRING
+====
+---- QUERY
+select aes_decrypt((aes_encrypt('impalaaaaaaaaaaaaaaaaaaaa',
'12345678901234567890123456789012','AES_256_CTR',
+'1234567890123456')),'12345678901234567890123456789012','AES_256_CTR','1234567890123456');
+---- RESULTS
+'impalaaaaaaaaaaaaaaaaaaaa'
+---- TYPES
+STRING
+====
+---- QUERY
+select base64encode(aes_encrypt(NULL,
'12345678901234567890123456789012','AES_256_CTR','1234567890123456'));
+---- RESULTS
+'NULL'
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(NULL,'12345678901234567890123456789012','AES_256_CTR','1234567890123456');
+---- RESULTS
+'NULL'
+---- TYPES
+STRING
+====
+---- QUERY
+# iv cannot be NULL for CTR mode.
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_CTR',NULL));
+---- RESULTS
+---- CATCH
+IV vector required for AES_256_CTR mode
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_CTR',NULL);
+---- RESULTS
+---- CATCH
+UDF ERROR: IV vector required for AES_256_CTR mode
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_ecb.test
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_ecb.test
new file mode 100644
index 000000000..5c1e9db4e
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_ecb.test
@@ -0,0 +1,15 @@
+====
+---- QUERY
+select
aes_decrypt(base64decode('E1zl+pDv/GY4JLk254KAIQ=='),'12345678901234567890123456789012','AES_256_ECB','');
+---- RESULTS
+'ABC'
+---- TYPES
+STRING
+====
+---- QUERY
+# ECB not supported for AES encryption.
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_ECB','1234567890123456'));
+---- RESULTS
+---- CATCH
+ECB mode is not supported for encryption.
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_gcm.test
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_gcm.test
new file mode 100644
index 000000000..d79371894
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_aes_256_gcm.test
@@ -0,0 +1,140 @@
+====
+---- QUERY
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM','1234567890123456'));
+---- RESULTS
+'F/DLkSwEikFOlqzXVCysy1JX7Q=='
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_GCM','1234567890123456');
+---- RESULTS
+'ABC'
+---- TYPES
+STRING
+====
+---- QUERY
+select aes_decrypt(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM','1234567890123456'),
+'12345678901234567890123456789012','AES_256_GCM','1234567890123456');
+---- RESULTS
+'ABC'
+---- TYPES
+STRING
+====
+---- QUERY
+select
base64encode(aes_encrypt(aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),
+'12345678901234567890123456789012','AES_256_GCM','1234567890123456'),'12345678901234567890123456789012','AES_256_GCM',
+'1234567890123456'));
+---- RESULTS
+'F/DLkSwEikFOlqzXVCysy1JX7Q=='
+---- TYPES
+STRING
+====
+---- QUERY
+select aes_decrypt(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM','1234567890123456'),
+'12345678901234567890123456789012',NULL,'1234567890123456');
+---- RESULTS
+'ABC'
+---- TYPES
+STRING
+====
+---- QUERY
+# Encryption/ decryption with expr as NULL.
+select base64encode(aes_encrypt(NULL,
'12345678901234567890123456789012','AES_256_GCM','1234567890123456'));
+---- RESULTS
+'NULL'
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(NULL,'12345678901234567890123456789012','AES_256_GCM','1234567890123456');
+---- RESULTS
+'NULL'
+---- TYPES
+STRING
+====
+---- QUERY
+# Key cannot be NULL, with the default mode GCM.
+select base64encode(aes_encrypt('ABC',NULL,'AES_256_GCM','1234567890123456'));
+---- RESULTS
+---- CATCH
+Key cannot be NULL.
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),NULL,'AES_256_GCM','1234567890123456');
+---- RESULTS
+---- CATCH
+UDF ERROR: Key cannot be NULL.
+====
+---- QUERY
+# iv cannot be NULL for GCM mode.
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM',NULL));
+---- RESULTS
+---- CATCH
+IV vector required for AES_256_GCM mode
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_GCM',NULL);
+---- RESULTS
+---- CATCH
+UDF ERROR: IV vector required for AES_256_GCM mode
+====
+---- QUERY
+# Error with incorrect key length.
+select base64encode(aes_encrypt('ABC',
'123456789012345678901234567890121','AES_256_GCM','1234567890123456'));
+---- RESULTS
+---- CATCH
+AES only supports 128 and 256 bit key lengths
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'123456789012345678901234567890121','AES_256_GCM',
+'1234567890123456');
+---- RESULTS
+---- CATCH
+UDF ERROR: AES only supports 128 and 256 bit key lengths
+====
+---- QUERY
+# Error with incorrect iv length.
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GCM','12345678901234567'));
+---- RESULTS
+---- CATCH
+IV vector size is greater than 16 bytes
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_GCM',
+'12345678901234567');
+---- RESULTS
+---- CATCH
+UDF ERROR: IV vector size is greater than 16 bytes
+====
+---- QUERY
+# Modes are case-insensitive.
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','aes_256_gcm','1234567890123456'));
+---- RESULTS
+'F/DLkSwEikFOlqzXVCysy1JX7Q=='
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','aes_256_gcm',
+'1234567890123456');
+---- RESULTS
+'ABC'
+---- TYPES
+STRING
+====
+---- QUERY
+# Encryption/ decryption when mode is NULL, defaulting to GCM mode.
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012',NULL,'1234567890123456'));
+---- RESULTS
+'F/DLkSwEikFOlqzXVCysy1JX7Q=='
+---- TYPES
+STRING
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012',NULL,'1234567890123456');
+---- RESULTS
+'ABC'
+---- TYPES
+STRING
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_errors.test
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_errors.test
new file mode 100644
index 000000000..869b120fd
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/encryption_exprs_errors.test
@@ -0,0 +1,55 @@
+====
+---- QUERY
+# Error resulting due to user entered incorrect mode.
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_CTB','1234567890123456'));
+---- RESULTS
+---- CATCH
+Invalid AES 'mode': AES_256_CTB
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_CTB',
+'1234567890123456');
+---- RESULTS
+---- CATCH
+UDF ERROR: Invalid AES 'mode': AES_256_CTB
+====
+---- QUERY
+# Prefixes of supported modes are not accepted
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','','1234567890123456'));
+---- RESULTS
+---- CATCH
+Invalid AES 'mode':
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','',
+'1234567890123456');
+---- RESULTS
+---- CATCH
+UDF ERROR: Invalid AES 'mode':
+====
+---- QUERY
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256','1234567890123456'));
+---- RESULTS
+---- CATCH
+Invalid AES 'mode': AES_256
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256',
+'1234567890123456');
+---- RESULTS
+---- CATCH
+UDF ERROR: Invalid AES 'mode': AES_256
+====
+---- QUERY
+select base64encode(aes_encrypt('ABC',
'12345678901234567890123456789012','AES_256_GC','1234567890123456'));
+---- RESULTS
+---- CATCH
+Invalid AES 'mode': AES_256_GC
+====
+---- QUERY
+select
aes_decrypt(base64decode('F/DLkSwEikFOlqzXVCysy1JX7Q=='),'12345678901234567890123456789012','AES_256_GC',
+'1234567890123456');
+---- RESULTS
+---- CATCH
+UDF ERROR: Invalid AES 'mode': AES_256_GC
+====
diff --git a/tests/query_test/test_exprs.py b/tests/query_test/test_exprs.py
index bdfa188bb..51f976d04 100644
--- a/tests/query_test/test_exprs.py
+++ b/tests/query_test/test_exprs.py
@@ -17,6 +17,7 @@
from __future__ import absolute_import, division, print_function
from builtins import range
+import logging
import pytest
import re
from random import randint
@@ -27,6 +28,8 @@ from tests.common.test_dimensions import
create_uncompressed_text_dimension
from tests.common.test_vector import ImpalaTestDimension
from tests.util.test_file_parser import QueryTestSectionReader
+LOG = logging.getLogger('test_exprs')
+
class TestExprs(ImpalaTestSuite):
@classmethod
def get_workload(cls):
@@ -73,10 +76,66 @@ class TestExprs(ImpalaTestSuite):
self.run_test_case('QueryTest/special-strings', vector)
def test_encryption_exprs(self, vector):
- """Test handling encryption/ decryption functionality"""
+ """Test handling encryption/ decryption functionality.
+ Some AES operation modes are not supported by all versions of the OpenSSL
+ library, therefore the tests are divided into separate .test files based on
+ the mode used in them. For modes that may not be supported, we run a
+ probing query first and only run the test file if it succeeds.
+ """
vector.get_value('exec_option')['enable_expr_rewrites'] = \
vector.get_value('enable_expr_rewrites')
- self.run_test_case('QueryTest/encryption_exprs', vector)
+
+ # Run queries that are expected to fail, e.g. trying invalid operation
modes etc.
+ self.run_test_case('QueryTest/encryption_exprs_errors', vector)
+
+ self.run_test_case('QueryTest/encryption_exprs_aes_128_ecb', vector)
+ self.run_test_case('QueryTest/encryption_exprs_aes_256_ecb', vector)
+ self.run_test_case('QueryTest/encryption_exprs_aes_256_cfb', vector)
+
+ aes_256_gcm_ok = self._check_aes_mode_supported("aes_256_gcm")
+ if aes_256_gcm_ok:
+ self.run_test_case('QueryTest/encryption_exprs_aes_256_gcm', vector)
+ self._log_whether_aes_tests_run("aes_256_gcm", aes_256_gcm_ok)
+
+ aes_128_gcm_ok = self._check_aes_mode_supported("aes_128_gcm")
+ if aes_128_gcm_ok:
+ self.run_test_case('QueryTest/encryption_exprs_aes_128_gcm', vector)
+ self._log_whether_aes_tests_run("aes_128_gcm", aes_128_gcm_ok)
+
+ aes_256_ctr_ok = self._check_aes_mode_supported("aes_256_ctr")
+ if aes_256_ctr_ok:
+ self.run_test_case('QueryTest/encryption_exprs_aes_256_ctr', vector)
+ self._log_whether_aes_tests_run("aes_256_ctr", aes_256_ctr_ok)
+
+ def _log_whether_aes_tests_run(self, mode, running):
+ msg = "{} {} tests because the OpenSSL version {} this mode.".format(
+ "Running" if running else "Not running",
+ mode,
+ "supports" if running else "does not support")
+ LOG.warning(msg)
+
+ def _check_aes_mode_supported(self, mode):
+ """Checks whether the given AES mode is supported in the current
+ environment (see "test_encryption_exprs()") by running a probing query."""
+ assert "ECB" not in mode.upper()
+
+ expr = "expr"
+ key_len_bytes = 32 if "256" in mode else 16
+ key = "A" * key_len_bytes
+
+ # GCM doesn't support an empty IV.
+ iv = "a"
+ query = 'select aes_encrypt("{expr}", "{key}", "{mode}", "{iv}")'.format(
+ expr=expr, key=key, mode=mode, iv=iv)
+
+ try:
+ res = self.execute_query(query)
+ assert res.success
+ return True
+ except Exception as e:
+ assert "not supported by OpenSSL" in str(e)
+ return False
+
# Tests very deep expression trees and expressions with many children. Impala
defines
# a 'safe' upper bound on the expr depth and the number of expr children in the
