Repository: incubator-hawq Updated Branches: refs/heads/master 71bb3676a -> 3bdc19464
HAWQ-1509. Support TDE read function. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/90f1c4de Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/90f1c4de Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/90f1c4de Branch: refs/heads/master Commit: 90f1c4ded4321adfa98a691a0b1b47f727316738 Parents: 71bb367 Author: amyrazz44 <a...@pivotal.io> Authored: Mon Aug 7 12:53:36 2017 +0800 Committer: amyrazz44 <a...@pivotal.io> Committed: Mon Aug 7 12:53:36 2017 +0800 ---------------------------------------------------------------------- depends/libhdfs3/src/client/CryptoCodec.cpp | 37 ++++++++++-------- depends/libhdfs3/src/client/CryptoCodec.h | 10 +++++ depends/libhdfs3/src/client/InputStreamImpl.cpp | 41 +++++++++++++++++++- depends/libhdfs3/src/client/InputStreamImpl.h | 26 +++++++++++++ 4 files changed, 97 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90f1c4de/depends/libhdfs3/src/client/CryptoCodec.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/CryptoCodec.cpp b/depends/libhdfs3/src/client/CryptoCodec.cpp index 77ccf09..417f51e 100644 --- a/depends/libhdfs3/src/client/CryptoCodec.cpp +++ b/depends/libhdfs3/src/client/CryptoCodec.cpp @@ -101,15 +101,15 @@ namespace Hdfs { } int CryptoCodec::init(CryptoMethod crypto_method, int64_t stream_offset) { - //check already init + // Check CryptoCodec init or not. if (is_init) return 0; - // Get decrypted key from KMS - std::string key = getDecryptedKeyFromKms(); + // Get decrypted key from KMS. + decryptedKey = getDecryptedKeyFromKms(); - // Select cipher method based on the key length - uint64_t AlgorithmBlockSize = key.length(); + // Select cipher method based on the decrypted key length. + AlgorithmBlockSize = decryptedKey.length(); if (AlgorithmBlockSize == KEY_LENGTH_256) { cipher = EVP_aes_256_ctr(); } else if (AlgorithmBlockSize == KEY_LENGTH_128) { @@ -119,7 +119,17 @@ namespace Hdfs { return -1; } - //calculate new IV when appending a existed file + // Calculate iv and counter in order to init cipher context with cipher method. Default value is 0. + resetStreamOffset(crypto_method, stream_offset); + + LOG(DEBUG3, "CryptoCodec init success, length of the decrypted key is : %llu, crypto method is : %d", AlgorithmBlockSize, crypto_method); + is_init = true; + return 1; + + } + + int CryptoCodec::resetStreamOffset(CryptoMethod crypto_method, int64_t stream_offset) { + // Calculate new IV when appending an existed file. std::string iv = encryptionInfo->getIv(); if (stream_offset > 0) { counter = stream_offset / AlgorithmBlockSize; @@ -127,25 +137,20 @@ namespace Hdfs { iv = this->calculateIV(iv, counter); } - //judge encrypt/decrypt - int enc = 0; - method = crypto_method; - if (method == CryptoMethod::ENCRYPT) - enc = 1; + // Judge the crypto method is encrypt or decrypt. + int enc = (method == CryptoMethod::ENCRYPT) ? 1 : 0; - // Init cipher context with cipher method + // Init cipher context with cipher method. if (!EVP_CipherInit_ex(cipherCtx, cipher, NULL, - (const unsigned char *) key.c_str(), (const unsigned char *) iv.c_str(), + (const unsigned char *) decryptedKey.c_str(), (const unsigned char *) iv.c_str(), enc)) { LOG(WARNING, "EVP_CipherInit_ex failed"); return -1; } - //AES/CTR/NoPadding + // AES/CTR/NoPadding, set padding to 0. EVP_CIPHER_CTX_set_padding(cipherCtx, 0); - LOG(DEBUG3, "CryptoCodec init success, key_length:%llu, is_encode:%d", AlgorithmBlockSize, enc); - is_init = true; return 1; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90f1c4de/depends/libhdfs3/src/client/CryptoCodec.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/CryptoCodec.h b/depends/libhdfs3/src/client/CryptoCodec.h index 45b1088..ee68637 100644 --- a/depends/libhdfs3/src/client/CryptoCodec.h +++ b/depends/libhdfs3/src/client/CryptoCodec.h @@ -71,6 +71,14 @@ namespace Hdfs { */ virtual int init(CryptoMethod crypto_method, int64_t stream_offset = 0); + /** + * Reset iv and padding value when seek file. + * @param crypto_method do encrypt/decrypt work according to crypto_method. + * @param stream_offset the offset of the current file. + * @return 1 sucess; 0 already initialized; -1 failed. + */ + virtual int resetStreamOffset(CryptoMethod crypto_method, int64_t stream_offset); + private: /** @@ -96,6 +104,8 @@ namespace Hdfs { int32_t bufSize; int64_t padding; int64_t counter; + std::string decryptedKey; + uint64_t AlgorithmBlockSize; }; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90f1c4de/depends/libhdfs3/src/client/InputStreamImpl.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/InputStreamImpl.cpp b/depends/libhdfs3/src/client/InputStreamImpl.cpp index 05f27a5..c899a83 100644 --- a/depends/libhdfs3/src/client/InputStreamImpl.cpp +++ b/depends/libhdfs3/src/client/InputStreamImpl.cpp @@ -432,6 +432,25 @@ void InputStreamImpl::openInternal(shared_ptr<FileSystemInter> fs, const char * peerCache = &fs->getPeerCache(); updateBlockInfos(); closed = false; + /* If file is encrypted , then initialize CryptoCodec. */ + fileStatus = fs->getFileStatus(this->path.c_str()); + FileEncryptionInfo *fileEnInfo = fileStatus.getFileEncryption(); + if (fileStatus.isFileEncrypted()) { + if (cryptoCodec == NULL) { + enAuth = shared_ptr<RpcAuth> ( + new RpcAuth(fs->getUserInfo(), RpcAuth::ParseMethod(conf->getKmsMethod()))); + kcp = shared_ptr<KmsClientProvider> ( + new KmsClientProvider(enAuth, conf)); + cryptoCodec = shared_ptr<CryptoCodec> ( + new CryptoCodec(fileEnInfo, kcp, conf->getCryptoBufferSize())); + + int64_t file_length = 0; + int ret = cryptoCodec->init(CryptoMethod::DECRYPT, file_length); + if (ret < 0) { + THROW(HdfsIOException, "init CryptoCodec failed, file:%s", this->path.c_str()); + } + } + } } catch (const HdfsCanceled & e) { throw; } catch (const FileNotFoundException & e) { @@ -626,6 +645,12 @@ int32_t InputStreamImpl::readInternal(char * buf, int32_t size) { continue; } + std::string bufDecode; + if (fileStatus.isFileEncrypted()) { + /* Decrypt buffer if the file is encrypted. */ + bufDecode = cryptoCodec->cipher_wrap(buf, size); + memcpy(buf, bufDecode.c_str(), size); + } return retval; } while (true); @@ -734,9 +759,17 @@ void InputStreamImpl::seekInternal(int64_t pos) { } try { - if (blockReader && pos > cursor && pos < endOfCurBlock) { + if (blockReader && pos > cursor && pos < endOfCurBlock && (pos - cursor) < blockReader->available()) { blockReader->skip(pos - cursor); cursor = pos; + if (cryptoCodec) { + int ret = cryptoCodec->resetStreamOffset(CryptoMethod::DECRYPT, + cursor); + if (ret < 0) { + THROW(HdfsIOException, "init CryptoCodec failed, file:%s", + this->path.c_str()); + } + } return; } } catch (const HdfsIOException & e) { @@ -758,6 +791,12 @@ void InputStreamImpl::seekInternal(int64_t pos) { endOfCurBlock = 0; blockReader.reset(); cursor = pos; + if (cryptoCodec) { + int ret = cryptoCodec->resetStreamOffset(CryptoMethod::DECRYPT, cursor); + if (ret < 0) { + THROW(HdfsIOException, "init CryptoCodec failed, file:%s", this->path.c_str()); + } + } } /** http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90f1c4de/depends/libhdfs3/src/client/InputStreamImpl.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/InputStreamImpl.h b/depends/libhdfs3/src/client/InputStreamImpl.h index e3c55ce..12a8e08 100644 --- a/depends/libhdfs3/src/client/InputStreamImpl.h +++ b/depends/libhdfs3/src/client/InputStreamImpl.h @@ -37,6 +37,8 @@ #include "server/LocatedBlocks.h" #include "SessionConfig.h" #include "Unordered.h" +#include "CryptoCodec.h" +#include "KmsClientProvider.h" #ifdef MOCK #include "TestDatanodeStub.h" @@ -101,6 +103,26 @@ public: * @return return a printable string */ std::string toString(); + + /** + * Get KmsClientProvider. + */ + shared_ptr<KmsClientProvider> getKmsClientProvider(); + + /** + * Set KmsClientProvider. + */ + void setKmsClientProvider(shared_ptr<KmsClientProvider> kcp); + + /** + * Get CryptoCodec. + */ + shared_ptr<CryptoCodec> getCryptoCodec(); + + /** + * Set CryptoCodec. + */ + void setCryptoCodec(shared_ptr<CryptoCodec> cryptoCodec); private: bool choseBestNode(); @@ -141,6 +163,10 @@ private: std::string path; std::vector<DatanodeInfo> failedNodes; std::vector<char> localReaderBuffer; + shared_ptr<CryptoCodec> cryptoCodec; + shared_ptr<KmsClientProvider> kcp; + shared_ptr<RpcAuth> enAuth; + FileStatus fileStatus; #ifdef MOCK private:
