This is an automated email from the ASF dual-hosted git repository.
abukor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new feaccee [security] KUDU-3316 Add encrypted file keys
feaccee is described below
commit feaccee2b82ebd47d8244a1fdf346c1394c8ae91
Author: Attila Bukor <[email protected]>
AuthorDate: Sun Jan 30 00:53:19 2022 +0100
[security] KUDU-3316 Add encrypted file keys
The previous patches in the data at rest encryption saga add the ability
to encrypt data at rest, but files were encrypted using the same hard-coded
key.
This patch adds an extra header to encrypted files to store the
encryption algorithm used and the encrypted file key. For now, the file
keys are encrypted with the same dummy encryption key which was
previously used to encrypt the files.
The header is a bit different from the one described in the design doc:
The encryption algorithm and key length was changed to be stored in
1 byte instead of 2 for easier handling and the magic string is
"kuduenc" instead of "kuduen".
This patch also introduces a new flag which is hidden for now:
--encryption_key_length. This can be set to any valid AES key length as
per its specification (128, 192, or 256 bits), as only AES encryption is
supported for now, and there are no plans to support anything else in
the foreseeable future.
As we add a 64-byte header to encrypted files, some changes had to be
made to code that handles files and relies on sizes and offsets,
including in tests. One of these changes is in the LogBlockManager,
which expects blocks to be aligned to file system block boundaries,
which is necessary for hole punching. With encryption enabled, extra
steps are necessary to align the first block, and aligning blocks makes
it impractical to hide encryption header size within Env and use
logical file sizes outside of it.
This commit also changes the PBC tool to check if a file is encrypted
based on the encryption header instead of the file name.
I ran the full test suite manually locally and on dist-test with
encryption enabled to make sure turning on encryption doesn't break
anything:
http://dist-test.cloudera.org/job?job_id=abukor.1643215963.60435
To make running dist-test with encryption enabled possible, this commit
also adds forwarding the KUDU_ENCRYPT_DATA_IN_TESTS environment variable
to dist_test.py.
Change-Id: Idb1282c117271fda63a8cc54c00add7cc96dcffd
Reviewed-on: http://gerrit.cloudera.org:8080/18025
Reviewed-by: Alexey Serbin <[email protected]>
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <[email protected]>
---
build-support/dist_test.py | 1 +
src/kudu/consensus/consensus_meta-test.cc | 2 +-
src/kudu/consensus/log-test.cc | 2 +-
src/kudu/consensus/log_index.cc | 4 +-
src/kudu/consensus/log_util.cc | 14 +-
src/kudu/fs/dir_util.cc | 3 +
src/kudu/fs/file_block_manager.cc | 3 +-
src/kudu/fs/log_block_manager-test-util.cc | 3 +-
src/kudu/fs/log_block_manager-test.cc | 29 +-
src/kudu/fs/log_block_manager.cc | 36 +-
src/kudu/mini-cluster/external_mini_cluster.cc | 3 +-
src/kudu/tools/kudu-tool-test.cc | 267 ++++++++-------
src/kudu/tools/tool_action_pbc.cc | 11 +-
src/kudu/tserver/tablet_copy_client-test.cc | 12 +-
src/kudu/tserver/tablet_copy_service-test.cc | 8 +-
.../tserver/tablet_copy_source_session-test.cc | 4 +-
src/kudu/tserver/tablet_copy_source_session.cc | 8 +-
src/kudu/util/env-test.cc | 51 +--
src/kudu/util/env.cc | 13 +
src/kudu/util/env.h | 23 +-
src/kudu/util/env_posix.cc | 364 ++++++++++++++++++---
src/kudu/util/env_util.cc | 23 +-
src/kudu/util/env_util.h | 4 +
src/kudu/util/file_cache-stress-test.cc | 16 +-
src/kudu/util/file_cache-test.cc | 27 +-
src/kudu/util/file_cache.cc | 12 +
src/kudu/util/pb_util-test.cc | 20 +-
src/kudu/util/pb_util.cc | 7 +-
28 files changed, 715 insertions(+), 255 deletions(-)
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index dae260d..00d4c78 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -389,6 +389,7 @@ def create_archive_input(staging, execution, dep_extractor,
command = ['../../build-support/run_dist_test.py',
'-e', 'KUDU_TEST_TIMEOUT=%d' % (TEST_TIMEOUT_SECS - 30),
'-e', 'KUDU_ALLOW_SLOW_TESTS=%s' %
os.environ.get('KUDU_ALLOW_SLOW_TESTS', 1),
+ '-e', 'KUDU_ENCRYPT_DATA_IN_TESTS=%s' %
os.environ.get('KUDU_ENCRYPT_DATA_IN_TESTS', 0),
'-e', 'KUDU_COMPRESS_TEST_OUTPUT=%s' % \
os.environ.get('KUDU_COMPRESS_TEST_OUTPUT', 0)]
for k, v in execution.env.items():
diff --git a/src/kudu/consensus/consensus_meta-test.cc
b/src/kudu/consensus/consensus_meta-test.cc
index 50e26ae..61d6f4f 100644
--- a/src/kudu/consensus/consensus_meta-test.cc
+++ b/src/kudu/consensus/consensus_meta-test.cc
@@ -80,7 +80,7 @@ class ConsensusMetadataTest : public KuduTest, public
::testing::WithParamInterf
void EnableEncryption(bool enable) {
- FLAGS_encrypt_data_at_rest = true;
+ FLAGS_encrypt_data_at_rest = enable;
}
FsManager fs_manager_;
diff --git a/src/kudu/consensus/log-test.cc b/src/kudu/consensus/log-test.cc
index 13db7eb..c3191cd 100644
--- a/src/kudu/consensus/log-test.cc
+++ b/src/kudu/consensus/log-test.cc
@@ -1073,7 +1073,7 @@ TEST_F(LogTest, TestGetGCableDataSize) {
const int kNumTotalSegments = 5;
const int kNumOpsPerSegment = 5;
- const int kSegmentSizeBytes = 331;
+ const int kSegmentSizeBytes = 331 + env_->GetEncryptionHeaderSize();
OpId op_id = MakeOpId(1, 10);
// Create 5 segments, starting from log index 10, with 5 ops per segment.
// [10-14], [15-19], [20-24], [25-29], [30-34]
diff --git a/src/kudu/consensus/log_index.cc b/src/kudu/consensus/log_index.cc
index 18d160d..56916c6 100644
--- a/src/kudu/consensus/log_index.cc
+++ b/src/kudu/consensus/log_index.cc
@@ -121,7 +121,7 @@ Status LogIndex::IndexChunk::GetEntry(int entry_index,
PhysicalEntry* ret) const
DCHECK_LT(entry_index, kEntriesPerIndexChunk);
Slice s(reinterpret_cast<const uint8_t*>(ret), sizeof(PhysicalEntry));
- return file_->Read(sizeof(PhysicalEntry) * entry_index, s);
+ return file_->Read(file_->GetEncryptionHeaderSize() + sizeof(PhysicalEntry)
* entry_index, s);
}
Status LogIndex::IndexChunk::SetEntry(int entry_index, const PhysicalEntry&
entry) {
@@ -129,7 +129,7 @@ Status LogIndex::IndexChunk::SetEntry(int entry_index,
const PhysicalEntry& entr
DCHECK_LT(entry_index, kEntriesPerIndexChunk);
Slice s(reinterpret_cast<const uint8_t*>(&entry), sizeof(PhysicalEntry));
- return file_->Write(sizeof(PhysicalEntry) * entry_index, s);
+ return file_->Write(file_->GetEncryptionHeaderSize() + sizeof(PhysicalEntry)
* entry_index, s);
}
////////////////////////////////////////////////////////////
diff --git a/src/kudu/consensus/log_util.cc b/src/kudu/consensus/log_util.cc
index 2455fa8..abcc0ee 100644
--- a/src/kudu/consensus/log_util.cc
+++ b/src/kudu/consensus/log_util.cc
@@ -438,9 +438,10 @@ Status ReadableLogSegment::ReadHeader() {
Slice header_slice(header_space, header_size);
LogSegmentHeaderPB header;
+ uint8_t offset = file_->GetEncryptionHeaderSize() +
kLogSegmentHeaderMagicAndHeaderLength;
+
// Read and parse the log segment header.
- RETURN_NOT_OK_PREPEND(file_->Read(kLogSegmentHeaderMagicAndHeaderLength,
- header_slice),
+ RETURN_NOT_OK_PREPEND(file_->Read(offset, header_slice),
"Unable to read fully");
RETURN_NOT_OK_PREPEND(pb_util::ParseFromArray(&header,
@@ -454,7 +455,7 @@ Status ReadableLogSegment::ReadHeader() {
}
header_ = std::move(header);
- first_entry_offset_ = header_size + kLogSegmentHeaderMagicAndHeaderLength;
+ first_entry_offset_ = header_size + offset;
return Status::OK();
}
@@ -463,7 +464,7 @@ Status ReadableLogSegment::ReadHeader() {
Status ReadableLogSegment::ReadHeaderMagicAndHeaderLength(uint32_t *len) const
{
uint8_t scratch[kLogSegmentHeaderMagicAndHeaderLength];
Slice slice(scratch, kLogSegmentHeaderMagicAndHeaderLength);
- RETURN_NOT_OK(file_->Read(0, slice));
+ RETURN_NOT_OK(file_->Read(file_->GetEncryptionHeaderSize(), slice));
RETURN_NOT_OK(ParseHeaderMagicAndHeaderLength(slice, len));
return Status::OK();
}
@@ -786,10 +787,11 @@ Status WritableLogSegment::WriteHeader(const
LogSegmentHeaderPB& new_header) {
PutFixed32(&buf, static_cast<uint32_t>(new_header.ByteSizeLong()));
// Then Serialize the PB.
pb_util::AppendToString(new_header, &buf);
- RETURN_NOT_OK(file_->Write(0, Slice(buf)));
+ uint8_t offset = file_->GetEncryptionHeaderSize();
+ RETURN_NOT_OK(file_->Write(offset, Slice(buf)));
header_.CopyFrom(new_header);
- first_entry_offset_ = buf.size();
+ first_entry_offset_ = buf.size() + offset;
written_offset_ = first_entry_offset_;
is_header_written_ = true;
diff --git a/src/kudu/fs/dir_util.cc b/src/kudu/fs/dir_util.cc
index 8d1a8a8..75aa764 100644
--- a/src/kudu/fs/dir_util.cc
+++ b/src/kudu/fs/dir_util.cc
@@ -70,6 +70,9 @@ Status CheckHolePunch(Env* env, const string& path) {
string filename = JoinPathSegments(path, "hole_punch_test_file");
unique_ptr<RWFile> file;
RWFileOptions opts;
+ // Encrypted files are larger due to the header size, which causes the file
+ // size calculations below to be off. We don't need the file to be encrypted
+ // for the hole punch test.
opts.is_sensitive = false;
RETURN_NOT_OK(env->NewRWFile(opts, filename, &file));
diff --git a/src/kudu/fs/file_block_manager.cc
b/src/kudu/fs/file_block_manager.cc
index 89cf6c8..ec2648a 100644
--- a/src/kudu/fs/file_block_manager.cc
+++ b/src/kudu/fs/file_block_manager.cc
@@ -519,6 +519,7 @@ Status FileReadableBlock::Size(uint64_t* sz) const {
DCHECK(!closed_.Load());
RETURN_NOT_OK_HANDLE_ERROR(reader_->Size(sz));
+ *sz -= reader_->GetEncryptionHeaderSize();
return Status::OK();
}
@@ -529,7 +530,7 @@ Status FileReadableBlock::Read(uint64_t offset, Slice
result) const {
Status FileReadableBlock::ReadV(uint64_t offset, ArrayView<Slice> results)
const {
DCHECK(!closed_.Load());
- RETURN_NOT_OK_HANDLE_ERROR(reader_->ReadV(offset, results));
+ RETURN_NOT_OK_HANDLE_ERROR(reader_->ReadV(offset +
reader_->GetEncryptionHeaderSize(), results));
if (block_manager_->metrics_) {
// Calculate the read amount of data
diff --git a/src/kudu/fs/log_block_manager-test-util.cc
b/src/kudu/fs/log_block_manager-test-util.cc
index f255698..0715444 100644
--- a/src/kudu/fs/log_block_manager-test-util.cc
+++ b/src/kudu/fs/log_block_manager-test-util.cc
@@ -92,8 +92,7 @@ Status LBMCorruptor::Init() {
// File size is an imprecise proxy for whether a container is full, but
// it should be good enough.
uint64_t data_file_size;
- RETURN_NOT_OK(env_->GetFileSize(e.second.data_filename,
- &data_file_size));
+ RETURN_NOT_OK(env_->GetFileSize(e.second.data_filename,
&data_file_size));
if (data_file_size >= FLAGS_log_container_max_size) {
full_containers.push_back(e.second);
}
diff --git a/src/kudu/fs/log_block_manager-test.cc
b/src/kudu/fs/log_block_manager-test.cc
index 638d310..e01dda3 100644
--- a/src/kudu/fs/log_block_manager-test.cc
+++ b/src/kudu/fs/log_block_manager-test.cc
@@ -350,7 +350,10 @@ TEST_P(LogBlockManagerTest, MetricsTest) {
// Lower the max container size so that we can more easily test full
// container metrics.
- FLAGS_log_container_max_size = 1024;
+ // TODO(abukor): If this is 1024, this becomes full when writing the first
+ // block because of alignments. If it is over 4k, it fails with encryption
+ // disabled due to having only 5 containers instead of 10. Investigate this.
+ FLAGS_log_container_max_size = GetParam() ? 8192 : 1024;
// One block --> one container.
unique_ptr<WritableBlock> writer;
@@ -821,9 +824,10 @@ TEST_P(LogBlockManagerTest, TestMetadataTruncation) {
uint64_t latest_meta_size;
ASSERT_OK(env_->GetFileSize(metadata_path, &latest_meta_size));
ASSERT_OK(env_->NewRandomAccessFile(raf_opts, metadata_path, &meta_file));
+ latest_meta_size -= meta_file->GetEncryptionHeaderSize();
unique_ptr<uint8_t[]> scratch(new uint8_t[latest_meta_size]);
Slice result(scratch.get(), latest_meta_size);
- ASSERT_OK(meta_file->Read(0, result));
+ ASSERT_OK(meta_file->Read(meta_file->GetEncryptionHeaderSize(), result));
string data = result.ToString();
// Flip the high bit of the length field, which is a 4-byte little endian
// unsigned integer. This will cause the length field to represent a large
@@ -892,7 +896,8 @@ TEST_P(LogBlockManagerTest, TestPreallocationAndTruncation)
{
ASSERT_OK(writer->Close());
uint64_t size_after_close;
ASSERT_OK(env_->GetFileSizeOnDisk(fname, &size_after_close));
- ASSERT_EQ(FLAGS_log_container_max_size, size_after_close);
+ ASSERT_GE(size_after_close, FLAGS_log_container_max_size);
+ ASSERT_LT(size_after_close, size_after_append);
// Now test the same startup behavior by artificially growing the file
// and reopening the block manager.
@@ -929,7 +934,7 @@ TEST_P(LogBlockManagerTest, TestPreallocationAndTruncation)
{
ASSERT_OK(ReopenBlockManager());
uint64_t size_after_reopen;
ASSERT_OK(env_->GetFileSizeOnDisk(fname, &size_after_reopen));
- ASSERT_EQ(FLAGS_log_container_max_size, size_after_reopen);
+ ASSERT_EQ(size_after_close, size_after_reopen);
}
}
@@ -1426,7 +1431,7 @@ TEST_P(LogBlockManagerTest, TestRepairUnpunchedBlocks) {
FLAGS_log_container_excess_space_before_cleanup_fraction = 0.0;
// Force our single container to become full once created.
- FLAGS_log_container_max_size = 0;
+ FLAGS_log_container_max_size = GetParam() ? 4096 : 0;
// Force the test to measure extra space in unpunched holes, not in the
// preallocation buffer.
@@ -1438,9 +1443,8 @@ TEST_P(LogBlockManagerTest, TestRepairUnpunchedBlocks) {
ASSERT_OK(block->Close());
string data_file;
NO_FATALS(GetOnlyContainerDataFile(&data_file));
- uint64_t file_size_on_disk;
- ASSERT_OK(env_->GetFileSizeOnDisk(data_file, &file_size_on_disk));
- ASSERT_EQ(0, file_size_on_disk);
+ uint64_t initial_file_size_on_disk;
+ ASSERT_OK(env_->GetFileSizeOnDisk(data_file, &initial_file_size_on_disk));
// Add some "unpunched blocks" to the container.
LBMCorruptor corruptor(env_, dd_manager_->GetDirs(), SeedRandom());
@@ -1449,8 +1453,9 @@ TEST_P(LogBlockManagerTest, TestRepairUnpunchedBlocks) {
ASSERT_OK(corruptor.AddUnpunchedBlockToFullContainer());
}
+ uint64_t file_size_on_disk;
ASSERT_OK(env_->GetFileSizeOnDisk(data_file, &file_size_on_disk));
- ASSERT_GT(file_size_on_disk, 0);
+ ASSERT_GT(file_size_on_disk, initial_file_size_on_disk);
// Check the report.
FsReport report;
@@ -1462,7 +1467,7 @@ TEST_P(LogBlockManagerTest, TestRepairUnpunchedBlocks) {
string container;
NO_FATALS(GetOnlyContainer(&container));
ASSERT_EQ(container, fcs.container);
- ASSERT_EQ(file_size_on_disk, fcs.excess_bytes);
+ ASSERT_EQ(file_size_on_disk, fcs.excess_bytes + initial_file_size_on_disk);
ASSERT_TRUE(fcs.repaired);
report.full_container_space_check->entries.clear();
NO_FATALS(AssertEmptyReport(report));
@@ -1475,7 +1480,7 @@ TEST_P(LogBlockManagerTest, TestRepairUnpunchedBlocks) {
// File size should be 0 post-repair.
ASSERT_OK(env_->GetFileSizeOnDisk(data_file, &file_size_on_disk));
- ASSERT_EQ(0, file_size_on_disk);
+ ASSERT_EQ(initial_file_size_on_disk, file_size_on_disk);
}
TEST_P(LogBlockManagerTest, TestRepairIncompleteContainer) {
@@ -1950,7 +1955,7 @@ TEST_P(LogBlockManagerTest,
TestDeleteDeadContainersByDeletionTransaction) {
}
{
// The last block makes a full container.
- FLAGS_log_container_max_size = 1;
+ FLAGS_log_container_max_size = GetParam() ? 4097 : 1;
unique_ptr<WritableBlock> writer;
ASSERT_OK(bm_->CreateBlock(test_block_opts_, &writer));
blocks.emplace_back(writer->id());
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index d5381fa..41275ba 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -660,8 +660,8 @@ class LogBlockContainer: public
RefCountedThreadSafe<LogBlockContainer> {
// Updates this container data file's position based on the offset and length
// of a block, marking this container as full if needed. Should only be
called
- // when a block is fully written, as it will round up the container data
file's
- // position.
+ // when a block is fully written, or after an encryption header is written,
as
+ // it will round up the container data file's position.
//
// This function is thread unsafe.
void UpdateNextBlockOffset(int64_t block_offset, int64_t block_length);
@@ -746,6 +746,15 @@ LogBlockContainer::LogBlockContainer(
blocks_being_written_(0),
dead_(false),
metrics_(block_manager->metrics()) {
+ // If we have an encryption header, we need to align the next offset to the
+ // next file system block.
+ if (auto encryption_header_size = data_file_->GetEncryptionHeaderSize();
+ encryption_header_size > 0) {
+ UpdateNextBlockOffset(0, encryption_header_size);
+ live_bytes_.Store(encryption_header_size);
+ total_bytes_.Store(next_block_offset_.Load());
+ live_bytes_aligned_.Store(next_block_offset_.Load());
+ }
}
LogBlockContainer::~LogBlockContainer() {
@@ -842,7 +851,6 @@ Status LogBlockContainer::Create(LogBlockManager*
block_manager,
unique_ptr<WritablePBContainerFile> metadata_file(new
WritablePBContainerFile(
std::move(metadata_writer)));
RETURN_NOT_OK_CONTAINER_DISK_FAILURE(metadata_file->CreateNew(BlockRecordPB()));
-
container->reset(new LogBlockContainer(block_manager,
dir,
std::move(metadata_file),
@@ -928,14 +936,17 @@ Status
LogBlockContainer::CheckContainerFiles(LogBlockManager* block_manager,
RETURN_NOT_OK_CONTAINER_DISK_FAILURE(s_meta);
}
+ const auto kEncryptionHeaderSize = env->GetEncryptionHeaderSize();
+ const auto kMinimumValidLength = pb_util::kPBContainerMinimumValidLength +
kEncryptionHeaderSize;
+
// Check that both the metadata and data files exist and have valid lengths.
// This covers a commonly seen case at startup, where the previous
incarnation
// of the server crashed due to "too many open files" just as it was trying
// to create a file. This orphans an empty or invalid length file, which we
can
// safely delete. And another case is that the metadata and data files exist,
// but the lengths are invalid.
- if (PREDICT_FALSE(metadata_size < pb_util::kPBContainerMinimumValidLength &&
- data_size == 0)) {
+ if (PREDICT_FALSE(metadata_size < kMinimumValidLength &&
+ data_size <= kEncryptionHeaderSize)) {
report->incomplete_container_check->entries.emplace_back(common_path);
return Status::Aborted(Substitute("orphaned empty or invalid length file
$0", common_path));
}
@@ -945,7 +956,7 @@ Status
LogBlockContainer::CheckContainerFiles(LogBlockManager* block_manager,
// metadata file will be deleted when repairing.
//
// Open the metadata file and quickly check whether or not there is any live
blocks.
- if (PREDICT_FALSE(metadata_size >= pb_util::kPBContainerMinimumValidLength &&
+ if (PREDICT_FALSE(metadata_size >= kMinimumValidLength &&
s_data.IsNotFound())) {
Status read_status;
BlockIdSet live_blocks;
@@ -1079,8 +1090,11 @@ Status LogBlockContainer::ProcessRecord(
RETURN_NOT_OK_HANDLE_ERROR(data_file_->Size(data_file_size));
}
- // If the record still extends beyond the end of the file, it is
malformed.
- if (PREDICT_FALSE(record->offset() + record->length() >
*data_file_size)) {
+ // If the record still extends beyond the end of the file, it is
+ // malformed, except if the length is 0, because in this case nothing has
+ // actually been written.
+ if (PREDICT_FALSE(record->offset() + record->length() > *data_file_size
&&
+ record->length() > 0)) {
// TODO(adar): treat as a different kind of inconsistency?
report->malformed_record_check->entries.emplace_back(ToString(),
record);
break;
@@ -1303,6 +1317,12 @@ Status LogBlockContainer::EnsurePreallocated(int64_t
block_start_offset,
next_append_length > preallocated_offset_ - block_start_offset) {
int64_t off = std::max(preallocated_offset_, block_start_offset);
int64_t len = FLAGS_log_container_preallocate_bytes;
+ // If encryption is enabled, preallocation happens after the encryption
+ // header is written. This keeps the file size a multiple of
+ // log_container_preallocate_bytes even in this case.
+ if (block_start_offset < FLAGS_log_container_preallocate_bytes) {
+ len -= block_start_offset;
+ }
RETURN_NOT_OK_HANDLE_ERROR(data_file_->PreAllocate(off, len,
RWFile::CHANGE_FILE_SIZE));
RETURN_NOT_OK_HANDLE_ERROR(data_dir_->RefreshAvailableSpace(Dir::RefreshMode::ALWAYS));
VLOG(2) << Substitute("Preallocated $0 bytes at offset $1 in container $2",
diff --git a/src/kudu/mini-cluster/external_mini_cluster.cc
b/src/kudu/mini-cluster/external_mini_cluster.cc
index a9f1ddc..9758805 100644
--- a/src/kudu/mini-cluster/external_mini_cluster.cc
+++ b/src/kudu/mini-cluster/external_mini_cluster.cc
@@ -101,6 +101,7 @@ using strings::Substitute;
typedef ListTabletsResponsePB::StatusAndSchemaPB StatusAndSchemaPB;
+DECLARE_bool(encrypt_data_at_rest);
DECLARE_string(block_manager);
DECLARE_string(dns_addr_resolution_override);
@@ -124,7 +125,7 @@ ExternalMiniClusterOptions::ExternalMiniClusterOptions()
principal("kudu"),
hms_mode(HmsMode::NONE),
enable_ranger(false),
- enable_encryption(false),
+ enable_encryption(FLAGS_encrypt_data_at_rest),
logtostderr(true),
start_process_timeout(MonoDelta::FromSeconds(70)),
rpc_negotiation_timeout(MonoDelta::FromSeconds(3))
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 69b5471..72898c2 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -312,6 +312,10 @@ class ToolTest : public KuduTest {
return RunTool(arg_str, stdout, stderr, nullptr, nullptr);
}
+ string GetEncryptionArgs() {
+ return "--encrypt_data_at_rest=true";
+ }
+
void RunActionStdoutLines(const string& arg_str, vector<string>*
stdout_lines) const {
string stderr;
Status s = RunTool(arg_str, nullptr, &stderr, stdout_lines, nullptr);
@@ -1390,9 +1394,11 @@ TEST_F(ToolTest, TestFsCheck) {
harness.tablet()->Shutdown();
}
+ string encryption_args = env_->IsEncryptionEnabled() ? GetEncryptionArgs() :
"";
+
// Check the filesystem; all the blocks should be accounted for, and there
// should be no blocks missing or orphaned.
- NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0", kTestDir),
+ NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0 $1", kTestDir,
encryption_args),
block_ids.size(), kTabletId, {}, 0));
// Delete half of the blocks. Upon the next check we can only find half, and
@@ -1412,7 +1418,7 @@ TEST_F(ToolTest, TestFsCheck) {
}
deletion_transaction->CommitDeletedBlocks(&missing_ids);
}
- NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0", kTestDir),
+ NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0 $1", kTestDir,
encryption_args),
block_ids.size() / 2, kTabletId, missing_ids, 0));
// Delete the tablet superblock. The next check finds half of the blocks,
@@ -1427,15 +1433,16 @@ TEST_F(ToolTest, TestFsCheck) {
ASSERT_OK(env_->DeleteFile(fs.GetTabletMetadataPath(kTabletId)));
}
for (int i = 0; i < 2; i++) {
- NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0", kTestDir),
+ NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0 $1", kTestDir,
encryption_args),
block_ids.size() / 2, kTabletId, {}, block_ids.size()
/ 2));
}
// Repair the filesystem. The remaining half of all blocks were found, deemed
// to be orphaned, and deleted. The next check shows no remaining blocks.
- NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0 --repair",
kTestDir),
+ NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0 $1 --repair",
kTestDir,
+ encryption_args),
block_ids.size() / 2, kTabletId, {}, block_ids.size() /
2));
- NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0", kTestDir),
+ NO_FATALS(RunFsCheck(Substitute("fs check --fs_wal_dir=$0 $1", kTestDir,
encryption_args),
0, kTabletId, {}, 0));
}
@@ -1727,11 +1734,13 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
metadata_path = metadata_files[0];
}
+ string encryption_args = env_->IsEncryptionEnabled() ? GetEncryptionArgs() :
"";
+
// Test default dump
{
vector<string> stdout;
NO_FATALS(RunActionStdoutLines(Substitute(
- "pbc dump $0", metadata_path), &stdout));
+ "pbc dump $0 $1", metadata_path, encryption_args), &stdout));
ASSERT_EQ(kNumCFileBlocks * 10 - 1, stdout.size());
ASSERT_EQ("Message 0", stdout[0]);
ASSERT_EQ("-------", stdout[1]);
@@ -1740,7 +1749,7 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
ASSERT_EQ("}", stdout[4]);
ASSERT_EQ("op_type: CREATE", stdout[5]);
ASSERT_STR_MATCHES(stdout[6], "^timestamp_us: [0-9]+$");
- ASSERT_EQ("offset: 0", stdout[7]);
+ ASSERT_STR_MATCHES(stdout[7], "^offset: [0-9]+$");
ASSERT_EQ("length: 153", stdout[8]);
ASSERT_EQ("", stdout[9]);
}
@@ -1749,7 +1758,7 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
{
vector<string> stdout;
NO_FATALS(RunActionStdoutLines(Substitute(
- "pbc dump $0 --debug", metadata_path), &stdout));
+ "pbc dump $0 $1 --debug", metadata_path, encryption_args), &stdout));
ASSERT_EQ(kNumCFileBlocks * 12 + 5, stdout.size());
// Header
ASSERT_EQ("File header", stdout[0]);
@@ -1768,7 +1777,7 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
ASSERT_EQ("}", stdout[12]);
ASSERT_EQ("op_type: CREATE", stdout[13]);
ASSERT_STR_MATCHES(stdout[14], "^timestamp_us: [0-9]+$");
- ASSERT_EQ("offset: 0", stdout[15]);
+ ASSERT_STR_MATCHES(stdout[15], "^offset: [0-9]+$");
ASSERT_EQ("length: 153", stdout[16]);
ASSERT_EQ("", stdout[17]);
}
@@ -1777,29 +1786,29 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
{
vector<string> stdout;
NO_FATALS(RunActionStdoutLines(Substitute(
- "pbc dump $0 --oneline", metadata_path), &stdout));
+ "pbc dump $0 $1 --oneline", metadata_path, encryption_args), &stdout));
ASSERT_EQ(kNumCFileBlocks, stdout.size());
ASSERT_STR_MATCHES(stdout[0],
"^0\tblock_id \\{ id: [0-9]+ \\} op_type: CREATE "
- "timestamp_us: [0-9]+ offset: 0 length: 153$");
+ "timestamp_us: [0-9]+ offset: [0-9]+ length: 153$");
}
// Test dump --json
{
vector<string> stdout;
NO_FATALS(RunActionStdoutLines(Substitute(
- "pbc dump $0 --json", metadata_path), &stdout));
+ "pbc dump $0 $1 --json", metadata_path, encryption_args), &stdout));
ASSERT_EQ(kNumCFileBlocks, stdout.size());
ASSERT_STR_MATCHES(stdout[0],
R"(^\{"blockId":\{"id":"[0-9]+"\},"opType":"CREATE",)"
- R"("timestampUs":"[0-9]+","offset":"0","length":"153"\}$$)");
+ R"("timestampUs":"[0-9]+","offset":"[0-9]+","length":"153"\}$$)");
}
// Test dump --json_pretty
{
vector<string> stdout;
NO_FATALS(RunActionStdoutLines(Substitute(
- "pbc dump $0 --json_pretty", metadata_path), &stdout));
+ "pbc dump $0 $1 --json_pretty", metadata_path, encryption_args),
&stdout));
ASSERT_EQ(kNumCFileBlocks * 10 - 1, stdout.size());
ASSERT_EQ("{", stdout[0]);
ASSERT_EQ(R"( "blockId": {)", stdout[1]);
@@ -1807,7 +1816,7 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
ASSERT_EQ(" },", stdout[3]);
ASSERT_EQ(R"( "opType": "CREATE",)", stdout[4]);
ASSERT_STR_MATCHES(stdout[5], R"( "timestampUs": "[0-9]+",)");
- ASSERT_EQ(R"( "offset": "0",)", stdout[6]);
+ ASSERT_STR_MATCHES(stdout[6], R"( "offset": "[0-9]+",)");
ASSERT_EQ(R"( "length": "153")", stdout[7]);
ASSERT_EQ(R"(})", stdout[8]);
ASSERT_EQ("", stdout[9]);
@@ -1822,7 +1831,7 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
editor_path));
chmod(editor_path.c_str(), 0755);
setenv("EDITOR", editor_path.c_str(), /* overwrite */1);
- return RunTool(Substitute("pbc edit $0 $1", extra_flags, metadata_path),
+ return RunTool(Substitute("pbc edit $0 $1 $2", extra_flags, metadata_path,
encryption_args),
stdout, stderr, nullptr, nullptr);
};
@@ -1836,11 +1845,11 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
// Dump to make sure the edit took place.
vector<string> stdout_lines;
NO_FATALS(RunActionStdoutLines(Substitute(
- "pbc dump $0 --oneline", metadata_path), &stdout_lines));
+ "pbc dump $0 $1 --oneline", metadata_path, encryption_args),
&stdout_lines));
ASSERT_EQ(kNumCFileBlocks, stdout_lines.size());
ASSERT_STR_MATCHES(stdout_lines[0],
"^0\tblock_id \\{ id: [0-9]+ \\} op_type: DELETE "
- "timestamp_us: [0-9]+ offset: 0 length: 153$");
+ "timestamp_us: [0-9]+ offset: [0-9]+ length: 153$");
// Make sure no backup file was written.
bool found_backup;
@@ -1858,11 +1867,11 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
// Dump to make sure the edit took place.
vector<string> stdout_lines;
NO_FATALS(RunActionStdoutLines(Substitute(
- "pbc dump $0 --oneline", metadata_path), &stdout_lines));
+ "pbc dump $0 $1 --oneline", metadata_path, encryption_args),
&stdout_lines));
ASSERT_EQ(kNumCFileBlocks, stdout_lines.size());
ASSERT_STR_MATCHES(stdout_lines[0],
"^0\tblock_id \\{ id: [0-9]+ \\} op_type: CREATE "
- "timestamp_us: [0-9]+ offset: 0 length: 153$");
+ "timestamp_us: [0-9]+ offset: [0-9]+ length: 153$");
// Make sure no backup file was written.
bool found_backup;
@@ -1916,11 +1925,11 @@ TEST_F(ToolTest, TestPbcToolsOnMultipleBlocks) {
{
vector<string> stdout;
NO_FATALS(RunActionStdoutLines(Substitute(
- "pbc dump $0 --oneline", metadata_path), &stdout));
+ "pbc dump $0 $1 --oneline", metadata_path, encryption_args), &stdout));
ASSERT_EQ(kNumCFileBlocks, stdout.size());
ASSERT_STR_MATCHES(stdout[0],
"^0\tblock_id \\{ id: [0-9]+ \\} op_type: DELETE "
- "timestamp_us: [0-9]+ offset: 0 length: 153$");
+ "timestamp_us: [0-9]+ offset: [0-9]+ length: 153$");
}
}
@@ -1944,16 +1953,18 @@ TEST_F(ToolTest, TestFsDumpCFile) {
ASSERT_OK_FAST(writer.AppendEntries(generator.values(), kNumEntries));
ASSERT_OK(writer.Finish());
+ string encryption_args = env_->IsEncryptionEnabled() ? GetEncryptionArgs() :
"";
+
{
NO_FATALS(RunActionStdoutNone(Substitute(
- "fs dump cfile --fs_wal_dir=$0 $1 --noprint_meta --noprint_rows",
- kTestDir, block_id.ToString())));
+ "fs dump cfile --fs_wal_dir=$0 $1 $2 --noprint_meta --noprint_rows",
+ kTestDir, block_id.ToString(), encryption_args)));
}
vector<string> stdout;
{
NO_FATALS(RunActionStdoutLines(Substitute(
- "fs dump cfile --fs_wal_dir=$0 $1 --noprint_rows",
- kTestDir, block_id.ToString()), &stdout));
+ "fs dump cfile --fs_wal_dir=$0 $1 $2 --noprint_rows",
+ kTestDir, block_id.ToString(), encryption_args), &stdout));
SCOPED_TRACE(stdout);
ASSERT_GE(stdout.size(), 4);
ASSERT_EQ(stdout[0], "Header:");
@@ -1961,15 +1972,15 @@ TEST_F(ToolTest, TestFsDumpCFile) {
}
{
NO_FATALS(RunActionStdoutLines(Substitute(
- "fs dump cfile --fs_wal_dir=$0 $1 --noprint_meta",
- kTestDir, block_id.ToString()), &stdout));
+ "fs dump cfile --fs_wal_dir=$0 $1 $2 --noprint_meta",
+ kTestDir, block_id.ToString(), encryption_args), &stdout));
SCOPED_TRACE(stdout);
ASSERT_EQ(kNumEntries, stdout.size());
}
{
NO_FATALS(RunActionStdoutLines(Substitute(
- "fs dump cfile --fs_wal_dir=$0 $1",
- kTestDir, block_id.ToString()), &stdout));
+ "fs dump cfile --fs_wal_dir=$0 $1 $2",
+ kTestDir, block_id.ToString(), encryption_args), &stdout));
SCOPED_TRACE(stdout);
ASSERT_GT(stdout.size(), kNumEntries);
ASSERT_EQ(stdout[0], "Header:");
@@ -1992,8 +2003,9 @@ TEST_F(ToolTest, TestFsDumpBlock) {
{
string stdout;
NO_FATALS(RunActionStdoutString(Substitute(
- "fs dump block --fs_wal_dir=$0 $1",
- kTestDir, block_id.ToString()), &stdout));
+ "fs dump block --fs_wal_dir=$0 $1 $2",
+ kTestDir, block_id.ToString(),
+ env_->IsEncryptionEnabled() ? GetEncryptionArgs() : ""), &stdout));
ASSERT_EQ("hello world", stdout);
}
}
@@ -2039,10 +2051,11 @@ TEST_F(ToolTest, TestWalDump) {
}
string wal_path = fs.GetWalSegmentFileName(kTestTablet, 1);
+ string encryption_args = env_->IsEncryptionEnabled() ? GetEncryptionArgs() :
"";
string stdout;
- for (const auto& args : { Substitute("wal dump $0", wal_path),
- Substitute("local_replica dump wals
--fs_wal_dir=$0 $1",
- kTestDir, kTestTablet)
+ for (const auto& args : { Substitute("wal dump $0 $1", wal_path,
encryption_args),
+ Substitute("local_replica dump wals
--fs_wal_dir=$0 $1 $2",
+ kTestDir, kTestTablet, encryption_args)
}) {
SCOPED_TRACE(args);
for (const auto& print_entries : { "true", "1", "yes", "decoded" }) {
@@ -2149,9 +2162,12 @@ TEST_F(ToolTest, TestLocalReplicaDumpDataDirs) {
string stdout;
NO_FATALS(RunActionStdoutString(Substitute("local_replica dump data_dirs $0 "
"--fs_wal_dir=$1 "
- "--fs_data_dirs=$2",
+ "--fs_data_dirs=$2 $3",
kTestTablet, opts.wal_root,
- JoinStrings(opts.data_roots,
",")),
+ JoinStrings(opts.data_roots, ","),
+ env_->IsEncryptionEnabled()
+ ? GetEncryptionArgs()
+ : ""),
&stdout));
vector<string> expected;
for (const auto& data_root : opts.data_roots) {
@@ -2187,9 +2203,12 @@ TEST_F(ToolTest, TestLocalReplicaDumpMeta) {
string stdout;
NO_FATALS(RunActionStdoutString(Substitute("local_replica dump meta $0 "
"--fs_wal_dir=$1 "
- "--fs_data_dirs=$2",
+ "--fs_data_dirs=$2 "
+ "$3",
kTestTablet, kTestDir,
- kTestDir), &stdout));
+ kTestDir,
+ env_->IsEncryptionEnabled()
+ ? GetEncryptionArgs() : ""),
&stdout));
// Verify the contents of the metadata output
SCOPED_TRACE(stdout);
@@ -2265,11 +2284,12 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
harness.tablet()->Shutdown();
string fs_paths = "--fs_wal_dir=" + kTestDir + " "
"--fs_data_dirs=" + kTestDir;
+ string encryption_args = env_->IsEncryptionEnabled() ? GetEncryptionArgs() :
"";
{
string stdout;
NO_FATALS(RunActionStdoutString(
- Substitute("local_replica dump block_ids $0 $1",
- kTestTablet, fs_paths), &stdout));
+ Substitute("local_replica dump block_ids $0 $1 $2",
+ kTestTablet, fs_paths, encryption_args), &stdout));
SCOPED_TRACE(stdout);
string tablet_out = "Listing all data blocks in tablet " + kTestTablet;
@@ -2283,8 +2303,8 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
{
string stdout;
NO_FATALS(RunActionStdoutString(
- Substitute("local_replica dump rowset $0 $1",
- kTestTablet, fs_paths), &stdout));
+ Substitute("local_replica dump rowset $0 $1 $2",
+ kTestTablet, fs_paths, encryption_args), &stdout));
SCOPED_TRACE(stdout);
ASSERT_STR_CONTAINS(stdout, "Dumping rowset 0");
@@ -2307,8 +2327,8 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
// This is expected to fail with Invalid argument for kRowId.
string stderr;
Status s = RunTool(
- Substitute("local_replica dump rowset $0 $1 --rowset_index=$2",
- kTestTablet, fs_paths, kRowId),
+ Substitute("local_replica dump rowset $0 $1 --rowset_index=$2 $3",
+ kTestTablet, fs_paths, kRowId, encryption_args),
&stdout, &stderr, nullptr, nullptr);
ASSERT_TRUE(s.IsRuntimeError());
SCOPED_TRACE(stderr);
@@ -2318,8 +2338,8 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
NO_FATALS(RunActionStdoutString(
Substitute("local_replica dump rowset --nodump_all_columns "
- "--nodump_metadata --nrows=15 $0 $1",
- kTestTablet, fs_paths), &stdout));
+ "--nodump_metadata --nrows=15 $0 $1 $2",
+ kTestTablet, fs_paths, encryption_args), &stdout));
SCOPED_TRACE(stdout);
ASSERT_STR_CONTAINS(stdout, "Dumping rowset 0");
@@ -2340,8 +2360,8 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
string stdout;
string debug_str;
NO_FATALS(RunActionStdoutString(
- Substitute("local_replica dump meta $0 $1",
- kTestTablet, fs_paths), &stdout));
+ Substitute("local_replica dump meta $0 $1 $2",
+ kTestTablet, fs_paths, encryption_args), &stdout));
SCOPED_TRACE(stdout);
debug_str = meta->partition_schema()
@@ -2368,8 +2388,8 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
{
string stdout;
NO_FATALS(RunActionStdoutString(
- Substitute("local_replica data_size $0 $1",
- kTestTablet, fs_paths), &stdout));
+ Substitute("local_replica data_size $0 $1 $2",
+ kTestTablet, fs_paths, encryption_args), &stdout));
SCOPED_TRACE(stdout);
string expected = R"(
@@ -2435,8 +2455,8 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
}
{
string stdout;
- NO_FATALS(RunActionStdoutString(Substitute("local_replica list $0",
- fs_paths), &stdout));
+ NO_FATALS(RunActionStdoutString(Substitute("local_replica list $0 $1",
+ fs_paths, encryption_args),
&stdout));
SCOPED_TRACE(stdout);
ASSERT_STR_MATCHES(stdout, kTestTablet);
@@ -2446,8 +2466,8 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
{
string stdout;
NO_FATALS(RunActionStdoutString(
- Substitute("fs list $0 --columns=table,tablet-id --format=csv",
- fs_paths),
+ Substitute("fs list $0 $1 --columns=table,tablet-id --format=csv",
+ fs_paths, encryption_args),
&stdout));
SCOPED_TRACE(stdout);
@@ -2458,8 +2478,8 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
{
vector<string> stdout;
NO_FATALS(RunActionStdoutLines(
- Substitute("fs list $0 --columns=table,tablet-id,rowset-id
--format=csv",
- fs_paths),
+ Substitute("fs list $0 $1 --columns=table,tablet-id,rowset-id
--format=csv",
+ fs_paths, encryption_args),
&stdout));
SCOPED_TRACE(stdout);
@@ -2474,10 +2494,10 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
{
vector<string> stdout;
NO_FATALS(RunActionStdoutLines(
- Substitute("fs list $0 "
+ Substitute("fs list $0 $1 "
"--columns=table,tablet-id,rowset-id,block-kind,column "
"--format=csv",
- fs_paths),
+ fs_paths, encryption_args),
&stdout));
SCOPED_TRACE(stdout);
@@ -2500,11 +2520,11 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
{
vector<string> stdout;
NO_FATALS(RunActionStdoutLines(
- Substitute("fs list $0 "
+ Substitute("fs list $0 $1 "
"--columns=table,tablet-id,rowset-id,block-kind,"
"column,cfile-encoding,cfile-num-values "
"--format=csv",
- fs_paths),
+ fs_paths, encryption_args),
&stdout));
SCOPED_TRACE(stdout);
@@ -3244,9 +3264,10 @@ TEST_F(ToolTest, TestPerfTabletScan) {
cluster_->Shutdown();
for (const string& tid : tablet_ids) {
const string args =
- Substitute("perf tablet_scan $0 --fs_wal_dir=$1 --fs_data_dirs=$2
--num_iters=2",
+ Substitute("perf tablet_scan $0 --fs_wal_dir=$1 --fs_data_dirs=$2
--num_iters=2 $3",
tid, cluster_->tablet_server(0)->wal_dir(),
- JoinStrings(cluster_->tablet_server(0)->data_dirs(), ","));
+ JoinStrings(cluster_->tablet_server(0)->data_dirs(), ","),
+ env_->IsEncryptionEnabled() ? GetEncryptionArgs() : "");
NO_FATALS(RunActionStdoutNone(args));
NO_FATALS(RunActionStdoutNone(args + " --ordered_scan"));
}
@@ -3302,16 +3323,18 @@ TEST_F(ToolTest, TestRemoteReplicaCopy) {
string stderr;
const string& src_ts_addr =
cluster_->tablet_server(kSrcTsIndex)->bound_rpc_addr().ToString();
const string& dst_ts_addr =
cluster_->tablet_server(kDstTsIndex)->bound_rpc_addr().ToString();
+ string encryption_args = env_->IsEncryptionEnabled() ? GetEncryptionArgs() :
"";
Status s = RunTool(
- Substitute("remote_replica copy $0 $1 $2",
- healthy_tablet_id, src_ts_addr, dst_ts_addr),
+ Substitute("remote_replica copy $0 $1 $2 $3",
+ healthy_tablet_id, src_ts_addr, dst_ts_addr, encryption_args),
nullptr, &stderr, nullptr, nullptr);
ASSERT_TRUE(s.IsRuntimeError());
SCOPED_TRACE(stderr);
ASSERT_STR_CONTAINS(stderr, "Rejecting tablet copy request");
- NO_FATALS(RunActionStdoutNone(Substitute("remote_replica copy $0 $1 $2
--force_copy",
- healthy_tablet_id, src_ts_addr,
dst_ts_addr)));
+ NO_FATALS(RunActionStdoutNone(Substitute("remote_replica copy $0 $1 $2 $3
--force_copy",
+ healthy_tablet_id, src_ts_addr,
dst_ts_addr,
+ encryption_args)));
ASSERT_OK(WaitUntilTabletInState(dst_ts, healthy_tablet_id,
tablet::RUNNING, kTimeout));
@@ -3332,9 +3355,10 @@ TEST_F(ToolTest, TestRemoteReplicaCopy) {
cluster_->tablet_server(kDstTsIndex)->Shutdown();
const string& deleted_tablet_id = tablets[1].tablet_status().tablet_id();
NO_FATALS(RunActionStdoutNone(Substitute(
- "local_replica delete $0 --fs_wal_dir=$1 --fs_data_dirs=$2
--clean_unsafe",
+ "local_replica delete $0 --fs_wal_dir=$1 --fs_data_dirs=$2
--clean_unsafe $3",
deleted_tablet_id, cluster_->tablet_server(kDstTsIndex)->wal_dir(),
- JoinStrings(cluster_->tablet_server(kDstTsIndex)->data_dirs(), ","))));
+ JoinStrings(cluster_->tablet_server(kDstTsIndex)->data_dirs(), ","),
+ encryption_args)));
// At this point, we expect only 2 tablets to show up on destination when
// we restart the destination tserver. deleted_tablet_id should not be found
on
@@ -3355,13 +3379,15 @@ TEST_F(ToolTest, TestRemoteReplicaCopy) {
{
TabletDataState::TABLET_DATA_TOMBSTONED },
kTimeout));
// Copy tombstoned_tablet_id from source to destination.
- NO_FATALS(RunActionStdoutNone(Substitute("remote_replica copy $0 $1 $2
--force_copy",
- tombstoned_tablet_id, src_ts_addr,
dst_ts_addr)));
+ NO_FATALS(RunActionStdoutNone(Substitute("remote_replica copy $0 $1 $2 $3
--force_copy",
+ tombstoned_tablet_id, src_ts_addr,
dst_ts_addr,
+ encryption_args)));
ASSERT_OK(WaitUntilTabletInState(dst_ts, tombstoned_tablet_id,
tablet::RUNNING, kTimeout));
// Copy deleted_tablet_id from source to destination.
- NO_FATALS(RunActionStdoutNone(Substitute("remote_replica copy $0 $1 $2",
- deleted_tablet_id, src_ts_addr,
dst_ts_addr)));
+ NO_FATALS(RunActionStdoutNone(Substitute("remote_replica copy $0 $1 $2 $3",
+ deleted_tablet_id, src_ts_addr,
dst_ts_addr,
+ encryption_args)));
ASSERT_OK(WaitUntilTabletInState(dst_ts, deleted_tablet_id,
tablet::RUNNING, kTimeout));
}
@@ -3511,12 +3537,13 @@ TEST_F(ToolTest, TestLocalReplicaDelete) {
ASSERT_OK(tablet->Flush());
tablet_id = tablet_replicas[0]->tablet_id();
}
+ string encryption_args = env_->IsEncryptionEnabled() ? GetEncryptionArgs() :
"";
const string& tserver_dir = ts->options()->fs_opts.wal_root;
// Using the delete tool with tablet server running fails.
string stderr;
Status s = RunTool(
- Substitute("local_replica delete $0 --fs_wal_dir=$1 --fs_data_dirs=$1 "
- "--clean_unsafe", tablet_id, tserver_dir),
+ Substitute("local_replica delete $0 --fs_wal_dir=$1 --fs_data_dirs=$1 $2
"
+ "--clean_unsafe", tablet_id, tserver_dir, encryption_args),
nullptr, &stderr, nullptr, nullptr);
ASSERT_TRUE(s.IsRuntimeError());
SCOPED_TRACE(stderr);
@@ -3528,9 +3555,9 @@ TEST_F(ToolTest, TestLocalReplicaDelete) {
const string& data_dir = JoinPathSegments(tserver_dir, "data");
uint64_t size_before_delete;
ASSERT_OK(env_->GetFileSizeOnDiskRecursively(data_dir, &size_before_delete));
- NO_FATALS(RunActionStdoutNone(Substitute("local_replica delete $0
--fs_wal_dir=$1 "
+ NO_FATALS(RunActionStdoutNone(Substitute("local_replica delete $0
--fs_wal_dir=$1 $2 "
"--fs_data_dirs=$1 --clean_unsafe",
- tablet_id, tserver_dir)));
+ tablet_id, tserver_dir,
encryption_args)));
// Verify metadata and WAL segments for the tablet_id are gone.
const string& wal_dir = JoinPathSegments(tserver_dir,
Substitute("wals/$0", tablet_id));
@@ -3541,8 +3568,8 @@ TEST_F(ToolTest, TestLocalReplicaDelete) {
// Try to remove the same tablet replica again.
s = RunTool(Substitute(
- "local_replica delete $0 --clean_unsafe --fs_wal_dir=$1
--fs_data_dirs=$1",
- tablet_id, tserver_dir),
+ "local_replica delete $0 --clean_unsafe --fs_wal_dir=$1
--fs_data_dirs=$1 $2",
+ tablet_id, tserver_dir, encryption_args),
nullptr, &stderr, nullptr, nullptr);
ASSERT_TRUE(s.IsRuntimeError());
ASSERT_STR_CONTAINS(stderr, "Not found: Could not load tablet metadata");
@@ -3553,8 +3580,8 @@ TEST_F(ToolTest, TestLocalReplicaDelete) {
// error ignored.
ASSERT_OK(RunActionStderrString(
Substitute("local_replica delete $0 --clean_unsafe --fs_wal_dir=$1 "
- "--fs_data_dirs=$1 --ignore_nonexistent",
- tablet_id, tserver_dir),
+ "--fs_data_dirs=$1 --ignore_nonexistent $2",
+ tablet_id, tserver_dir, encryption_args),
&stderr));
ASSERT_STR_CONTAINS(stderr, Substitute("ignoring error for tablet replica $0
"
"because of the --ignore_nonexistent
flag",
@@ -3611,7 +3638,8 @@ TEST_F(ToolTest, TestLocalReplicaDeleteMultiple) {
const string tablet_ids_csv_str = JoinStrings(tablet_ids, ",");
NO_FATALS(RunActionStdoutNone(Substitute(
"local_replica delete --fs_wal_dir=$0 --fs_data_dirs=$0 "
- "--clean_unsafe $1", tserver_dir, tablet_ids_csv_str)));
+ "--clean_unsafe $1 $2", tserver_dir, tablet_ids_csv_str,
+ env_->IsEncryptionEnabled() ? GetEncryptionArgs() : "")));
ASSERT_OK(ts->Start());
ASSERT_OK(ts->WaitStarted());
@@ -3665,8 +3693,11 @@ TEST_F(ToolTest, TestLocalReplicaTombstoneDelete) {
uint64_t size_before_delete;
ASSERT_OK(env_->GetFileSizeOnDiskRecursively(data_dir, &size_before_delete));
NO_FATALS(RunActionStdoutNone(Substitute("local_replica delete $0
--fs_wal_dir=$1 "
- "--fs_data_dirs=$1",
- tablet_id, tserver_dir)));
+ "--fs_data_dirs=$1 $2",
+ tablet_id, tserver_dir,
+ env_->IsEncryptionEnabled()
+ ? GetEncryptionArgs()
+ : "")));
// Verify WAL segments for the tablet_id are gone and
// the data_dir size on tserver is reduced.
const string& wal_dir = JoinPathSegments(tserver_dir,
@@ -3715,12 +3746,15 @@ TEST_F(ToolTest, TestLocalReplicaCMetaOps) {
for (int i = 0; i < kNumTabletServers; ++i) {
ts_uuids.emplace_back(mini_cluster_->mini_tablet_server(i)->uuid());
}
+ string encryption_args = env_->IsEncryptionEnabled() ? GetEncryptionArgs() :
"";
const string& flags =
- Substitute("-fs-wal-dir $0",
-
mini_cluster_->mini_tablet_server(0)->options()->fs_opts.wal_root);
+ Substitute("-fs-wal-dir $0 $1",
+
mini_cluster_->mini_tablet_server(0)->options()->fs_opts.wal_root,
+ encryption_args);
vector<string> tablet_ids;
{
- NO_FATALS(RunActionStdoutLines(Substitute("local_replica list $0", flags),
&tablet_ids));
+ NO_FATALS(RunActionStdoutLines(Substitute("local_replica list $0 $1",
flags, encryption_args),
+ &tablet_ids));
ASSERT_EQ(kNumTablets, tablet_ids.size());
}
vector<string> cmeta_paths;
@@ -3737,8 +3771,9 @@ TEST_F(ToolTest, TestLocalReplicaCMetaOps) {
// We have kNumTablets replicas, so we expect kNumTablets lines, with our 3
servers' UUIDs.
{
vector<string> lines;
- NO_FATALS(RunActionStdoutLines(Substitute("local_replica cmeta
print_replica_uuids $0 $1",
- flags, JoinStrings(tablet_ids,
",")), &lines));
+ NO_FATALS(RunActionStdoutLines(Substitute("local_replica cmeta
print_replica_uuids $0 $1 $2",
+ flags, JoinStrings(tablet_ids,
","), encryption_args),
+ &lines));
ASSERT_EQ(kNumTablets, lines.size());
for (int i = 0; i < lines.size(); ++i) {
ASSERT_STR_MATCHES(lines[i],
@@ -3752,11 +3787,11 @@ TEST_F(ToolTest, TestLocalReplicaCMetaOps) {
// Test using set-term to bump the term to 123.
for (int i = 0; i < tablet_ids.size(); ++i) {
- NO_FATALS(RunActionStdoutNone(Substitute("local_replica cmeta set-term $0
$1 123",
- flags, tablet_ids[i])));
+ NO_FATALS(RunActionStdoutNone(Substitute("local_replica cmeta set-term $0
$1 $2 123",
+ flags, tablet_ids[i],
encryption_args)));
string stdout;
- NO_FATALS(RunActionStdoutString(Substitute("pbc dump $0", cmeta_paths[i]),
+ NO_FATALS(RunActionStdoutString(Substitute("pbc dump $0 $1",
cmeta_paths[i], encryption_args),
&stdout));
ASSERT_STR_CONTAINS(stdout, "current_term: 123");
}
@@ -3764,8 +3799,8 @@ TEST_F(ToolTest, TestLocalReplicaCMetaOps) {
// Test that set-term refuses to decrease the term.
for (const auto& tablet_id : tablet_ids) {
string stdout, stderr;
- Status s = RunTool(Substitute("local_replica cmeta set-term $0 $1 10",
- flags, tablet_id),
+ Status s = RunTool(Substitute("local_replica cmeta set-term $0 $1 $2 10",
+ flags, tablet_id, encryption_args),
&stdout, &stderr,
/* stdout_lines = */ nullptr,
/* stderr_lines = */ nullptr);
@@ -3778,19 +3813,21 @@ TEST_F(ToolTest, TestLocalReplicaCMetaOps) {
// Test using rewrite_raft_config to set all tablets' raft config with only
1 member.
{
NO_FATALS(RunActionStdoutNone(
- Substitute("local_replica cmeta rewrite_raft_config $0 $1 $2:$3",
+ Substitute("local_replica cmeta rewrite_raft_config $0 $1 $2:$3 $4",
flags,
JoinStrings(tablet_ids, ","),
ts_uuids[0],
- ts_host_port)));
+ ts_host_port,
+ encryption_args)));
}
// We have kNumTablets replicas, so we expect kNumTablets lines, with our the
// first tservers' UUIDs.
{
vector<string> lines;
- NO_FATALS(RunActionStdoutLines(Substitute("local_replica cmeta
print_replica_uuids $0 $1",
- flags, JoinStrings(tablet_ids,
",")), &lines));
+ NO_FATALS(RunActionStdoutLines(Substitute("local_replica cmeta
print_replica_uuids $0 $1 $2",
+ flags, JoinStrings(tablet_ids,
","), encryption_args),
+ &lines));
ASSERT_EQ(kNumTablets, lines.size());
for (int i = 0; i < lines.size(); ++i) {
ASSERT_STR_MATCHES(lines[i], Substitute("tablet: $0, peers: $1",
@@ -6727,8 +6764,9 @@ TEST_F(ToolTest, TestFsRemoveDataDirWithTombstone) {
mts->Shutdown();
// KUDU-2680: tombstones shouldn't prevent us from removing a directory.
NO_FATALS(RunActionStdoutNone(Substitute(
- "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1",
- mts->options()->fs_opts.wal_root, data_root)));
+ "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 $2",
+ mts->options()->fs_opts.wal_root, data_root,
+ env_->IsEncryptionEnabled() ? GetEncryptionArgs() : "")));
ASSERT_OK(mts->Start());
ASSERT_OK(mts->WaitStarted());
@@ -6764,9 +6802,10 @@ TEST_F(ToolTest, TestFsAddRemoveDataDirEndToEnd) {
string to_add = JoinPathSegments(DirName(data_roots.back()), "data-new");
data_roots.emplace_back(to_add);
mts->Shutdown();
+ string encryption_args = env_->IsEncryptionEnabled() ? GetEncryptionArgs() :
"";
NO_FATALS(RunActionStdoutNone(Substitute(
- "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1",
- wal_root, JoinStrings(data_roots, ","))));
+ "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 $2",
+ wal_root, JoinStrings(data_roots, ","), encryption_args)));
// Reconfigure the tserver to use the newly added data directory and restart
it.
//
@@ -6790,8 +6829,8 @@ TEST_F(ToolTest, TestFsAddRemoveDataDirEndToEnd) {
data_roots.pop_back();
string stderr;
ASSERT_TRUE(RunActionStderrString(Substitute(
- "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1",
- wal_root, JoinStrings(data_roots, ",")), &stderr).IsRuntimeError());
+ "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 $2",
+ wal_root, JoinStrings(data_roots, ","), encryption_args),
&stderr).IsRuntimeError());
ASSERT_STR_CONTAINS(
stderr, "Not found: cannot update data directories: at least one "
"tablet is configured to use removed data directory. Retry with --force "
@@ -6805,8 +6844,8 @@ TEST_F(ToolTest, TestFsAddRemoveDataDirEndToEnd) {
// If we force the removal it'll succeed, but the tserver will fail to
// bootstrap some tablets when restarted.
NO_FATALS(RunActionStdoutNone(Substitute(
- "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 --force",
- wal_root, JoinStrings(data_roots, ","))));
+ "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 $2 --force",
+ wal_root, JoinStrings(data_roots, ","), encryption_args)));
mts->options()->fs_opts.data_roots = data_roots;
ASSERT_OK(mts->Start());
Status s = mts->WaitStarted();
@@ -6835,8 +6874,8 @@ TEST_F(ToolTest, TestFsAddRemoveDataDirEndToEnd) {
mts->Shutdown();
data_roots.emplace_back(to_add);
NO_FATALS(RunActionStdoutNone(Substitute(
- "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1",
- wal_root, JoinStrings(data_roots, ","))));
+ "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 $2",
+ wal_root, JoinStrings(data_roots, ","), encryption_args)));
mts->options()->fs_opts.data_roots = data_roots;
ASSERT_OK(mts->Start());
ASSERT_OK(mts->WaitStarted());
@@ -6845,8 +6884,8 @@ TEST_F(ToolTest, TestFsAddRemoveDataDirEndToEnd) {
// Remove it again so that the second table's tablets fail once again.
data_roots.pop_back();
NO_FATALS(RunActionStdoutNone(Substitute(
- "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 --force",
- wal_root, JoinStrings(data_roots, ","))));
+ "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 $2 --force",
+ wal_root, JoinStrings(data_roots, ","), encryption_args)));
mts->options()->fs_opts.data_roots = data_roots;
ASSERT_OK(mts->Start());
s = mts->WaitStarted();
@@ -6871,8 +6910,8 @@ TEST_F(ToolTest, TestFsAddRemoveDataDirEndToEnd) {
mts->Shutdown();
data_roots.emplace_back(JoinPathSegments(DirName(data_roots.back()),
"data-new2"));
NO_FATALS(RunActionStdoutNone(Substitute(
- "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1",
- wal_root, JoinStrings(data_roots, ","))));
+ "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 $2",
+ wal_root, JoinStrings(data_roots, ","), encryption_args)));
mts->options()->fs_opts.data_roots = data_roots;
ASSERT_OK(mts->Start());
ASSERT_OK(mts->WaitStarted());
@@ -6883,8 +6922,8 @@ TEST_F(ToolTest, TestFsAddRemoveDataDirEndToEnd) {
mts->Shutdown();
data_roots.pop_back();
NO_FATALS(RunActionStdoutNone(Substitute(
- "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1",
- wal_root, JoinStrings(data_roots, ","))));
+ "fs update_dirs --fs_wal_dir=$0 --fs_data_dirs=$1 $2",
+ wal_root, JoinStrings(data_roots, ","), encryption_args)));
mts->options()->fs_opts.data_roots = data_roots;
ASSERT_OK(mts->Start());
ASSERT_OK(mts->WaitStarted());
diff --git a/src/kudu/tools/tool_action_pbc.cc
b/src/kudu/tools/tool_action_pbc.cc
index 81c97ff..4c3e51c 100644
--- a/src/kudu/tools/tool_action_pbc.cc
+++ b/src/kudu/tools/tool_action_pbc.cc
@@ -104,10 +104,13 @@ namespace {
const char* const kPathArg = "path";
bool IsFileEncrypted(Env* env, const std::string& fname) {
- // TODO(abukor): replace with real encryption check. As instance files are
the
- // only PBC files that are unencrypted right now, this check will suffice
- // until we can tell if a file is encrypted based on an encryption header.
- return fname.length() < 8 || fname.compare(fname.length() - 8, 8,
"instance") != 0;
+ if (!env->IsEncryptionEnabled()) {
+ return false;
+ }
+ RandomAccessFileOptions opts;
+ opts.is_sensitive = true;
+ unique_ptr<RandomAccessFile> reader;
+ return env->NewRandomAccessFile(opts, fname, &reader).ok();
}
Status DumpPBContainerFile(const RunnerContext& context) {
diff --git a/src/kudu/tserver/tablet_copy_client-test.cc
b/src/kudu/tserver/tablet_copy_client-test.cc
index e6c1b9c..fbe0228 100644
--- a/src/kudu/tserver/tablet_copy_client-test.cc
+++ b/src/kudu/tserver/tablet_copy_client-test.cc
@@ -168,12 +168,16 @@ class TabletCopyClientTest : public TabletCopyTest {
Status TabletCopyClientTest::CompareFileContents(const string& path1, const
string& path2) {
shared_ptr<RandomAccessFile> file1, file2;
- RETURN_NOT_OK(env_util::OpenFileForRandom(fs_manager_->env(), path1,
&file1));
- RETURN_NOT_OK(env_util::OpenFileForRandom(fs_manager_->env(), path2,
&file2));
+ RandomAccessFileOptions opts;
+ opts.is_sensitive = true;
+ RETURN_NOT_OK(env_util::OpenFileForRandom(opts, fs_manager_->env(), path1,
&file1));
+ RETURN_NOT_OK(env_util::OpenFileForRandom(opts, fs_manager_->env(), path2,
&file2));
uint64_t size1, size2;
RETURN_NOT_OK(file1->Size(&size1));
RETURN_NOT_OK(file2->Size(&size2));
+ size1 -= file1->GetEncryptionHeaderSize();
+ size2 -= file2->GetEncryptionHeaderSize();
if (size1 != size2) {
return Status::Corruption("Sizes of files don't match",
Substitute("$0 vs $1 bytes", size1, size2));
@@ -184,8 +188,8 @@ Status TabletCopyClientTest::CompareFileContents(const
string& path1, const stri
scratch2.resize(size2);
Slice slice1(scratch1.data(), size1);
Slice slice2(scratch2.data(), size2);
- RETURN_NOT_OK(file1->Read(0, slice1));
- RETURN_NOT_OK(file2->Read(0, slice2));
+ RETURN_NOT_OK(file1->Read(file1->GetEncryptionHeaderSize(), slice1));
+ RETURN_NOT_OK(file2->Read(file2->GetEncryptionHeaderSize(), slice2));
int result = strings::fastmemcmp_inlined(slice1.data(), slice2.data(),
size1);
if (result != 0) {
return Status::Corruption("Files do not match");
diff --git a/src/kudu/tserver/tablet_copy_service-test.cc
b/src/kudu/tserver/tablet_copy_service-test.cc
index 38ebd2a..97c6578 100644
--- a/src/kudu/tserver/tablet_copy_service-test.cc
+++ b/src/kudu/tserver/tablet_copy_service-test.cc
@@ -391,7 +391,8 @@ TEST_F(TabletCopyServiceTest, TestFetchInvalidBlockOffset) {
FetchDataResponsePB resp;
RpcController controller;
// Impossible offset.
- uint64_t offset = std::numeric_limits<uint64_t>::max();
+ uint64_t offset = std::numeric_limits<uint64_t>::max() -
+ mini_server_->server()->fs_manager()->env()->GetEncryptionHeaderSize();
Status status = DoFetchData(session_id,
AsDataTypeId(FirstColumnBlockId(superblock)),
&offset, nullptr, &resp, &controller);
NO_FATALS(AssertRemoteError(status, controller.error_response(),
@@ -482,10 +483,11 @@ TEST_F(TabletCopyServiceTest, TestFetchLog) {
<< " and " << first_seg_seqno;
const scoped_refptr<ReadableLogSegment>& segment = local_segments[0];
faststring scratch;
- int64_t size = segment->file_size();
+ auto file = segment->file();
+ int64_t size = segment->file_size() - file->GetEncryptionHeaderSize();
scratch.resize(size);
Slice slice(scratch.data(), size);
- ASSERT_OK(segment->file()->Read(0, slice));
+ ASSERT_OK(file->Read(file->GetEncryptionHeaderSize(), slice));
AssertDataEqual(slice.data(), slice.size(), resp.chunk());
}
diff --git a/src/kudu/tserver/tablet_copy_source_session-test.cc
b/src/kudu/tserver/tablet_copy_source_session-test.cc
index eb12fe1..bf57e45 100644
--- a/src/kudu/tserver/tablet_copy_source_session-test.cc
+++ b/src/kudu/tserver/tablet_copy_source_session-test.cc
@@ -337,13 +337,13 @@ TEST_F(TabletCopyTest, TestBlocksEqual) {
buf.resize(tablet_block_size);
Slice data2(buf.data(), tablet_block_size);
ASSERT_OK(tablet_block->Read(0, data2));
- uint32_t tablet_crc = crc::Crc32c(data.data(), data.size());
+ uint32_t tablet_crc = crc::Crc32c(data2.data(), data2.size());
LOG(INFO) << "tablet block file has size of " << tablet_block_size
<< " and CRC32C of " << tablet_crc
<< ": " << block_id;
// Compare the blocks.
- ASSERT_EQ(tablet_block_size, session_block_size);
+ ASSERT_EQ(tablet_block_size, session_block_size -
Env::Default()->GetEncryptionHeaderSize());
ASSERT_EQ(tablet_crc, session_crc);
}
}
diff --git a/src/kudu/tserver/tablet_copy_source_session.cc
b/src/kudu/tserver/tablet_copy_source_session.cc
index f48f036..999157d 100644
--- a/src/kudu/tserver/tablet_copy_source_session.cc
+++ b/src/kudu/tserver/tablet_copy_source_session.cc
@@ -330,9 +330,15 @@ Status
TabletCopySourceSession::GetLogSegmentPiece(uint64_t segment_seqno,
"Tablet copy source could not get log segment");
ImmutableRWFileInfo* file_info;
RETURN_NOT_OK(FindLogSegment(segment_seqno, &file_info, error_code));
- RETURN_NOT_OK(ReadFileChunkToBuf(file_info, offset, client_maxlen,
+ const auto& kHeaderSize = file_info->readable->GetEncryptionHeaderSize();
+ // To make sure unencrypted tservers can copy tablets from encrypted tservers
+ // and vice versa (which is a valid scenario when encrypting an existing
+ // cluster in place), we add the encryption header size here and start at 0
at
+ // the client side.
+ RETURN_NOT_OK(ReadFileChunkToBuf(file_info, offset + kHeaderSize,
client_maxlen,
Substitute("log segment $0", segment_seqno),
data, log_file_size, error_code));
+ *log_file_size -= kHeaderSize;
// Note: We do not eagerly close log segment files, since we share ownership
// of the LogSegment objects with the Log itself.
diff --git a/src/kudu/util/env-test.cc b/src/kudu/util/env-test.cc
index 88b31b1..f3e2343 100644
--- a/src/kudu/util/env-test.cc
+++ b/src/kudu/util/env-test.cc
@@ -74,9 +74,11 @@
DECLARE_bool(never_fsync);
DECLARE_bool(crash_on_eio);
+DECLARE_bool(encrypt_data_at_rest);
DECLARE_double(env_inject_eio);
DECLARE_int32(env_inject_short_read_bytes);
DECLARE_int32(env_inject_short_write_bytes);
+DECLARE_int32(encryption_key_length);
DECLARE_string(env_inject_eio_globs);
namespace kudu {
@@ -271,7 +273,7 @@ TEST_F(TestEnv, TestPreallocate) {
// the writable file size should now report 1 MB
ASSERT_EQ(file->Size(), kOneMb);
ASSERT_OK(file->Close());
- // and the real size for the file on disk should match ony the
+ // and the real size for the file on disk should match only the
// written size
ASSERT_OK(env_->GetFileSize(test_path, &size));
ASSERT_EQ(kOneMb, size);
@@ -459,8 +461,8 @@ TEST_F(TestEnv, TestTruncate) {
// Write 'size' bytes of data to a file, with a simple pattern stored in it.
static void WriteTestFile(Env* env, const string& path, size_t size) {
- shared_ptr<WritableFile> wf;
- ASSERT_OK(env_util::OpenFileForWrite(env, path, &wf));
+ unique_ptr<WritableFile> wf;
+ ASSERT_OK(env->NewWritableFile(path, &wf));
faststring data;
data.resize(size);
for (int i = 0; i < data.size(); i++) {
@@ -567,7 +569,7 @@ TEST_F(TestEnv, TestIOVMax) {
FLAGS_env_inject_short_read_bytes = 3;
// Verify all the data is read
- ASSERT_OK(file->ReadV(0, results));
+ ASSERT_OK(file->ReadV(file->GetEncryptionHeaderSize(), results));
VerifyTestData(Slice(scratch, data_size), 0);
}
@@ -600,7 +602,7 @@ TEST_F(TestEnv, TestOpenEmptyRandomAccessFile) {
ASSERT_OK(env->NewRandomAccessFile(test_file, &readable_file));
uint64_t size;
ASSERT_OK(readable_file->Size(&size));
- ASSERT_EQ(0, size);
+ ASSERT_EQ(readable_file->GetEncryptionHeaderSize(), size);
}
TEST_F(TestEnv, TestOverwrite) {
@@ -629,8 +631,7 @@ TEST_F(TestEnv, TestReopen) {
// Create the file and write to it.
shared_ptr<WritableFile> writer;
- ASSERT_OK(env_util::OpenFileForWrite(WritableFileOptions(),
- env_, test_path, &writer));
+ ASSERT_OK(env_util::OpenFileForWrite(env_, test_path, &writer));
ASSERT_OK(writer->Append(first));
ASSERT_EQ(first.length(), writer->Size());
ASSERT_OK(writer->Close());
@@ -1004,7 +1005,7 @@ TEST_F(TestEnv, TestCopyFile) {
ASSERT_OK(env_util::CopyFile(env, orig_path, copy_path,
WritableFileOptions()));
unique_ptr<RandomAccessFile> copy;
ASSERT_OK(env->NewRandomAccessFile(copy_path, ©));
- NO_FATALS(ReadAndVerifyTestData(copy.get(), 0, kFileSize));
+ NO_FATALS(ReadAndVerifyTestData(copy.get(), copy->GetEncryptionHeaderSize(),
kFileSize));
}
// Simple regression test for NewTempRWFile().
@@ -1120,6 +1121,7 @@ TEST_F(TestEnv, TestGetExtentMap) {
// Punch out a hole and split the extent.
s = f->PunchHole(found_offset, fs_block_size);
+ LOG(INFO) << Substitute("Punched a $0 byte sized hole at offset $1",
fs_block_size, found_offset);
if (s.IsNotSupported()) {
LOG(INFO) << "PunchHole() not supported, skipping this part of the test";
return;
@@ -1249,7 +1251,18 @@ TEST_F(TestEnv, TestCreateFifo) {
ASSERT_OK(env_->NewFifo(kFifo, &fifo));
}
-TEST_F(TestEnv, TestEncryption) {
+class TestEncryptedEnv : public TestEnv, public
::testing::WithParamInterface<int> {
+ public:
+ void SetUp() override {
+ FLAGS_encrypt_data_at_rest = true;
+ FLAGS_encryption_key_length = GetParam();
+ }
+};
+
+INSTANTIATE_TEST_SUITE_P(TestEncryption, TestEncryptedEnv,
::testing::Values(128, 192, 256));
+
+TEST_P(TestEncryptedEnv, TestEncryption) {
+ FLAGS_encrypt_data_at_rest = true;
const string kFile = JoinPathSegments(test_dir_, "encrypted_file");
unique_ptr<RWFile> rw;
RWFileOptions opts;
@@ -1265,8 +1278,8 @@ TEST_F(TestEnv, TestEncryption) {
"We also need to append to this file to make sure initialization vectors
are calculated "
"correctly.";
- ASSERT_OK(rw->Write(0, kTestData));
- ASSERT_OK(rw->Write(kTestData.length(), kTestData2));
+ ASSERT_OK(rw->Write(env_->GetEncryptionHeaderSize(), kTestData));
+ ASSERT_OK(rw->Write(env_->GetEncryptionHeaderSize() + kTestData.length(),
kTestData2));
// Setup read parameters
constexpr size_t size1 = 9;
@@ -1278,7 +1291,7 @@ TEST_F(TestEnv, TestEncryption) {
vector<Slice> results = { result1, result2 };
// Reading back from the RWFile should succeed
- ASSERT_OK(rw->ReadV(13, results));
+ ASSERT_OK(rw->ReadV(env_->GetEncryptionHeaderSize() + 13, results));
ASSERT_EQ(result1, "This text");
ASSERT_EQ(result2, " is slightly longer");
@@ -1292,7 +1305,7 @@ TEST_F(TestEnv, TestEncryption) {
// Treating it as an unencrypted file should yield garbage and not contain
the
// cleartext written to the file.
string unencrypted_string;
- ASSERT_OK(unencrpyted->Read(0, unencrypted_string));
+ ASSERT_OK(unencrpyted->Read(env_->GetEncryptionHeaderSize(),
unencrypted_string));
ASSERT_EQ(string::npos, unencrypted_string.find("This text is slightly
longer"));
// Check if the file can be read into a SequentialFile.
@@ -1315,7 +1328,7 @@ TEST_F(TestEnv, TestEncryption) {
size_t size = kTestData.length() + kTestData2.length();
uint8_t scratch[size];
Slice result(scratch, size);
- ASSERT_OK(random->Read(0, result));
+ ASSERT_OK(random->Read(env_->GetEncryptionHeaderSize(), result));
ASSERT_EQ(kTestData + kTestData2, result);
// Read a SequentialFile until EOF.
@@ -1326,7 +1339,7 @@ TEST_F(TestEnv, TestEncryption) {
ASSERT_EQ(kTestData + kTestData2, result3);
}
-TEST_F(TestEnv, TestPreallocatedReadEncryptedFile) {
+TEST_P(TestEncryptedEnv, TestPreallocatedReadEncryptedFile) {
const string kFile = JoinPathSegments(test_dir_, "encrypted_file");
unique_ptr<RWFile> rw;
RWFileOptions opts;
@@ -1339,11 +1352,11 @@ TEST_F(TestEnv, TestPreallocatedReadEncryptedFile) {
uint8_t scratch[size];
Slice result(scratch, size);
vector<Slice> results = {result};
- ASSERT_OK(rw->ReadV(0, results));
+ ASSERT_OK(rw->ReadV(env_->GetEncryptionHeaderSize(), results));
ASSERT_TRUE(IsAllZeros(result));
}
-TEST_F(TestEnv, TestEncryptionMultipleSlices) {
+TEST_P(TestEncryptedEnv, TestEncryptionMultipleSlices) {
const string kFile = JoinPathSegments(test_dir_, "encrypted_file");
unique_ptr<RWFile> rw;
RWFileOptions opts;
@@ -1352,11 +1365,11 @@ TEST_F(TestEnv, TestEncryptionMultipleSlices) {
vector<Slice> data = {"foo", "bar", "hello", "world"};
- ASSERT_OK(rw->WriteV(0, data));
+ ASSERT_OK(rw->WriteV(env_->GetEncryptionHeaderSize(), data));
uint8_t scratch[16];
Slice result(scratch, 16);
- ASSERT_OK(rw->Read(0, result));
+ ASSERT_OK(rw->Read(env_->GetEncryptionHeaderSize(), result));
ASSERT_EQ("foobarhelloworld", result);
}
diff --git a/src/kudu/util/env.cc b/src/kudu/util/env.cc
index 2935fcf..c4c954f 100644
--- a/src/kudu/util/env.cc
+++ b/src/kudu/util/env.cc
@@ -60,6 +60,15 @@ Status WriteStringToFileSync(Env* env, const Slice& data,
const std::string& fname) {
return DoWriteStringToFile(env, data, fname, true, false);
}
+Status WriteStringToFileEncrypted(Env* env, const Slice& data,
+ const std::string& fname) {
+ return DoWriteStringToFile(env, data, fname, false, true);
+}
+
+Status WriteStringToFileEncryptedSync(Env* env, const Slice& data,
+ const std::string& fname) {
+ return DoWriteStringToFile(env, data, fname, true, true);
+}
Status DoReadFileToString(Env* env, const std::string& fname, faststring*
data, bool is_sensitive) {
data->clear();
@@ -90,4 +99,8 @@ Status ReadFileToString(Env* env, const std::string& fname,
faststring* data) {
return DoReadFileToString(env, fname, data, false);
}
+Status ReadFileToStringEncrypted(Env* env, const std::string& fname,
faststring* data) {
+ return DoReadFileToString(env, fname, data, true);
+}
+
} // namespace kudu
diff --git a/src/kudu/util/env.h b/src/kudu/util/env.h
index e3813e8..aa95b0b 100644
--- a/src/kudu/util/env.h
+++ b/src/kudu/util/env.h
@@ -190,7 +190,7 @@ class Env {
virtual Status DeleteRecursively(const std::string &dirname) = 0;
// Store the logical size of fname in *file_size.
- virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) =
0;
+ virtual Status GetFileSize(const std::string& fname, uint64_t* file_size)= 0;
// Store the physical size of fname in *file_size.
//
@@ -374,13 +374,16 @@ class Env {
// Creates symlink 'dst' that points to 'source'.
virtual Status CreateSymLink(const std::string& src, const std::string& dst)
= 0;
+ // Returns the encryption header size.
+ virtual size_t GetEncryptionHeaderSize() const = 0;
+
// Special string injected into file-growing operations' random failures
// (if enabled).
//
// Only useful for tests.
static const char* const kInjectedFailureStatusMsg;
- virtual const bool IsEncryptionEnabled() = 0;
+ virtual bool IsEncryptionEnabled() const = 0;
private:
DISALLOW_COPY_AND_ASSIGN(Env);
@@ -391,6 +394,9 @@ class File {
public:
virtual ~File();
+ // Returns the encryption header size.
+ virtual size_t GetEncryptionHeaderSize() const = 0;
+
// Returns the filename provided at construction time.
virtual const std::string& filename() const = 0;
};
@@ -689,6 +695,8 @@ class RWFile : public File {
// Retrieves the file's size.
virtual Status Size(uint64_t* size) const = 0;
+ virtual bool IsEncrypted() const = 0;
+
// Retrieve a map of the file's live extents.
//
// Each map entry is an offset and size representing a section of live file
@@ -722,10 +730,21 @@ extern Status WriteStringToFile(Env* env, const Slice&
data,
extern Status WriteStringToFileSync(Env* env, const Slice& data,
const std::string& fname);
+// A utility routine: write "data" to the named encrypted file.
+extern Status WriteStringToFileEncrypted(Env* env, const Slice& data,
+ const std::string& fname);
+// Like above but also fsyncs the new file.
+extern Status WriteStringToFileEncryptedSync(Env* env, const Slice& data,
+ const std::string& fname);
+
// A utility routine: read contents of named file into *data
extern Status ReadFileToString(Env* env, const std::string& fname,
faststring* data);
+// A utility routine: read contents of named encrypted file into *data
+extern Status ReadFileToStringEncrypted(Env* env, const std::string& fname,
+ faststring* data);
+
// Overloaded operator for printing Env::ResourceLimitType.
std::ostream& operator<<(std::ostream& o, Env::ResourceLimitType t);
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index 04514bb..3226389 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -7,6 +7,7 @@
#include <fnmatch.h>
#include <fts.h>
#include <glob.h>
+#include <openssl/rand.h>
#include <openssl/ssl.h>
#include <pthread.h>
#include <sys/resource.h>
@@ -41,6 +42,7 @@
#include "kudu/gutil/atomicops.h"
#include "kudu/gutil/basictypes.h"
+#include "kudu/gutil/integral_types.h"
#include "kudu/gutil/macros.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/once.h"
@@ -203,6 +205,10 @@ TAG_FLAG(env_inject_lock_failure_globs, hidden);
DEFINE_bool(encrypt_data_at_rest, false,
"Whether sensitive files should be encrypted on the file system.");
TAG_FLAG(encrypt_data_at_rest, hidden);
+DEFINE_int32(encryption_key_length, 128, "Encryption key length.");
+TAG_FLAG(encryption_key_length, hidden);
+DEFINE_validator(encryption_key_length,
+ [](const char* /*n*/, int32 v) { return v == 128 || v == 192
|| v == 256; });
static __thread uint64_t thread_local_id;
static Atomic64 cur_thread_local_id_;
@@ -213,21 +219,84 @@ const char* const Env::kInjectedFailureStatusMsg =
"INJECTED FAILURE";
const uint8_t kEncryptionBlockSize = 16;
+const uint8_t kEncryptionHeaderSize = 64;
+
+const char* const kEncryptionHeaderMagic = "kuduenc";
+
using evp_ctx_unique_ptr = std::unique_ptr<EVP_CIPHER_CTX,
decltype(&EVP_CIPHER_CTX_free)>;
namespace {
+
struct FreeDeleter {
inline void operator()(void* ptr) const {
free(ptr);
}
};
+enum class EncryptionAlgorithm {
+ AES128CTR = 0x00,
+ AES192CTR = 0x01,
+ AES256CTR = 0x02,
+ // ECB mode below only used to encrypt keys.
+ AES128ECB = 0xFD,
+ AES192ECB = 0XFE,
+ AES256ECB = 0xFF,
+};
+
+// The encryption header is stored on disk as follows:
+//
+// *----------------------------------------------*
+// | "kuduenc" magic string (7 bytes) |
+// *----------------------------------------------*
+// | Algorithm and key length (1 byte) |
+// *----------------------------------------------*
+// | Encrypted File Key (right-padded) (32 bytes) |
+// *----------------------------------------------*
+// | Reserved for future use (24 bytes) |
+// *----------------------------------------------*
+//
+// The algorithm and key length mapping is:
+//
+// *------*-------------*
+// | 0x00 | AES-128-CTR |
+// *------*-------------*
+// | 0x01 | AES-192-CTR |
+// *------*-------------*
+// | 0x02 | AES-256-CTR |
+// *------*-------------*
+struct EncryptionHeader {
+ EncryptionAlgorithm algorithm;
+ uint8_t key[32];
+};
+
// KUDU-3316: This is the key temporarily used for all encrypion. Obviously,
// this is not secure and MUST be removed and replaced with real keys once the
// key infra is in place.
// TODO(abukor): delete this.
-const uint8_t kDummyEncryptionKey[16] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 42};
+const struct EncryptionHeader kDummyEncryptionKey = {
+ EncryptionAlgorithm::AES128ECB,
+ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 42},
+};
+
+const EVP_CIPHER* GetEVPCipher(EncryptionAlgorithm algorithm) {
+ switch (algorithm) {
+ case EncryptionAlgorithm::AES128CTR:
+ return EVP_aes_128_ctr();
+ case EncryptionAlgorithm::AES192CTR:
+ return EVP_aes_192_ctr();
+ case EncryptionAlgorithm::AES256CTR:
+ return EVP_aes_256_ctr();
+ case EncryptionAlgorithm::AES128ECB:
+ return EVP_aes_128_ecb();
+ case EncryptionAlgorithm::AES192ECB:
+ return EVP_aes_192_ecb();
+ case EncryptionAlgorithm::AES256ECB:
+ return EVP_aes_256_ecb();
+ default:
+ return nullptr;
+ }
+}
#if defined(__APPLE__)
// Simulates Linux's fallocate file preallocation API on OS X.
@@ -383,7 +452,7 @@ Status DoOpen(const string& filename, int flags, const
string& reason, int* fd)
// Encrypts the data in 'cleartext' and writes it to 'ciphertext'. It requires
// 'offset' to be set in the file as it's used to set the initialization
vector.
-Status DoEncryptV(const uint8_t* key,
+Status DoEncryptV(const EncryptionHeader* eh,
uint64_t offset,
ArrayView<const Slice> cleartext,
ArrayView<Slice> ciphertext) {
@@ -394,7 +463,9 @@ Status DoEncryptV(const uint8_t* key,
InlineBigEndianEncodeFixed64(&iv[8], offset / kEncryptionBlockSize);
evp_ctx_unique_ptr ctx(EVP_CIPHER_CTX_new(), EVP_CIPHER_CTX_free);
- OPENSSL_RET_NOT_OK(EVP_EncryptInit_ex(ctx.get(), EVP_aes_128_ctr(), nullptr,
key, iv),
+
+ OPENSSL_RET_NOT_OK(EVP_EncryptInit_ex(ctx.get(), GetEVPCipher(eh->algorithm),
+ nullptr, eh->key, iv),
"Failed to initialize encryption");
size_t offset_mod = offset % kEncryptionBlockSize;
if (offset_mod) {
@@ -425,14 +496,15 @@ Status DoEncryptV(const uint8_t* key,
}
// Decrypts 'data'. Uses 'offset' in the file to set the initialization vector.
-Status DoDecryptV(const uint8_t* key, uint64_t offset, ArrayView<Slice> data) {
+Status DoDecryptV(const EncryptionHeader* eh, uint64_t offset,
ArrayView<Slice> data) {
// Set the initialization vector based on the offset.
uint8_t iv[16];
InlineBigEndianEncodeFixed64(&iv[0], 0);
InlineBigEndianEncodeFixed64(&iv[8], offset / kEncryptionBlockSize);
evp_ctx_unique_ptr ctx(EVP_CIPHER_CTX_new(), EVP_CIPHER_CTX_free);
- OPENSSL_RET_NOT_OK(EVP_DecryptInit_ex(ctx.get(), EVP_aes_128_ctr(), nullptr,
key, iv),
+ OPENSSL_RET_NOT_OK(EVP_DecryptInit_ex(ctx.get(), GetEVPCipher(eh->algorithm),
+ nullptr, eh->key, iv),
"Failed to initialize decryption");
size_t offset_mod = offset % kEncryptionBlockSize;
if (offset_mod) {
@@ -460,7 +532,6 @@ Status DoDecryptV(const uint8_t* key, uint64_t offset,
ArrayView<Slice> data) {
ciphertext_slice.data(),
in_length),
"Failed to decrypt data");
- DCHECK_EQ(out_length, in_length);
}
return Status::OK();
}
@@ -494,7 +565,11 @@ Status DoOpen(const string& filename, Env::OpenMode mode,
int* fd) {
}
Status DoReadV(
- int fd, const string& filename, uint64_t offset, ArrayView<Slice> results,
bool decrypt) {
+ int fd,
+ const string& filename,
+ uint64_t offset,
+ ArrayView<Slice> results,
+ const EncryptionHeader* eh) {
MAYBE_RETURN_EIO(filename, IOError(Env::kInjectedFailureStatusMsg, EIO));
ThreadRestrictions::AssertIOAllowed();
@@ -533,14 +608,16 @@ Status DoReadV(
}
if (PREDICT_FALSE(r == 0)) {
// EOF.
- RETURN_NOT_OK(DoDecryptV(kDummyEncryptionKey, offset, results));
+ if (eh) {
+ RETURN_NOT_OK(DoDecryptV(eh, offset, results));
+ }
return Status::EndOfFile(
Substitute("EOF trying to read $0 bytes at offset $1", bytes_req,
offset));
}
if (PREDICT_TRUE(r == rem)) {
// All requested bytes were read. This is almost always the case.
- if (decrypt) {
- RETURN_NOT_OK(DoDecryptV(kDummyEncryptionKey, offset, results));
+ if (eh) {
+ RETURN_NOT_OK(DoDecryptV(eh, offset, results));
}
return Status::OK();
}
@@ -563,15 +640,19 @@ Status DoReadV(
cur_offset += r;
rem -= r;
}
- if (decrypt) {
- RETURN_NOT_OK(DoDecryptV(kDummyEncryptionKey, offset, results));
+ if (eh) {
+ RETURN_NOT_OK(DoDecryptV(eh, offset, results));
}
DCHECK_EQ(0, rem);
return Status::OK();
}
Status DoWriteV(
- int fd, const string& filename, uint64_t offset, ArrayView<const Slice>
data, bool encrypt) {
+ int fd,
+ const string& filename,
+ uint64_t offset,
+ ArrayView<const Slice> data,
+ const EncryptionHeader* eh) {
MAYBE_RETURN_EIO(filename, IOError(Env::kInjectedFailureStatusMsg, EIO));
ThreadRestrictions::AssertIOAllowed();
@@ -582,11 +663,11 @@ Status DoWriteV(
struct iovec iov[iov_size];
std::vector<Slice> encrypted_data(iov_size);
SCOPED_CLEANUP({
- if (encrypt) {
+ if (eh) {
delete[] encrypted_data[0].mutable_data();
}
});
- if (encrypt) {
+ if (eh) {
for (size_t i = 0; i < iov_size; i++) {
bytes_req += data[i].size();
}
@@ -598,7 +679,7 @@ Status DoWriteV(
buffer_offset += size;
iov[i] = {const_cast<uint8_t*>(encrypted_data[i].data()), size};
}
- RETURN_NOT_OK(DoEncryptV(kDummyEncryptionKey, offset, data,
encrypted_data));
+ RETURN_NOT_OK(DoEncryptV(eh, offset, data, encrypted_data));
encrypted_buf.release();
} else {
for (size_t i = 0; i < iov_size; i++) {
@@ -659,6 +740,68 @@ Status DoWriteV(
return Status::OK();
}
+Status GenerateHeader(EncryptionHeader* eh) {
+ switch (FLAGS_encryption_key_length) {
+ case 128:
+ eh->algorithm = EncryptionAlgorithm::AES128CTR;
+ break;
+ case 192:
+ eh->algorithm = EncryptionAlgorithm::AES192CTR;
+ break;
+ case 256:
+ eh->algorithm = EncryptionAlgorithm::AES256CTR;
+ break;
+ default:
+ return Status::InvalidArgument(
+ "Supported key lengths for AES encryption are 128, 192, and 256.");
+ }
+ OPENSSL_RET_NOT_OK(RAND_bytes(eh->key, FLAGS_encryption_key_length / 8),
+ "Failed to generate random key");
+
+ return Status::OK();
+}
+
+Status WriteEncryptionHeader(int fd, const string& filename, const
EncryptionHeader& eh) {
+ vector<Slice> headerv = { kEncryptionHeaderMagic };
+ uint32_t key_size;
+ uint8_t algorithm[1];
+ switch (eh.algorithm) {
+ case EncryptionAlgorithm::AES128CTR:
+ algorithm[0] = 0;
+ key_size = 16;
+ break;
+ case EncryptionAlgorithm::AES192CTR:
+ algorithm[0] = 1;
+ // As the keys are encrypted in ECB mode which requires padding, we need
+ // 32 bytes instead of 24 to encrypt and write a 192-bit key.
+ key_size = 32;
+ break;
+ case EncryptionAlgorithm::AES256CTR:
+ algorithm[0] = 2;
+ key_size = 32;
+ break;
+ default:
+ return Status::InvalidArgument(Substitute("Unknown encryption algorithm:
$0", algorithm));
+ }
+ headerv.emplace_back(Slice(algorithm, 1));
+ Slice file_key(eh.key, key_size);
+
+ uint8_t encrypted_file_key[32];
+ Slice efk(encrypted_file_key, key_size);
+ vector<Slice> clear = {file_key};
+ vector<Slice> cipher = {efk};
+ RETURN_NOT_OK(DoEncryptV(&kDummyEncryptionKey, 0, clear, cipher));
+
+ // Add the encrypted file key and trailing zeros to the header.
+ headerv.emplace_back(efk);
+ static const uint8_t padding[40] = {0};
+ // 7 bytes of magic + 1 byte of algorithm and key length.
+ constexpr int kMagicAndAlgorithmSize = 8;
+ Slice padding_slice(padding, kEncryptionHeaderSize - kMagicAndAlgorithmSize
- key_size);
+ headerv.emplace_back(padding_slice);
+ return DoWriteV(fd, filename, 0, headerv, nullptr);
+}
+
Status DoIsOnXfsFilesystem(const string& path, bool* result) {
#ifdef __APPLE__
*result = false;
@@ -676,6 +819,39 @@ Status DoIsOnXfsFilesystem(const string& path, bool*
result) {
return Status::OK();
}
+Status ReadEncryptionHeader(int fd, const string& filename, EncryptionHeader*
eh) {
+ char magic[7];
+ uint8_t algorithm[1];
+ char file_key[32];
+ vector<Slice> headerv({ Slice(magic, 7), Slice(algorithm, 1),
Slice(file_key, 32) });
+ RETURN_NOT_OK(DoReadV(fd, filename, 0, headerv, nullptr));
+ if (strncmp(magic, kEncryptionHeaderMagic, 7) != 0) {
+ return Status::Corruption(Substitute("Invalid encryption header: $0",
magic));
+ }
+ uint16_t key_size;
+ eh->algorithm = EncryptionAlgorithm(algorithm[0]);
+ switch (eh->algorithm) {
+ case EncryptionAlgorithm::AES128CTR:
+ key_size = 16;
+ break;
+ case EncryptionAlgorithm::AES192CTR:
+ key_size = 24;
+ break;
+ case EncryptionAlgorithm::AES256CTR:
+ key_size = 32;
+ break;
+ default:
+ return Status::Corruption(Substitute("Unknown encryption algorithm: $0",
algorithm));
+ }
+ // Round up to the nearest multiple of 16 bytes when reading and decrypting
+ // the file. The actual key size can be used when storing the key in memory.
+ // See WriteEncryptionHeader for more info.
+ vector<Slice> v = {Slice(file_key, (key_size + 15) & -16)};
+ RETURN_NOT_OK(DoDecryptV(&kDummyEncryptionKey, 0, v));
+ memcpy(&eh->key, file_key, key_size);
+ return Status::OK();
+}
+
const char* ResourceLimitTypeToString(Env::ResourceLimitType t) {
switch (t) {
case Env::ResourceLimitType::OPEN_FILES_PER_PROCESS:
@@ -713,6 +889,10 @@ class PosixFifo : public Fifo {
public:
explicit PosixFifo(string fname) : filename_(std::move(fname)) {}
+ size_t GetEncryptionHeaderSize() const override {
+ return 0;
+ }
+
const string& filename() const override {
return filename_;
}
@@ -758,10 +938,16 @@ class PosixSequentialFile: public SequentialFile {
FILE* const file_;
const bool encrypted_;
size_t offset_;
+ const EncryptionHeader encryption_header_;
public:
- PosixSequentialFile(string fname, bool encrypted, FILE* f)
- : filename_(std::move(fname)), file_(f), encrypted_(encrypted),
offset_(0) {}
+ PosixSequentialFile(string fname, bool encrypted, FILE* f, EncryptionHeader
eh)
+ : filename_(std::move(fname)),
+ file_(f),
+ encrypted_(encrypted),
+ offset_(encrypted ? kEncryptionHeaderSize : 0),
+ encryption_header_(eh) {}
+
~PosixSequentialFile() {
int err;
RETRY_ON_EINTR(err, fclose(file_));
@@ -787,7 +973,7 @@ class PosixSequentialFile: public SequentialFile {
}
}
if (encrypted_) {
- RETURN_NOT_OK(DoDecryptV(kDummyEncryptionKey, offset_,
ArrayView<Slice>(result, 1)));
+ RETURN_NOT_OK(DoDecryptV(&encryption_header_, offset_,
ArrayView<Slice>(result, 1)));
}
offset_ += r;
return Status::OK();
@@ -805,6 +991,10 @@ class PosixSequentialFile: public SequentialFile {
}
virtual const string& filename() const OVERRIDE { return filename_; }
+
+ size_t GetEncryptionHeaderSize() const override {
+ return encrypted_ ? kEncryptionHeaderSize : 0;
+ }
};
// pread() based random-access
@@ -813,20 +1003,28 @@ class PosixRandomAccessFile: public RandomAccessFile {
const string filename_;
const int fd_;
const bool encrypted_;
+ const EncryptionHeader encryption_header_;
public:
- PosixRandomAccessFile(string fname, int fd, bool encrypted)
- : filename_(std::move(fname)), fd_(fd), encrypted_(encrypted) {}
+ PosixRandomAccessFile(string fname, int fd, bool encrypted, EncryptionHeader
eh)
+ : filename_(std::move(fname)),
+ fd_(fd),
+ encrypted_(encrypted),
+ encryption_header_(eh) {}
~PosixRandomAccessFile() {
DoClose(fd_);
}
virtual Status Read(uint64_t offset, Slice result) const OVERRIDE {
- return DoReadV(fd_, filename_, offset, ArrayView<Slice>(&result, 1),
encrypted_);
+ DCHECK_GE(offset, GetEncryptionHeaderSize());
+ return DoReadV(fd_, filename_, offset, ArrayView<Slice>(&result, 1),
+ encrypted_ ? &encryption_header_ : nullptr);
}
virtual Status ReadV(uint64_t offset, ArrayView<Slice> results) const
OVERRIDE {
- return DoReadV(fd_, filename_, offset, results, encrypted_);
+ DCHECK_GE(offset, GetEncryptionHeaderSize());
+ return DoReadV(fd_, filename_, offset, results,
+ encrypted_ ? &encryption_header_ : nullptr);
}
virtual Status Size(uint64_t *size) const OVERRIDE {
@@ -843,6 +1041,10 @@ class PosixRandomAccessFile: public RandomAccessFile {
virtual const string& filename() const OVERRIDE { return filename_; }
+ size_t GetEncryptionHeaderSize() const override {
+ return encrypted_ ? kEncryptionHeaderSize : 0;
+ }
+
virtual size_t memory_footprint() const OVERRIDE {
return kudu_malloc_usable_size(this) + filename_.capacity();
}
@@ -854,7 +1056,8 @@ class PosixRandomAccessFile: public RandomAccessFile {
// order to further improve Sync() performance.
class PosixWritableFile : public WritableFile {
public:
- PosixWritableFile(string fname, int fd, uint64_t file_size, bool
sync_on_close, bool encrypted)
+ PosixWritableFile(string fname, int fd, uint64_t file_size, bool
sync_on_close,
+ bool encrypted, EncryptionHeader eh)
: filename_(std::move(fname)),
fd_(fd),
sync_on_close_(sync_on_close),
@@ -862,7 +1065,8 @@ class PosixWritableFile : public WritableFile {
pre_allocated_size_(0),
pending_sync_(false),
closed_(false),
- encrypted_(encrypted) {}
+ encrypted_(encrypted),
+ encryption_header_(eh) {}
~PosixWritableFile() {
WARN_NOT_OK(Close(), "Failed to close " + filename_);
@@ -874,7 +1078,8 @@ class PosixWritableFile : public WritableFile {
virtual Status AppendV(ArrayView<const Slice> data) OVERRIDE {
ThreadRestrictions::AssertIOAllowed();
- RETURN_NOT_OK(DoWriteV(fd_, filename_, filesize_, data, encrypted_));
+ RETURN_NOT_OK(DoWriteV(fd_, filename_, filesize_, data,
+ encrypted_ ? &encryption_header_ : nullptr));
// Calculate the amount of data written
size_t bytes_written = accumulate(data.begin(), data.end(),
static_cast<size_t>(0),
[&](int sum, const Slice& curr) {
@@ -987,6 +1192,10 @@ class PosixWritableFile : public WritableFile {
virtual const string& filename() const OVERRIDE { return filename_; }
+ size_t GetEncryptionHeaderSize() const override {
+ return encrypted_ ? kEncryptionHeaderSize : 0;
+ }
+
private:
const string filename_;
const int fd_;
@@ -997,28 +1206,35 @@ class PosixWritableFile : public WritableFile {
bool pending_sync_;
bool closed_;
const bool encrypted_;
+ const EncryptionHeader encryption_header_;
};
class PosixRWFile : public RWFile {
public:
- PosixRWFile(string fname, int fd, bool sync_on_close, bool encrypted)
+ PosixRWFile(string fname, int fd, bool sync_on_close, bool encrypted,
+ EncryptionHeader eh)
: filename_(std::move(fname)),
fd_(fd),
sync_on_close_(sync_on_close),
is_on_xfs_(false),
closed_(false),
- encrypted_(encrypted) {}
+ encrypted_(encrypted),
+ encryption_header_(eh) {}
~PosixRWFile() {
WARN_NOT_OK(Close(), "Failed to close " + filename_);
}
virtual Status Read(uint64_t offset, Slice result) const OVERRIDE {
- return DoReadV(fd_, filename_, offset, ArrayView<Slice>(&result, 1),
encrypted_);
+ DCHECK_GE(offset, GetEncryptionHeaderSize());
+ return DoReadV(fd_, filename_, offset, ArrayView<Slice>(&result, 1),
+ encrypted_ ? &encryption_header_ : nullptr);
}
virtual Status ReadV(uint64_t offset, ArrayView<Slice> results) const
OVERRIDE {
- return DoReadV(fd_, filename_, offset, results, encrypted_);
+ DCHECK_GE(offset, GetEncryptionHeaderSize());
+ return DoReadV(fd_, filename_, offset, results,
+ encrypted_ ? &encryption_header_ : nullptr);
}
virtual Status Write(uint64_t offset, const Slice& data) OVERRIDE {
@@ -1026,7 +1242,9 @@ class PosixRWFile : public RWFile {
}
virtual Status WriteV(uint64_t offset, ArrayView<const Slice> data) OVERRIDE
{
- return DoWriteV(fd_, filename_, offset, data, encrypted_);
+ DCHECK_GE(offset, GetEncryptionHeaderSize());
+ return DoWriteV(fd_, filename_, offset, data,
+ encrypted_ ? &encryption_header_ : nullptr);
}
virtual Status PreAllocate(uint64_t offset,
@@ -1096,7 +1314,8 @@ class PosixRWFile : public RWFile {
} else {
int ret;
RETRY_ON_EINTR(ret, fallocate(
- fd_, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, length));
+ fd_, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+ offset, length));
if (ret != 0) {
return IOError(filename_, errno);
}
@@ -1234,10 +1453,18 @@ class PosixRWFile : public RWFile {
#endif
}
+ bool IsEncrypted() const override {
+ return encrypted_;
+ }
+
virtual const string& filename() const OVERRIDE {
return filename_;
}
+ size_t GetEncryptionHeaderSize() const override {
+ return encrypted_ ? kEncryptionHeaderSize : 0;
+ }
+
private:
static void InitIsOnXFS(void* arg) {
PosixRWFile* rwf = reinterpret_cast<PosixRWFile*>(arg);
@@ -1260,6 +1487,7 @@ class PosixRWFile : public RWFile {
bool is_on_xfs_;
bool closed_;
const bool encrypted_;
+ const EncryptionHeader encryption_header_;
};
int LockOrUnlock(int fd, bool lock) {
@@ -1304,7 +1532,17 @@ class PosixEnv : public Env {
if (f == nullptr) {
return IOError(fname, errno);
}
- result->reset(new PosixSequentialFile(fname, opts.is_sensitive &&
IsEncryptionEnabled(), f));
+ bool encrypted = opts.is_sensitive && IsEncryptionEnabled();
+ EncryptionHeader header;
+ if (encrypted) {
+ int fd;
+ RETURN_NOT_OK(DoOpen(fname, OpenMode::MUST_EXIST, &fd));
+ RETURN_NOT_OK(ReadEncryptionHeader(fd, fname, &header));
+ if (fseek(f, kEncryptionHeaderSize, SEEK_CUR)) {
+ return IOError(fname, errno);
+ }
+ }
+ result->reset(new PosixSequentialFile(fname, encrypted, f, header));
return Status::OK();
}
@@ -1324,9 +1562,13 @@ class PosixEnv : public Env {
if (fd < 0) {
return IOError(fname, errno);
}
-
+ EncryptionHeader header;
+ bool encrypted = opts.is_sensitive && IsEncryptionEnabled();
+ if (encrypted) {
+ RETURN_NOT_OK(ReadEncryptionHeader(fd, fname, &header));
+ }
result->reset(new PosixRandomAccessFile(fname, fd,
- opts.is_sensitive && IsEncryptionEnabled()));
+ encrypted, header));
return Status::OK();
}
@@ -1367,9 +1609,26 @@ class PosixEnv : public Env {
unique_ptr<RWFile>* result) OVERRIDE {
TRACE_EVENT1("io", "PosixEnv::NewRWFile", "path", fname);
int fd;
+ bool encrypt = opts.is_sensitive && IsEncryptionEnabled();
+ uint64_t size = 0;
+ if (opts.mode == MUST_EXIST) {
+ RETURN_NOT_OK(GetFileSize(fname, &size));
+ } else if (encrypt) {
+ GetFileSize(fname, &size);
+ }
+
RETURN_NOT_OK(DoOpen(fname, opts.mode, &fd));
+ EncryptionHeader eh;
+ if (encrypt) {
+ if (size >= kEncryptionHeaderSize) {
+ RETURN_NOT_OK(ReadEncryptionHeader(fd, fname, &eh));
+ } else {
+ RETURN_NOT_OK(GenerateHeader(&eh));
+ RETURN_NOT_OK(WriteEncryptionHeader(fd, fname, eh));
+ }
+ }
result->reset(new PosixRWFile(fname, fd, opts.sync_on_close,
- opts.is_sensitive && IsEncryptionEnabled()));
+ encrypt, eh));
return Status::OK();
}
@@ -1378,8 +1637,14 @@ class PosixEnv : public Env {
TRACE_EVENT1("io", "PosixEnv::NewTempRWFile", "template", name_template);
int fd = 0;
RETURN_NOT_OK(MkTmpFile(name_template, &fd, created_filename));
+ bool encrypt = opts.is_sensitive && IsEncryptionEnabled();
+ EncryptionHeader eh;
+ if (encrypt) {
+ RETURN_NOT_OK(GenerateHeader(&eh));
+ RETURN_NOT_OK(WriteEncryptionHeader(fd, *created_filename, eh));
+ }
res->reset(new PosixRWFile(*created_filename, fd, opts.sync_on_close,
- opts.is_sensitive && IsEncryptionEnabled()));
+ encrypt, eh));
return Status::OK();
}
@@ -1499,7 +1764,7 @@ class PosixEnv : public Env {
});
}
- virtual Status GetFileSize(const string& fname, uint64_t* size) OVERRIDE {
+ virtual Status GetFileSize(const string& fname, uint64_t* size) override {
TRACE_EVENT1("io", "PosixEnv::GetFileSize", "path", fname);
MAYBE_RETURN_EIO(fname, IOError(Env::kInjectedFailureStatusMsg, EIO));
ThreadRestrictions::AssertIOAllowed();
@@ -1995,7 +2260,7 @@ class PosixEnv : public Env {
return result;
}
- const bool IsEncryptionEnabled() override { return
FLAGS_encrypt_data_at_rest; }
+ bool IsEncryptionEnabled() const override { return
FLAGS_encrypt_data_at_rest; }
private:
// unique_ptr Deleter implementation for fts_close
@@ -2038,11 +2303,24 @@ class PosixEnv : public Env {
const WritableFileOptions& opts,
unique_ptr<WritableFile>* result) {
uint64_t file_size = 0;
+ Status s = GetFileSize(fname, &file_size);
if (opts.mode == MUST_EXIST) {
- RETURN_NOT_OK(GetFileSize(fname, &file_size));
+ RETURN_NOT_OK(s);
+ }
+ bool encrypt = opts.is_sensitive && IsEncryptionEnabled();
+ EncryptionHeader eh;
+ if (encrypt) {
+ if (file_size < kEncryptionHeaderSize) {
+ RETURN_NOT_OK(GenerateHeader(&eh));
+ RETURN_NOT_OK(WriteEncryptionHeader(fd, fname, eh));
+ file_size = kEncryptionHeaderSize;
+ } else {
+ RETURN_NOT_OK(ReadEncryptionHeader(fd, fname, &eh));
+ }
}
result->reset(new PosixWritableFile(fname, fd, file_size,
opts.sync_on_close,
- opts.is_sensitive &&
IsEncryptionEnabled()));
+ encrypt, eh));
+
return Status::OK();
}
@@ -2064,6 +2342,10 @@ class PosixEnv : public Env {
}
}
+ size_t GetEncryptionHeaderSize() const override {
+ return IsEncryptionEnabled() ? kEncryptionHeaderSize : 0;
+ }
+
Status GetFileSizeOnDiskRecursivelyCb(uint64_t* bytes_used,
FileType type,
const string& dirname,
diff --git a/src/kudu/util/env_util.cc b/src/kudu/util/env_util.cc
index 45a9ddd..f481e0c 100644
--- a/src/kudu/util/env_util.cc
+++ b/src/kudu/util/env_util.cc
@@ -117,8 +117,6 @@ namespace env_util {
Status OpenFileForWrite(Env* env, const string& path,
shared_ptr<WritableFile>* file) {
- WritableFileOptions opts;
- opts.is_sensitive = true;
return OpenFileForWrite(WritableFileOptions(), env, path, file);
}
@@ -133,9 +131,13 @@ Status OpenFileForWrite(const WritableFileOptions& opts,
Status OpenFileForRandom(Env *env, const string &path,
shared_ptr<RandomAccessFile> *file) {
+ return OpenFileForRandom(RandomAccessFileOptions(), env, path, file);
+}
+
+Status OpenFileForRandom(const RandomAccessFileOptions& opts,
+ Env* env, const string& path,
+ shared_ptr<RandomAccessFile>* file) {
unique_ptr<RandomAccessFile> r;
- RandomAccessFileOptions opts;
- opts.is_sensitive = true;
RETURN_NOT_OK(env->NewRandomAccessFile(opts, path, &r));
file->reset(r.release());
return Status::OK();
@@ -144,9 +146,7 @@ Status OpenFileForRandom(Env *env, const string &path,
Status OpenFileForSequential(Env *env, const string &path,
shared_ptr<SequentialFile> *file) {
unique_ptr<SequentialFile> r;
- SequentialFileOptions opts;
- opts.is_sensitive = true;
- RETURN_NOT_OK(env->NewSequentialFile(opts, path, &r));
+ RETURN_NOT_OK(env->NewSequentialFile(SequentialFileOptions(), path, &r));
file->reset(r.release());
return Status::OK();
}
@@ -240,11 +240,18 @@ Status CreateDirsRecursively(Env* env, const string&
path) {
Status CopyFile(Env* env, const string& source_path, const string& dest_path,
WritableFileOptions opts) {
unique_ptr<SequentialFile> source;
- RETURN_NOT_OK(env->NewSequentialFile(source_path, &source));
+ // Both the source and the destination files are treated as insensitive,
+ // because if they're encrypted, it would be unnecessary to decrypt and
+ // re-encrypt it. This way, we make a byte for byte copy of the file
+ // regardless if it's encrypted.
+ SequentialFileOptions source_opts;
+ source_opts.is_sensitive = false;
+ RETURN_NOT_OK(env->NewSequentialFile(source_opts, source_path, &source));
uint64_t size;
RETURN_NOT_OK(env->GetFileSize(source_path, &size));
unique_ptr<WritableFile> dest;
+ opts.is_sensitive = false;
RETURN_NOT_OK(env->NewWritableFile(opts, dest_path, &dest));
RETURN_NOT_OK(dest->PreAllocate(size));
diff --git a/src/kudu/util/env_util.h b/src/kudu/util/env_util.h
index 00007d6..39c3201 100644
--- a/src/kudu/util/env_util.h
+++ b/src/kudu/util/env_util.h
@@ -30,6 +30,7 @@ class Env;
class RandomAccessFile;
class SequentialFile;
class WritableFile;
+struct RandomAccessFileOptions;
struct WritableFileOptions;
namespace env_util {
@@ -41,6 +42,9 @@ Status OpenFileForWrite(const WritableFileOptions& opts,
Env *env, const std::string &path,
std::shared_ptr<WritableFile> *file);
+Status OpenFileForRandom(const RandomAccessFileOptions& opts,
+ Env *env, const std::string &path,
+ std::shared_ptr<RandomAccessFile> *file);
Status OpenFileForRandom(Env *env, const std::string &path,
std::shared_ptr<RandomAccessFile> *file);
diff --git a/src/kudu/util/file_cache-stress-test.cc
b/src/kudu/util/file_cache-stress-test.cc
index 35afd0f..205cc01 100644
--- a/src/kudu/util/file_cache-stress-test.cc
+++ b/src/kudu/util/file_cache-stress-test.cc
@@ -126,7 +126,9 @@ class FileCacheStressTest : public KuduTest {
oid_generator.Next());
{
unique_ptr<WritableFile> next_file;
- CHECK_OK(env_->NewWritableFile(next_file_name, &next_file));
+ WritableFileOptions opts;
+ opts.is_sensitive = true;
+ CHECK_OK(env_->NewWritableFile(opts, next_file_name, &next_file));
uint8_t buf[rand.Uniform((32 * 1024) - 1) + 1];
CHECK_OK(next_file->Append(GenerateRandomChunk(buf, sizeof(buf),
&rand)));
CHECK_OK(next_file->Close());
@@ -286,8 +288,11 @@ class FileCacheStressTest : public KuduTest {
uint64_t file_size;
RETURN_NOT_OK(file->Size(&file_size));
- uint64_t off = file_size > 0 ? rand->Uniform(file_size) : 0;
- size_t len = file_size > 0 ? rand->Uniform(file_size - off) : 0;
+ const uint8_t kHeaderSize = file->GetEncryptionHeaderSize();;
+ uint64_t off = file_size > kHeaderSize
+ ? rand->Uniform(file_size - kHeaderSize) + kHeaderSize
+ : kHeaderSize;
+ size_t len = file_size > kHeaderSize ? rand->Uniform(file_size - off) : 0;
unique_ptr<uint8_t[]> scratch(new uint8_t[len]);
RETURN_NOT_OK(file->Read(off, Slice(scratch.get(), len)));
@@ -307,7 +312,10 @@ class FileCacheStressTest : public KuduTest {
uint64_t file_size;
RETURN_NOT_OK(file->Size(&file_size));
- uint64_t off = file_size > 0 ? rand->Uniform(file_size) : 0;
+ const uint8_t kHeaderSize = file->GetEncryptionHeaderSize();
+ uint64_t off = file_size > kHeaderSize
+ ? rand->Uniform(file_size - kHeaderSize) + kHeaderSize
+ : kHeaderSize;
uint8_t buf[64];
RETURN_NOT_OK(file->Write(off, GenerateRandomChunk(buf, sizeof(buf),
rand)));
(*metrics)[BaseName(file->filename())]["write"]++;
diff --git a/src/kudu/util/file_cache-test.cc b/src/kudu/util/file_cache-test.cc
index 70e0ef1..b0af030 100644
--- a/src/kudu/util/file_cache-test.cc
+++ b/src/kudu/util/file_cache-test.cc
@@ -104,7 +104,7 @@ class FileCacheTest : public KuduTest {
RWFileOptions opts;
opts.is_sensitive = true;
RETURN_NOT_OK(env_->NewRWFile(opts, name, &f));
- RETURN_NOT_OK(f->Write(0, data));
+ RETURN_NOT_OK(f->Write(f->GetEncryptionHeaderSize(), data));
return Status::OK();
}
@@ -155,7 +155,7 @@ TYPED_TEST(FileCacheTest, TestBasicOperations) {
for (int i = 0; i < 3; i++) {
uint64_t size;
ASSERT_OK(f1->Size(&size));
- ASSERT_EQ(kData1.size(), size);
+ ASSERT_EQ(kData1.size(), size -
Env::Default()->GetEncryptionHeaderSize());
NO_FATALS(this->AssertFdsAndDescriptors(1, 1));
}
@@ -262,10 +262,12 @@ TYPED_TEST(FileCacheTest, TestInvalidation) {
ASSERT_OK(this->WriteTestFile(kFile2, kData2));
ASSERT_OK(this->env_->RenameFile(kFile2, kFile1));
+ const uint8_t kHeaderSize = Env::Default()->GetEncryptionHeaderSize();
+
// We should still be able to access the file, since it has a cached fd.
uint64_t size;
ASSERT_OK(f->Size(&size));
- ASSERT_EQ(kData1.size(), size);
+ ASSERT_EQ(kData1.size(), size - kHeaderSize);
// If we invalidate it from the cache and try again, it should crash because
// the existing descriptor was invalidated.
@@ -277,7 +279,7 @@ TYPED_TEST(FileCacheTest, TestInvalidation) {
shared_ptr<TypeParam> f2;
ASSERT_OK(this->cache_->template OpenFile<Env::MUST_EXIST>(kFile1, &f2));
ASSERT_OK(f2->Size(&size));
- ASSERT_EQ(kData2.size(), size);
+ ASSERT_EQ(kData2.size(), size - kHeaderSize);
}
@@ -311,8 +313,8 @@ TYPED_TEST(FileCacheTest, TestHeavyReads) {
const auto& f = opened_files[idx];
uint64_t size;
ASSERT_OK(f->Size(&size));
- Slice s(buf.get(), size);
- ASSERT_OK(f->Read(0, s));
+ Slice s(buf.get(), size - f->GetEncryptionHeaderSize());
+ ASSERT_OK(f->Read(f->GetEncryptionHeaderSize(), s));
ASSERT_EQ(data, s);
ASSERT_LE(this->CountOpenFds(),
this->initial_open_fds_ + kCacheCapacity);
@@ -444,12 +446,13 @@ TEST_P(MixedFileCacheTest, TestBothFileTypes) {
// Create the two test files.
{
unique_ptr<RWFile> f;
+ const uint8_t kHeaderSize = Env::Default()->GetEncryptionHeaderSize();
RWFileOptions opts;
opts.is_sensitive = true;
ASSERT_OK(env_->NewRWFile(opts, kFile1, &f));
- ASSERT_OK(f->Write(0, kData1));
+ ASSERT_OK(f->Write(kHeaderSize, kData1));
ASSERT_OK(env_->NewRWFile(opts, kFile2, &f));
- ASSERT_OK(f->Write(0, kData2));
+ ASSERT_OK(f->Write(kHeaderSize, kData2));
}
FileCache cache("test", env_, 1, /*entity=*/ nullptr);
@@ -465,12 +468,12 @@ TEST_P(MixedFileCacheTest, TestBothFileTypes) {
uint64_t size;
ASSERT_OK(rwf->Size(&size));
uint8_t buf[16];
- Slice s1(buf, size);
- ASSERT_OK(rwf->Read(0, s1));
+ Slice s1(buf, size - rwf->GetEncryptionHeaderSize());
+ ASSERT_OK(rwf->Read(rwf->GetEncryptionHeaderSize(), s1));
ASSERT_EQ(kData1, s1);
ASSERT_OK(raf->Size(&size));
- Slice s2(buf, size);
- ASSERT_OK(raf->Read(0, s2));
+ Slice s2(buf, size - raf->GetEncryptionHeaderSize());
+ ASSERT_OK(raf->Read(raf->GetEncryptionHeaderSize(), s2));
ASSERT_EQ(kData2, s2);
// It's okay to reopen the test file using the same file type, but not with a
diff --git a/src/kudu/util/file_cache.cc b/src/kudu/util/file_cache.cc
index 7be762c..a59274b 100644
--- a/src/kudu/util/file_cache.cc
+++ b/src/kudu/util/file_cache.cc
@@ -310,10 +310,18 @@ class Descriptor<RWFile> : public RWFile {
return opened.file()->GetExtentMap(out);
}
+ bool IsEncrypted() const override {
+ return true;
+ }
+
const string& filename() const override {
return base_.filename();
}
+ size_t GetEncryptionHeaderSize() const override {
+ return base_.env()->GetEncryptionHeaderSize();
+ }
+
private:
friend class ::kudu::FileCache;
@@ -392,6 +400,10 @@ class Descriptor<RandomAccessFile> : public
RandomAccessFile {
return base_.filename();
}
+ size_t GetEncryptionHeaderSize() const override {
+ return base_.env()->GetEncryptionHeaderSize();
+ }
+
size_t memory_footprint() const override {
// Normally we would use kudu_malloc_usable_size(this). However, that's
// not safe because 'this' was allocated via std::make_shared(), which
diff --git a/src/kudu/util/pb_util-test.cc b/src/kudu/util/pb_util-test.cc
index 63f3a08..7b88b70 100644
--- a/src/kudu/util/pb_util-test.cc
+++ b/src/kudu/util/pb_util-test.cc
@@ -44,6 +44,8 @@
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
+DECLARE_bool(encrypt_data_at_rest);
+
namespace kudu {
namespace pb_util {
@@ -95,6 +97,10 @@ class TestPBUtil : public KuduTest {
// Truncate the specified file to the specified length.
Status TruncateFile(const string& path, uint64_t size);
+ void EnableEncryption(bool enable) {
+ FLAGS_encrypt_data_at_rest = enable;
+ }
+
// Output file name for most unit tests.
string path_;
};
@@ -158,7 +164,9 @@ Status TestPBUtil::BitFlipFileByteRange(const string& path,
uint64_t offset, uin
// Read the data from disk.
{
unique_ptr<RandomAccessFile> file;
- RETURN_NOT_OK(env_->NewRandomAccessFile(path, &file));
+ RandomAccessFileOptions opts;
+ opts.is_sensitive = false;
+ RETURN_NOT_OK(env_->NewRandomAccessFile(opts, path, &file));
uint64_t size;
RETURN_NOT_OK(file->Size(&size));
faststring scratch;
@@ -176,7 +184,9 @@ Status TestPBUtil::BitFlipFileByteRange(const string& path,
uint64_t offset, uin
// Write the data back to disk.
unique_ptr<WritableFile> file;
- RETURN_NOT_OK(env_->NewWritableFile(path, &file));
+ WritableFileOptions opts;
+ opts.is_sensitive = false;
+ RETURN_NOT_OK(env_->NewWritableFile(opts, path, &file));
RETURN_NOT_OK(file->Append(buf));
RETURN_NOT_OK(file->Close());
@@ -187,6 +197,7 @@ Status TestPBUtil::TruncateFile(const string& path,
uint64_t size) {
unique_ptr<RWFile> file;
RWFileOptions opts;
opts.mode = Env::MUST_EXIST;
+ opts.is_sensitive = false;
RETURN_NOT_OK(env_->NewRWFile(opts, path, &file));
RETURN_NOT_OK(file->Truncate(size));
return Status::OK();
@@ -239,6 +250,7 @@ TEST_F(TestPBUtil, TestWritableFileOutputStream) {
// Basic read/write test.
TEST_F(TestPBUtil, TestPBContainerSimple) {
+ EnableEncryption(true);
// Exercise both the SYNC and NO_SYNC codepaths, along with SENSITIVE and
// NOT_SENSITIVE, despite the fact that we aren't able to observe a
difference
// in the test.
@@ -272,7 +284,9 @@ TEST_P(TestPBContainerVersions, TestCorruption) {
{
// Create the empty file.
unique_ptr<WritableFile> file;
- ASSERT_OK(env_->NewWritableFile(path_, &file));
+ WritableFileOptions opts;
+ opts.is_sensitive = false;
+ ASSERT_OK(env_->NewWritableFile(opts, path_, &file));
ASSERT_OK(file->Close());
}
s = ReadPBContainerFromPath(env_, path_, &test_pb, NOT_SENSITIVE);
diff --git a/src/kudu/util/pb_util.cc b/src/kudu/util/pb_util.cc
index f049508..49bca1d 100644
--- a/src/kudu/util/pb_util.cc
+++ b/src/kudu/util/pb_util.cc
@@ -668,10 +668,9 @@ string SecureShortDebugString(const Message& msg) {
WritablePBContainerFile::WritablePBContainerFile(shared_ptr<RWFile> writer)
: state_(FileState::NOT_INITIALIZED),
- offset_(0),
+ offset_(writer->GetEncryptionHeaderSize()),
version_(kPBContainerDefaultVersion),
- writer_(std::move(writer)) {
-}
+ writer_(std::move(writer)) {}
WritablePBContainerFile::~WritablePBContainerFile() {
WARN_NOT_OK(Close(), "Could not Close() when destroying file");
@@ -889,7 +888,7 @@ void WritablePBContainerFile::PopulateDescriptorSet(
ReadablePBContainerFile::ReadablePBContainerFile(shared_ptr<RandomAccessFile>
reader)
: state_(FileState::NOT_INITIALIZED),
version_(kPBContainerInvalidVersion),
- offset_(0),
+ offset_(reader->GetEncryptionHeaderSize()),
reader_(std::move(reader)) {
}
