fs: allow format with user-specified uuid The CLI tool is going to use this for the "handling permanent failure" workflow.
Change-Id: Ib4189d6150a263b7dde304dd19a449d3c0ab6c8c Reviewed-on: http://gerrit.cloudera.org:8080/3968 Reviewed-by: Todd Lipcon <[email protected]> Tested-by: Adar Dembo <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1bce7312 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1bce7312 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1bce7312 Branch: refs/heads/master Commit: 1bce73126799cb046755c0555bbe5be7622c9ace Parents: aa94185 Author: Adar Dembo <[email protected]> Authored: Thu Aug 11 21:14:36 2016 -0700 Committer: Adar Dembo <[email protected]> Committed: Tue Aug 16 01:32:58 2016 +0000 ---------------------------------------------------------------------- src/kudu/fs/fs_manager-test.cc | 25 ++++++++++++++-- src/kudu/fs/fs_manager.cc | 17 ++++++++--- src/kudu/fs/fs_manager.h | 11 +++++-- src/kudu/util/CMakeLists.txt | 1 + src/kudu/util/oid_generator-test.cc | 50 ++++++++++++++++++++++++++++++++ src/kudu/util/oid_generator.cc | 37 +++++++++++++++++++---- src/kudu/util/oid_generator.h | 12 ++++++++ 7 files changed, 139 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/fs/fs_manager-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/fs/fs_manager-test.cc b/src/kudu/fs/fs_manager-test.cc index 922e617..21bb460 100644 --- a/src/kudu/fs/fs_manager-test.cc +++ b/src/kudu/fs/fs_manager-test.cc @@ -21,12 +21,15 @@ #include "kudu/fs/block_manager.h" #include "kudu/fs/fs_manager.h" +#include "kudu/gutil/strings/substitute.h" #include "kudu/gutil/strings/util.h" #include "kudu/util/metrics.h" +#include "kudu/util/oid_generator.h" #include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" using std::shared_ptr; +using strings::Substitute; namespace kudu { @@ -155,7 +158,7 @@ TEST_F(FsManagerTestBase, TestCannotUseNonEmptyFsRoot) { } TEST_F(FsManagerTestBase, TestEmptyWALPath) { - ReinitFsManager("", vector<string>()); + ReinitFsManager("", {}); Status s = fs_manager()->CreateInitialFileSystemLayout(); ASSERT_TRUE(s.IsIOError()); ASSERT_STR_CONTAINS(s.ToString(), "directory (fs_wal_dir) not provided"); @@ -165,7 +168,7 @@ TEST_F(FsManagerTestBase, TestOnlyWALPath) { string path = GetTestPath("new_fs_root"); ASSERT_OK(env_->CreateDir(path)); - ReinitFsManager(path, vector<string>()); + ReinitFsManager(path, {}); ASSERT_OK(fs_manager()->CreateInitialFileSystemLayout()); ASSERT_TRUE(HasPrefixString(fs_manager()->GetWalsRootDir(), path)); ASSERT_TRUE(HasPrefixString(fs_manager()->GetConsensusMetadataDir(), path)); @@ -175,4 +178,22 @@ TEST_F(FsManagerTestBase, TestOnlyWALPath) { ASSERT_TRUE(HasPrefixString(data_dirs[0], path)); } +TEST_F(FsManagerTestBase, TestFormatWithSpecificUUID) { + string path = GetTestPath("new_fs_root"); + ReinitFsManager(path, {}); + + // Use an invalid uuid at first. + string uuid = "not_a_valid_uuid"; + Status s = fs_manager()->CreateInitialFileSystemLayout(uuid); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_STR_CONTAINS(s.ToString(), Substitute("invalid uuid $0", uuid)); + + // Now use a valid one. + ObjectIdGenerator oid_generator; + uuid = oid_generator.Next(); + ASSERT_OK(fs_manager()->CreateInitialFileSystemLayout(uuid)); + ASSERT_OK(fs_manager()->Open()); + ASSERT_EQ(uuid, fs_manager()->uuid()); +} + } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/fs/fs_manager.cc ---------------------------------------------------------------------- diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc index 117389f..86e5d4f 100644 --- a/src/kudu/fs/fs_manager.cc +++ b/src/kudu/fs/fs_manager.cc @@ -22,6 +22,7 @@ #include <map> #include <unordered_set> +#include <boost/optional.hpp> #include <glog/logging.h> #include <glog/stl_logging.h> #include <google/protobuf/message.h> @@ -244,7 +245,7 @@ Status FsManager::Open() { return Status::OK(); } -Status FsManager::CreateInitialFileSystemLayout() { +Status FsManager::CreateInitialFileSystemLayout(boost::optional<string> uuid) { CHECK(!read_only_); RETURN_NOT_OK(Init()); @@ -271,7 +272,7 @@ Status FsManager::CreateInitialFileSystemLayout() { ElementDeleter d(&delete_on_failure); InstanceMetadataPB metadata; - CreateInstanceMetadata(&metadata); + RETURN_NOT_OK(CreateInstanceMetadata(std::move(uuid), &metadata)); unordered_set<string> to_sync; for (const string& root : canonicalized_all_fs_roots_) { bool created; @@ -319,9 +320,16 @@ Status FsManager::CreateInitialFileSystemLayout() { return Status::OK(); } -void FsManager::CreateInstanceMetadata(InstanceMetadataPB* metadata) { +Status FsManager::CreateInstanceMetadata(boost::optional<string> uuid, + InstanceMetadataPB* metadata) { ObjectIdGenerator oid_generator; - metadata->set_uuid(oid_generator.Next()); + if (uuid) { + string canonicalized_uuid; + RETURN_NOT_OK(oid_generator.Canonicalize(uuid.get(), &canonicalized_uuid)); + metadata->set_uuid(canonicalized_uuid); + } else { + metadata->set_uuid(oid_generator.Next()); + } string time_str; StringAppendStrftime(&time_str, "%Y-%m-%d %H:%M:%S", time(nullptr), false); @@ -330,6 +338,7 @@ void FsManager::CreateInstanceMetadata(InstanceMetadataPB* metadata) { hostname = "<unknown host>"; } metadata->set_format_stamp(Substitute("Formatted at $0 on $1", time_str, hostname)); + return Status::OK(); } Status FsManager::WriteInstanceMetadata(const InstanceMetadataPB& metadata, http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/fs/fs_manager.h ---------------------------------------------------------------------- diff --git a/src/kudu/fs/fs_manager.h b/src/kudu/fs/fs_manager.h index fab0436..3c4420a 100644 --- a/src/kudu/fs/fs_manager.h +++ b/src/kudu/fs/fs_manager.h @@ -18,6 +18,8 @@ #ifndef KUDU_FS_FS_MANAGER_H #define KUDU_FS_FS_MANAGER_H +#include <boost/none.hpp> +#include <boost/optional/optional.hpp> #include <gtest/gtest_prod.h> #include <iosfwd> #include <memory> @@ -107,10 +109,12 @@ class FsManager { // the on-disk structures. Status Open(); - // Create the initial filesystem layout. + // Create the initial filesystem layout. If 'uuid' is provided, uses it as + // uuid of the filesystem. Otherwise generates one at random. // // Returns an error if the file system is already initialized. - Status CreateInitialFileSystemLayout(); + Status CreateInitialFileSystemLayout( + boost::optional<std::string> uuid = boost::none); void DumpFileSystemTree(std::ostream& out); @@ -212,7 +216,8 @@ class FsManager { void InitBlockManager(); // Create a new InstanceMetadataPB. - void CreateInstanceMetadata(InstanceMetadataPB* metadata); + Status CreateInstanceMetadata(boost::optional<std::string> uuid, + InstanceMetadataPB* metadata); // Save a InstanceMetadataPB to the filesystem. // Does not mutate the current state of the fsmanager. http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt index 4a000b6..a4527b9 100644 --- a/src/kudu/util/CMakeLists.txt +++ b/src/kudu/util/CMakeLists.txt @@ -309,6 +309,7 @@ ADD_KUDU_TEST(mt-threadlocal-test RUN_SERIAL true) ADD_KUDU_TEST(net/dns_resolver-test) ADD_KUDU_TEST(net/net_util-test) ADD_KUDU_TEST(object_pool-test) +ADD_KUDU_TEST(oid_generator-test) ADD_KUDU_TEST(once-test) ADD_KUDU_TEST(os-util-test) ADD_KUDU_TEST(path_util-test) http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/oid_generator-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/oid_generator-test.cc b/src/kudu/util/oid_generator-test.cc new file mode 100644 index 0000000..a38b496 --- /dev/null +++ b/src/kudu/util/oid_generator-test.cc @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/util/oid_generator.h" + +#include <gtest/gtest.h> +#include <string> + +#include "kudu/util/test_util.h" + +using std::string; + +namespace kudu { + +TEST(ObjectIdGeneratorTest, TestCanoicalizeUuid) { + ObjectIdGenerator gen; + const string kExpectedCanonicalized = "0123456789abcdef0123456789abcdef"; + string canonicalized; + Status s = gen.Canonicalize("not_a_uuid", &canonicalized); + { + SCOPED_TRACE(s.ToString()); + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_STR_CONTAINS(s.ToString(), "invalid uuid"); + } + ASSERT_OK(gen.Canonicalize( + "01234567-89ab-cdef-0123-456789abcdef", &canonicalized)); + ASSERT_EQ(kExpectedCanonicalized, canonicalized); + ASSERT_OK(gen.Canonicalize( + "0123456789abcdef0123456789abcdef", &canonicalized)); + ASSERT_EQ(kExpectedCanonicalized, canonicalized); + ASSERT_OK(gen.Canonicalize( + "0123456789AbCdEf0123456789aBcDeF", &canonicalized)); + ASSERT_EQ(kExpectedCanonicalized, canonicalized); +} + +} // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/oid_generator.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/oid_generator.cc b/src/kudu/util/oid_generator.cc index da7acd8..580463c 100644 --- a/src/kudu/util/oid_generator.cc +++ b/src/kudu/util/oid_generator.cc @@ -15,21 +15,48 @@ // specific language governing permissions and limitations // under the License. +#include "kudu/util/oid_generator.h" + +#include <boost/uuid/uuid_generators.hpp> +#include <exception> #include <mutex> #include <string> #include "kudu/gutil/stringprintf.h" -#include "kudu/util/oid_generator.h" +#include "kudu/gutil/strings/substitute.h" +#include "kudu/util/status.h" + +using strings::Substitute; namespace kudu { -string ObjectIdGenerator::Next() { - std::lock_guard<LockType> l(oid_lock_); - boost::uuids::uuid oid = oid_generator_(); - const uint8_t *uuid = oid.data; +namespace { + +string ConvertUuidToString(const boost::uuids::uuid& to_convert) { + const uint8_t* uuid = to_convert.data; return StringPrintf("%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", uuid[0], uuid[1], uuid[2], uuid[3], uuid[4], uuid[5], uuid[6], uuid[7], uuid[8], uuid[9], uuid[10], uuid[11], uuid[12], uuid[13], uuid[14], uuid[15]); } +} // anonymous namespace + +string ObjectIdGenerator::Next() { + std::lock_guard<LockType> l(oid_lock_); + boost::uuids::uuid uuid = oid_generator_(); + return ConvertUuidToString(uuid); +} + +Status ObjectIdGenerator::Canonicalize(const string& input, + string* output) const { + try { + boost::uuids::uuid uuid = oid_validator_(input); + *output = ConvertUuidToString(uuid); + return Status::OK(); + } catch (std::exception& e) { + return Status::InvalidArgument(Substitute("invalid uuid $0: $1", + input, e.what())); + } +} + } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/oid_generator.h ---------------------------------------------------------------------- diff --git a/src/kudu/util/oid_generator.h b/src/kudu/util/oid_generator.h index 85a7412..7acccc9 100644 --- a/src/kudu/util/oid_generator.h +++ b/src/kudu/util/oid_generator.h @@ -23,6 +23,7 @@ #include "kudu/gutil/macros.h" #include "kudu/util/locks.h" +#include "kudu/util/status.h" namespace kudu { @@ -33,15 +34,26 @@ class ObjectIdGenerator { ObjectIdGenerator() {} ~ObjectIdGenerator() {} + // Generates and returns a new UUID. std::string Next(); + // Validates an existing UUID and converts it into the format used by Kudu + // (that is, 16 hexadecimal bytes without any dashes). + Status Canonicalize(const std::string& input, std::string* output) const; + private: DISALLOW_COPY_AND_ASSIGN(ObjectIdGenerator); typedef simple_spinlock LockType; + // Protects 'oid_generator_'. LockType oid_lock_; + + // Generates new UUIDs. boost::uuids::random_generator oid_generator_; + + // Validates provided UUIDs. + boost::uuids::string_generator oid_validator_; }; } // namespace kudu
