Repository: kudu Updated Branches: refs/heads/master e94556ef7 -> 2a5a12fbd
fs: clarify error message at startup Currently the error messages you get when you start a Kudu instance without specifying all the expected directories is: I0205 09:26:39.961467 25499 server_base.cc:229] Could not load existing FS layout: Not found: /data/01/kudu/data/instance: No such file or directory (error 2) I0205 09:26:39.961493 25499 server_base.cc:230] Creating new FS layout F0205 09:26:39.963004 25499 tablet_server_main.cc:80] Check failed: _s.ok() Bad status: Already present: Could not create new FS layout: unable to create file system roots: FSManager root is not empty. See https://kudu.apache.org/releases/1.7.0/docs/troubleshooting.html: /data/01/kudu/wal This can be misleading for users, who might think, "Well if it's not starting because this path isn't empty, I'll just delete the contents of that path!" This would be a terrible senario, as such actions could easily lead to data loss. This patch attempts to make it more obvious that Kudu may already exist in some (possibly incomplete) form at the specified locations, and that if the user truly wants to change directories, they should run the appropriate tools. Now, starting Kudu will log. I0221 18:36:14.403893 35003 server_base.cc:431] Could not load existing FS layout: Not found: /data/04/kudu/data/instance: No such file or directory (error 2) I0221 18:36:14.403903 35003 server_base.cc:432] Attempting to create new FS layout instead I0221 18:36:14.404762 35003 server_base.cc:438] To start Kudu with a different FS layout, the `kudu fs update_dirs` tool must be run first F0221 18:36:14.404858 35003 tablet_server_main.cc:80] Check failed: _s.ok() Bad status: Already present: FS layout already exists; not overwriting existing layout. See https://kudu.apache.org/releases/1.7.0/docs/troubleshooting.html: unable to create file system roots: FSManager roots already exist: /data/01/kudu/wal,/data/02/kudu/data,/data/03/kudu/data Change-Id: I72294036e9aa78b285803d5d78b685cf444d9662 Reviewed-on: http://gerrit.cloudera.org:8080/9281 Tested-by: Kudu Jenkins Reviewed-by: Andrew Wong <aw...@cloudera.com> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2a5a12fb Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2a5a12fb Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2a5a12fb Branch: refs/heads/master Commit: 2a5a12fbd69882b387f83e1da71d8a0fd19b6a63 Parents: e94556e Author: Andrew Wong <aw...@cloudera.com> Authored: Fri Feb 9 14:56:39 2018 -0800 Committer: Andrew Wong <aw...@cloudera.com> Committed: Fri Feb 23 19:09:02 2018 +0000 ---------------------------------------------------------------------- docs/troubleshooting.adoc | 2 +- src/kudu/fs/fs_manager.cc | 11 +++++++---- src/kudu/server/server_base.cc | 16 +++++++++++++--- 3 files changed, 21 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/2a5a12fb/docs/troubleshooting.adoc ---------------------------------------------------------------------- diff --git a/docs/troubleshooting.adoc b/docs/troubleshooting.adoc index a0a0b23..7c48ef3 100644 --- a/docs/troubleshooting.adoc +++ b/docs/troubleshooting.adoc @@ -490,7 +490,7 @@ When Kudu starts, it checks each configured data directory, expecting either for initialized or for all to be empty. If a server fails to start with a log message like ---- -Check failed: _s.ok() Bad status: Already present: Could not create new FS layout: FSManager root is not empty: /data0/kudu/data +Check failed: _s.ok() Bad status: Already present: FS layout already exists; not overwriting existing layout: FSManager roots already exist: /data0/kudu/data ---- then this precondition has failed. This could be because Kudu was configured with non-empty data http://git-wip-us.apache.org/repos/asf/kudu/blob/2a5a12fb/src/kudu/fs/fs_manager.cc ---------------------------------------------------------------------- diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc index 380169d..e7757f9 100644 --- a/src/kudu/fs/fs_manager.cc +++ b/src/kudu/fs/fs_manager.cc @@ -57,7 +57,6 @@ #include "kudu/util/pb_util.h" #include "kudu/util/scoped_cleanup.h" #include "kudu/util/stopwatch.h" -#include "kudu/util/website_util.h" DEFINE_bool(enable_data_block_fsync, true, "Whether to enable fsync() of data blocks, metadata, and their parent directories. " @@ -520,6 +519,7 @@ Status FsManager::CreateFileSystemRoots( CHECK(!opts_.read_only); // It's OK if a root already exists as long as there's nothing in it. + vector<string> non_empty_roots; for (const auto& root : canonicalized_roots) { if (!root.status.ok()) { return Status::IOError("cannot create FS layout; at least one directory " @@ -533,12 +533,15 @@ Status FsManager::CreateFileSystemRoots( RETURN_NOT_OK_PREPEND(env_util::IsDirectoryEmpty(env_, root.path, &is_empty), "unable to check if FSManager root is empty"); if (!is_empty) { - return Status::AlreadyPresent( - Substitute("FSManager root is not empty. See $0", KuduDocsTroubleshootingUrl()), - root.path); + non_empty_roots.emplace_back(root.path); } } + if (!non_empty_roots.empty()) { + return Status::AlreadyPresent( + Substitute("FSManager roots already exist: $0", JoinStrings(non_empty_roots, ","))); + } + // All roots are either empty or non-existent. Create missing roots and all // subdirectories. for (const auto& root : canonicalized_roots) { http://git-wip-us.apache.org/repos/asf/kudu/blob/2a5a12fb/src/kudu/server/server_base.cc ---------------------------------------------------------------------- diff --git a/src/kudu/server/server_base.cc b/src/kudu/server/server_base.cc index c996f70..fd2d420 100644 --- a/src/kudu/server/server_base.cc +++ b/src/kudu/server/server_base.cc @@ -79,6 +79,7 @@ #include "kudu/util/thread.h" #include "kudu/util/user.h" #include "kudu/util/version_info.h" +#include "kudu/util/website_util.h" DEFINE_int32(num_reactor_threads, 4, "Number of libev reactor threads to start."); TAG_FLAG(num_reactor_threads, advanced); @@ -428,10 +429,19 @@ Status ServerBase::Init() { Status s = fs_manager_->Open(&report); if (s.IsNotFound()) { LOG(INFO) << "Could not load existing FS layout: " << s.ToString(); - LOG(INFO) << "Creating new FS layout"; + LOG(INFO) << "Attempting to create new FS layout instead"; is_first_run_ = true; - RETURN_NOT_OK_PREPEND(fs_manager_->CreateInitialFileSystemLayout(), - "Could not create new FS layout"); + s = fs_manager_->CreateInitialFileSystemLayout(); + if (s.IsAlreadyPresent()) { + // The operator is likely trying to start up with an extra entry in their + // `fs_data_dirs` configuration. + LOG(INFO) << "To start Kudu with a different FS layout, the `kudu fs " + "update_dirs` tool must be run first"; + return s.CloneAndPrepend( + strings::Substitute("FS layout already exists; not overwriting existing layout. See $0", + KuduDocsTroubleshootingUrl())); + } + RETURN_NOT_OK_PREPEND(s, "Could not create new FS layout"); s = fs_manager_->Open(&report); } RETURN_NOT_OK_PREPEND(s, "Failed to load FS layout");
