This is an automated email from the ASF dual-hosted git repository.
awong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new bd5f72e KUDU-1959 - Implement server startup progress page for tablet
and master servers
bd5f72e is described below
commit bd5f72e0ed03b614d49b8125160bf555d11dcf9c
Author: Abhishek Chennaka <[email protected]>
AuthorDate: Tue Jul 27 11:11:06 2021 -0400
KUDU-1959 - Implement server startup progress page for tablet and master
servers
This patch implements adjusting the server startup sequence to start the
web server and load the startup page which shows the progress of the
startup.
This progress is broken down into the below steps:
Initializing
Reading Filesystem
Reading instance metadata files
Opening container files
Bootstrapping and opening the tablets
Starting RPC server
Sample screenshot of the page can be found here:
https://i.imgur.com/J4NJD5i.png
Each of the steps above has a progress status of either 0 or 100 except
for the step “Bootstrapping and opening the tablets” which tracks the
processing of each of the tablet and the step “Opening container files”.
For log block manager this step shows the progress of opening the log
block containers. For file block manager, this is renamed to
“Reporting Filesystem” and has only 0 or 100 progress status. All of
the steps have time elapsed presented in seconds.
For the master startup page, the step “Bootstrapping and opening the
tablets” is replaced by “Initializing master catalog”.
Along with the above startup page the web server will also display
the pages - /home, /config, /logs, /mem-trackers, /memz, /threadz, /varz.
During the initialization since we expect the user's primary reason to
open the web UI to be to check the startup progress, /home will be
redirected to /startup until the startup is completed. The usual homepage
contents and the other remaining web pages will be available once the rpc
server is started due to dependency. Manually validated the startup when
security is enabled and filesystem is not present.
The footer of the WebUI has the UUID by default. In case of new
instances, since we are starting the Webserver before the UUID is
generated the footer doesn’t contain UUID but it is set later when
the rpc server is started.
The next steps for this patch include writing tests for the startup
page. So far all the testing has been manual.
Change-Id: I1db1fcf16261d4ced1b3657a697766a5335271b4
Reviewed-on: http://gerrit.cloudera.org:8080/17730
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <[email protected]>
---
src/kudu/fs/block_manager.h | 7 +-
src/kudu/fs/file_block_manager.cc | 3 +-
src/kudu/fs/file_block_manager.h | 4 +-
src/kudu/fs/fs_manager.cc | 25 ++++++-
src/kudu/fs/fs_manager.h | 17 ++++-
src/kudu/fs/log_block_manager.cc | 23 ++++--
src/kudu/fs/log_block_manager.h | 15 +++-
src/kudu/master/master.cc | 8 ++-
src/kudu/server/CMakeLists.txt | 4 +-
src/kudu/server/default_path_handlers.cc | 10 +--
src/kudu/server/default_path_handlers.h | 8 ++-
src/kudu/server/server_base.cc | 57 ++++++++++++---
src/kudu/server/server_base.h | 2 +
src/kudu/server/startup_path_handler.cc | 120 +++++++++++++++++++++++++++++++
src/kudu/server/startup_path_handler.h | 97 +++++++++++++++++++++++++
src/kudu/server/tracing_path_handlers.h | 4 +-
src/kudu/server/webserver-test.cc | 6 +-
src/kudu/server/webserver.cc | 40 +++++++----
src/kudu/server/webserver.h | 14 ++++
src/kudu/tserver/tablet_server.cc | 22 ++++--
src/kudu/tserver/ts_tablet_manager.cc | 51 ++++++++++---
src/kudu/tserver/ts_tablet_manager.h | 24 ++++++-
src/kudu/util/timer.h | 67 +++++++++++++++++
src/kudu/util/web_callback_registry.h | 1 +
www/startup.mustache | 79 ++++++++++++++++++++
25 files changed, 637 insertions(+), 71 deletions(-)
diff --git a/src/kudu/fs/block_manager.h b/src/kudu/fs/block_manager.h
index 87239c3..1c74e40 100644
--- a/src/kudu/fs/block_manager.h
+++ b/src/kudu/fs/block_manager.h
@@ -219,7 +219,12 @@ class BlockManager {
//
// Returns an error if an on-disk representation does not exist or cannot be
// opened.
- virtual Status Open(FsReport* report) = 0;
+ //
+ // If 'containers_processed' and 'containers_total' are not nullptr, they
will
+ // be populated with total containers attempted to be opened/processed and
+ // total containers present respectively.
+ virtual Status Open(FsReport* report, std::atomic<int>* containers_processed
= nullptr,
+ std::atomic<int>* containers_total = nullptr) = 0;
// Creates a new block using the provided options and opens it for
// writing. The block's ID will be generated.
diff --git a/src/kudu/fs/file_block_manager.cc
b/src/kudu/fs/file_block_manager.cc
index 06b83ed..1e9db05 100644
--- a/src/kudu/fs/file_block_manager.cc
+++ b/src/kudu/fs/file_block_manager.cc
@@ -715,7 +715,8 @@ FileBlockManager::FileBlockManager(Env* env,
FileBlockManager::~FileBlockManager() {
}
-Status FileBlockManager::Open(FsReport* report) {
+Status FileBlockManager::Open(FsReport* report, std::atomic<int>*
containers_processed,
+ std::atomic<int>* containers_total) {
// Prepare the filesystem report and either return or log it.
FsReport local_report;
set<int> failed_dirs = dd_manager_->GetFailedDirs();
diff --git a/src/kudu/fs/file_block_manager.h b/src/kudu/fs/file_block_manager.h
index 38b0653..f3cadb8 100644
--- a/src/kudu/fs/file_block_manager.h
+++ b/src/kudu/fs/file_block_manager.h
@@ -18,6 +18,7 @@
#ifndef KUDU_FS_FILE_BLOCK_MANAGER_H
#define KUDU_FS_FILE_BLOCK_MANAGER_H
+#include <atomic>
#include <cstdint>
#include <memory>
#include <string>
@@ -79,7 +80,8 @@ class FileBlockManager : public BlockManager {
virtual ~FileBlockManager();
- Status Open(FsReport* report) override;
+ Status Open(FsReport* report, std::atomic<int>* containers_processed =
nullptr,
+ std::atomic<int>* containers_total = nullptr) override;
Status CreateBlock(const CreateBlockOptions& opts,
std::unique_ptr<WritableBlock>* block) override;
diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc
index 3a747d6..2ecca51 100644
--- a/src/kudu/fs/fs_manager.cc
+++ b/src/kudu/fs/fs_manager.cc
@@ -57,6 +57,7 @@
#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/slice.h"
#include "kudu/util/stopwatch.h"
+#include "kudu/util/timer.h"
DEFINE_bool(enable_data_block_fsync, true,
"Whether to enable fsync() of data blocks, metadata, and their
parent directories. "
@@ -359,7 +360,13 @@ Status FsManager::PartialOpen(CanonicalizedRootsList*
missing_roots) {
return Status::OK();
}
-Status FsManager::Open(FsReport* report) {
+Status FsManager::Open(FsReport* report, Timer* read_instance_metadata_files,
+ Timer* read_data_directories,
+ std::atomic<int>* containers_processed,
+ std::atomic<int>* containers_total) {
+ if (read_instance_metadata_files) {
+ read_instance_metadata_files->Start();
+ }
// Load and verify the instance metadata files.
//
// Done first to minimize side effects in the case that the configured roots
@@ -410,6 +417,9 @@ Status FsManager::Open(FsReport* report) {
RETURN_NOT_OK_PREPEND(s, kUnableToCreateMsg);
}
}
+ if (read_instance_metadata_files) {
+ read_instance_metadata_files->Stop();
+ }
// Open the directory manager if it has not been opened already.
if (!dd_manager_) {
@@ -440,12 +450,21 @@ Status FsManager::Open(FsReport* report) {
// Finally, initialize and open the block manager if needed.
if (!opts_.skip_block_manager) {
+ if (read_data_directories) {
+ read_data_directories->Start();
+ }
InitBlockManager();
LOG_TIMING(INFO, "opening block manager") {
- RETURN_NOT_OK(block_manager_->Open(report));
+ if (opts_.block_manager_type == "file") {
+ RETURN_NOT_OK(block_manager_->Open(report));
+ } else {
+ RETURN_NOT_OK(block_manager_->Open(report, containers_processed,
containers_total));
+ }
+ }
+ if (read_data_directories) {
+ read_data_directories->Stop();
}
}
-
// Report wal and metadata directories.
if (report) {
report->wal_dir = canonicalized_wal_fs_root_.path;
diff --git a/src/kudu/fs/fs_manager.h b/src/kudu/fs/fs_manager.h
index 52e72bf..ff04514 100644
--- a/src/kudu/fs/fs_manager.h
+++ b/src/kudu/fs/fs_manager.h
@@ -17,6 +17,7 @@
#pragma once
+#include <atomic>
#include <cstdint>
#include <iosfwd>
#include <memory>
@@ -46,6 +47,7 @@ class BlockId;
class FileCache;
class InstanceMetadataPB;
class MemTracker;
+class Timer;
namespace fs {
@@ -181,7 +183,20 @@ class FsManager {
// If the filesystem has not been initialized, returns NotFound. In that
// case, CreateInitialFileSystemLayout() may be used to initialize the
// on-disk and in-memory structures.
- Status Open(fs::FsReport* report = nullptr);
+ //
+ // If 'read_instance_metadata_files' and 'read_data_directories' are not
nullptr,
+ // they will be populated with time spent reading the instance metadata files
+ // and time spent reading data directories respectively.
+ //
+ // If 'containers_processed' and 'containers_total' are not nullptr, they
will
+ // be populated with total containers attempted to be opened/processed and
+ // total containers present respectively in the subsequent calls made to
+ // the block manager.
+ Status Open(fs::FsReport* report = nullptr,
+ Timer* read_instance_metadata_files = nullptr,
+ Timer* read_data_directories = nullptr,
+ std::atomic<int>* containers_processed = nullptr,
+ std::atomic<int>* containers_total = nullptr );
// Create the initial filesystem layout. If 'uuid' is provided, uses it as
// uuid of the filesystem. Otherwise generates one at random.
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index a94e619..8fcae4a 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -2029,7 +2029,8 @@ LogBlockManager::~LogBlockManager() {
} \
} while (false)
-Status LogBlockManager::Open(FsReport* report) {
+Status LogBlockManager::Open(FsReport* report, std::atomic<int>*
containers_processed,
+ std::atomic<int>* containers_total) {
// Establish (and log) block limits for each data directory using kernel,
// filesystem, and gflags information.
for (const auto& dd : dd_manager_->dirs()) {
@@ -2089,8 +2090,8 @@ Status LogBlockManager::Open(FsReport* report) {
auto* dd_raw = dd.get();
auto* results = &container_results[i];
auto* s = &statuses[i];
- dd->ExecClosure([this, dd_raw, results, s]() {
- this->OpenDataDir(dd_raw, results, s);
+ dd->ExecClosure([this, dd_raw, results, s, containers_processed,
containers_total]() {
+ this->OpenDataDir(dd_raw, results, s, containers_processed,
containers_total);
WARN_NOT_OK(*s, Substitute("failed to open dir $0", dd_raw->dir()));
});
}
@@ -2515,7 +2516,9 @@ Status LogBlockManager::RemoveLogBlock(const BlockId&
block_id,
void LogBlockManager::OpenDataDir(
Dir* dir,
vector<unique_ptr<internal::LogBlockContainerLoadResult>>* results,
- Status* result_status) {
+ Status* result_status,
+ std::atomic<int>* containers_processed,
+ std::atomic<int>* containers_total) {
vector<string> children;
Status s = env_->GetChildren(dir->dir(), &children);
if (!s.ok()) {
@@ -2537,15 +2540,21 @@ void LogBlockManager::OpenDataDir(
child, LogBlockManager::kContainerMetadataFileSuffix,
&container_name)) {
continue;
}
- if (!InsertIfNotPresent(&containers_seen, container_name)) {
- continue;
- }
+ InsertIfNotPresent(&containers_seen, container_name);
+ }
+ if (containers_total) {
+ *containers_total += containers_seen.size();
+ }
+ for (const string& container_name : containers_seen) {
// Add a new result for the container.
results->emplace_back(new internal::LogBlockContainerLoadResult());
LogBlockContainerRefPtr container;
s = LogBlockContainer::Open(
this, dir, &results->back()->report, container_name, &container);
+ if (containers_processed) {
+ ++*containers_processed;
+ }
if (!s.ok()) {
if (s.IsAborted()) {
// Skip the container. Open() added a record of it to
'results->back()->report' for us.
diff --git a/src/kudu/fs/log_block_manager.h b/src/kudu/fs/log_block_manager.h
index b077947..ae62c8f 100644
--- a/src/kudu/fs/log_block_manager.h
+++ b/src/kudu/fs/log_block_manager.h
@@ -17,6 +17,7 @@
#pragma once
+#include <atomic>
#include <cstdint>
#include <deque>
#include <map>
@@ -194,7 +195,8 @@ class LogBlockManager : public BlockManager {
~LogBlockManager();
- Status Open(FsReport* report) override;
+ Status Open(FsReport* report, std::atomic<int>* containers_processed =
nullptr,
+ std::atomic<int>* containers_total = nullptr) override;
Status CreateBlock(const CreateBlockOptions& opts,
std::unique_ptr<WritableBlock>* block) override;
@@ -365,9 +367,18 @@ class LogBlockManager : public BlockManager {
// results of consistency checking are written to 'results'.
//
// Success or failure is set in 'result_status'.
+ //
+ // If 'containers_processed' and 'containers_total' are not nullptr, they
will
+ // be populated with total containers attempted to be opened/processed and
+ // total containers present respectively in the subsequent calls made to
+ // the block manager.
+ // TODO(achennaka): Implement a cleaner way to pass the atomic values to
startup page
+ // probably by using metrics.
void OpenDataDir(Dir* dir,
std::vector<std::unique_ptr<internal::LogBlockContainerLoadResult>>* results,
- Status* result_status);
+ Status* result_status,
+ std::atomic<int>* containers_processed = nullptr,
+ std::atomic<int>* containers_total = nullptr);
// Reads records from one log block container in the data directory.
// The result details will be collected into 'result'.
diff --git a/src/kudu/master/master.cc b/src/kudu/master/master.cc
index 1c9800a..c34821d 100644
--- a/src/kudu/master/master.cc
+++ b/src/kudu/master/master.cc
@@ -53,6 +53,7 @@
#include "kudu/rpc/service_if.h"
#include "kudu/security/token_signer.h"
#include "kudu/server/rpc_server.h"
+#include "kudu/server/startup_path_handler.h"
#include "kudu/server/webserver.h"
#include "kudu/tserver/tablet_copy_service.h"
#include "kudu/tserver/tablet_service.h"
@@ -64,6 +65,7 @@
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/status.h"
#include "kudu/util/threadpool.h"
+#include "kudu/util/timer.h"
#include "kudu/util/version_info.h"
DEFINE_int32(master_registration_rpc_timeout_ms, 1500,
@@ -195,7 +197,7 @@ Status Master::Init() {
metric_entity(), opts_.block_cache_metrics_policy());
RETURN_NOT_OK(ThreadPoolBuilder("init").set_max_threads(1).Build(&init_pool_));
-
+ startup_path_handler_->set_is_tablet_server(false);
RETURN_NOT_OK(KuduServer::Init());
if (web_server_) {
@@ -223,9 +225,13 @@ Status Master::Init() {
}
Status Master::Start() {
+ Timer* init_master_catalog =
+ startup_path_handler_->initialize_master_catalog_progress();
+ init_master_catalog->Start();
RETURN_NOT_OK(StartAsync());
RETURN_NOT_OK(WaitForCatalogManagerInit());
google::FlushLogFiles(google::INFO); // Flush the startup messages.
+ init_master_catalog->Stop();
return Status::OK();
}
diff --git a/src/kudu/server/CMakeLists.txt b/src/kudu/server/CMakeLists.txt
index ce1d5dd..78ff965 100644
--- a/src/kudu/server/CMakeLists.txt
+++ b/src/kudu/server/CMakeLists.txt
@@ -48,12 +48,12 @@ set(SERVER_PROCESS_SRCS
rpc_server.cc
server_base.cc
server_base_options.cc
+ startup_path_handler.cc
tcmalloc_metrics.cc
tracing_path_handlers.cc
webserver.cc
webserver_options.cc
- webui_util.cc
-)
+ webui_util.cc)
add_library(server_process ${SERVER_PROCESS_SRCS})
target_link_libraries(server_process
diff --git a/src/kudu/server/default_path_handlers.cc
b/src/kudu/server/default_path_handlers.cc
index 0da303f..16c6a45 100644
--- a/src/kudu/server/default_path_handlers.cc
+++ b/src/kudu/server/default_path_handlers.cc
@@ -425,18 +425,20 @@ static void ConfigurationHandler(const
Webserver::WebRequest& /* req */,
FillSecurityConfigs(output);
FillTimeSourceConfigs(output);
}
-
-void AddDefaultPathHandlers(Webserver* webserver) {
+void AddPreInitializedDefaultPathHandlers(Webserver* webserver) {
bool styled = true;
bool on_nav_bar = true;
webserver->RegisterPathHandler("/logs", "Logs", LogsHandler, styled,
on_nav_bar);
webserver->RegisterPrerenderedPathHandler("/varz", "Flags", FlagsHandler,
styled, on_nav_bar);
+ webserver->RegisterPathHandler("/config", "Configuration",
ConfigurationHandler,
+ styled, on_nav_bar);
webserver->RegisterPrerenderedPathHandler("/memz", "Memory (total)",
MemUsageHandler,
styled, on_nav_bar);
webserver->RegisterPrerenderedPathHandler("/mem-trackers", "Memory
(detail)", MemTrackersHandler,
styled, on_nav_bar);
- webserver->RegisterPathHandler("/config", "Configuration",
ConfigurationHandler,
- styled, on_nav_bar);
+}
+
+void AddPostInitializedDefaultPathHandlers(Webserver* webserver) {
webserver->RegisterPrerenderedPathHandler("/stacks", "Stacks", StacksHandler,
/*is_styled=*/false,
/*is_on_nav_bar=*/true);
diff --git a/src/kudu/server/default_path_handlers.h
b/src/kudu/server/default_path_handlers.h
index 82f3e2b..1a58915 100644
--- a/src/kudu/server/default_path_handlers.h
+++ b/src/kudu/server/default_path_handlers.h
@@ -24,8 +24,12 @@ class MetricRegistry;
class Webserver;
// Adds a set of default path handlers to the webserver to display
-// logs and configuration flags.
-void AddDefaultPathHandlers(Webserver* webserver);
+// logs and configuration flags before the server is initialized.
+void AddPreInitializedDefaultPathHandlers(Webserver* webserver);
+
+// Adds a set of default path handlers to the webserver to display
+// stacks and support pprof profiling after the server is initialized.
+void AddPostInitializedDefaultPathHandlers(Webserver* webserver);
// Adds an endpoint to get metrics in JSON format.
void RegisterMetricsJsonHandler(Webserver* webserver, const MetricRegistry*
const metrics);
diff --git a/src/kudu/server/server_base.cc b/src/kudu/server/server_base.cc
index 43dc69e..bbbcefa 100644
--- a/src/kudu/server/server_base.cc
+++ b/src/kudu/server/server_base.cc
@@ -64,6 +64,7 @@
#include "kudu/server/rpcz-path-handler.h"
#include "kudu/server/server_base.pb.h"
#include "kudu/server/server_base_options.h"
+#include "kudu/server/startup_path_handler.h"
#include "kudu/server/tcmalloc_metrics.h"
#include "kudu/server/tracing_path_handlers.h"
#include "kudu/server/webserver.h"
@@ -84,6 +85,7 @@
#include "kudu/util/net/net_util.h"
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/pb_util.h"
+#include "kudu/util/timer.h"
#ifdef TCMALLOC_ENABLED
#include "kudu/util/process_memory.h"
#endif
@@ -493,6 +495,7 @@ ServerBase::ServerBase(string name, const
ServerBaseOptions& options,
file_cache_(new FileCache("file cache", options.env,
GetFileCacheCapacity(options.env),
metric_entity_)),
rpc_server_(new RpcServer(options.rpc_opts)),
+ startup_path_handler_(new StartupPathHandler),
result_tracker_(new rpc::ResultTracker(shared_ptr<MemTracker>(
MemTracker::CreateTracker(-1, "result-tracker", mem_tracker_)))),
is_first_run_(false),
@@ -564,6 +567,9 @@ void ServerBase::GenerateInstanceID() {
}
Status ServerBase::Init() {
+ Timer* init = startup_path_handler_->init_progress();
+ Timer* read_filesystem = startup_path_handler_->read_filesystem_progress();
+ init->Start();
glog_metrics_.reset(new ScopedGLogMetrics(metric_entity_));
tcmalloc::RegisterMetrics(metric_entity_);
RegisterSpinLockContentionMetrics(metric_entity_);
@@ -577,8 +583,30 @@ Status ServerBase::Init() {
RETURN_NOT_OK(security::InitKerberosForServer(FLAGS_principal,
FLAGS_keytab_file));
RETURN_NOT_OK(file_cache_->Init());
+ // Register the startup web handler and start the web server to make the web
UI
+ // available while the server is initializing, loading the file system, etc.
+ //
+ // NOTE: unlike the other path handlers, we register this path handler
+ // separately, as the startup handler is meant to be displayed before all of
+ // Kudu's subsystems have finished initializing.
+ if (options_.fs_opts.block_manager_type == "file") {
+ startup_path_handler_->set_is_using_lbm(false);
+ }
+ if (web_server_) {
+ startup_path_handler_->RegisterStartupPathHandler(web_server_.get());
+ AddPreInitializedDefaultPathHandlers(web_server_.get());
+ web_server_->set_footer_html(FooterHtml());
+ RETURN_NOT_OK(web_server_->Start());
+ }
+
fs::FsReport report;
- Status s = fs_manager_->Open(&report);
+ init->Stop();
+ read_filesystem->Start();
+ Status s = fs_manager_->Open(&report,
+
startup_path_handler_->read_instance_metadata_files_progress(),
+
startup_path_handler_->read_data_directories_progress(),
+ startup_path_handler_->containers_processed(),
+ startup_path_handler_->containers_total());
// No instance files existed. Try creating a new FS layout.
if (s.IsNotFound()) {
LOG(INFO) << "This appears to be a new deployment of Kudu; creating new FS
layout";
@@ -588,11 +616,12 @@ Status ServerBase::Init() {
return s.CloneAndPrepend("FS layout already exists; not overwriting
existing layout");
}
RETURN_NOT_OK_PREPEND(s, "Could not create new FS layout");
- s = fs_manager_->Open(&report);
+ s = fs_manager_->Open(&report,
startup_path_handler_->read_instance_metadata_files_progress(),
+
startup_path_handler_->read_data_directories_progress());
}
RETURN_NOT_OK_PREPEND(s, "Failed to load FS layout");
RETURN_NOT_OK(report.LogAndCheckForFatalErrors());
-
+ read_filesystem->Stop();
RETURN_NOT_OK(InitAcls());
vector<string> rpc_tls_excluded_protocols = strings::Split(
@@ -664,7 +693,7 @@ Status ServerBase::InitAcls() {
// If we aren't logged in from a keytab, then just assume that the services
// will be running as the same Unix user as we are.
RETURN_NOT_OK_PREPEND(GetLoggedInUser(&service_user),
- "could not deterine local username");
+ "could not determine local username");
}
// If the user has specified a superuser acl, use that. Otherwise, assume
@@ -863,9 +892,14 @@ void ServerBase::TcmallocMemoryGcThread() {
#endif
std::string ServerBase::FooterHtml() const {
- return Substitute("<pre>$0\nserver uuid $1</pre>",
- VersionInfo::GetVersionInfo(),
- instance_pb_->permanent_uuid());
+ if (instance_pb_) {
+ return Substitute("<pre>$0\nserver uuid $1</pre>",
+ VersionInfo::GetVersionInfo(),
+ instance_pb_->permanent_uuid());
+ }
+ // Load the footer with UUID only if the UUID is available
+ return Substitute("<pre>$0\n</pre>",
+ VersionInfo::GetVersionInfo());
}
Status ServerBase::Start() {
@@ -918,16 +952,17 @@ Status ServerBase::Start() {
#ifdef TCMALLOC_ENABLED
RETURN_NOT_OK(StartTcmallocMemoryGcThread());
#endif
-
+ Timer* start_rpc_server = startup_path_handler_->start_rpc_server_progress();
+ start_rpc_server->Start();
RETURN_NOT_OK(rpc_server_->Start());
-
+ start_rpc_server->Stop();
if (web_server_) {
- AddDefaultPathHandlers(web_server_.get());
+ AddPostInitializedDefaultPathHandlers(web_server_.get());
AddRpczPathHandlers(messenger_, web_server_.get());
RegisterMetricsJsonHandler(web_server_.get(), metric_registry_.get());
TracingPathHandlers::RegisterHandlers(web_server_.get());
web_server_->set_footer_html(FooterHtml());
- RETURN_NOT_OK(web_server_->Start());
+ web_server_->SetStartupComplete(true);
}
if (!options_.dump_info_path.empty()) {
diff --git a/src/kudu/server/server_base.h b/src/kudu/server/server_base.h
index a8a16e9..42cc2d6 100644
--- a/src/kudu/server/server_base.h
+++ b/src/kudu/server/server_base.h
@@ -66,6 +66,7 @@ class TokenVerifier;
namespace server {
class DiagnosticsLog;
class ServerStatusPB;
+class StartupPathHandler;
// Base class for tablet server and master.
// Handles starting and stopping the RPC server and web server,
@@ -201,6 +202,7 @@ class ServerBase {
std::unique_ptr<FsManager> fs_manager_;
std::unique_ptr<RpcServer> rpc_server_;
std::unique_ptr<Webserver> web_server_;
+ std::unique_ptr<StartupPathHandler> startup_path_handler_;
std::shared_ptr<rpc::Messenger> messenger_;
scoped_refptr<rpc::ResultTracker> result_tracker_;
diff --git a/src/kudu/server/startup_path_handler.cc
b/src/kudu/server/startup_path_handler.cc
new file mode 100644
index 0000000..93f839e
--- /dev/null
+++ b/src/kudu/server/startup_path_handler.cc
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/server/startup_path_handler.h"
+
+#include <functional>
+#include <iosfwd>
+#include <string>
+
+#include "kudu/gutil/strings/human_readable.h"
+#include "kudu/server/webserver.h"
+#include "kudu/util/easy_json.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/timer.h"
+#include "kudu/util/web_callback_registry.h"
+
+using std::ifstream;
+using std::ostringstream;
+using std::string;
+
+namespace kudu {
+
+namespace server {
+
+void SetWebResponse(EasyJson* output, const string& step,
+ const Timer& startup_step, const int percent = -1) {
+ output->Set(step + "_status", percent == -1 ? (startup_step.IsStopped() ?
100 : 0) : percent);
+ output->Set(step + "_time", HumanReadableElapsedTime::ToShortString(
+ (startup_step.TimeElapsed()).ToSeconds()));
+}
+
+StartupPathHandler::StartupPathHandler():
+ tablets_processed_(0),
+ tablets_total_(0),
+ containers_processed_(0),
+ containers_total_(0),
+ is_tablet_server_(false),
+ is_using_lbm_(true) {
+}
+
+void StartupPathHandler::Startup(const Webserver::WebRequest& /*req*/,
+ Webserver::WebResponse* resp) {
+
+ auto* output = &resp->output;
+
+ output->Set("is_tablet_server", is_tablet_server_);
+ output->Set("is_master_server", !is_tablet_server_);
+ output->Set("is_log_block_manager", is_using_lbm_);
+
+ // Populate the different startup steps with their progress
+ SetWebResponse(output, "init", init_progress_);
+ SetWebResponse(output, "read_filesystem", read_filesystem_progress_);
+ SetWebResponse(output, "read_instance_metadatafiles",
read_instance_metadata_files_progress_);
+
+ // Populate the progress percentage of opening of container files in case of
lbm and non-lbm
+ if (is_using_lbm_) {
+ if (containers_total_ == 0) {
+ SetWebResponse(output, "read_data_directories",
read_data_directories_progress_);
+ } else {
+ SetWebResponse(output, "read_data_directories",
read_data_directories_progress_,
+ containers_processed_ * 100 / containers_total_);
+ }
+ output->Set("containers_processed", containers_processed_.load());
+ output->Set("containers_total", containers_total_.load());
+ } else {
+ SetWebResponse(output, "read_data_directories",
read_data_directories_progress_);
+ }
+
+ // Set the bootstrapping and opening tablets step and handle the case of
zero tablets
+ // present in the server
+ if (tablets_total_ == 0) {
+ SetWebResponse(output, "start_tablets", start_tablets_progress_);
+ } else {
+ SetWebResponse(output, "start_tablets", start_tablets_progress_,
+ tablets_processed_ * 100 / tablets_total_);
+ }
+
+ if (is_tablet_server_) {
+ output->Set("tablets_processed", tablets_processed_.load());
+ output->Set("tablets_total", tablets_total_.load());
+ }
+
+ SetWebResponse(output, "initialize_master_catalog",
initialize_master_catalog_progress_);
+ SetWebResponse(output, "start_rpc_server", start_rpc_server_progress_);
+}
+
+void StartupPathHandler::RegisterStartupPathHandler(Webserver *webserver) {
+ bool styled = true;
+ bool on_nav_bar = true;
+ webserver->RegisterPathHandler("/startup", "Startup",
+ [this](const Webserver::WebRequest& req,
+ Webserver::WebResponse* resp) {
+ this->Startup(req, resp);
+ },
+ styled, on_nav_bar);
+}
+
+void StartupPathHandler::set_is_tablet_server(bool is_tablet_server) {
+ is_tablet_server_ = is_tablet_server;
+}
+
+void StartupPathHandler::set_is_using_lbm(bool is_using_lbm) {
+ is_using_lbm_ = is_using_lbm;
+}
+} // namespace server
+} // namespace kudu
diff --git a/src/kudu/server/startup_path_handler.h
b/src/kudu/server/startup_path_handler.h
new file mode 100644
index 0000000..1a32281
--- /dev/null
+++ b/src/kudu/server/startup_path_handler.h
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <atomic>
+
+#include "kudu/server/webserver.h"
+#include "kudu/util/timer.h"
+
+namespace kudu {
+
+namespace server {
+
+class StartupPathHandler {
+public:
+
+ StartupPathHandler();
+
+ // Populate the response output with the current information
+ void Startup(const Webserver::WebRequest &req, Webserver::WebResponse *resp);
+
+ // Startup page path handler
+ void RegisterStartupPathHandler(Webserver *webserver);
+
+ Timer* init_progress() {return &init_progress_;}
+ Timer* read_filesystem_progress() {return &read_filesystem_progress_;}
+ Timer* read_instance_metadata_files_progress() {return
&read_instance_metadata_files_progress_;}
+ Timer* read_data_directories_progress() {return
&read_data_directories_progress_;}
+ Timer* start_tablets_progress() {return &start_tablets_progress_;}
+ Timer* initialize_master_catalog_progress() {return
&initialize_master_catalog_progress_;}
+ Timer* start_rpc_server_progress() {return &start_rpc_server_progress_;}
+ std::atomic<int>* tablets_processed() {return &tablets_processed_;}
+ std::atomic<int>* tablets_total() {return &tablets_total_;}
+ std::atomic<int>* containers_processed() {return &containers_processed_;}
+ std::atomic<int>* containers_total() {return &containers_total_;}
+ void set_is_tablet_server(bool is_tablet_server);
+ void set_is_using_lbm(bool is_using_lbm);
+
+private:
+ // Hold the initialization step progress information like the status, start
and end time.
+ Timer init_progress_;
+
+ // Hold the read filesystem step progress information.
+ Timer read_filesystem_progress_;
+
+ // Hold the reading of instance metadata files progress in all the
configured directories.
+ Timer read_instance_metadata_files_progress_;
+
+ // Hold the reading of data directories progress.
+ Timer read_data_directories_progress_;
+
+ // Hold the progress of bootstrapping and starting the tablets.
+ Timer start_tablets_progress_;
+
+ // Hold the progress of initializing the master catalog.
+ Timer initialize_master_catalog_progress_;
+
+ // Hold the progress of starting of the rpc server.
+ Timer start_rpc_server_progress_;
+
+ // Hold the number of tablets which were attempted to be bootstrapped and
started.
+ std::atomic<int> tablets_processed_;
+
+ // Hold the total tablets present on the server.
+ std::atomic<int> tablets_total_;
+
+ // Hold the number of containers which were opened and processed
+ std::atomic<int> containers_processed_;
+
+ // Hold the total containers present on the server.
+ std::atomic<int> containers_total_;
+
+ // To display different web page contents for masters and tablet servers, we
need to know
+ // if the particular instance is a master of tablet server.
+ bool is_tablet_server_;
+
+ // We do not open containers if file block manager is being used and hence
display different
+ // webpage contents if file block manager is being used.
+ bool is_using_lbm_;
+};
+
+} // namespace server
+} // namespace kudu
diff --git a/src/kudu/server/tracing_path_handlers.h
b/src/kudu/server/tracing_path_handlers.h
index 7e1feef..66a10ff 100644
--- a/src/kudu/server/tracing_path_handlers.h
+++ b/src/kudu/server/tracing_path_handlers.h
@@ -14,8 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-#ifndef KUDU_SERVER_TRACING_PATH_HANDLERS_H
-#define KUDU_SERVER_TRACING_PATH_HANDLERS_H
+#pragma once
#include "kudu/gutil/macros.h"
@@ -37,4 +36,3 @@ class TracingPathHandlers {
} // namespace server
} // namespace kudu
-#endif /* KUDU_SERVER_TRACING_PATH_HANDLERS_H */
diff --git a/src/kudu/server/webserver-test.cc
b/src/kudu/server/webserver-test.cc
index 27847a7..9cddc5c 100644
--- a/src/kudu/server/webserver-test.cc
+++ b/src/kudu/server/webserver-test.cc
@@ -101,7 +101,8 @@ class WebserverTest : public KuduTest {
MaybeSetupSpnego(&opts);
server_.reset(new Webserver(opts));
- AddDefaultPathHandlers(server_.get());
+ AddPreInitializedDefaultPathHandlers(server_.get());
+ AddPostInitializedDefaultPathHandlers(server_.get());
if (!use_htpasswd() || !FIPS_mode()) {
ASSERT_OK(server_->Start());
@@ -111,6 +112,9 @@ class WebserverTest : public KuduTest {
ASSERT_TRUE(addrs[0].IsWildcard());
ASSERT_OK(addr_.ParseString("127.0.0.1", addrs[0].port()));
url_ = Substitute("http://$0";, addr_.ToString());
+ // For testing purposes, we assume the server has been initialized.
Typically this
+ // is set to true after the rpc server is started in the server startup
process.
+ server_->SetStartupComplete(true);
}
}
diff --git a/src/kudu/server/webserver.cc b/src/kudu/server/webserver.cc
index 518d08a..2f57490 100644
--- a/src/kudu/server/webserver.cc
+++ b/src/kudu/server/webserver.cc
@@ -103,6 +103,8 @@ string HttpStatusCodeToString(kudu::HttpStatusCode code) {
switch (code) {
case kudu::HttpStatusCode::Ok:
return "200 OK";
+ case kudu::HttpStatusCode::TemporaryRedirect:
+ return "307 Temporary Redirect";
case kudu::HttpStatusCode::BadRequest:
return "400 Bad Request";
case kudu::HttpStatusCode::AuthenticationRequired:
@@ -175,7 +177,8 @@ Status RunSpnegoStep(const char* authz_header,
Webserver::Webserver(const WebserverOptions& opts)
: opts_(opts),
- context_(nullptr) {
+ context_(nullptr),
+ is_started_(false) {
string host = opts.bind_interface.empty() ? "0.0.0.0" : opts.bind_interface;
http_address_ = host + ":" + std::to_string(opts.port);
}
@@ -185,17 +188,22 @@ Webserver::~Webserver() {
STLDeleteValues(&path_handlers_);
}
-void Webserver::RootHandler(const WebRequest& /* args */,
+void Webserver::RootHandler(const WebRequest& req,
WebResponse* resp) {
- EasyJson path_handlers = resp->output.Set("path_handlers", EasyJson::kArray);
- for (const PathHandlerMap::value_type& handler : path_handlers_) {
- if (handler.second->is_on_nav_bar()) {
- EasyJson path_handler = path_handlers.PushBack(EasyJson::kObject);
- path_handler["path"] = handler.first;
- path_handler["alias"] = handler.second->alias();
+ if (is_started_) {
+ EasyJson path_handlers = resp->output.Set("path_handlers",
EasyJson::kArray);
+ for (const PathHandlerMap::value_type& handler : path_handlers_) {
+ if (handler.second->is_on_nav_bar()) {
+ EasyJson path_handler = path_handlers.PushBack(EasyJson::kObject);
+ path_handler["path"] = handler.first;
+ path_handler["alias"] = handler.second->alias();
+ }
}
+ resp->output["version_info"] =
EscapeForHtmlToString(VersionInfo::GetAllVersionInfo());
+ } else {
+ resp->status_code = HttpStatusCode::TemporaryRedirect;
+ resp->response_headers.insert({"Location", "/startup"});
}
- resp->output["version_info"] =
EscapeForHtmlToString(VersionInfo::GetAllVersionInfo());
}
void Webserver::BuildArgumentMap(const string& args, ArgumentMap* output) {
@@ -711,12 +719,16 @@ void Webserver::RegisterPathHandler(const string& path,
const string& alias,
auto wrapped_cb = [=](const WebRequest& req, PrerenderedWebResponse*
rendered_resp) {
WebResponse resp;
callback(req, &resp);
- stringstream out;
AddKnoxVariables(req, &resp.output);
- Render(render_path, resp.output, is_styled, &out);
rendered_resp->status_code = resp.status_code;
rendered_resp->response_headers = std::move(resp.response_headers);
- rendered_resp->output << out.rdbuf();
+ // As the home page is redirected to startup until the server's
initialization is complete,
+ // do not render the page
+ if (render_path != "/home" || is_started_) {
+ stringstream out;
+ Render(render_path, resp.output, is_styled, &out);
+ rendered_resp->output << out.rdbuf();
+ }
};
RegisterPrerenderedPathHandler(path, alias, wrapped_cb, is_styled,
is_on_nav_bar);
}
@@ -830,4 +842,8 @@ void Webserver::set_footer_html(const std::string& html) {
footer_html_ = html;
}
+void Webserver::SetStartupComplete(bool state) {
+ is_started_ = state;
+}
+
} // namespace kudu
diff --git a/src/kudu/server/webserver.h b/src/kudu/server/webserver.h
index 8eddfcd..a8fd840 100644
--- a/src/kudu/server/webserver.h
+++ b/src/kudu/server/webserver.h
@@ -17,6 +17,7 @@
#pragma once
+#include <atomic>
#include <iosfwd>
#include <map>
#include <string>
@@ -85,6 +86,15 @@ class Webserver : public WebCallbackRegistry {
// True if serving all traffic over SSL, false otherwise
bool IsSecure() const;
+ // Change the status to true once the webserver's startup is completed. The
startup
+ // of a kudu server is split into two parts initialization and starting
phase. In the
+ // initialization phase we start the webserver, register only the path
handlers which are ready
+ // and in the startup phase the rest of them. Even though we start the
webserver in the
+ // initialization phase, once all the path handlers are registered, we
consider the web server
+ // to be started.
+ // Note: This only reflects the webserver's startup state and not the entire
kudu server.
+ void SetStartupComplete(bool state);
+
private:
// Container class for a list of path handler callbacks for a single URL.
class PathHandler {
@@ -207,6 +217,10 @@ class Webserver : public WebCallbackRegistry {
// Handle to Mongoose context; owned and freed by Mongoose internally
struct sq_context* context_;
+
+ // Hold the status of the webserver's startup status.
+ std::atomic<bool> is_started_;
+
};
} // namespace kudu
diff --git a/src/kudu/tserver/tablet_server.cc
b/src/kudu/tserver/tablet_server.cc
index 92f727d..43f6014 100644
--- a/src/kudu/tserver/tablet_server.cc
+++ b/src/kudu/tserver/tablet_server.cc
@@ -31,6 +31,7 @@
#include "kudu/gutil/strings/substitute.h"
#include "kudu/rpc/service_if.h"
#include "kudu/server/rpc_server.h"
+#include "kudu/server/startup_path_handler.h"
#include "kudu/transactions/txn_system_client.h"
#include "kudu/tserver/heartbeater.h"
#include "kudu/tserver/scanners.h"
@@ -43,6 +44,10 @@
#include "kudu/util/net/net_util.h"
#include "kudu/util/status.h"
+namespace kudu {
+class Timer;
+} // namespace kudu
+
using kudu::fs::ErrorHandlerType;
using kudu::rpc::ServiceIf;
using kudu::transactions::TxnSystemClientInitializer;
@@ -70,7 +75,7 @@ TabletServer::~TabletServer() {
Status TabletServer::Init() {
CHECK_EQ(kStopped, state_);
-
+ startup_path_handler_->set_is_tablet_server(true);
cfile::BlockCache::GetSingleton()->StartInstrumentation(metric_entity());
UnorderedHostPortSet master_addrs;
@@ -97,9 +102,6 @@ Status TabletServer::Init() {
}
RETURN_NOT_OK(KuduServer::Init());
- if (web_server_) {
- RETURN_NOT_OK(path_handlers_->Register(web_server_.get()));
- }
maintenance_manager_ = std::make_shared<MaintenanceManager>(
MaintenanceManager::kDefaultOptions, fs_manager_->uuid(),
metric_entity());
@@ -109,9 +111,12 @@ Status TabletServer::Init() {
heartbeater_.reset(new Heartbeater(std::move(master_addrs), this));
- RETURN_NOT_OK_PREPEND(tablet_manager_->Init(),
+ Timer* start_tablets = startup_path_handler_->start_tablets_progress();
+ std::atomic<int>* tablets_processed =
startup_path_handler_->tablets_processed();
+ std::atomic<int>* total_tablets = startup_path_handler_->tablets_total();
+ RETURN_NOT_OK_PREPEND(tablet_manager_->Init(start_tablets, tablets_processed,
+ total_tablets),
"Could not init Tablet Manager");
-
RETURN_NOT_OK_PREPEND(scanner_manager_->StartRemovalThread(),
"Could not start expired Scanner removal thread");
@@ -138,7 +143,6 @@ Status TabletServer::Start() {
ErrorHandlerType::KUDU_2233_CORRUPTION, [this](const string& uuid) {
this->tablet_manager_->FailTabletAndScheduleShutdown(uuid);
});
-
unique_ptr<ServiceIf> ts_service(new TabletServiceImpl(this));
unique_ptr<ServiceIf> admin_service(new TabletServiceAdminImpl(this));
unique_ptr<ServiceIf> consensus_service(new ConsensusServiceImpl(this,
tablet_manager_.get()));
@@ -151,6 +155,10 @@ Status TabletServer::Start() {
RETURN_NOT_OK(RegisterService(std::move(tablet_copy_service)));
RETURN_NOT_OK(KuduServer::Start());
+ if (web_server_) {
+ RETURN_NOT_OK(path_handlers_->Register(web_server_.get()));
+ }
+
RETURN_NOT_OK(heartbeater_->Start());
RETURN_NOT_OK(maintenance_manager_->Start());
diff --git a/src/kudu/tserver/ts_tablet_manager.cc
b/src/kudu/tserver/ts_tablet_manager.cc
index c3b745d..d64041f 100644
--- a/src/kudu/tserver/ts_tablet_manager.cc
+++ b/src/kudu/tserver/ts_tablet_manager.cc
@@ -72,6 +72,7 @@
#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/stopwatch.h"
#include "kudu/util/threadpool.h"
+#include "kudu/util/timer.h"
#include "kudu/util/trace.h"
DEFINE_int32(num_tablets_to_copy_simultaneously, 10,
@@ -385,7 +386,9 @@ protected:
TSTabletManager::~TSTabletManager() {
}
-Status TSTabletManager::Init() {
+Status TSTabletManager::Init(Timer* start_tablets,
+ std::atomic<int>* tablets_processed,
+ std::atomic<int>* tablets_total) {
CHECK_EQ(state(), MANAGER_INITIALIZING);
// Start the tablet copy thread pool. We set a max queue size of 0 so that if
@@ -447,6 +450,8 @@ Status TSTabletManager::Init() {
this->TxnStalenessTrackerTask();
}));
+ start_tablets->Start();
+
// Search for tablets in the metadata dir.
vector<string> tablet_ids;
RETURN_NOT_OK(fs_manager_->ListTabletIds(&tablet_ids));
@@ -476,6 +481,8 @@ Status TSTabletManager::Init() {
loaded_count, metas.size());
// Now submit the "Open" task for each.
+ *tablets_total = metas.size();
+ *tablets_processed = 0;
int registered_count = 0;
for (const auto& meta : metas) {
KLOG_EVERY_N_SECS(INFO, 1) << Substitute("Registering tablets ($0/$1
complete)",
@@ -488,13 +495,18 @@ Status TSTabletManager::Init() {
scoped_refptr<TabletReplica> replica;
RETURN_NOT_OK(CreateAndRegisterTabletReplica(meta, NEW_REPLICA, &replica));
- RETURN_NOT_OK(open_tablet_pool_->Submit([this, replica, deleter]() {
- this->OpenTablet(replica, deleter);
- }));
+ RETURN_NOT_OK(open_tablet_pool_->Submit([this, replica, deleter,
tablets_processed,
+ tablets_total, start_tablets]() {
+ this->OpenTablet(replica, deleter, tablets_processed, tablets_total,
start_tablets);
+ }));
registered_count++;
}
LOG(INFO) << Substitute("Registered $0 tablets", registered_count);
+ if (registered_count == 0) {
+ start_tablets->Stop();
+ }
+
{
std::lock_guard<RWMutex> lock(lock_);
state_ = MANAGER_RUNNING;
@@ -581,8 +593,8 @@ Status TSTabletManager::CreateNewTablet(const string&
table_id,
// We can run this synchronously since there is nothing to bootstrap.
RETURN_NOT_OK(open_tablet_pool_->Submit([this, new_replica, deleter]() {
- this->OpenTablet(new_replica, deleter);
- }));
+ this->OpenTablet(new_replica, deleter);
+ }));
if (replica) {
*replica = new_replica;
@@ -1228,8 +1240,11 @@ Status TSTabletManager::OpenTabletMeta(const string&
tablet_id,
return Status::OK();
}
-void TSTabletManager::OpenTablet(const scoped_refptr<TabletReplica>& replica,
- const
scoped_refptr<TransitionInProgressDeleter>& deleter) {
+void TSTabletManager::OpenTablet(const scoped_refptr<tablet::TabletReplica>&
replica,
+ const
scoped_refptr<TransitionInProgressDeleter>& deleter,
+ std::atomic<int>* tablets_processed,
+ std::atomic<int>* tablets_total,
+ Timer* start_tablets) {
const string& tablet_id = replica->tablet_id();
TRACE_EVENT1("tserver", "TSTabletManager::OpenTablet",
"tablet_id", tablet_id);
@@ -1256,6 +1271,8 @@ void TSTabletManager::OpenTablet(const
scoped_refptr<TabletReplica>& replica,
});
if (PREDICT_FALSE(!s.ok())) {
LOG(ERROR) << LogPrefix(tablet_id) << "Failed to load consensus metadata:
" << s.ToString();
+ IncrementTabletsProcessed((tablets_total != nullptr) ?
tablets_total->load() : 0,
+ tablets_processed, start_tablets);
return;
}
@@ -1284,6 +1301,8 @@ void TSTabletManager::OpenTablet(const
scoped_refptr<TabletReplica>& replica,
if (!s.ok()) {
LOG(ERROR) << LogPrefix(tablet_id) << "Tablet failed to bootstrap: "
<< s.ToString();
+ IncrementTabletsProcessed((tablets_total != nullptr) ?
tablets_total->load() : 0,
+ tablets_processed, start_tablets);
return;
}
}
@@ -1302,6 +1321,8 @@ void TSTabletManager::OpenTablet(const
scoped_refptr<TabletReplica>& replica,
if (!s.ok()) {
LOG(ERROR) << LogPrefix(tablet_id) << "Tablet failed to start: "
<< s.ToString();
+ IncrementTabletsProcessed((tablets_total != nullptr) ?
tablets_total->load() : 0,
+ tablets_processed, start_tablets);
return;
}
@@ -1325,11 +1346,23 @@ void TSTabletManager::OpenTablet(const
scoped_refptr<TabletReplica>& replica,
<< Trace::CurrentTrace()->DumpToString();
}
}
-
+ IncrementTabletsProcessed((tablets_total != nullptr) ? tablets_total->load()
: 0,
+ tablets_processed, start_tablets);
deleter->Destroy();
MarkTabletDirty(tablet_id, "Open tablet completed");
}
+void TSTabletManager::IncrementTabletsProcessed(int tablets_total,
+ std::atomic<int>*
tablets_processed,
+ Timer* start_tablets) {
+ if (tablets_processed) {
+ ++*tablets_processed;
+ if (*tablets_processed == tablets_total) {
+ start_tablets->Stop();
+ }
+ }
+}
+
void TSTabletManager::Shutdown() {
{
std::lock_guard<RWMutex> lock(lock_);
diff --git a/src/kudu/tserver/ts_tablet_manager.h
b/src/kudu/tserver/ts_tablet_manager.h
index 0508563..e7e3a6c 100644
--- a/src/kudu/tserver/ts_tablet_manager.h
+++ b/src/kudu/tserver/ts_tablet_manager.h
@@ -17,6 +17,7 @@
#pragma once
+#include <atomic>
#include <cstdint>
#include <functional>
#include <map>
@@ -58,6 +59,7 @@ class Partition;
class PartitionSchema;
class Schema;
class ThreadPool;
+class Timer;
namespace transactions {
class TxnSystemClient;
@@ -102,9 +104,13 @@ class TSTabletManager : public
tserver::TabletReplicaLookupIf {
virtual ~TSTabletManager();
// Load all tablet metadata blocks from disk, and open their respective
tablets.
+ // Starts the timer, 'start_tablets' and populates the 'tablets_total'. The
subsequent
+ // function call to OpenTablet() updates 'tablets_processed' and stops the
timer.
// Upon return of this method all existing tablets are registered, but
// the bootstrap is performed asynchronously.
- Status Init();
+ Status Init(Timer* start_tablets,
+ std::atomic<int>* tablets_processed,
+ std::atomic<int>* tablets_total);
// Waits for all the bootstraps to complete.
// Returns Status::OK if all tablets bootstrapped successfully. If
@@ -322,8 +328,14 @@ class TSTabletManager : public
tserver::TabletReplicaLookupIf {
// method. A TransitionInProgressDeleter must be passed as 'deleter' into
// this method in order to remove that transition-in-progress entry when
// opening the tablet is complete (in either a success or a failure case).
- void OpenTablet(const scoped_refptr<tablet::TabletReplica>& replica,
- const scoped_refptr<TransitionInProgressDeleter>& deleter);
+ //
+ // In the subsequent call made to UpdateStartupProgress, 'tablets_processed'
+ // will be updated and the timer is stopped once all the tablets are
processed.
+ void OpenTablet(const scoped_refptr<tablet::TabletReplica> &replica,
+ const scoped_refptr<TransitionInProgressDeleter> &deleter,
+ std::atomic<int>* tablets_processed = nullptr,
+ std::atomic<int>* tablets_total = nullptr,
+ Timer* bootstrap_tablets = nullptr);
// Open a tablet whose metadata has already been loaded.
void BootstrapAndInitTablet(const scoped_refptr<tablet::TabletMetadata>&
meta,
@@ -380,6 +392,12 @@ class TSTabletManager : public
tserver::TabletReplicaLookupIf {
// Just for tests.
void SetNextUpdateTimeForTests();
+ // If 'tablets_processed' is not nullptr, 'tablets_processed' will be
incremented
+ // after every tablet is attempted to be opened and the timer is stopped
once all
+ // the tablets are processed.
+ void IncrementTabletsProcessed(int tablets_total, std::atomic<int>*
tablets_processed,
+ Timer* start_tablets);
+
FsManager* const fs_manager_;
const scoped_refptr<consensus::ConsensusMetadataManager> cmeta_manager_;
diff --git a/src/kudu/util/timer.h b/src/kudu/util/timer.h
new file mode 100644
index 0000000..5f756c8
--- /dev/null
+++ b/src/kudu/util/timer.h
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <mutex>
+
+#include "kudu/util/monotime.h"
+#include "kudu/util/locks.h"
+
+namespace kudu {
+
+// Using this class we can keep measure how long something took to run.
+// Use the Start() and Stop() to start the stop the timer and TimeElapsed()
+// to get the time taken to run and if still running, time elapsed until that
point
+class Timer {
+
+ public:
+ Timer()
+ : start_time_(MonoTime::Min()),
+ end_time_(MonoTime::Min()) {
+ }
+
+ void Start() {
+ if (start_time_ == MonoTime::Min()) {
+ start_time_ = MonoTime::Now();
+ }
+ }
+
+ void Stop() {
+ DCHECK_NE(start_time_, MonoTime::Min());
+ end_time_ = MonoTime::Now();
+ }
+
+ MonoDelta TimeElapsed() const {
+ if (start_time_ == MonoTime::Min() && end_time_ == MonoTime::Min()) {
+ return MonoDelta::FromSeconds(0);
+ }
+ if (end_time_ == MonoTime::Min()) {
+ return (MonoTime::Now() - start_time_);
+ }
+ return (end_time_ - start_time_);
+ }
+
+ bool IsStopped() const {
+ return (end_time_ != MonoTime::Min());
+ }
+
+ private:
+ std::atomic<MonoTime> start_time_;
+ std::atomic<MonoTime> end_time_;
+};
+
+} // namespace kudu
diff --git a/src/kudu/util/web_callback_registry.h
b/src/kudu/util/web_callback_registry.h
index ee72e4d..39a8062 100644
--- a/src/kudu/util/web_callback_registry.h
+++ b/src/kudu/util/web_callback_registry.h
@@ -27,6 +27,7 @@ namespace kudu {
enum class HttpStatusCode {
Ok, // 200
+ TemporaryRedirect, //307
BadRequest, // 400
AuthenticationRequired, // 401
NotFound, // 404
diff --git a/www/startup.mustache b/www/startup.mustache
new file mode 100644
index 0000000..fb026ee
--- /dev/null
+++ b/www/startup.mustache
@@ -0,0 +1,79 @@
+{{!
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+}}
+{{#error}}
+ <div class="text-error">{{.}}</div>
+{{/error}}
+{{^error}}
+ <h2>Server Startup Progress</h2>
+ <table class='table table-striped'>
+ <tr style="background-color: #eee">
+ <th>Step</th>
+ <th>Progress (%)</th>
+ <th>Elapsed Time</th>
+ </tr>
+ <tr>
+ <th>Initializing</th>
+ <th>{{init_status}}%</th>
+ <th>{{init_time}}</th>
+ </tr>
+ <tr>
+ <th>Reading Filesystem</th>
+ <th>{{read_filesystem_status}}%</th>
+ <th>{{read_filesystem_time}}</th>
+ </tr>
+ <tr>
+ <td style="padding-left:50px">Reading instance metadata files</td>
+ <td>{{read_instance_metadatafiles_status}}%</td>
+ <td>{{read_instance_metadatafiles_time}}</td>
+ </tr>
+ {{#is_log_block_manager}}
+ <tr>
+ <td style="padding-left:50px">Opening container files 
 ({{containers_processed}}/{{containers_total}})</td>
+ <td>{{read_data_directories_status}}%</td>
+ <td>{{read_data_directories_time}}</td>
+ </tr>
+ {{/is_log_block_manager}}
+ {{^is_log_block_manager}}
+ <tr>
+ <td style="padding-left:50px">Reporting filesystem</td>
+ <td>{{read_data_directories_status}}%</td>
+ <td>{{read_data_directories_time}}</td>
+ </tr>
+ {{/is_log_block_manager}}
+ {{#is_tablet_server}}
+ <tr>
+ <th>Bootstrapping and opening the tablets  
 ({{tablets_processed}}/{{tablets_total}})</th>
+ <th>{{start_tablets_status}}%</th>
+ <th>{{start_tablets_time}}</th>
+ </tr>
+ {{/is_tablet_server}}
+ {{#is_master_server}}
+ <tr>
+ <th>Initializing master catalog</th>
+ <th>{{initialize_master_catalog_status}}%</th>
+ <th>{{initialize_master_catalog_time}}</th>
+ </tr>
+ {{/is_master_server}}
+ <tr>
+ <th>Starting RPC server</th>
+ <th>{{start_rpc_server_status}}%</th>
+ <th>{{start_rpc_server_time}}</th>
+ </tr>
+ </table>
+{{/error}}
