Repository: incubator-quickstep Updated Branches: refs/heads/multi-shiftboss-test d4f4b4717 -> d2862e578 (forced update)
Moved FLAGS_num_workers into a file for shared flags. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e666a81f Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e666a81f Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e666a81f Branch: refs/heads/multi-shiftboss-test Commit: e666a81f472e4c8ed81e98995c1914ce655e247a Parents: 212e6cd Author: Zuyu Zhang <zu...@apache.org> Authored: Sun Nov 20 22:42:49 2016 -0800 Committer: Zuyu Zhang <zu...@apache.org> Committed: Sun Nov 20 22:42:49 2016 -0800 ---------------------------------------------------------------------- cli/CMakeLists.txt | 1 + cli/Flags.cpp | 29 +++++++++++++++++++++++++++++ cli/Flags.hpp | 2 ++ cli/QuickstepCli.cpp | 31 +++++-------------------------- 4 files changed, 37 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt index 9928f3f..9847787 100644 --- a/cli/CMakeLists.txt +++ b/cli/CMakeLists.txt @@ -112,6 +112,7 @@ if(QUICKSTEP_HAVE_LIBNUMA) ${LIBNUMA_LIBRARY}) endif() target_link_libraries(quickstep_cli_Flags + quickstep_cli_DefaultsConfigurator quickstep_storage_StorageConstants ${GFLAGS_LIB_NAME}) target_link_libraries(quickstep_cli_InputParserUtil http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.cpp ---------------------------------------------------------------------- diff --git a/cli/Flags.cpp b/cli/Flags.cpp index 384ef71..87f9f73 100644 --- a/cli/Flags.cpp +++ b/cli/Flags.cpp @@ -19,14 +19,43 @@ #include "cli/Flags.hpp" +#include <cstddef> +#include <cstdio> #include <string> +#include "cli/DefaultsConfigurator.hpp" #include "storage/StorageConstants.hpp" #include "gflags/gflags.h" +using std::fprintf; + namespace quickstep { +static bool ValidateNumWorkers(const char *flagname, int value) { + if (value > 0) { + return true; + } + + // Detect the hardware concurrency level. + const std::size_t num_hw_threads = + DefaultsConfigurator::GetNumHardwareThreads(); + + // Use the command-line value if that was supplied, else use the value + // that we computed above, provided it did return a valid value. + // TODO(jmp): May need to change this at some point to keep one thread + // available for the OS if the hardware concurrency level is high. + FLAGS_num_workers = num_hw_threads != 0 ? num_hw_threads : 1; + + return FLAGS_num_workers > 0; +} +DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is " + "specified and is greater than 0, then this user-supplied value is " + "used. Else (i.e. the default case), we examine the reported " + "hardware concurrency level, and use that."); +static const volatile bool num_workers_dummy + = gflags::RegisterFlagValidator(&FLAGS_num_workers, &ValidateNumWorkers); + static bool ValidateStoragePath(const char *flagname, const std::string &value) { if (!value.empty() && value.back() != kPathSeparator) { http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.hpp ---------------------------------------------------------------------- diff --git a/cli/Flags.hpp b/cli/Flags.hpp index a623448..b020a3e 100644 --- a/cli/Flags.hpp +++ b/cli/Flags.hpp @@ -32,6 +32,8 @@ namespace quickstep { * @brief A collection of common flags shared by Quickstep CLIs in both the * single-node and the distributed version. **/ +DECLARE_int32(num_workers); + DECLARE_string(storage_path); /** @} */ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/QuickstepCli.cpp ---------------------------------------------------------------------- diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp index 33a9ff4..6a7c500 100644 --- a/cli/QuickstepCli.cpp +++ b/cli/QuickstepCli.cpp @@ -109,6 +109,7 @@ using quickstep::AdmitRequestMessage; using quickstep::CatalogRelation; using quickstep::DefaultsConfigurator; using quickstep::DropRelation; +using quickstep::FLAGS_num_workers; using quickstep::FLAGS_storage_path; using quickstep::ForemanSingleNode; using quickstep::InputParserUtil; @@ -136,11 +137,6 @@ using tmb::client_id; namespace quickstep { -DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is " - "specified and is greater than 0, then this " - "user-supplied value is used. Else (i.e. the" - "default case), we examine the reported " - "hardware concurrency level, and use that."); DEFINE_bool(preload_buffer_pool, false, "If true, pre-load all known blocks into buffer pool before " "accepting queries (should also set --buffer_pool_slots to be " @@ -189,25 +185,8 @@ int main(int argc, char* argv[]) { google::InitGoogleLogging(argv[0]); gflags::ParseCommandLineFlags(&argc, &argv, true); - // Detect the hardware concurrency level. - const std::size_t num_hw_threads = - DefaultsConfigurator::GetNumHardwareThreads(); - - // Use the command-line value if that was supplied, else use the value - // that we computed above, provided it did return a valid value. - // TODO(jmp): May need to change this at some point to keep one thread - // available for the OS if the hardware concurrency level is high. - if (quickstep::FLAGS_num_workers <= 0) { - LOG(INFO) << "Quickstep expects at least one worker thread, switching to " - "the default number of worker threads"; - } - const int real_num_workers = quickstep::FLAGS_num_workers > 0 - ? quickstep::FLAGS_num_workers - : (num_hw_threads != 0 ? num_hw_threads : 1); - - DCHECK_GT(real_num_workers, 0); - printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool\n", - real_num_workers, + printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool.\n", + FLAGS_num_workers, (static_cast<double>(quickstep::FLAGS_buffer_pool_slots) * quickstep::kSlotSizeBytes)/quickstep::kAGigaByte); #ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS @@ -290,7 +269,7 @@ int main(int argc, char* argv[]) { // Parse the CPU affinities for workers and the preloader thread, if enabled // to warm up the buffer pool. const vector<int> worker_cpu_affinities = - InputParserUtil::ParseWorkerAffinities(real_num_workers, + InputParserUtil::ParseWorkerAffinities(FLAGS_num_workers, quickstep::FLAGS_worker_affinities); const std::size_t num_numa_nodes_system = DefaultsConfigurator::GetNumNUMANodes(); @@ -323,7 +302,7 @@ int main(int argc, char* argv[]) { vector<client_id> worker_client_ids; // Initialize the worker threads. - DCHECK_EQ(static_cast<std::size_t>(real_num_workers), + DCHECK_EQ(static_cast<std::size_t>(FLAGS_num_workers), worker_cpu_affinities.size()); for (std::size_t worker_thread_index = 0; worker_thread_index < worker_cpu_affinities.size();
