Moved hdfs-related flags into a file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/dda085c7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/dda085c7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/dda085c7 Branch: refs/heads/reorder-partitioned-hash-join Commit: dda085c7aeaa4f7f353c3392b547ca788d1507b8 Parents: 43a9b39 Author: Zuyu Zhang <zu...@apache.org> Authored: Sat Feb 4 14:46:44 2017 -0800 Committer: Zuyu Zhang <zu...@apache.org> Committed: Sat Feb 4 16:25:03 2017 -0800 ---------------------------------------------------------------------- CMakeLists.txt | 1 + cli/QuickstepCli.cpp | 1 + storage/CMakeLists.txt | 13 ++++++--- storage/FileManagerHdfs.cpp | 35 +---------------------- storage/FileManagerHdfs.hpp | 6 ---- storage/Flags.cpp | 62 ++++++++++++++++++++++++++++++++++++++++ storage/Flags.hpp | 51 +++++++++++++++++++++++++++++++++ storage/StorageManager.cpp | 5 +--- storage/StorageManager.hpp | 4 --- 9 files changed, 126 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 85210ef..e74a274 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -789,6 +789,7 @@ target_link_libraries(quickstep_cli_shell quickstep_queryexecution_WorkerDirectory quickstep_queryoptimizer_QueryHandle quickstep_queryoptimizer_QueryProcessor + quickstep_storage_Flags quickstep_storage_PreloaderThread quickstep_storage_StorageConstants quickstep_storage_StorageManager http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/cli/QuickstepCli.cpp ---------------------------------------------------------------------- diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp index 9db7577..02ec4ec 100644 --- a/cli/QuickstepCli.cpp +++ b/cli/QuickstepCli.cpp @@ -58,6 +58,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl; #include "query_execution/WorkerDirectory.hpp" #include "query_optimizer/QueryHandle.hpp" #include "query_optimizer/QueryProcessor.hpp" +#include "storage/Flags.hpp" #include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS. #ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index fddea1f..a44c3a7 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -207,6 +207,7 @@ if (QUICKSTEP_HAVE_FILE_MANAGER_POSIX) elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS) add_library(quickstep_storage_FileManagerWindows FileManagerWindows.cpp FileManagerWindows.hpp) endif() +add_library(quickstep_storage_Flags Flags.cpp Flags.hpp) add_library(quickstep_storage_HashTable ../empty_src.cpp HashTable.hpp) add_library(quickstep_storage_HashTable_proto ${storage_HashTable_proto_srcs}) add_library(quickstep_storage_HashTableBase ../empty_src.cpp HashTableBase.hpp) @@ -679,8 +680,8 @@ target_link_libraries(quickstep_storage_FileManager if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS) target_link_libraries(quickstep_storage_FileManagerHdfs glog - ${GFLAGS_LIB_NAME} quickstep_storage_FileManager + quickstep_storage_Flags quickstep_storage_StorageBlockInfo quickstep_storage_StorageConstants quickstep_storage_StorageErrors @@ -711,6 +712,8 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS) quickstep_utility_Macros quickstep_utility_StringUtil) endif() +target_link_libraries(quickstep_storage_Flags + ${GFLAGS_LIB_NAME}) target_link_libraries(quickstep_storage_HashTable quickstep_catalog_CatalogTypedefs quickstep_storage_HashTableBase @@ -1000,6 +1003,7 @@ target_link_libraries(quickstep_storage_StorageManager quickstep_storage_EvictionPolicy quickstep_storage_FileManager quickstep_storage_FileManagerLocal + quickstep_storage_Flags quickstep_storage_StorageBlob quickstep_storage_StorageBlock quickstep_storage_StorageBlockBase @@ -1121,17 +1125,18 @@ target_link_libraries(quickstep_storage quickstep_storage_CompressedTupleStorageSubBlock quickstep_storage_CountedReference quickstep_storage_EvictionPolicy - quickstep_storage_FileManager - quickstep_storage_FileManagerLocal quickstep_storage_FastHashTable quickstep_storage_FastHashTableFactory quickstep_storage_FastSeparateChainingHashTable + quickstep_storage_FileManager + quickstep_storage_FileManagerLocal + quickstep_storage_Flags quickstep_storage_HashTable - quickstep_storage_HashTable_proto quickstep_storage_HashTableBase quickstep_storage_HashTableFactory quickstep_storage_HashTableKeyManager quickstep_storage_HashTablePool + quickstep_storage_HashTable_proto quickstep_storage_IndexSubBlock quickstep_storage_IndexSubBlockDescriptionFactory quickstep_storage_InsertDestination http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/FileManagerHdfs.cpp ---------------------------------------------------------------------- diff --git a/storage/FileManagerHdfs.cpp b/storage/FileManagerHdfs.cpp index e28ed3b..937750a 100644 --- a/storage/FileManagerHdfs.cpp +++ b/storage/FileManagerHdfs.cpp @@ -31,12 +31,12 @@ #include <cstring> #include <string> +#include "storage/Flags.hpp" #include "storage/StorageBlockInfo.hpp" #include "storage/StorageConstants.hpp" #include "storage/StorageErrors.hpp" #include "utility/StringUtil.hpp" -#include "gflags/gflags.h" #include "glog/logging.h" using std::size_t; @@ -46,41 +46,8 @@ using std::string; namespace quickstep { -DEFINE_string(hdfs_namenode_host, "localhost", "Host name or IP address of HDFS namenode."); - -static bool ValidateHdfsNameNodePort(const char *flagname, - std::int32_t value) { - if (value > 0 && value < 65536) { - return true; - } else { - std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname); - return false; - } -} -DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode."); -static const bool hdfs_namenode_port_dummy - = gflags::RegisterFlagValidator(&FLAGS_hdfs_namenode_port, &ValidateHdfsNameNodePort); - -static bool ValidateHdfsNumReplications(const char *flagname, - std::int32_t value) { - // The default dfs.namenode.replication.min is 1, while the default - // dfs.replication.max is 512. - if (value > 0) { - return true; - } else { - std::fprintf(stderr, "--%s must be at least 1\n", flagname); - return false; - } -} -DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications."); -static const bool hdfs_num_replications_dummy - = gflags::RegisterFlagValidator(&FLAGS_hdfs_num_replications, &ValidateHdfsNumReplications); - FileManagerHdfs::FileManagerHdfs(const string &storage_path) : FileManager(storage_path) { - DCHECK(hdfs_namenode_port_dummy); - DCHECK(hdfs_num_replications_dummy); - struct hdfsBuilder *builder = hdfsNewBuilder(); hdfsBuilderSetNameNode(builder, FLAGS_hdfs_namenode_host.c_str()); hdfsBuilderSetNameNodePort(builder, FLAGS_hdfs_namenode_port); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/FileManagerHdfs.hpp ---------------------------------------------------------------------- diff --git a/storage/FileManagerHdfs.hpp b/storage/FileManagerHdfs.hpp index 3c751b5..f47e4a8 100644 --- a/storage/FileManagerHdfs.hpp +++ b/storage/FileManagerHdfs.hpp @@ -29,14 +29,8 @@ #include "storage/StorageBlockInfo.hpp" #include "utility/Macros.hpp" -#include "gflags/gflags.h" - namespace quickstep { -DECLARE_string(hdfs_namenode_host); -DECLARE_int32(hdfs_namenode_port); -DECLARE_int32(hdfs_num_replications); - /** \addtogroup Storage * @{ */ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/Flags.cpp ---------------------------------------------------------------------- diff --git a/storage/Flags.cpp b/storage/Flags.cpp new file mode 100644 index 0000000..4672f81 --- /dev/null +++ b/storage/Flags.cpp @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +#include "storage/Flags.hpp" + +#include "storage/StorageConfig.h" + +#include "gflags/gflags.h" + +namespace quickstep { + +#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS +DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the local disk."); + +DEFINE_string(hdfs_namenode_host, "localhost", "Host name or IP address of HDFS namenode."); + +static bool ValidateHdfsNameNodePort(const char *flagname, + std::int32_t value) { + if (value > 0 && value < 65536) { + return true; + } else { + std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname); + return false; + } +} +DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode."); +static const bool hdfs_namenode_port_dummy + = gflags::RegisterFlagValidator(&FLAGS_hdfs_namenode_port, &ValidateHdfsNameNodePort); + +static bool ValidateHdfsNumReplications(const char *flagname, + std::int32_t value) { + // The default dfs.namenode.replication.min is 1, while the default + // dfs.replication.max is 512. + if (value > 0) { + return true; + } else { + std::fprintf(stderr, "--%s must be at least 1\n", flagname); + return false; + } +} +DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications."); +static const bool hdfs_num_replications_dummy + = gflags::RegisterFlagValidator(&FLAGS_hdfs_num_replications, &ValidateHdfsNumReplications); +#endif + +} // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/Flags.hpp ---------------------------------------------------------------------- diff --git a/storage/Flags.hpp b/storage/Flags.hpp new file mode 100644 index 0000000..1d5527c --- /dev/null +++ b/storage/Flags.hpp @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +#ifndef QUICKSTEP_STORAGE_FLAGS_HPP_ +#define QUICKSTEP_STORAGE_FLAGS_HPP_ + +#include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS. + +#include "gflags/gflags_declare.h" + +namespace quickstep { + +/** \addtogroup STORAGE + * @{ + */ + +/** + * @brief A collection of common flags shared by Quickstep STORAGEs in both the + * single-node and the distributed version. + **/ + +#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS +DECLARE_bool(use_hdfs); + +DECLARE_string(hdfs_namenode_host); +DECLARE_int32(hdfs_namenode_port); +DECLARE_int32(hdfs_num_replications); + +#endif + +/** @} */ + +} // namespace quickstep + +#endif // QUICKSTEP_STORAGE_FLAGS_HPP_ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/StorageManager.cpp ---------------------------------------------------------------------- diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp index 6299cda..6f7d38b 100644 --- a/storage/StorageManager.cpp +++ b/storage/StorageManager.cpp @@ -76,6 +76,7 @@ #include "storage/EvictionPolicy.hpp" #include "storage/FileManagerLocal.hpp" +#include "storage/Flags.hpp" #include "storage/StorageBlob.hpp" #include "storage/StorageBlock.hpp" #include "storage/StorageBlockBase.hpp" @@ -189,10 +190,6 @@ DEFINE_uint64(buffer_pool_slots, 0, static const volatile bool buffer_pool_slots_dummy = gflags::RegisterFlagValidator(&FLAGS_buffer_pool_slots, &SetOrValidateBufferPoolSlots); -#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS -DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the local disk."); -#endif - StorageManager::StorageManager( const std::string &storage_path, const block_id_domain block_domain, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/StorageManager.hpp ---------------------------------------------------------------------- diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp index b61f10a..42176ee 100644 --- a/storage/StorageManager.hpp +++ b/storage/StorageManager.hpp @@ -63,10 +63,6 @@ namespace quickstep { DECLARE_int32(block_domain); DECLARE_uint64(buffer_pool_slots); -#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS -DECLARE_bool(use_hdfs); -#endif - class CatalogRelationSchema; #ifdef QUICKSTEP_DISTRIBUTED
