Repository: incubator-quickstep Updated Branches: refs/heads/fix-admit-queries-bug 59c2184bd -> 9a991bb00 (forced update)
Fixed the catalog relation schema reference bug in the preloading in the distributed version. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4a95738d Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4a95738d Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4a95738d Branch: refs/heads/fix-admit-queries-bug Commit: 4a95738de730b03ccfb2026843f5a4e6838cfe2f Parents: a5c68dd Author: Zuyu Zhang <zu...@apache.org> Authored: Fri Mar 10 14:58:39 2017 -0800 Committer: Zuyu Zhang <zu...@apache.org> Committed: Fri Mar 10 14:58:39 2017 -0800 ---------------------------------------------------------------------- catalog/tests/Catalog_unittest.cpp | 5 ++++- query_execution/Shiftboss.cpp | 4 +++- storage/CMakeLists.txt | 1 + storage/PreloaderThread.cpp | 17 +++++++++++++---- storage/PreloaderThread.hpp | 12 ++++++++++-- 5 files changed, 31 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/catalog/tests/Catalog_unittest.cpp ---------------------------------------------------------------------- diff --git a/catalog/tests/Catalog_unittest.cpp b/catalog/tests/Catalog_unittest.cpp index 276bfa9..f761026 100644 --- a/catalog/tests/Catalog_unittest.cpp +++ b/catalog/tests/Catalog_unittest.cpp @@ -571,7 +571,10 @@ TEST_F(CatalogTest, CatalogIndexTest) { } TEST_F(CatalogTest, CatalogDatabaseCacheTest) { - CatalogRelationSchema* const rel = createCatalogRelation("rel"); + CatalogRelation * const rel = createCatalogRelation("rel"); + + rel->addBlock(BlockIdUtil::GetBlockId(0 /* domain */, 7)); + rel->addBlock(BlockIdUtil::GetBlockId(0 /* domain */, 14)); rel->addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt), -1 /* id */, "int")); rel->addAttribute(new CatalogAttribute(nullptr, "attr_long", TypeFactory::GetType(kLong))); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/query_execution/Shiftboss.cpp ---------------------------------------------------------------------- diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp index 905464f..72d8913 100644 --- a/query_execution/Shiftboss.cpp +++ b/query_execution/Shiftboss.cpp @@ -340,9 +340,11 @@ void Shiftboss::processShiftbossRegistrationResponseMessage() { storage_manager_->sendBlockDomainToShiftbossIndexMessage(shiftboss_index_); if (FLAGS_preload_buffer_pool) { + database_cache_.update(proto.catalog_database()); + const CatalogDatabase catalog_database(proto.catalog_database()); - PreloaderThread preloader(catalog_database, storage_manager_, cpu_id_); + PreloaderThread preloader(catalog_database, storage_manager_, cpu_id_, &database_cache_); printf("Preloading the buffer pool ... \n"); const std::chrono::time_point<std::chrono::steady_clock> preload_start = std::chrono::steady_clock::now(); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/storage/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 8b68150..6fe6436 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -827,6 +827,7 @@ target_link_libraries(quickstep_storage_PartitionedHashTablePool target_link_libraries(quickstep_storage_PreloaderThread glog quickstep_catalog_CatalogDatabase + quickstep_catalog_CatalogDatabaseLite quickstep_catalog_CatalogRelation quickstep_catalog_CatalogTypedefs quickstep_storage_StorageBlock http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/storage/PreloaderThread.cpp ---------------------------------------------------------------------- diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp index edaa143..354e468 100644 --- a/storage/PreloaderThread.cpp +++ b/storage/PreloaderThread.cpp @@ -27,6 +27,7 @@ #endif #include "catalog/CatalogDatabase.hpp" +#include "catalog/CatalogDatabaseLite.hpp" #include "catalog/CatalogRelation.hpp" #include "catalog/CatalogTypedefs.hpp" #include "storage/StorageBlock.hpp" @@ -60,8 +61,12 @@ void PreloaderThread::run() { std::vector<block_id> blocks = relation.getBlocksSnapshot(); for (block_id current_block_id : blocks) { try { - BlockReference current_block = - storage_manager_->getBlock(current_block_id, relation); + const CatalogRelationSchema &relation_schema = + database_for_schemas_ ? database_for_schemas_->getRelationSchemaById(relation.getID()) + : relation; + + const BlockReference current_block = + storage_manager_->getBlock(current_block_id, relation_schema); } catch (...) { LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks"; throw; @@ -104,8 +109,12 @@ std::size_t PreloaderThread::preloadNUMAAware( placement_scheme->getNUMANodeForPartition(part_id); for (block_id curr_block_id : part_scheme.getBlocksInPartition(part_id)) { try { - BlockReference current_block = storage_manager_->getBlock( - curr_block_id, relation, partition_numa_node_id); + const CatalogRelationSchema &relation_schema = + database_for_schemas_ ? database_for_schemas_->getRelationSchemaById(relation.getID()) + : relation; + + const BlockReference current_block = storage_manager_->getBlock( + curr_block_id, relation_schema, partition_numa_node_id); } catch (...) { LOG(ERROR) << "Error while preloading: After loading total " << blocks_loaded + num_previously_loaded_blocks http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/storage/PreloaderThread.hpp ---------------------------------------------------------------------- diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp index d9853b8..af3f6ba 100644 --- a/storage/PreloaderThread.hpp +++ b/storage/PreloaderThread.hpp @@ -29,6 +29,7 @@ namespace quickstep { class CatalogDatabase; +class CatalogDatabaseLite; class CatalogRelation; class StorageManager; @@ -55,13 +56,18 @@ class PreloaderThread : public Thread { * evicted and the database will not be fully preloaded. * @param cpu_id The ID of the CPU to affinitize this thread to, or -1 to * indicate no affinity. + * @param database_for_schemas All non-temporary relations in this database + * will be used for referencing the schema in a block for the + * distributed version. **/ PreloaderThread(const CatalogDatabase &database, StorageManager *storage_manager, - const int cpu_id = -1) + const int cpu_id = -1, + const CatalogDatabaseLite *database_for_schemas = nullptr) : database_(database), storage_manager_(storage_manager), - cpu_id_(cpu_id) { + cpu_id_(cpu_id), + database_for_schemas_(database_for_schemas) { } ~PreloaderThread() override { @@ -102,6 +108,8 @@ class PreloaderThread : public Thread { const int cpu_id_; + const CatalogDatabaseLite *database_for_schemas_; + DISALLOW_COPY_AND_ASSIGN(PreloaderThread); };
