incubator-quickstep git commit: Fix glog build issue for Mac OS X 10.14.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master f5c84fd6c -> 468f2a6aa
Fix glog build issue for Mac OS X 10.14.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/468f2a6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/468f2a6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/468f2a6a
Branch: refs/heads/master
Commit: 468f2a6aaffd7defc742c4cfb9e8b14a09dd540e
Parents: f5c84fd
Author: Zuyu Zhang
Authored: Mon Oct 22 21:31:29 2018 -0500
Committer: Zuyu Zhang
Committed: Mon Oct 22 21:31:29 2018 -0500
--
CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/468f2a6a/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 434019e..f66a51a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -320,7 +320,7 @@ else()
# builds to fail. As a short-term workaround for now, we turn off deprecated
# warnings so that they do not cause build failures anymore.
# TODO: Remove this workaround by fixing the protobuf_cmake and glog_cmake.
- if (${CMAKE_SYSTEM} MATCHES "Darwin-1[67].[0-9]*.[0-9]*")
+ if (${CMAKE_SYSTEM} MATCHES "Darwin-1[678].[0-9]*.[0-9]*")
if (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
CHECK_CXX_COMPILER_FLAG("-Wno-error=deprecated-declarations"
COMPILER_HAS_WNO_DEPRECATED)
if (COMPILER_HAS_WNO_DEPRECATED)
incubator-quickstep git commit: Add virtual destructors where they are needed to remove compilations errors.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 5cbaa7ef8 -> f5c84fd6c
Add virtual destructors where they are needed to remove compilations errors.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f5c84fd6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f5c84fd6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f5c84fd6
Branch: refs/heads/master
Commit: f5c84fd6c8211c161f9862f4ac5c117568427580
Parents: 5cbaa7e
Author: johnchronis
Authored: Mon Oct 8 18:01:27 2018 -0500
Committer: Yannis Chronis
Committed: Thu Oct 11 12:52:55 2018 -0500
--
.../table_generator/GenerateSeriesHandle.hpp| 2 ++
.../table_generator/GeneratorFunctionHandle.hpp | 2 ++
.../src/tmb/tests/message_bus_unittest_common.h | 22 +++-
utility/lip_filter/BitVectorExactFilter.hpp | 2 ++
utility/lip_filter/LIPFilter.hpp| 2 ++
utility/lip_filter/SingleIdentityHashFilter.hpp | 2 ++
utility/tests/TemplateUtil_unittest.cpp | 4
7 files changed, 31 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c84fd6/expressions/table_generator/GenerateSeriesHandle.hpp
--
diff --git a/expressions/table_generator/GenerateSeriesHandle.hpp
b/expressions/table_generator/GenerateSeriesHandle.hpp
index 594f12b..bfb4ab5 100644
--- a/expressions/table_generator/GenerateSeriesHandle.hpp
+++ b/expressions/table_generator/GenerateSeriesHandle.hpp
@@ -46,6 +46,8 @@ namespace quickstep {
*/
class GenerateSeriesHandle : public GeneratorFunctionHandle {
public:
+ ~GenerateSeriesHandle() override {};
+
int getNumberOfOutputColumns() const override {
return 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c84fd6/expressions/table_generator/GeneratorFunctionHandle.hpp
--
diff --git a/expressions/table_generator/GeneratorFunctionHandle.hpp
b/expressions/table_generator/GeneratorFunctionHandle.hpp
index b2fc068..394749f 100644
--- a/expressions/table_generator/GeneratorFunctionHandle.hpp
+++ b/expressions/table_generator/GeneratorFunctionHandle.hpp
@@ -59,6 +59,8 @@ typedef std::shared_ptr
GeneratorFunctionHandlePt
**/
class GeneratorFunctionHandle {
public:
+ virtual ~GeneratorFunctionHandle() {}
+
/**
* @brief Get the number of output columns of this generator function.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c84fd6/third_party/src/tmb/tests/message_bus_unittest_common.h
--
diff --git a/third_party/src/tmb/tests/message_bus_unittest_common.h
b/third_party/src/tmb/tests/message_bus_unittest_common.h
index e0994da..69df005 100644
--- a/third_party/src/tmb/tests/message_bus_unittest_common.h
+++ b/third_party/src/tmb/tests/message_bus_unittest_common.h
@@ -61,6 +61,8 @@ class Thread {
Thread() {
}
+ virtual ~Thread() {}
+
void Start() {
internal_thread_ = std::thread(Thread::ExecuteRunMethodOfThread, this);
}
@@ -196,6 +198,8 @@ class ConnectorThread : public Thread {
num_clients_(num_clients) {
}
+ ~ConnectorThread() override {};
+
const std::unordered_set& assigned_ids() const {
return assigned_ids_;
}
@@ -205,7 +209,7 @@ class ConnectorThread : public Thread {
}
protected:
- void Run() {
+ void Run() override {
for (std::size_t client_idx = 0; client_idx < num_clients_; ++client_idx) {
client_id current_client = message_bus_ptr_->Connect();
EXPECT_EQ(assigned_ids_.end(), assigned_ids_.find(current_client));
@@ -656,6 +660,8 @@ class SimpleReceiverThread : public Thread {
receiver_id_(sender_id) {
}
+ ~SimpleReceiverThread() override {};
+
// Spins until the receiver id is actually available.
client_id GetReceiverID() const {
client_id receiver_id = receiver_id_.load();
@@ -671,7 +677,7 @@ class SimpleReceiverThread : public Thread {
}
protected:
- void Run() {
+ void Run() override {
const client_id receiver_id = message_bus_ptr_->Connect();
EXPECT_TRUE(message_bus_ptr_->RegisterClientAsReceiver(receiver_id, 0));
receiver_id_.store(receiver_id);
@@ -780,6 +786,8 @@ class SimpleSenderThread : public Thread {
sender_id_(receiver_id) {
}
+ ~SimpleSenderThread() override {}
+
// Spins until the sender id is actually available.
client_id GetSenderID() const {
client_id sender_id = sender_id_.load();
@@ -791,7 +799,7 @@ class SimpleSenderThread : public Thread {
}
protected:
- void Run() {
+ void Run() override {
const c
incubator-quickstep git commit: Fix a bug of filter side in HashJoinOperator
Repository: incubator-quickstep
Updated Branches:
refs/heads/master dfefe6293 -> 97bef178f
Fix a bug of filter side in HashJoinOperator
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/97bef178
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/97bef178
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/97bef178
Branch: refs/heads/master
Commit: 97bef178fe403e14ebb304e60006fda2e988aac4
Parents: dfefe62
Author: Jianqiao Zhu
Authored: Sun Jun 3 15:38:36 2018 -0500
Committer: Jianqiao Zhu
Committed: Tue Jun 5 00:15:38 2018 -0500
--
autofix_cmakelists.py | 11 ++--
.../tests/execution_generator/Join.test | 59
relational_operators/HashJoinOperator.cpp | 12 ++--
storage/CMakeLists.txt | 13 ++---
4 files changed, 77 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/97bef178/autofix_cmakelists.py
--
diff --git a/autofix_cmakelists.py b/autofix_cmakelists.py
index 17ef588..ec19408 100755
--- a/autofix_cmakelists.py
+++ b/autofix_cmakelists.py
@@ -19,6 +19,7 @@
import os
import re
+import sys
from collections import defaultdict
@@ -1356,7 +1357,7 @@ class CMakeLists:
self.path = path
if "CMakeLists.txt" in os.listdir(path):
- with open(self.path + "/CMakeLists.txt", "rb") as file:
+ with open(self.path + "/CMakeLists.txt", "r") as file:
self.root = CMakeGroup(LineStream(file.readlines()))
self.newfile = False
else:
@@ -1524,8 +1525,8 @@ class CMakeLists:
continue
if name in subdirectories:
-print "Warning: Possibly duplicated add_subdirectory(" + \
- name + ") in", self.path
+sys.stdout.write("Warning: Possibly duplicated add_subdirectory(" +
+ name + ") in" + self.path)
subdirectories[name] = node
@@ -1690,7 +1691,7 @@ class CMakeLists:
if len(output) == 0:
return
-with open(self.path + "/CMakeLists.txt", "wb") as file:
+with open(self.path + "/CMakeLists.txt", "w") as file:
file.write("".join(output))
file.close()
@@ -1754,7 +1755,7 @@ class SourceFile:
self.includes = []
# Parse include items
-with open(self.path + "/" + self.name, "rb") as file:
+with open(self.path + "/" + self.name, "r") as file:
dependencies = []
for line in file:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/97bef178/query_optimizer/tests/execution_generator/Join.test
--
diff --git a/query_optimizer/tests/execution_generator/Join.test
b/query_optimizer/tests/execution_generator/Join.test
index c26b915..5ef8e99 100644
--- a/query_optimizer/tests/execution_generator/Join.test
+++ b/query_optimizer/tests/execution_generator/Join.test
@@ -231,3 +231,62 @@ WHERE b.x > 10
+---++++
| 6| 61| 601| C6|
+---++++
+==
+
+# Semi-join w/o residual predicates
+SELECT SUM(x)
+FROM a
+WHERE w IN (SELECT w FROM b);
+--
+++
+|SUM(x) |
+++
+| 900|
+++
+==
+
+# Semi-join w/ residual predicates
+SELECT SUM(x)
+FROM a
+WHERE EXISTS (
+ SELECT *
+ FROM b
+ WHERE a.w = b.w
+AND a.x < b.x
+);
+--
+++
+|SUM(x) |
+++
+| 500|
+++
+==
+
+# Anti-join w/o residual predicates
+SELECT SUM(x)
+FROM a
+WHERE w NOT IN (SELECT w FROM b);
+--
+++
+|SUM(x) |
+++
+|1000|
+++
+==
+
+# Anti-join w residual predicates
+SELECT SUM(x)
+FROM a
+WHERE NOT EXISTS (
+ SELECT *
+ FROM b
+ WHERE a.w = b.w
+AND a.x < b.x
+);
+--
+++
+|SUM(x) |
+++
+|1400|
+++
+==
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/97bef178/relational_operators/HashJoinOperator.cpp
--
diff --git a/relational_operators/HashJoinOperator.cpp
b/relational_operators/HashJoinOperator.cpp
index f08de9d..a6283a8 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -733,9 +733,9 @@ void HashSemiJoinWorkOrder::executeWithResidualPredicate() {
build
incubator-quickstep git commit: Minor fix in EliminateEmptyNode.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 666102fff -> 3a7b9c796
Minor fix in EliminateEmptyNode.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3a7b9c79
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3a7b9c79
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3a7b9c79
Branch: refs/heads/master
Commit: 3a7b9c796df3f0c2e37b5d69d41aed0162502892
Parents: 666102f
Author: Zuyu Zhang
Authored: Sat May 5 14:28:59 2018 -0500
Committer: Zuyu Zhang
Committed: Sat May 5 14:28:59 2018 -0500
--
query_optimizer/rules/EliminateEmptyNode.cpp | 18 +-
1 file changed, 9 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3a7b9c79/query_optimizer/rules/EliminateEmptyNode.cpp
--
diff --git a/query_optimizer/rules/EliminateEmptyNode.cpp
b/query_optimizer/rules/EliminateEmptyNode.cpp
index c025ebf..fd1c383 100644
--- a/query_optimizer/rules/EliminateEmptyNode.cpp
+++ b/query_optimizer/rules/EliminateEmptyNode.cpp
@@ -328,6 +328,15 @@ P::PhysicalPtr EliminateEmptyNode::apply(const
P::PhysicalPtr &input) {
}
}
+ auto output = Apply(plan);
+ if (output == plan) {
+return input;
+ }
+
+ if (output) {
+return input->copyWithNewChildren({output});
+ }
+
#ifdef QUICKSTEP_DEBUG
{
CHECK(!project_expressions.empty());
@@ -341,15 +350,6 @@ P::PhysicalPtr EliminateEmptyNode::apply(const
P::PhysicalPtr &input) {
}
#endif
- auto output = Apply(plan);
- if (output == plan) {
-return input;
- }
-
- if (output) {
-return input->copyWithNewChildren({output});
- }
-
auto catalog_relation = std::make_unique(
catalog_database_, GetNewRelationName(catalog_database_->size()), -1,
true);
incubator-quickstep git commit: Refactored ScalarCaseExpression.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 77287a788 -> 666102fff
Refactored ScalarCaseExpression.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/666102ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/666102ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/666102ff
Branch: refs/heads/master
Commit: 666102fff32d258a5a2c85e33bf8d8ebb5d3a9cf
Parents: 77287a7
Author: Zuyu Zhang
Authored: Wed May 2 16:06:59 2018 -0500
Committer: Zuyu Zhang
Committed: Fri May 4 14:40:28 2018 -0500
--
expressions/scalar/ScalarCaseExpression.cpp | 265
expressions/scalar/ScalarCaseExpression.hpp | 26 --
.../tests/ScalarCaseExpression_unittest.cpp | 307 +++
3 files changed, 450 insertions(+), 148 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/666102ff/expressions/scalar/ScalarCaseExpression.cpp
--
diff --git a/expressions/scalar/ScalarCaseExpression.cpp
b/expressions/scalar/ScalarCaseExpression.cpp
index 6847425..c2af83b 100644
--- a/expressions/scalar/ScalarCaseExpression.cpp
+++ b/expressions/scalar/ScalarCaseExpression.cpp
@@ -41,6 +41,102 @@
namespace quickstep {
+namespace {
+
+// Merge the values in the NativeColumnVector 'case_result' into '*output' at
+// the positions specified by 'case_matches'. If '*source_sequence' is
+// non-NULL, it indicates which positions actually have tuples in the input,
+// otherwise it is assumed that there are no holes in the input.
+void MultiplexNativeColumnVector(
+const TupleIdSequence *source_sequence,
+const TupleIdSequence &case_matches,
+const NativeColumnVector &case_result,
+NativeColumnVector *output) {
+ if (source_sequence == nullptr) {
+if (case_result.typeIsNullable()) {
+ TupleIdSequence::const_iterator output_pos_it = case_matches.begin();
+ for (std::size_t input_pos = 0;
+ input_pos < case_result.size();
+ ++input_pos, ++output_pos_it) {
+const void *value = case_result.getUntypedValue(input_pos);
+if (value) {
+ output->positionalWriteUntypedValue(*output_pos_it, value);
+} else {
+ output->positionalWriteNullValue(*output_pos_it);
+}
+ }
+} else {
+ TupleIdSequence::const_iterator output_pos_it = case_matches.begin();
+ for (std::size_t input_pos = 0;
+ input_pos < case_result.size();
+ ++input_pos, ++output_pos_it) {
+output->positionalWriteUntypedValue(*output_pos_it,
+
case_result.getUntypedValue(input_pos));
+ }
+}
+ } else {
+if (case_result.typeIsNullable()) {
+ std::size_t input_pos = 0;
+ TupleIdSequence::const_iterator source_sequence_it =
source_sequence->begin();
+ for (std::size_t output_pos = 0;
+ output_pos < output->size();
+ ++output_pos, ++source_sequence_it) {
+if (case_matches.get(*source_sequence_it)) {
+ const void *value = case_result.getUntypedValue(input_pos++);
+ if (value) {
+output->positionalWriteUntypedValue(output_pos, value);
+ } else {
+output->positionalWriteNullValue(output_pos);
+ }
+}
+ }
+} else {
+ std::size_t input_pos = 0;
+ TupleIdSequence::const_iterator source_sequence_it =
source_sequence->begin();
+ for (std::size_t output_pos = 0;
+ output_pos < output->size();
+ ++output_pos, ++source_sequence_it) {
+if (case_matches.get(*source_sequence_it)) {
+ output->positionalWriteUntypedValue(output_pos,
+
case_result.getUntypedValue(input_pos++));
+}
+ }
+}
+ }
+}
+
+// Same as MultiplexNativeColumnVector(), but works on IndirectColumnVectors
+// instead of NativeColumnVectors.
+void MultiplexIndirectColumnVector(
+const TupleIdSequence *source_sequence,
+const TupleIdSequence &case_matches,
+const IndirectColumnVector &case_result,
+IndirectColumnVector *output) {
+ if (source_sequence == nullptr) {
+TupleIdSequence::const_iterator output_pos_it = case_matches.begin();
+for (std::size_t input_pos = 0;
+ input_pos < case_result.size();
+ ++input_pos, ++output_pos_it) {
+ output->positionalWriteTypedValue(*output_pos_it,
+case_result.getTypedValue(input_pos));
+}
+ } else {
+std::size_t input_pos = 0;
+TupleIdSequence::const_iterator source_sequence_it =
source_sequence->begin();
+for (std::size_t output_pos = 0;
+ output_po
incubator-quickstep git commit: Add a python script to auto fix CMakeLists files
Repository: incubator-quickstep
Updated Branches:
refs/heads/master b17856474 -> 73d796dee
Add a python script to auto fix CMakeLists files
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/73d796de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/73d796de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/73d796de
Branch: refs/heads/master
Commit: 73d796dee760a03a91d55cb0fe4d8f073f831237
Parents: b178564
Author: Jianqiao Zhu
Authored: Fri Apr 27 17:28:51 2018 -0500
Committer: Jianqiao Zhu
Committed: Fri Apr 27 20:39:58 2018 -0500
--
autofix_cmakelists.py | 2054
1 file changed, 2054 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/73d796de/autofix_cmakelists.py
--
diff --git a/autofix_cmakelists.py b/autofix_cmakelists.py
new file mode 100755
index 000..17ef588
--- /dev/null
+++ b/autofix_cmakelists.py
@@ -0,0 +1,2054 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+
+from collections import defaultdict
+
+
+# Configurations
###
+
+
+# Project name.
+PROJECT_NAME = "quickstep"
+
+# Don't scan these directories.
+EXCLUDED_DIRECTORIES = frozenset(
+[ "./build",
+ "./cmake",
+ "./parser/preprocessed",
+ "./release",
+ "./storage/tests",
+ "./third_party" ])
+
+# Don't scan these files.
+EXCLUDED_FILES = frozenset(
+[ "./empty_src.cpp" ])
+
+# Third-party libraries.
+THIRD_PARTY_LIBRARIES = \
+{ "farmhash" : [ "farmhash" ],
+ "gflags" : [ "${GFLAGS_LIB_NAME}" ],
+ "glog" : [ "glog" ],
+ "gtest/gtest.h" : [ "gtest", "gtest_main" ],
+ "gtest/gtest_prod.h" : [ "gtest" ],
+ "re2" : [ "re2" ],
+ "tmb" : [ "tmb" ] }
+
+# Explicitly ignored targets.
+IGNORED_TARGETS = frozenset(
+[ "quickstep_cli_LineReader",
+ "quickstep_cli_shell",
+ "quickstep_parser_SqlLexer",
+ "quickstep_parser_SqlParser",
+ "quickstep_parser_SqlParserWrapper",
+ "quickstep_threading_ConditionVariable",
+ "quickstep_threading_Mutex",
+ "quickstep_threading_SharedMutex",
+ "quickstep_threading_Thread" ])
+
+# Do not move these targets around.
+ANCHORED_TARGETS = frozenset(
+[ "quickstep_cli_shell",
+ "quickstep_utility_TextBasedTestDriver"] )
+
+# Do not move these link dependencies around.
+ANCHORED_LINKS = frozenset(
+[ "quickstep_cli_shell" ] )
+
+# Source code file extensions under consideration.
+SOURCE_CODE_FILE_EXTENSIONS = [ ".hpp", ".cpp" ]
+
+# Maximum number of characters in one line.
+MAX_LINE_WIDTH = 120
+
+# Dummy source file name.
+DUMMY_CPP_FILE_NAME = "empty_src.cpp"
+
+# Whether to auto-fix target names according to default naming rule.
+# E.g. ./types/containers/ColumnVector.*
+# -> quickstep_types_containers_ColumnVector
+FORCE_DEFAULT_NAMING_RULE_FOR_LIBRARY = True
+FORCE_DEFAULT_NAMING_RULE_FOR_EXECUTABLE = False
+
+
+
+### Utility Functions
##
+
+
+def TopDownVisitor(functor):
+ def Wrapper(*args):
+# First invoke functor.
+functor(*args)
+
+# Recursively process all child nodes.
+for node in args[0].getChildren():
+ getattr(node, functor.__name__)(*list(args)[1:])
+
+ return Wrapper
+
+def BottomUpVisitor(functor):
+ def Wrapper(*args):
+# Recursively process all child nodes.
+for node in args[0].getChildren():
+ getattr(node, functor.__name__)(*list(args)[1:])
+
+# Then invoke functor.
+functor(*args)
+
+
[3/4] incubator-quickstep git commit: Fix and rearrange all CMakeLists.txt so that they are ready to be processed and regenrated by an automation tool.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b1785647/query_optimizer/expressions/CMakeLists.txt -- diff --git a/query_optimizer/expressions/CMakeLists.txt b/query_optimizer/expressions/CMakeLists.txt index 3e7f8e4..7032f05 100644 --- a/query_optimizer/expressions/CMakeLists.txt +++ b/query_optimizer/expressions/CMakeLists.txt @@ -28,9 +28,9 @@ add_library(quickstep_queryoptimizer_expressions_ComparisonExpression ComparisonExpression.cpp ComparisonExpression.hpp) add_library(quickstep_queryoptimizer_expressions_Exists Exists.cpp Exists.hpp) +add_library(quickstep_queryoptimizer_expressions_ExprId ../../empty_src.cpp ExprId.hpp) add_library(quickstep_queryoptimizer_expressions_Expression ../../empty_src.cpp Expression.hpp) add_library(quickstep_queryoptimizer_expressions_ExpressionType ../../empty_src.cpp ExpressionType.hpp) -add_library(quickstep_queryoptimizer_expressions_ExprId ../../empty_src.cpp ExprId.hpp) add_library(quickstep_queryoptimizer_expressions_ExpressionUtil ExpressionUtil.cpp ExpressionUtil.hpp) add_library(quickstep_queryoptimizer_expressions_InTableQuery InTableQuery.cpp InTableQuery.hpp) add_library(quickstep_queryoptimizer_expressions_InValueList InValueList.cpp InValueList.hpp) @@ -142,6 +142,7 @@ target_link_libraries(quickstep_queryoptimizer_expressions_ComparisonExpression quickstep_types_operations_comparisons_ComparisonID quickstep_utility_Macros) target_link_libraries(quickstep_queryoptimizer_expressions_Exists + glog quickstep_queryoptimizer_OptimizerTree quickstep_queryoptimizer_expressions_AttributeReference quickstep_queryoptimizer_expressions_ExprId @@ -149,9 +150,9 @@ target_link_libraries(quickstep_queryoptimizer_expressions_Exists quickstep_queryoptimizer_expressions_ExpressionType quickstep_queryoptimizer_expressions_Predicate quickstep_queryoptimizer_expressions_SubqueryExpression - quickstep_utility_Macros - glog) + quickstep_utility_Macros) target_link_libraries(quickstep_queryoptimizer_expressions_Expression + glog quickstep_queryoptimizer_OptimizerTree quickstep_queryoptimizer_expressions_ExpressionType quickstep_types_Type @@ -163,6 +164,7 @@ target_link_libraries(quickstep_queryoptimizer_expressions_ExpressionUtil quickstep_queryoptimizer_expressions_NamedExpression quickstep_queryoptimizer_expressions_PatternMatcher) target_link_libraries(quickstep_queryoptimizer_expressions_InTableQuery + glog quickstep_queryoptimizer_OptimizerTree quickstep_queryoptimizer_expressions_AttributeReference quickstep_queryoptimizer_expressions_ExprId @@ -173,9 +175,10 @@ target_link_libraries(quickstep_queryoptimizer_expressions_InTableQuery quickstep_queryoptimizer_expressions_SubqueryExpression quickstep_utility_Macros) target_link_libraries(quickstep_queryoptimizer_expressions_InValueList - quickstep_queryoptimizer_OptimizerTree + glog quickstep_expressions_predicate_DisjunctionPredicate quickstep_expressions_predicate_Predicate + quickstep_queryoptimizer_OptimizerTree quickstep_queryoptimizer_expressions_AttributeReference quickstep_queryoptimizer_expressions_ComparisonExpression quickstep_queryoptimizer_expressions_ExprId @@ -251,7 +254,6 @@ target_link_libraries(quickstep_queryoptimizer_expressions_PredicateLiteral quickstep_queryoptimizer_expressions_Predicate quickstep_utility_Macros) target_link_libraries(quickstep_queryoptimizer_expressions_Scalar - glog quickstep_queryoptimizer_expressions_ExprId quickstep_queryoptimizer_expressions_Expression quickstep_utility_HashError @@ -271,6 +273,7 @@ target_link_libraries(quickstep_queryoptimizer_expressions_ScalarLiteral quickstep_utility_HashPair quickstep_utility_Macros) target_link_libraries(quickstep_queryoptimizer_expressions_SearchedCase + glog quickstep_expressions_predicate_Predicate quickstep_expressions_scalar_Scalar quickstep_expressions_scalar_ScalarCaseExpression @@ -286,6 +289,7 @@ target_link_libraries(qui
[4/4] incubator-quickstep git commit: Fix and rearrange all CMakeLists.txt so that they are ready to be processed and regenrated by an automation tool.
Fix and rearrange all CMakeLists.txt so that they are ready to be processed and
regenrated by an automation tool.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b1785647
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b1785647
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b1785647
Branch: refs/heads/master
Commit: b17856474bf5f60dbee8d1874c2d11c55fa26f8d
Parents: c2e2691
Author: Jianqiao Zhu
Authored: Wed Apr 18 00:54:31 2018 -0500
Committer: Jianqiao Zhu
Committed: Fri Apr 27 17:18:25 2018 -0500
--
CMakeLists.txt | 35 +-
catalog/CMakeLists.txt | 74 +--
cli/CMakeLists.txt | 87 +--
cli/distributed/CMakeLists.txt | 4 +-
cli/tests/CMakeLists.txt| 10 +-
compression/CMakeLists.txt | 4 +-
expressions/CMakeLists.txt | 7 +-
expressions/aggregation/CMakeLists.txt | 12 +-
expressions/predicate/CMakeLists.txt| 1 +
expressions/scalar/CMakeLists.txt | 4 +
expressions/table_generator/CMakeLists.txt | 35 +-
expressions/window_aggregation/CMakeLists.txt | 10 +-
parser/CMakeLists.txt | 37 +-
parser/tests/CMakeLists.txt | 9 +-
query_execution/CMakeLists.txt | 332 ++-
query_optimizer/CMakeLists.txt | 17 +-
query_optimizer/cost_model/CMakeLists.txt | 1 +
query_optimizer/expressions/CMakeLists.txt | 17 +-
query_optimizer/logical/CMakeLists.txt | 12 +-
query_optimizer/physical/CMakeLists.txt | 5 +-
query_optimizer/resolver/CMakeLists.txt | 2 +-
query_optimizer/rules/CMakeLists.txt| 60 +-
query_optimizer/rules/Partition.hpp | 2 -
query_optimizer/rules/tests/CMakeLists.txt | 6 +-
query_optimizer/strategy/tests/CMakeLists.txt | 4 +-
query_optimizer/tests/CMakeLists.txt| 17 +-
.../tests/logical_generator/CMakeLists.txt | 6 +-
.../tests/physical_generator/CMakeLists.txt | 6 +-
query_optimizer/tests/resolver/CMakeLists.txt | 6 +-
relational_operators/CMakeLists.txt | 66 ++-
storage/CMakeLists.txt | 565 ++-
storage/TupleStorageSubBlock.cpp| 11 +-
threading/CMakeLists.txt| 44 +-
transaction/CMakeLists.txt | 19 +-
types/CMakeLists.txt| 21 +-
types/containers/CMakeLists.txt | 8 +-
types/operations/CMakeLists.txt | 4 +-
.../ArithmeticBinaryOperators.hpp | 4 -
.../operations/binary_operations/CMakeLists.txt | 1 +
.../comparisons/AsciiStringComparators-inl.hpp | 4 -
types/operations/comparisons/CMakeLists.txt | 8 +-
types/operations/comparisons/Comparison-inl.hpp | 4 -
types/operations/comparisons/Comparison.cpp | 2 -
.../comparisons/LiteralComparators-inl.hpp | 4 -
.../PatternMatchingComparators-inl.hpp | 4 -
.../ArithmeticUnaryOperators.hpp| 4 -
.../operations/unary_operations/CMakeLists.txt | 21 +-
.../unary_operations/DateExtractOperation.cpp | 6 +-
.../unary_operations/DateExtractOperation.hpp | 6 +-
.../unary_operations/UnaryOperation.hpp | 6 +-
types/port/CMakeLists.txt | 8 +-
types/port/gmtime_r.hpp | 3 -
types/port/localtime_r.hpp | 3 -
utility/CMakeLists.txt | 124 ++--
utility/lip_filter/CMakeLists.txt | 9 +-
yarn/CMakeLists.txt | 1 -
56 files changed, 912 insertions(+), 870 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b1785647/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1343a18..3f1c6cd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -625,8 +625,8 @@ include_directories(SYSTEM ${PROTOBUF_INCLUDE_DIR})
# Add gtest unit-testing framework.
set(gtest_force_shared_crt ON CACHE BOOL "Link gtest against shared DLLs on
Windows")
-add_subdirectory ("${THIRD_PARTY_SOURCE_DIR}/googletest/googletest"
-
"${CMAKE_CURRENT_BINARY_DIR}/third_party/googletest/googletest")
+add_subdirectory("${THIRD_PARTY_SOURCE_DIR}/googletest/googletest"
+
"${CMAKE_CURRENT_BINARY_DIR}/third_party/googletest/googletest")
include_directories(SYSTEM
"${THIRD_PARTY_SOURCE_DIR}/googletest/googletest/include")
enable_tes
[1/4] incubator-quickstep git commit: Fix and rearrange all CMakeLists.txt so that they are ready to be processed and regenrated by an automation tool.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master c2e269119 -> b17856474
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b1785647/utility/CMakeLists.txt
--
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index c5c9dd8..a429947 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -162,6 +162,9 @@ add_subdirectory(lip_filter)
# Declare micro-libs:
add_library(quickstep_utility_Alignment ../empty_src.cpp Alignment.hpp)
+add_library(quickstep_utility_BarrieredReadWriteConcurrentBitVector
+../empty_src.cpp
+BarrieredReadWriteConcurrentBitVector.hpp)
add_library(quickstep_utility_BitManipulation ../empty_src.cpp
BitManipulation.hpp)
add_library(quickstep_utility_BitVector ../empty_src.cpp BitVector.hpp)
add_library(quickstep_utility_BloomFilter ../empty_src.cpp BloomFilter.hpp)
@@ -174,9 +177,6 @@ add_library(quickstep_utility_Cast ../empty_src.cpp
Cast.hpp)
add_library(quickstep_utility_CheckSnprintf ../empty_src.cpp CheckSnprintf.hpp)
add_library(quickstep_utility_ColumnVectorCache ../empty_src.cpp
ColumnVectorCache.hpp)
add_library(quickstep_utility_CompositeHash ../empty_src.cpp CompositeHash.hpp)
-add_library(quickstep_utility_BarrieredReadWriteConcurrentBitVector
-../empty_src.cpp
-BarrieredReadWriteConcurrentBitVector.hpp)
add_library(quickstep_utility_DAG ../empty_src.cpp DAG.hpp)
add_library(quickstep_utility_DisjointTreeForest ../empty_src.cpp
DisjointTreeForest.hpp)
add_library(quickstep_utility_EqualsAnyConstant ../empty_src.cpp
EqualsAnyConstant.hpp)
@@ -205,6 +205,10 @@ add_library(quickstep_utility_SortConfiguration_proto
add_library(quickstep_utility_SqlError SqlError.cpp SqlError.hpp)
add_library(quickstep_utility_StringUtil StringUtil.cpp StringUtil.hpp)
add_library(quickstep_utility_TemplateUtil ../empty_src.cpp TemplateUtil.hpp)
+add_library(quickstep_utility_ThreadSafeQueue ../empty_src.cpp
ThreadSafeQueue.hpp)
+add_library(quickstep_utility_TreeStringSerializable ../empty_src.cpp
TreeStringSerializable.hpp)
+add_library(quickstep_utility_VectorUtil ../empty_src.cpp VectorUtil.hpp)
+
# Note that TextBasedTest.{hpp, cpp} are not in this static library.
# Any tests that use them need to include them in the
# executable.
@@ -212,17 +216,15 @@ add_library(quickstep_utility_TextBasedTestDriver
textbased_test/TextBasedTestDriver.cpp
textbased_test/TextBasedTestDriver.hpp
textbased_test/TextBasedTestRunner.hpp)
-add_library(quickstep_utility_ThreadSafeQueue ../empty_src.cpp
ThreadSafeQueue.hpp)
-add_library(quickstep_utility_TreeStringSerializable ../empty_src.cpp
TreeStringSerializable.hpp)
-add_library(quickstep_utility_VectorUtil ../empty_src.cpp VectorUtil.hpp)
# Link dependencies:
target_link_libraries(quickstep_utility_Alignment
+ glog
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_BarrieredReadWriteConcurrentBitVector
+ glog
+ quickstep_utility_BitManipulation
quickstep_utility_Macros)
-if (QUICKSTEP_HAVE_TCMALLOC)
- target_link_libraries(quickstep_utility_Alignment
-libtcmalloc_minimal)
-endif()
target_link_libraries(quickstep_utility_BitVector
glog
quickstep_utility_BitManipulation
@@ -237,28 +239,29 @@ target_link_libraries(quickstep_utility_BloomFilter
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_BloomFilter_proto
${PROTOBUF_LIBRARY})
+target_link_libraries(quickstep_utility_BulkIoConfiguration
+ glog
+ quickstep_utility_Macros)
target_link_libraries(quickstep_utility_CalculateInstalledMemory
glog)
target_link_libraries(quickstep_utility_CheckSnprintf
glog)
target_link_libraries(quickstep_utility_ColumnVectorCache
+ glog
quickstep_types_containers_ColumnVector
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_CompositeHash
+ glog
quickstep_types_TypedValue
- quickstep_utility_HashPair
- glog)
-target_link_libraries(quickstep_utility_BarrieredReadWriteConcurrentBitVector
- quickstep_utility_BitManipulation
- quickstep_utility_Macros)
-target_link_libraries(quickstep_utility_BulkIoConfiguration
- quickstep_utility_Macros)
+ quickstep_utility_HashPair)
target_link_libraries(quickstep_utility_DAG
glog
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_DisjointTreeForest
[2/4] incubator-quickstep git commit: Fix and rearrange all CMakeLists.txt so that they are ready to be processed and regenrated by an automation tool.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b1785647/storage/CMakeLists.txt
--
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index e67269f..832e667 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -152,19 +152,7 @@ add_library(quickstep_storage_BasicColumnStoreValueAccessor
../empty_src.cpp
BasicColumnStoreValueAccessor.hpp)
add_library(quickstep_storage_BloomFilterIndexSubBlock
BloomFilterIndexSubBlock.cpp BloomFilterIndexSubBlock.hpp)
-# CMAKE_VALIDATE_IGNORE_BEGIN
-if(QUICKSTEP_HAVE_BITWEAVING)
- add_library(quickstep_storage_BitWeavingIndexSubBlock
- bitweaving/BitWeavingIndexSubBlock.cpp
- bitweaving/BitWeavingIndexSubBlock.hpp)
- add_library(quickstep_storage_BitWeavingHIndexSubBlock
- bitweaving/BitWeavingHIndexSubBlock.cpp
- bitweaving/BitWeavingHIndexSubBlock.hpp)
- add_library(quickstep_storage_BitWeavingVIndexSubBlock
- bitweaving/BitWeavingVIndexSubBlock.cpp
- bitweaving/BitWeavingVIndexSubBlock.hpp)
-endif()
-# CMAKE_VALIDATE_IGNORE_END
+add_library(quickstep_storage_CSBTreeIndexSubBlock CSBTreeIndexSubBlock.cpp
CSBTreeIndexSubBlock.hpp)
add_library(quickstep_storage_CollisionFreeVectorTable
CollisionFreeVectorTable.cpp
CollisionFreeVectorTable.hpp)
@@ -187,33 +175,16 @@
add_library(quickstep_storage_CompressedTupleStorageSubBlock
CompressedTupleStorageSubBlock.cpp
CompressedTupleStorageSubBlock.hpp)
add_library(quickstep_storage_CountedReference ../empty_src.cpp
CountedReference.hpp)
-add_library(quickstep_storage_CSBTreeIndexSubBlock CSBTreeIndexSubBlock.cpp
CSBTreeIndexSubBlock.hpp)
-
-if (ENABLE_DISTRIBUTED)
- add_library(quickstep_storage_DataExchange_proto
- ${storage_DataExchange_proto_srcs}
- ${storage_DataExchange_proto_hdrs})
- add_library(quickstep_storage_DataExchangerAsync DataExchangerAsync.cpp
DataExchangerAsync.hpp)
-endif()
-
add_library(quickstep_storage_EvictionPolicy EvictionPolicy.cpp
EvictionPolicy.hpp)
add_library(quickstep_storage_FileManager ../empty_src.cpp FileManager.hpp)
-if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
- add_library(quickstep_storage_FileManagerHdfs FileManagerHdfs.cpp
FileManagerHdfs.hpp)
-endif()
add_library(quickstep_storage_FileManagerLocal ../empty_src.cpp
FileManagerLocal.hpp)
-if (QUICKSTEP_HAVE_FILE_MANAGER_POSIX)
- add_library(quickstep_storage_FileManagerPosix FileManagerPosix.cpp
FileManagerPosix.hpp)
-elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
- add_library(quickstep_storage_FileManagerWindows FileManagerWindows.cpp
FileManagerWindows.hpp)
-endif()
add_library(quickstep_storage_Flags Flags.cpp Flags.hpp)
add_library(quickstep_storage_HashTable ../empty_src.cpp HashTable.hpp)
-add_library(quickstep_storage_HashTable_proto ${storage_HashTable_proto_srcs})
add_library(quickstep_storage_HashTableBase ../empty_src.cpp HashTableBase.hpp)
add_library(quickstep_storage_HashTableFactory HashTableFactory.cpp
HashTableFactory.hpp)
add_library(quickstep_storage_HashTableKeyManager ../empty_src.cpp
HashTableKeyManager.hpp)
add_library(quickstep_storage_HashTablePool ../empty_src.cpp HashTablePool.hpp)
+add_library(quickstep_storage_HashTable_proto ${storage_HashTable_proto_srcs})
add_library(quickstep_storage_IndexSubBlock ../empty_src.cpp IndexSubBlock.hpp)
add_library(quickstep_storage_IndexSubBlockDescriptionFactory ../empty_src.cpp
IndexSubBlockDescriptionFactory.hpp)
add_library(quickstep_storage_InsertDestination InsertDestination.cpp
InsertDestination.hpp)
@@ -260,10 +231,40 @@ add_library(quickstep_storage_ValueAccessor
../empty_src.cpp ValueAccessor.hpp)
add_library(quickstep_storage_ValueAccessorMultiplexer ../empty_src.cpp
ValueAccessorMultiplexer.hpp)
add_library(quickstep_storage_ValueAccessorUtil ../empty_src.cpp
ValueAccessorUtil.hpp)
add_library(quickstep_storage_WindowAggregationOperationState
-WindowAggregationOperationState.hpp
-WindowAggregationOperationState.cpp)
-add_library(quickstep_storage_WindowAggregationOperationState_proto
${storage_WindowAggregationOperationState_proto_srcs})
+WindowAggregationOperationState.cpp
+WindowAggregationOperationState.hpp)
+add_library(quickstep_storage_WindowAggregationOperationState_proto
+${storage_WindowAggregationOperationState_proto_srcs})
+# CMAKE_VALIDATE_IGNORE_BEGIN
+if(QUICKSTEP_HAVE_BITWEAVING)
+ add_library(quickstep_storage_BitWeavingIndexSubBlock
+ bitweaving/BitWeavingIndexSubBlock.cpp
+ bitweaving/BitWeavingIndexSubBlock.hpp)
+ add_library(quickstep_storage_BitWeavingHIndexSubBlock
+ bitweaving/BitWeavingHIndexSubBlock.cpp
+ bitweaving/BitWeavingHIndexSubBlock.hpp)
+ add_library(quickstep_storage_BitWeavingVIndexSubBlock
+
incubator-quickstep git commit: Code style fix.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master cc9ab901f -> 3c7a89c26
Code style fix.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3c7a89c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3c7a89c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3c7a89c2
Branch: refs/heads/master
Commit: 3c7a89c265506a7a90c3b75d4165f212cb0cc49a
Parents: cc9ab90
Author: Zuyu Zhang
Authored: Thu Apr 26 21:39:18 2018 -0500
Committer: Zuyu Zhang
Committed: Thu Apr 26 23:22:00 2018 -0500
--
query_optimizer/ExecutionGenerator.cpp | 11 +++--
.../expressions/AggregateFunction.hpp | 2 +-
query_optimizer/expressions/Alias.cpp | 2 +-
query_optimizer/expressions/Alias.hpp | 4 +-
.../expressions/AttributeReference.hpp | 10 ++---
.../expressions/BinaryExpression.hpp| 4 +-
query_optimizer/expressions/Cast.hpp| 2 +-
.../expressions/ComparisonExpression.hpp| 4 +-
query_optimizer/expressions/Exists.hpp | 2 +-
query_optimizer/expressions/ExpressionUtil.cpp | 8 ++--
query_optimizer/expressions/ExpressionUtil.hpp | 2 +
query_optimizer/expressions/InTableQuery.hpp| 4 +-
query_optimizer/expressions/InValueList.hpp | 4 +-
query_optimizer/expressions/LogicalAnd.cpp | 1 +
query_optimizer/expressions/LogicalAnd.hpp | 1 +
query_optimizer/expressions/LogicalNot.hpp | 2 +-
query_optimizer/expressions/LogicalOr.cpp | 1 +
query_optimizer/expressions/LogicalOr.hpp | 1 +
query_optimizer/expressions/NamedExpression.hpp | 8 ++--
.../expressions/PredicateLiteral.hpp| 2 +-
query_optimizer/expressions/ScalarLiteral.hpp | 1 +
query_optimizer/expressions/SearchedCase.hpp| 6 +--
query_optimizer/expressions/SimpleCase.hpp | 8 ++--
.../expressions/SubqueryExpression.hpp | 2 +-
query_optimizer/expressions/UnaryExpression.hpp | 2 +-
.../expressions/WindowAggregateFunction.hpp | 2 +-
query_optimizer/logical/Aggregate.hpp | 10 ++---
query_optimizer/logical/BinaryJoin.hpp | 4 +-
query_optimizer/logical/DeleteTuples.hpp| 4 +-
query_optimizer/logical/HashJoin.hpp| 6 +--
query_optimizer/logical/InsertSelection.hpp | 4 +-
query_optimizer/logical/InsertTuple.hpp | 4 +-
.../logical/MultiwayCartesianJoin.hpp | 3 +-
query_optimizer/logical/NestedLoopsJoin.hpp | 2 +-
query_optimizer/logical/Project.hpp | 4 +-
query_optimizer/logical/Sample.hpp | 2 +-
.../logical/SharedSubplanReference.hpp | 10 ++---
query_optimizer/logical/Sort.hpp| 10 ++---
query_optimizer/logical/TableGenerator.hpp | 46 ++--
query_optimizer/logical/TableReference.cpp | 1 +
query_optimizer/logical/TableReference.hpp | 2 +-
query_optimizer/logical/TopLevelPlan.hpp| 6 +--
query_optimizer/logical/UpdateTable.hpp | 8 ++--
query_optimizer/physical/BinaryJoin.hpp | 4 +-
.../CrossReferenceCoalesceAggregate.hpp | 18
query_optimizer/physical/DeleteTuples.hpp | 4 +-
query_optimizer/physical/HashJoin.hpp | 10 ++---
query_optimizer/physical/InsertSelection.hpp| 4 +-
query_optimizer/physical/InsertTuple.hpp| 4 +-
query_optimizer/physical/Join.hpp | 4 +-
query_optimizer/physical/NestedLoopsJoin.hpp| 2 +-
.../physical/SharedSubplanReference.hpp | 10 ++---
query_optimizer/physical/Sort.hpp | 12 ++---
query_optimizer/physical/TableGenerator.hpp | 6 +--
query_optimizer/physical/TopLevelPlan.hpp | 8 ++--
query_optimizer/physical/UpdateTable.hpp| 8 ++--
56 files changed, 164 insertions(+), 152 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c7a89c2/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp
b/query_optimizer/ExecutionGenerator.cpp
index 8dd93d0..f8f7c9d 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -687,7 +687,7 @@ bool ExecutionGenerator::convertSimpleProjection(
void ExecutionGenerator::convertSelection(
const P::SelectionPtr &physical_selection) {
- const P::PhysicalPtr input = physical_selection->input();
+ const P::PhysicalPtr &input = physical_selection->input();
const CatalogRelationInfo *input_relation_info =
findRelationInfoOutputByPhysical(input);
DCHECK(input_relation_info != nullptr);
@@ -829,7 +829,7 @@ void
ExecutionGenerator::convertSharedSubplanReferenc
incubator-quickstep git commit: Fixed the bug that Executor / Cli does not create directory for StorageManager.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master d27733908 -> 7cc928429
Fixed the bug that Executor / Cli does not create directory for StorageManager.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7cc92842
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7cc92842
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7cc92842
Branch: refs/heads/master
Commit: 7cc92842966e0dcf6402f56687df2af0ca2ebcc5
Parents: d277339
Author: Zuyu Zhang
Authored: Wed Feb 28 21:03:46 2018 -0600
Committer: Zuyu Zhang
Committed: Wed Mar 14 14:42:11 2018 -0500
--
CMakeLists.txt | 1 +
cli/DefaultsConfigurator.cpp | 14 +-
cli/DefaultsConfigurator.hpp | 7 +++
cli/distributed/CMakeLists.txt | 1 +
cli/distributed/Cli.cpp| 3 +++
cli/distributed/Executor.cpp | 3 +++
6 files changed, 24 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7cc92842/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c777a6a..3965851 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -792,6 +792,7 @@ if (ENABLE_DISTRIBUTED)
glog
quickstep_catalog_CatalogRelation
quickstep_cli_Constants
+quickstep_cli_DefaultsConfigurator
quickstep_cli_Flags
quickstep_cli_LineReader
quickstep_cli_PrintToScreen
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7cc92842/cli/DefaultsConfigurator.cpp
--
diff --git a/cli/DefaultsConfigurator.cpp b/cli/DefaultsConfigurator.cpp
index 21b0af6..0e1f64c 100644
--- a/cli/DefaultsConfigurator.cpp
+++ b/cli/DefaultsConfigurator.cpp
@@ -41,10 +41,7 @@ using std::string;
namespace quickstep {
-void DefaultsConfigurator::InitializeDefaultDatabase(const string
&storage_path, const string &catalog_path) {
- // TODO(jmp): Refactor the code in this file!
- LOG(INFO) << "Initializing the database, creating a new catalog file and
storage directory";
-
+void DefaultsConfigurator::CreateDirectory(const std::string &storage_path) {
// Create the directory
// TODO(jmp): At some point, likely in C++-17, we will just have the
//filesystem path, and we can clean this up
@@ -54,11 +51,18 @@ void DefaultsConfigurator::InitializeDefaultDatabase(const
string &storage_path,
<< "\nCheck if the directory already exists. If so, delete it or move it
before initializing";
#else
{
-const string path_name = "mkdir " + storage_path;
+const string path_name = "mkdir -p " + storage_path;
CHECK(!std::system(path_name.c_str()))
<< "Failed when attempting to create the directory: " << storage_path;
}
#endif // QUICKSTEP_OS_WINDOWS
+}
+
+void DefaultsConfigurator::InitializeDefaultDatabase(const string
&storage_path, const string &catalog_path) {
+ // TODO(jmp): Refactor the code in this file!
+ LOG(INFO) << "Initializing the database, creating a new catalog file and
storage directory";
+
+ CreateDirectory(storage_path);
// Create the default catalog file.
std::ofstream catalog_file(catalog_path.c_str());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7cc92842/cli/DefaultsConfigurator.hpp
--
diff --git a/cli/DefaultsConfigurator.hpp b/cli/DefaultsConfigurator.hpp
index 4b534d6..28d84b8 100644
--- a/cli/DefaultsConfigurator.hpp
+++ b/cli/DefaultsConfigurator.hpp
@@ -116,6 +116,13 @@ class DefaultsConfigurator {
static void InitializeDefaultDatabase(const std::string &storage_path,
const std::string &catalog_path);
+ /**
+ * @brief Create the directory.
+ *
+ * @param storage_path The filesystem directory for StorageManager.
+ **/
+ static void CreateDirectory(const std::string &storage_path);
+
private:
/**
* @brief Private constructor to disable instantiation of the class.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7cc92842/cli/distributed/CMakeLists.txt
--
diff --git a/cli/distributed/CMakeLists.txt b/cli/distributed/CMakeLists.txt
index 6a31895..93d3b5e 100644
--- a/cli/distributed/CMakeLists.txt
+++ b/cli/distributed/CMakeLists.txt
@@ -55,6 +55,7 @@ target_link_libraries(quickstep_cli_distributed_Conductor
target_link_libraries(quickstep_cli_distributed_Executor
glog
incubator-quickstep git commit: Fixed the gRPC Problem for Data Exchange
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 934f3ab87 -> d27733908
Fixed the gRPC Problem for Data Exchange
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d2773390
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d2773390
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d2773390
Branch: refs/heads/master
Commit: d27733908cf90751bcf572309aaf8123c2f98698
Parents: 934f3ab
Author: Yuanchen Li
Authored: Tue Mar 13 14:58:00 2018 -0500
Committer: Yuanchen Li
Committed: Wed Mar 14 14:10:53 2018 -0500
--
storage/StorageConstants.hpp | 2 ++
storage/StorageManager.cpp | 7 ++-
2 files changed, 8 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d2773390/storage/StorageConstants.hpp
--
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index c4debe0..6046ab6 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -53,6 +53,8 @@ const std::size_t kSlotSizeBytes = 0x20;
const std::uint64_t kAGigaByte = (1 << 30);
// A MegaByte.
const std::uint64_t kAMegaByte = (1 << 20);
+// 4 MegaByte + 9 Bytes
+const std::uint32_t kGrpcChanelSize = (4 << 20) + 9;
// Constants for the minimum and maximum user-specifiable BLOCKSIZEMB in
// the SQL clause BLOCKPROPERTIES.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d2773390/storage/StorageManager.cpp
--
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 12a96d1..8f17455 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -687,8 +687,13 @@ StorageManager::BlockHandle
StorageManager::loadBlockOrBlob(
#ifdef QUICKSTEP_DISTRIBUTED
const string domain_network_address =
getPeerDomainNetworkAddress(BlockIdUtil::Domain(block));
DLOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from "
<< domain_network_address;
+
+// Customize the grpc channel
+grpc::ChannelArguments channelArgs;
+channelArgs.SetMaxReceiveMessageSize(kGrpcChanelSize);
+
DataExchangerClientAsync client(
-grpc::CreateChannel(domain_network_address,
grpc::InsecureChannelCredentials()), this);
+grpc::CreateCustomChannel(domain_network_address,
grpc::InsecureChannelCredentials(), channelArgs), this);
while (!client.Pull(block, numa_node, &loaded_handle)) {
LOG(INFO) << "Retry pulling Block " << BlockIdUtil::ToString(block) << "
from " << domain_network_address;
}
incubator-quickstep git commit: Check File Handle in TextScanWorkOrder to avoid segfault.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master c2ed5c69b -> 934f3ab87
Check File Handle in TextScanWorkOrder to avoid segfault.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/934f3ab8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/934f3ab8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/934f3ab8
Branch: refs/heads/master
Commit: 934f3ab87b642fa6230a951f051cd44828b96f57
Parents: c2ed5c6
Author: Zuyu Zhang
Authored: Tue Mar 6 03:00:10 2018 -0600
Committer: Zuyu Zhang
Committed: Tue Mar 6 03:00:10 2018 -0600
--
relational_operators/TextScanOperator.cpp | 5 +
1 file changed, 5 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/934f3ab8/relational_operators/TextScanOperator.cpp
--
diff --git a/relational_operators/TextScanOperator.cpp
b/relational_operators/TextScanOperator.cpp
index 66137d8..9e3bcb1 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -273,6 +273,11 @@ void TextScanWorkOrder::execute() {
// Read text segment into buffer.
file = std::fopen(filename_.c_str(), "rb");
+if (file == nullptr) {
+ LOG(ERROR) << "Failed to open file " << filename_
+ << " with error: " << strerror(errno);
+ return;
+}
std::fseek(file, text_offset_, SEEK_SET);
bytes_read = std::fread(buffer, 1, text_segment_size_, file);
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/fix-iwyu [deleted] c2ed5c69b
incubator-quickstep git commit: Fix iwyu include path
Repository: incubator-quickstep Updated Branches: refs/heads/master 023c43a40 -> c2ed5c69b Fix iwyu include path Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c2ed5c69 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c2ed5c69 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c2ed5c69 Branch: refs/heads/master Commit: c2ed5c69b6b8dad07d7410beb0c8292ea1a746e0 Parents: 023c43a Author: Jianqiao Zhu Authored: Fri Sep 1 15:07:41 2017 -0500 Committer: Jianqiao Zhu Committed: Mon Feb 26 13:15:06 2018 -0600 -- third_party/src/iwyu/iwyu_helper.py | 12 ++-- 1 file changed, 6 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c2ed5c69/third_party/src/iwyu/iwyu_helper.py -- diff --git a/third_party/src/iwyu/iwyu_helper.py b/third_party/src/iwyu/iwyu_helper.py index dff4d55..42bf84c 100755 --- a/third_party/src/iwyu/iwyu_helper.py +++ b/third_party/src/iwyu/iwyu_helper.py @@ -22,12 +22,12 @@ QUICKSTEP_INCLUDES = [ '.', './build/third_party/gflags/include', './build/third_party/protobuf/include', './build/third_party/tmb/include', - './third_party/benchmark/include', - './third_party/glog/src', - './third_party/googletest/googletest/include', - './third_party/protobuf/src', - './third_party/re2', - './third_party/tmb/include'] + './third_party/src/benchmark/include', + './third_party/src/glog/src', + './third_party/src/googletest/googletest/include', + './third_party/src/protobuf/src', + './third_party/src/re2', + './third_party/src/tmb/include'] QUICKSTEP_DEFINES = [ '-DQUICKSTEP_DEBUG', '-DQUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION', ] CLANG_FLAGS = [ '-std=c++14', '-x', 'c++', ]
incubator-quickstep git commit: Small adjust in star schema cost model for # distinct values
Repository: incubator-quickstep
Updated Branches:
refs/heads/master d1dbb0d9b -> 023c43a40
Small adjust in star schema cost model for # distinct values
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/023c43a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/023c43a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/023c43a4
Branch: refs/heads/master
Commit: 023c43a40b145c225c6deab103655156a034fa90
Parents: d1dbb0d
Author: Jianqiao Zhu
Authored: Wed Feb 7 15:42:15 2018 -0600
Committer: Jianqiao Zhu
Committed: Fri Feb 23 14:59:06 2018 -0600
--
.../cost_model/StarSchemaSimpleCostModel.cpp| 4 ++--
.../tests/execution_generator/Partition.test| 16
2 files changed, 10 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/023c43a4/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
--
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index 5aec4b4..6ab86e5 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -493,7 +493,7 @@ std::size_t StarSchemaSimpleCostModel::getNumDistinctValues(
return stat.getNumDistinctValues(rel_attr_id);
}
}
- return estimateCardinalityForTableReference(table_reference);
+ return estimateCardinalityForTableReference(table_reference) * 0.1;
}
bool StarSchemaSimpleCostModel::impliesUniqueAttributes(
@@ -527,7 +527,7 @@ bool StarSchemaSimpleCostModel::impliesUniqueAttributes(
std::static_pointer_cast(physical_plan);
const CatalogRelationStatistics &stat =
table_reference->relation()->getStatistics();
- if (stat.hasNumTuples()) {
+ if (stat.isExact() && stat.hasNumTuples()) {
const std::size_t num_tuples = stat.getNumTuples();
for (const auto &attr : attributes) {
const attribute_id rel_attr_id =
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/023c43a4/query_optimizer/tests/execution_generator/Partition.test
--
diff --git a/query_optimizer/tests/execution_generator/Partition.test
b/query_optimizer/tests/execution_generator/Partition.test
index 747b969..4b11a04 100644
--- a/query_optimizer/tests/execution_generator/Partition.test
+++ b/query_optimizer/tests/execution_generator/Partition.test
@@ -116,16 +116,16 @@ WHERE dim_2_hash_partitions.id = fact.id
+---++
|id |char_col|
+---++
-| 2| 2 1.414214|
| 4| 4 2.00|
-| 6| 6 2.449490|
| 8| 8 2.828427|
| 12| 12 3.464102|
-| 14| 14 3.741657|
| 16| 16 4.00|
+| 24| 24 4.898979|
+| 2| 2 1.414214|
+| 6| 6 2.449490|
+| 14| 14 3.741657|
| 18| 18 4.242641|
| 22| 22 4.690416|
-| 24| 24 4.898979|
+---++
==
@@ -193,15 +193,15 @@ GROUP BY fact.score;
+++
|score |COUNT(*)|
+++
+| 8| 1|
| 41.569219381653056| 1|
-| 76.367532368147124| 1|
| 64| 1|
| 52.38320341483518| 1|
-| 8| 1|
-| 2.8284271247461903| 1|
-| 14.696938456699067| 1|
+| 76.367532368147124| 1|
| 22.627416997969522| 1|
| 117.57550765359254| 1|
+| 2.8284271247461903| 1|
+| 14.696938456699067| 1|
| 103.18914671611546| 1|
+++
==
incubator-quickstep git commit: Fix the hanging problem of SeparateChainingHashTable::resize()
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 539e1ebe0 -> d1dbb0d9b
Fix the hanging problem of SeparateChainingHashTable::resize()
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d1dbb0d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d1dbb0d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d1dbb0d9
Branch: refs/heads/master
Commit: d1dbb0d9bc2d1f001deee4039157b0be464870f4
Parents: 539e1eb
Author: Jianqiao Zhu
Authored: Sun Feb 18 01:16:07 2018 -0600
Committer: Jianqiao Zhu
Committed: Sun Feb 18 01:16:07 2018 -0600
--
storage/SeparateChainingHashTable.hpp | 15 +--
1 file changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d1dbb0d9/storage/SeparateChainingHashTable.hpp
--
diff --git a/storage/SeparateChainingHashTable.hpp
b/storage/SeparateChainingHashTable.hpp
index 2403623..2594f61 100644
--- a/storage/SeparateChainingHashTable.hpp
+++ b/storage/SeparateChainingHashTable.hpp
@@ -243,9 +243,10 @@ class SeparateChainingHashTable : public HashTable key_manager_;
@@ -1131,7 +1132,7 @@ void SeparateChainingHashTable
bool SeparateChainingHashTable
-::isFull(const std::size_t extra_variable_storage) const {
- if (header_->buckets_allocated.load(std::memory_order_relaxed) >=
header_->num_buckets) {
-// All buckets are allocated.
+::isFull(const std::size_t extra_buckets,
+ const std::size_t extra_variable_storage) const {
+ if (header_->buckets_allocated.load(std::memory_order_relaxed)
+ + extra_buckets >= header_->num_buckets) {
+// Not enough buckets.
return true;
}
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/fix-travis-timeout [deleted] 539e1ebe0
incubator-quickstep git commit: Add a flag to allow disabling of Comparison inline expansion to enable acceleration of Quickstep build.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 4a945a6b9 -> 539e1ebe0
Add a flag to allow disabling of Comparison inline expansion to enable
acceleration of Quickstep build.
(for development productivity as well as solving the Travis CI timeout problem)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/539e1ebe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/539e1ebe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/539e1ebe
Branch: refs/heads/master
Commit: 539e1ebe09b5d1a2d86069ed1fdc6e9fb38c5ce7
Parents: 4a945a6
Author: Jianqiao Zhu
Authored: Fri Feb 2 17:27:59 2018 -0600
Committer: Jianqiao Zhu
Committed: Fri Feb 2 17:31:24 2018 -0600
--
.travis.yml | 3 ++-
CMakeLists.txt | 14 ++
.../comparisons/AsciiStringComparators-inl.hpp | 2 ++
.../comparisons/AsciiStringComparators.hpp | 4
types/operations/comparisons/Comparison.cpp | 20
.../comparisons/LiteralComparators-inl.hpp | 2 ++
.../comparisons/LiteralComparators.hpp | 4
7 files changed, 28 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/.travis.yml
--
diff --git a/.travis.yml b/.travis.yml
index 4e7833f..6517ae8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -76,7 +76,8 @@ before_script:
-D CMAKE_CXX_COMPILER=$CXX
-D CMAKE_LINKER=$CLINKER
-D USE_TCMALLOC=0
- -D VECTOR_COPY_ELISION_LEVEL=$VECTOR_COPY_ELISION_LEVEL ..)
+ -D VECTOR_COPY_ELISION_LEVEL=$VECTOR_COPY_ELISION_LEVEL
+ -D ENABLE_COMPARISON_INLINE_EXPANSION=OFF ..)
script:
- ./lint_everything.py
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 88835ac..c777a6a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -145,6 +145,20 @@ if (ENABLE_VECTOR_PREDICATE_SHORT_CIRCUIT)
)
endif()
+set(COMPARISON_INLINE_EXPANSION_MSG_LIST
+"This option controls whether to enable inlined template expansion "
+"of comparison predicates. WARNING: This option should only be "
+"turned off for development use. Turning off this option will greatly "
+"reduce Quickstep compile time but incur drastic performance degradation.")
+string(REPLACE ";" "" COMPARISON_INLINE_EXPANSION_MSG
${COMPARISON_INLINE_EXPANSION_MSG_LIST})
+option(ENABLE_COMPARISON_INLINE_EXPANSION ${COMPARISON_INLINE_EXPANSION_MSG}
ON)
+if (ENABLE_COMPARISON_INLINE_EXPANSION)
+ set_property(
+DIRECTORY
+APPEND PROPERTY COMPILE_DEFINITIONS
QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
+ )
+endif()
+
option(ENABLE_NETWORK_CLI "Allows use of the network cli" OFF)
option(ENABLE_DISTRIBUTED "Use the distributed version of Quickstep" OFF)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/types/operations/comparisons/AsciiStringComparators-inl.hpp
--
diff --git a/types/operations/comparisons/AsciiStringComparators-inl.hpp
b/types/operations/comparisons/AsciiStringComparators-inl.hpp
index fd0d17d..8b2c1bf 100644
--- a/types/operations/comparisons/AsciiStringComparators-inl.hpp
+++ b/types/operations/comparisons/AsciiStringComparators-inl.hpp
@@ -46,6 +46,7 @@
namespace quickstep {
+#ifdef QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
template class ComparisonFunctor,
bool left_nullable, bool left_null_terminated, bool left_longer,
bool right_nullable, bool right_null_terminated, bool right_longer>
@@ -586,6 +587,7 @@ TypedValue AsciiStringUncheckedComparatorhttp://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/types/operations/comparisons/AsciiStringComparators.hpp
--
diff --git a/types/operations/comparisons/AsciiStringComparators.hpp
b/types/operations/comparisons/AsciiStringComparators.hpp
index 936fd1f..2aec8c4 100644
--- a/types/operations/comparisons/AsciiStringComparators.hpp
+++ b/types/operations/comparisons/AsciiStringComparators.hpp
@@ -122,6 +122,7 @@ class AsciiStringUncheckedComparator : public
UncheckedComparator {
0);
}
+#ifdef QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
TupleIdSequence* compareColumnVectors(
const ColumnVector &left,
const ColumnVector &right,
@@ -202,6 +203,7 @@ class AsciiStringUncheckedComparator : public
UncheckedComparator
[1/2] incubator-quickstep git commit: Upgrade cpplint
Repository: incubator-quickstep Updated Branches: refs/heads/master c43107d1f -> d886ddb09 http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d886ddb0/third_party/src/cpplint/cpplint.py -- diff --git a/third_party/src/cpplint/cpplint.py b/third_party/src/cpplint/cpplint.py index 469283f..431c112 100755 --- a/third_party/src/cpplint/cpplint.py +++ b/third_party/src/cpplint/cpplint.py @@ -1,15 +1,4 @@ -#!/usr/bin/env python2 -# -# Original version: svn revision 141 -# -# This file modified for quickstep as follows: -# - Allow no copyright message at the top of the file. -# - Allow line length up to 120 characters by default. -# - Recognize .hpp files as C++ source. -# - Allow use of C++11 header. -# - Supress IWYU warnings for std::tuple (since we use "tuple" as a variable -# name in many places). -# - Allow C++11 rvalue references anywhere. +#!/usr/bin/env python # # Copyright (c) 2009 Google Inc. All rights reserved. # @@ -52,6 +41,13 @@ We do a small hack, which is to ignore //'s with "'s after them on the same line, but it is far from perfect (in either direction). """ +""" +Additional notes from Quickstep developers: +commit ID: e8ffd7ce60e5c0e98ba37745749fed0222982668 +Repo: https://github.com/google/styleguide +Changes: Remove chrono from the banned C++11 headers' list as it appears to be +just a matter of preference for the Google chromium team. +""" import codecs import copy import getopt @@ -67,11 +63,12 @@ import unicodedata _USAGE = """ Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] [--counting=total|toplevel|detailed] [--root=subdir] - [--linelength=digits] + [--linelength=digits] [--headers=x,y,...] + [--quiet] [file] ... The style guidelines this tries to follow are those in -http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml +https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml Every problem is given a confidence score from 1-5, with 5 meaning we are certain of the problem, and 1 meaning it could be a legitimate construct. @@ -94,6 +91,9 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] verbose=# Specify a number 0-5 to restrict errors to certain verbosity levels. +quiet + Don't print anything if no errors are found. + filter=-x,+y,... Specify a comma-separated list of category-filters to apply: only error messages whose category names pass the filters will be printed. @@ -125,12 +125,13 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] ignored. Examples: -Assuming that src/.git exists, the header guard CPP variables for -src/chrome/browser/ui/browser.h are: +Assuming that top/src/.git exists (and cwd=top/src), the header guard +CPP variables for top/src/chrome/browser/ui/browser.h are: No flag => CHROME_BROWSER_UI_BROWSER_H_ --root=chrome => BROWSER_UI_BROWSER_H_ --root=chrome/browser => UI_BROWSER_H_ +--root=.. => SRC_CHROME_BROWSER_UI_BROWSER_H_ linelength=digits This is the allowed line length for the project. The default value is @@ -145,6 +146,14 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] Examples: --extensions=hpp,cpp +headers=x,y,... + The header extensions that cpplint will treat as .h in checks. Values are + automatically added to --extensions list. + + Examples: +--headers=hpp,hxx +--headers=hpp + cpplint.py supports per-directory configurations specified in CPPLINT.cfg files. CPPLINT.cfg file can contain a number of key=value pairs. Currently the following options are supported: @@ -153,6 +162,8 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] filter=+filter1,-filter2,... exclude_files=regex linelength=80 + root=subdir + headers=x,y,... "set noparent" option prevents cpplint from traversing directory tree upwards looking for more .cfg files in parent directories. This option @@ -168,6 +179,12 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] "linelength" allows to specify the allowed line length for the project. +The "root" option is similar in function to the --root flag (see example +above). Paths are relative to the directory of the CPPLINT.cfg. + +The "headers" option is similar in function to the --headers flag +(see example above). + CPPLINT.cfg has an effect on files in the same directory and all sub-directories, unless overridden by a nested configuration file. @@ -188,6 +205,8 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] _ERROR_CATEGORIES = [ 'build/class', 'build/c++1
[2/2] incubator-quickstep git commit: Upgrade cpplint
Upgrade cpplint - Added several missing headers. - Modified the lint_everything file to add the command line flags. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d886ddb0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d886ddb0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d886ddb0 Branch: refs/heads/master Commit: d886ddb095d3eaafde00bdfb5f85f97aabe5e7e2 Parents: c43107d Author: Harshad Deshmukh Authored: Tue Dec 19 12:56:36 2017 -0600 Committer: Harshad Deshmukh Committed: Thu Dec 21 14:30:47 2017 -0600 -- .travis.yml |2 +- catalog/CatalogRelation.hpp |1 + catalog/IndexScheme.hpp |1 + catalog/tests/Catalog_unittest.cpp |1 + cli/tests/NetworkIO_unittest.cpp|6 +- .../aggregation/AggregationHandleSum.cpp|1 + expressions/scalar/ScalarAttribute.cpp |1 + .../WindowAggregateFunctionAvg.cpp |1 + .../WindowAggregateFunctionAvg.hpp |1 + .../WindowAggregateFunctionCount.cpp|1 + .../WindowAggregateFunctionCount.hpp|1 + .../WindowAggregateFunctionMax.cpp |1 + .../WindowAggregateFunctionMax.hpp |1 + .../WindowAggregateFunctionMin.cpp |1 + .../WindowAggregateFunctionMin.hpp |1 + .../WindowAggregateFunctionSum.cpp |1 + .../WindowAggregateFunctionSum.hpp |1 + lint_everything.py | 47 + parser/ParseBlockProperties.cpp |8 +- parser/ParseBlockProperties.hpp |8 +- parser/ParseString.hpp |2 +- query_optimizer/LIPFilterGenerator.cpp |1 + .../expressions/BinaryExpression.cpp|1 + query_optimizer/expressions/Exists.cpp |4 +- query_optimizer/expressions/InTableQuery.cpp|1 + query_optimizer/expressions/InValueList.cpp |2 + query_optimizer/expressions/LogicalAnd.cpp |1 + query_optimizer/expressions/LogicalOr.cpp |1 + query_optimizer/expressions/SearchedCase.cpp|1 + query_optimizer/expressions/SimpleCase.cpp |2 + .../expressions/SubqueryExpression.cpp |1 + query_optimizer/logical/SetOperation.hpp|1 + query_optimizer/logical/Sort.hpp|1 + .../physical/SharedSubplanReference.hpp |1 + query_optimizer/physical/Sort.hpp |1 + query_optimizer/physical/TableReference.hpp |1 + query_optimizer/resolver/NameResolver.cpp |1 + query_optimizer/resolver/NameResolver.hpp |1 + query_optimizer/rules/AttachLIPFilters.cpp |5 +- .../StarSchemaHashJoinOrderOptimization.cpp |2 + query_optimizer/rules/UnnestSubqueries.cpp |1 + .../rules/tests/PushDownFilter_unittest.cpp |2 + .../rules/tests/UpdateExpression_unittest.cpp |2 + .../strategy/tests/Aggregate_unittest.cpp |1 + .../strategy/tests/Join_unittest.cpp|1 + .../strategy/tests/OneToOne_unittest.cpp|2 + .../strategy/tests/Selection_unittest.cpp |2 + query_optimizer/strategy/tests/StrategyTest.hpp |2 + .../tests/ExecutionGeneratorTestRunner.cpp |2 +- .../tests/ExecutionGeneratorTestRunner.hpp |2 +- relational_operators/AggregationOperator.hpp|1 + relational_operators/InsertOperator.cpp |1 + relational_operators/InsertOperator.hpp |1 + .../WindowAggregationOperator.cpp |1 + .../WindowAggregationOperator.hpp |1 + .../BasicColumnStoreTupleStorageSubBlock.hpp|4 +- storage/BasicColumnStoreValueAccessor.hpp | 13 +- storage/CollisionFreeVectorTable.hpp|1 + storage/CompressedTupleStorageSubBlock.hpp |4 +- storage/LinearOpenAddressingHashTable.hpp |1 + storage/PackedPayloadHashTable.cpp |1 + storage/SplitRowStoreTupleStorageSubBlock.cpp |1 + storage/SplitRowStoreTupleStorageSubBlock.hpp |1 + storage/StorageManager.cpp |1 + storage/SubBlockTypeRegistry.cpp|1 + storage/ThreadPrivateCompactKeyHashTable.cpp|2 + .../tests/BloomFilterIndexSubBlock_unittest.cpp |1 + storage/tests/FileManager_unittest_common.hpp |4 - storage/tests/StorageBlockSort_unittest.cpp |1 + third_party/src/cpplint/cpplint.py | 1205 +- third_party/src/cpplint/lint_everything.py | 40 - transaction/LockManager.cpp |1 + .../comparisons/AsciiStringComparators-inl.hpp |1 +
incubator-quickstep git commit: Fixed gcc compiler warning.
Repository: incubator-quickstep Updated Branches: refs/heads/master e79b520ec -> b5130feab Fixed gcc compiler warning. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b5130fea Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b5130fea Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b5130fea Branch: refs/heads/master Commit: b5130feabc571010924cacd1fa6287f4518be8d6 Parents: e79b520 Author: Zuyu Zhang Authored: Thu Oct 12 23:01:22 2017 -0500 Committer: Zuyu Zhang Committed: Thu Oct 12 23:01:22 2017 -0500 -- storage/Flags.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b5130fea/storage/Flags.cpp -- diff --git a/storage/Flags.cpp b/storage/Flags.cpp index 4672f81..7312cd3 100644 --- a/storage/Flags.cpp +++ b/storage/Flags.cpp @@ -40,7 +40,7 @@ static bool ValidateHdfsNameNodePort(const char *flagname, } } DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode."); -static const bool hdfs_namenode_port_dummy +static volatile const bool hdfs_namenode_port_dummy = gflags::RegisterFlagValidator(&FLAGS_hdfs_namenode_port, &ValidateHdfsNameNodePort); static bool ValidateHdfsNumReplications(const char *flagname, @@ -55,7 +55,7 @@ static bool ValidateHdfsNumReplications(const char *flagname, } } DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications."); -static const bool hdfs_num_replications_dummy +static volatile const bool hdfs_num_replications_dummy = gflags::RegisterFlagValidator(&FLAGS_hdfs_num_replications, &ValidateHdfsNumReplications); #endif
incubator-quickstep git commit: Added ProbabilityStore class
Repository: incubator-quickstep
Updated Branches:
refs/heads/master f820c45ee -> 8f094a1c0
Added ProbabilityStore class
- Used to store probabilities of objects.
- Probabilities are of two kinds: Individual and cumulative.
- All the individual probabilities within the store add up to one.
- Support for finding the object with given cumulative probability.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8f094a1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8f094a1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8f094a1c
Branch: refs/heads/master
Commit: 8f094a1c086445b79d6dba36f81326ac06050209
Parents: f820c45
Author: Harshad Deshmukh
Authored: Fri Sep 29 15:38:42 2017 -0500
Committer: Harshad Deshmukh
Committed: Wed Oct 11 10:38:36 2017 -0500
--
query_execution/CMakeLists.txt | 13 +
query_execution/ProbabilityStore.hpp| 263 +++
.../tests/ProbabilityStore_unittest.cpp | 106
3 files changed, 382 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f094a1c/query_execution/CMakeLists.txt
--
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 8f797f7..791434a 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -40,6 +40,7 @@ if (ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_PolicyEnforcerDistributed
PolicyEnforcerDistributed.cpp PolicyEnforcerDistributed.hpp)
endif(ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_PolicyEnforcerSingleNode
PolicyEnforcerSingleNode.cpp PolicyEnforcerSingleNode.hpp)
+add_library(quickstep_queryexecution_ProbabilityStore ../empty_src.cpp
ProbabilityStore.hpp)
add_library(quickstep_queryexecution_QueryContext QueryContext.cpp
QueryContext.hpp)
add_library(quickstep_queryexecution_QueryContext_proto
${queryexecution_QueryContext_proto_srcs}
@@ -201,6 +202,9 @@
target_link_libraries(quickstep_queryexecution_PolicyEnforcerSingleNode
quickstep_utility_Macros
tmb
${GFLAGS_LIB_NAME})
+target_link_libraries(quickstep_queryexecution_ProbabilityStore
+ glog
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryexecution_QueryContext
glog
quickstep_catalog_CatalogDatabaseLite
@@ -372,6 +376,7 @@ target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_ForemanSingleNode
quickstep_queryexecution_PolicyEnforcerBase
quickstep_queryexecution_PolicyEnforcerSingleNode
+ quickstep_queryexecution_ProbabilityStore
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
quickstep_queryexecution_QueryExecutionMessages_proto
@@ -425,6 +430,14 @@ if (ENABLE_DISTRIBUTED)
add_test(BlockLocator_unittest BlockLocator_unittest)
endif(ENABLE_DISTRIBUTED)
+add_executable(ProbabilityStore_unittest
+"${CMAKE_CURRENT_SOURCE_DIR}/tests/ProbabilityStore_unittest.cpp")
+target_link_libraries(ProbabilityStore_unittest
+ gtest
+ gtest_main
+ quickstep_queryexecution_ProbabilityStore)
+add_test(ProbabilityStore_unittest ProbabilityStore_unittest)
+
add_executable(QueryManagerSingleNode_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/QueryManagerSingleNode_unittest.cpp")
target_link_libraries(QueryManagerSingleNode_unittest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f094a1c/query_execution/ProbabilityStore.hpp
--
diff --git a/query_execution/ProbabilityStore.hpp
b/query_execution/ProbabilityStore.hpp
new file mode 100644
index 000..079f60b
--- /dev/null
+++ b/query_execution/ProbabilityStore.hpp
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BAS
incubator-quickstep git commit: Removed unused argument always_mark_full.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master ffb8e055a -> f820c45ee
Removed unused argument always_mark_full.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f820c45e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f820c45e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f820c45e
Branch: refs/heads/master
Commit: f820c45ee56a9e74671fb1c22c5e6b7e13471b5d
Parents: ffb8e05
Author: Zuyu Zhang
Authored: Thu Oct 5 17:18:05 2017 -0500
Committer: Zuyu Zhang
Committed: Tue Oct 10 14:14:29 2017 -0500
--
storage/InsertDestination.cpp | 73 ++---
storage/InsertDestination.hpp | 21 -
storage/InsertDestinationInterface.hpp | 12 ++---
3 files changed, 36 insertions(+), 70 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f820c45e/storage/InsertDestination.cpp
--
diff --git a/storage/InsertDestination.cpp b/storage/InsertDestination.cpp
index 8821019..c2b44d8 100644
--- a/storage/InsertDestination.cpp
+++ b/storage/InsertDestination.cpp
@@ -199,7 +199,7 @@ void InsertDestination::insertTupleInBatch(const Tuple
&tuple) {
returnBlock(std::move(output_block), false);
}
-void InsertDestination::bulkInsertTuples(ValueAccessor *accessor, bool
always_mark_full) {
+void InsertDestination::bulkInsertTuples(ValueAccessor *accessor, const bool
always_mark_full) {
InvokeOnAnyValueAccessor(
accessor,
[&](auto *accessor) -> void { // NOLINT(build/c++11)
@@ -207,23 +207,17 @@ void InsertDestination::bulkInsertTuples(ValueAccessor
*accessor, bool always_ma
while (!accessor->iterationFinished()) {
MutableBlockReference output_block = this->getBlockForInsertion();
// FIXME(chasseur): Deal with TupleTooLargeForBlock exception.
- if (output_block->bulkInsertTuples(accessor) == 0) {
-// output_block is full.
-this->returnBlock(std::move(output_block), true);
- } else {
-// Bulk insert into output_block was successful. output_block
-// will be rebuilt when there won't be any more insertions to it.
-this->returnBlock(std::move(output_block),
- always_mark_full || !accessor->iterationFinished());
- }
+ const auto num_tuples_inserted =
output_block->bulkInsertTuples(accessor);
+ this->returnBlock(std::move(output_block),
+num_tuples_inserted == 0 ||
!accessor->iterationFinished() ||
+always_mark_full);
}
});
}
void InsertDestination::bulkInsertTuplesWithRemappedAttributes(
const std::vector &attribute_map,
-ValueAccessor *accessor,
-bool always_mark_full) {
+ValueAccessor *accessor) {
InvokeOnAnyValueAccessor(
accessor,
[&](auto *accessor) -> void { // NOLINT(build/c++11)
@@ -231,17 +225,10 @@ void
InsertDestination::bulkInsertTuplesWithRemappedAttributes(
while (!accessor->iterationFinished()) {
MutableBlockReference output_block = this->getBlockForInsertion();
// FIXME(chasseur): Deal with TupleTooLargeForBlock exception.
- if (output_block->bulkInsertTuplesWithRemappedAttributes(
- attribute_map,
- accessor) == 0) {
-// output_block is full.
-this->returnBlock(std::move(output_block), true);
- } else {
-// Bulk insert into output_block was successful. output_block
-// will be rebuilt when there won't be any more insertions to it.
-this->returnBlock(std::move(output_block),
- always_mark_full || !accessor->iterationFinished());
- }
+ const auto num_tuples_inserted =
+ output_block->bulkInsertTuplesWithRemappedAttributes(attribute_map,
accessor);
+ this->returnBlock(std::move(output_block),
+num_tuples_inserted == 0 ||
!accessor->iterationFinished());
}
});
}
@@ -267,8 +254,7 @@ void removeGapOnlyAccessors(
}
void InsertDestination::bulkInsertTuplesFromValueAccessors(
-const std::vector>>
&accessor_attribute_map,
-bool always_mark_full) {
+const std::vector>>
&accessor_attribute_map) {
// Handle pathological corner case where there are no accessors
if (accessor_attribute_map.size() == 0)
return;
@@ -323,9 +309,7 @@ void InsertDestination::bulkInsertTuplesFromValueAccessors(
// Update the header for output_block and then return it.
output_block->bulkInsertPartialTuplesFinalize(num_tuples_inserted);
-const bool mark_full = always_mark_full
- || !first_accessor->iterationFinishedVirtual();
-this->returnBlock
incubator-quickstep git commit: Relax the sort requirement in columnstore.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 69fd94b89 -> ffb8e055a
Relax the sort requirement in columnstore.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ffb8e055
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ffb8e055
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ffb8e055
Branch: refs/heads/master
Commit: ffb8e055a890a9002235a8516e5f2dece2d6228a
Parents: 69fd94b
Author: Zuyu Zhang
Authored: Mon Oct 9 11:23:08 2017 -0500
Committer: Zuyu Zhang
Committed: Tue Oct 10 14:10:03 2017 -0500
--
parser/CMakeLists.txt | 1 +
parser/ParseBlockProperties.hpp| 10 --
parser/tests/Create.test | 45 +
query_optimizer/OptimizerTree.hpp | 9 +++--
query_optimizer/resolver/Resolver.cpp | 15 +
query_optimizer/tests/resolver/Create.test | 38 +
6 files changed, 99 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ffb8e055/parser/CMakeLists.txt
--
diff --git a/parser/CMakeLists.txt b/parser/CMakeLists.txt
index b3ddf30..d4aaab4 100644
--- a/parser/CMakeLists.txt
+++ b/parser/CMakeLists.txt
@@ -150,6 +150,7 @@ target_link_libraries(quickstep_parser_ParseBlockProperties
quickstep_parser_ParseTreeNode
quickstep_utility_Macros
quickstep_utility_PtrList
+ quickstep_utility_SqlError
quickstep_utility_StringUtil)
target_link_libraries(quickstep_parser_ParseCaseExpressions
quickstep_parser_ParseExpression
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ffb8e055/parser/ParseBlockProperties.hpp
--
diff --git a/parser/ParseBlockProperties.hpp b/parser/ParseBlockProperties.hpp
index ce0cd92..fa176b1 100644
--- a/parser/ParseBlockProperties.hpp
+++ b/parser/ParseBlockProperties.hpp
@@ -31,6 +31,7 @@
#include "parser/ParseTreeNode.hpp"
#include "utility/Macros.hpp"
#include "utility/PtrList.hpp"
+#include "utility/SqlError.hpp"
#include "utility/StringUtil.hpp"
#include "glog/logging.h"
@@ -143,10 +144,13 @@ class ParseBlockProperties : public ParseTreeNode {
if (sort_key_value == nullptr) {
return nullptr;
}
-if (sort_key_value->getKeyValueType() !=
-ParseKeyValue::KeyValueType::kStringString) {
- return nullptr;
+if (sort_key_value->getKeyValueType() ==
+ParseKeyValue::KeyValueType::kStringStringList) {
+ THROW_SQL_ERROR_AT(sort_key_value)
+ << "The SORT property must be a string, not a string list.";
}
+
+DCHECK(sort_key_value->getKeyValueType() ==
ParseKeyValue::KeyValueType::kStringString);
return static_cast(sort_key_value)->value();
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ffb8e055/parser/tests/Create.test
--
diff --git a/parser/tests/Create.test b/parser/tests/Create.test
index 8c11054..3923c13 100644
--- a/parser/tests/Create.test
+++ b/parser/tests/Create.test
@@ -259,6 +259,51 @@ CreateTableStatement[relation_name=test]
+-value=String[value=rowstore]
==
+CREATE TABLE test (attr INT, attr2 INT) WITH BLOCKPROPERTIES
+(TYPE columnstore)
+--
+CreateTableStatement[relation_name=test]
++-attribute_list=
+| +-AttributeDefinition[name=attr,type=Int]
+| +-AttributeDefinition[name=attr2,type=Int]
++-block_properties=
+ +-BlockProperties
++-block_property=KeyStringValue[key=TYPE]
+ +-value=String[value=columnstore]
+==
+
+CREATE TABLE test (attr INT, attr2 INT) WITH BLOCKPROPERTIES
+(TYPE columnstore, SORT attr)
+--
+CreateTableStatement[relation_name=test]
++-attribute_list=
+| +-AttributeDefinition[name=attr,type=Int]
+| +-AttributeDefinition[name=attr2,type=Int]
++-block_properties=
+ +-BlockProperties
++-block_property=KeyStringValue[key=TYPE]
+| +-value=String[value=columnstore]
++-block_property=KeyStringValue[key=SORT]
+ +-value=String[value=attr]
+==
+
+CREATE TABLE test (attr INT, attr2 INT) WITH BLOCKPROPERTIES
+(TYPE columnstore, SORT (attr, attr2))
+--
+CreateTableStatement[relation_name=test]
++-attribute_list=
+| +-AttributeDefinition[name=attr,type=Int]
+| +-AttributeDefinition[name=attr2,type=Int]
++-block_properties=
+ +-BlockProperties
++-block_property=KeyStringValue[key=TYPE]
+| +-value=String[value=columnstore]
++-block_property=KeyStringList[key=SORT]
+ +-value_list=
++-String[value=attr]
++-String[v
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/get-touched-blocks [deleted] a61b03dcc
incubator-quickstep git commit: Moved InsertDestination::getTouchedBlocks as a private method.
Repository: incubator-quickstep
Updated Branches:
refs/heads/get-touched-blocks [created] a61b03dcc
Moved InsertDestination::getTouchedBlocks as a private method.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a61b03dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a61b03dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a61b03dc
Branch: refs/heads/get-touched-blocks
Commit: a61b03dcc2446a5bd276a0117f493ba56d8a3ffe
Parents: 79710ca
Author: Zuyu Zhang
Authored: Thu Oct 5 16:41:38 2017 -0500
Committer: Zuyu Zhang
Committed: Mon Oct 9 12:00:08 2017 -0500
--
.../tests/HashJoinOperator_unittest.cpp | 24 +++
storage/InsertDestination.cpp | 27 +++-
storage/InsertDestination.hpp | 70
3 files changed, 65 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a61b03dc/relational_operators/tests/HashJoinOperator_unittest.cpp
--
diff --git a/relational_operators/tests/HashJoinOperator_unittest.cpp
b/relational_operators/tests/HashJoinOperator_unittest.cpp
index 1fc84fc..cfd4314 100644
--- a/relational_operators/tests/HashJoinOperator_unittest.cpp
+++ b/relational_operators/tests/HashJoinOperator_unittest.cpp
@@ -485,7 +485,7 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -640,7 +640,7 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -795,7 +795,7 @@ TEST_P(HashJoinOperatorTest,
CharKeyCartesianProductHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -944,7 +944,7 @@ TEST_P(HashJoinOperatorTest,
VarCharDuplicateKeyHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -1123,7 +1123,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -1313,7 +1313,7 @@ TEST_P(HashJoinOperatorTest,
CompositeKeyHashJoinWithResidualPredicateTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(
incubator-quickstep git commit: Moved InsertDestination::getTouchedBlocks as a private method.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 79710ca6c -> a61b03dcc
Moved InsertDestination::getTouchedBlocks as a private method.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a61b03dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a61b03dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a61b03dc
Branch: refs/heads/master
Commit: a61b03dcc2446a5bd276a0117f493ba56d8a3ffe
Parents: 79710ca
Author: Zuyu Zhang
Authored: Thu Oct 5 16:41:38 2017 -0500
Committer: Zuyu Zhang
Committed: Mon Oct 9 12:00:08 2017 -0500
--
.../tests/HashJoinOperator_unittest.cpp | 24 +++
storage/InsertDestination.cpp | 27 +++-
storage/InsertDestination.hpp | 70
3 files changed, 65 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a61b03dc/relational_operators/tests/HashJoinOperator_unittest.cpp
--
diff --git a/relational_operators/tests/HashJoinOperator_unittest.cpp
b/relational_operators/tests/HashJoinOperator_unittest.cpp
index 1fc84fc..cfd4314 100644
--- a/relational_operators/tests/HashJoinOperator_unittest.cpp
+++ b/relational_operators/tests/HashJoinOperator_unittest.cpp
@@ -485,7 +485,7 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -640,7 +640,7 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -795,7 +795,7 @@ TEST_P(HashJoinOperatorTest,
CharKeyCartesianProductHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -944,7 +944,7 @@ TEST_P(HashJoinOperatorTest,
VarCharDuplicateKeyHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -1123,7 +1123,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
- const std::vector &result_blocks =
insert_destination->getTouchedBlocks();
+ const std::vector result_blocks =
insert_destination->getTouchedBlocks();
for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
BlockReference result_block =
storage_manager_->getBlock(result_blocks[bid],
insert_destination->getRelation());
@@ -1313,7 +1313,7 @@ TEST_P(HashJoinOperatorTest,
CompositeKeyHashJoinWithResidualPredicateTest) {
InsertDestination *insert_destination =
query_context_->getInsertDestination(prober->getInsertDestinationID());
DCHECK(insert_destination);
incubator-quickstep git commit: Removed the virtual function call in InvokeOnAnyValueAccessor.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 696a783e5 -> 79710ca6c
Removed the virtual function call in InvokeOnAnyValueAccessor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/79710ca6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/79710ca6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/79710ca6
Branch: refs/heads/master
Commit: 79710ca6c6b75410bf2c26b4646acbfc5d554d7c
Parents: 696a783
Author: Zuyu Zhang
Authored: Fri Oct 6 14:34:21 2017 -0500
Committer: Zuyu Zhang
Committed: Mon Oct 9 11:37:16 2017 -0500
--
storage/SplitRowStoreTupleStorageSubBlock.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/79710ca6/storage/SplitRowStoreTupleStorageSubBlock.cpp
--
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.cpp
b/storage/SplitRowStoreTupleStorageSubBlock.cpp
index 5060208..9f5a839 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.cpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.cpp
@@ -343,7 +343,7 @@ tuple_id
SplitRowStoreTupleStorageSubBlock::bulkInsertPartialTuplesImpl(
InvokeOnAnyValueAccessor(
accessor,
- [&](auto *accessor) -> void { // NOLINT(build/c++11
+ [&](auto *accessor) -> void { // NOLINT(build/c++11)
BitVector tuple_null_bitmap(tuple_slot, num_null_attrs_);
const std::size_t nullmap_size =
BitVector::BytesNeeded(num_null_attrs_);
@@ -410,7 +410,7 @@ tuple_id
SplitRowStoreTupleStorageSubBlock::bulkInsertPartialTuplesImpl(
max_num_tuples_to_insert += additional_tuples_insert;
}
}
-} while (fill_to_capacity && !accessor->iterationFinishedVirtual() &&
+} while (fill_to_capacity && !accessor->iterationFinished() &&
num_tuples_inserted < max_num_tuples_to_insert);
});
incubator-quickstep git commit: Fixed a flaky case in Catalog test.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master e496cb58e -> 696a783e5
Fixed a flaky case in Catalog test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/696a783e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/696a783e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/696a783e
Branch: refs/heads/master
Commit: 696a783e5d8adb3ca62ca9044a8d7ccd89f67b3a
Parents: e496cb5
Author: Zuyu Zhang
Authored: Sun Oct 8 14:20:41 2017 -0500
Committer: Zuyu Zhang
Committed: Mon Oct 9 11:34:50 2017 -0500
--
catalog/tests/Catalog_unittest.cpp | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/696a783e/catalog/tests/Catalog_unittest.cpp
--
diff --git a/catalog/tests/Catalog_unittest.cpp
b/catalog/tests/Catalog_unittest.cpp
index f761026..7b121fa 100644
--- a/catalog/tests/Catalog_unittest.cpp
+++ b/catalog/tests/Catalog_unittest.cpp
@@ -552,13 +552,15 @@ TEST_F(CatalogTest, CatalogIndexTest) {
IndexSubBlockDescription index_description;
index_description.set_sub_block_type(IndexSubBlockDescription::CSB_TREE);
index_description.add_indexed_attribute_ids(rel->getAttributeByName("attr_idx1")->getID());
+ IndexSubBlockDescription index_description_copy;
+ index_description_copy.MergeFrom(index_description);
EXPECT_TRUE(rel->addIndex("idx1", std::move(index_description)));
EXPECT_TRUE(rel->hasIndexWithName("idx1"));
// Adding an index with duplicate name should return false.
- EXPECT_FALSE(rel->addIndex("idx1", std::move(index_description)));
+ EXPECT_FALSE(rel->addIndex("idx1", IndexSubBlockDescription()));
// Adding an index of same type with different name on the same attribute
should return false.
- EXPECT_FALSE(rel->addIndex("idx2", std::move(index_description)));
+ EXPECT_FALSE(rel->addIndex("idx2", std::move(index_description_copy)));
index_description.Clear();
index_description.set_sub_block_type(IndexSubBlockDescription::CSB_TREE);
incubator-quickstep git commit: Fixed the distributed version due to query execution engine simplification.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 0898a77be -> e496cb58e
Fixed the distributed version due to query execution engine simplification.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e496cb58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e496cb58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e496cb58
Branch: refs/heads/master
Commit: e496cb58e10d32de9dc83d69ece84df3f5b62747
Parents: 0898a77
Author: Zuyu Zhang
Authored: Fri Oct 6 22:33:02 2017 -0500
Committer: Zuyu Zhang
Committed: Fri Oct 6 22:33:02 2017 -0500
--
query_execution/QueryManagerDistributed.cpp | 24 +---
relational_operators/WorkOrderFactory.cpp | 4 ++--
2 files changed, 11 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e496cb58/query_execution/QueryManagerDistributed.cpp
--
diff --git a/query_execution/QueryManagerDistributed.cpp
b/query_execution/QueryManagerDistributed.cpp
index 30a1396..97b451f 100644
--- a/query_execution/QueryManagerDistributed.cpp
+++ b/query_execution/QueryManagerDistributed.cpp
@@ -70,8 +70,11 @@ QueryManagerDistributed::QueryManagerDistributed(QueryHandle
*query_handle,
// Collect all the workorders from all the non-blocking relational operators
in the DAG.
for (const dag_node_index index : non_dependent_operators_) {
if (!fetchNormalWorkOrders(index)) {
- DCHECK(!checkRebuildRequired(index) || initiateRebuild(index));
- markOperatorFinished(index);
+ if (checkRebuildRequired(index)) {
+initiateRebuild(index);
+ } else {
+markOperatorFinished(index);
+ }
}
}
@@ -201,21 +204,12 @@ void
QueryManagerDistributed::processInitiateRebuildResponseMessage(const dag_no
const
std::size_t shiftboss_index) {
query_exec_state_->updateRebuildStatus(op_index, num_rebuild_work_orders,
shiftboss_index);
- if (!query_exec_state_->hasRebuildFinished(op_index, num_shiftbosses_)) {
-// Wait for the rebuild work orders to finish.
-return;
+ if (query_exec_state_->hasRebuildFinished(op_index, num_shiftbosses_)) {
+// No needs for rebuilds, or the rebuild has finished.
+markOperatorFinished(op_index);
}
- // No needs for rebuilds, or the rebuild has finished.
- markOperatorFinished(op_index);
-
- for (const std::pair &dependent_link :
- query_dag_->getDependents(op_index)) {
-const dag_node_index dependent_op_index = dependent_link.first;
-if (checkAllBlockingDependenciesMet(dependent_op_index)) {
- fetchNormalWorkOrders(dependent_op_index);
-}
- }
+ // Wait for the rebuild work orders to finish.
}
bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e496cb58/relational_operators/WorkOrderFactory.cpp
--
diff --git a/relational_operators/WorkOrderFactory.cpp
b/relational_operators/WorkOrderFactory.cpp
index 5baa21b..25cc81a 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -741,7 +741,7 @@ bool WorkOrderFactory::ProtoIsValid(const
serialization::WorkOrder &proto,
proto.GetExtension(serialization::DeleteWorkOrder::predicate_index)) &&
proto.HasExtension(serialization::DeleteWorkOrder::block_id) &&
proto.HasExtension(serialization::DeleteWorkOrder::operator_index) &&
- proto.GetExtension(serialization::DeleteWorkOrder::partition_id);
+ proto.HasExtension(serialization::DeleteWorkOrder::partition_id);
}
case serialization::DESTROY_AGGREGATION_STATE: {
return
proto.HasExtension(serialization::DestroyAggregationStateWorkOrder::aggr_state_index)
&&
@@ -1033,7 +1033,7 @@ bool WorkOrderFactory::ProtoIsValid(const
serialization::WorkOrder &proto,
proto.GetExtension(serialization::UpdateWorkOrder::update_group_index)) &&
proto.HasExtension(serialization::UpdateWorkOrder::operator_index) &&
proto.HasExtension(serialization::UpdateWorkOrder::block_id) &&
- proto.GetExtension(serialization::UpdateWorkOrder::partition_id);
+ proto.HasExtension(serialization::UpdateWorkOrder::partition_id);
}
case serialization::WINDOW_AGGREGATION: {
return
proto.HasExtension(serialization::WindowAggregationWorkOrder::window_aggr_state_index)
&&
incubator-quickstep git commit: Created a class to track execution statistics
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 9cbb930b5 -> 1b2698d22
Created a class to track execution statistics
- Stats are maintained for active operators in the query.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1b2698d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1b2698d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1b2698d2
Branch: refs/heads/master
Commit: 1b2698d2225bfab59fb675da5f92a2285dd5650c
Parents: 9cbb930
Author: Harshad Deshmukh
Authored: Wed Sep 27 14:55:16 2017 -0500
Committer: Harshad Deshmukh
Committed: Fri Sep 29 12:10:52 2017 -0500
--
query_execution/CMakeLists.txt | 5 +
query_execution/ExecutionStats.hpp | 211
2 files changed, 216 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b2698d2/query_execution/CMakeLists.txt
--
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 9394c00..8f797f7 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -29,6 +29,7 @@ if (ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp
BlockLocator.hpp)
add_library(quickstep_queryexecution_BlockLocatorUtil BlockLocatorUtil.cpp
BlockLocatorUtil.hpp)
endif(ENABLE_DISTRIBUTED)
+add_library(quickstep_queryexecution_ExecutionStats ../empty_src.cpp
ExecutionStats.hpp)
add_library(quickstep_queryexecution_ForemanBase ../empty_src.cpp
ForemanBase.hpp)
if (ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_ForemanDistributed
ForemanDistributed.cpp ForemanDistributed.hpp)
@@ -123,6 +124,9 @@ if (ENABLE_DISTRIBUTED)
tmb
${GFLAGS_LIB_NAME})
endif(ENABLE_DISTRIBUTED)
+target_link_libraries(quickstep_queryexecution_ExecutionStats
+ glog
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryexecution_ForemanSingleNode
glog
quickstep_queryexecution_AdmitRequestMessage
@@ -363,6 +367,7 @@
target_link_libraries(quickstep_queryexecution_WorkerSelectionPolicy
add_library(quickstep_queryexecution ../empty_src.cpp QueryExecutionModule.hpp)
target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_AdmitRequestMessage
+ quickstep_queryexecution_ExecutionStats
quickstep_queryexecution_ForemanBase
quickstep_queryexecution_ForemanSingleNode
quickstep_queryexecution_PolicyEnforcerBase
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b2698d2/query_execution/ExecutionStats.hpp
--
diff --git a/query_execution/ExecutionStats.hpp
b/query_execution/ExecutionStats.hpp
new file mode 100644
index 000..8d19651
--- /dev/null
+++ b/query_execution/ExecutionStats.hpp
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_EXECUTION_STATS_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_EXECUTION_STATS_HPP_
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief Record the execution stats of a query.
+ *
+ * @note The time is measured in microseconds.
+ **/
+class ExecutionStats {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param max_entries The maximum number of entries we remember for each
+ *operator.
+ **/
+ explicit ExecutionStats(const std::size_t max_entries)
+ : max_entries_(max_entries) {}
+
+ /**
+ * @brief Get the number of active operators in stats.
+ **/
+ std::size_t getNumActiveOperator
incubator-quickstep git commit: Bug fix in LockManager loop
Repository: incubator-quickstep
Updated Branches:
refs/heads/master bf455e26e -> 9cbb930b5
Bug fix in LockManager loop
- Added a false condition for acquire lock
- Added clarifying comment.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9cbb930b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9cbb930b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9cbb930b
Branch: refs/heads/master
Commit: 9cbb930b5adb589f7a2ba8140d8d59227c9a570e
Parents: bf455e2
Author: Harshad Deshmukh
Authored: Wed Sep 27 10:02:10 2017 -0500
Committer: Harshad Deshmukh
Committed: Fri Sep 29 10:42:41 2017 -0500
--
transaction/LockManager.cpp | 15 ---
1 file changed, 8 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9cbb930b/transaction/LockManager.cpp
--
diff --git a/transaction/LockManager.cpp b/transaction/LockManager.cpp
index 2a3760f..c917b4b 100644
--- a/transaction/LockManager.cpp
+++ b/transaction/LockManager.cpp
@@ -80,21 +80,22 @@ void LockManager::run() {
if (request.getRequestType() == RequestType::kReleaseLocks) {
CHECK(releaseAllLocks(request.getTransactionId()))
<< "Unexpected condition occured.";
-
} else if (acquireLock(request.getTransactionId(),
request.getResourceId(),
request.getAccessMode())) {
+ // Lock has been acquired.
LOG(INFO) << "Transaction "
<< std::to_string(request.getTransactionId())
-<< " is waiting " + request.getResourceId().toString();
+<< " acquired " + request.getResourceId().toString();
-inner_pending_requests_.push(request);
+ permitted_requests_.push(request);
} else {
-LOG(INFO) << "Transaction "
- << std::to_string(request.getTransactionId())
- << " acquired " + request.getResourceId().toString();
+ // We are unable to acquire lock at this point.
+ LOG(INFO) << "Transaction "
+<< std::to_string(request.getTransactionId())
+<< " is waiting " + request.getResourceId().toString();
-permitted_requests_.push(request);
+ inner_pending_requests_.push(request);
}
}
}
incubator-quickstep git commit: Simplified the work order generation.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 77960a42d -> 8d7284dec
Simplified the work order generation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8d7284de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8d7284de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8d7284de
Branch: refs/heads/master
Commit: 8d7284decb7ebf5c0eaac232f39027ddd8bf6144
Parents: 77960a4
Author: Zuyu Zhang
Authored: Mon Aug 21 19:51:55 2017 -0500
Committer: Zuyu Zhang
Committed: Fri Sep 22 13:43:08 2017 -0500
--
query_execution/CMakeLists.txt | 2 -
query_execution/ForemanDistributed.cpp | 5 +-
query_execution/ForemanSingleNode.cpp | 16 +--
query_execution/QueryManagerBase.cpp| 136 ---
query_execution/QueryManagerBase.hpp| 79 ++-
query_execution/QueryManagerDistributed.cpp | 54 +++-
query_execution/QueryManagerDistributed.hpp | 3 +-
query_execution/QueryManagerSingleNode.cpp | 58
query_execution/QueryManagerSingleNode.hpp | 7 +-
query_execution/WorkOrdersContainer.hpp | 1 +
.../tests/QueryManagerSingleNode_unittest.cpp | 58
11 files changed, 152 insertions(+), 267 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8d7284de/query_execution/CMakeLists.txt
--
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 5c750f0..9394c00 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -119,7 +119,6 @@ if (ENABLE_DISTRIBUTED)
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageManager
quickstep_threading_ThreadUtil
-quickstep_utility_EqualsAnyConstant
quickstep_utility_Macros
tmb
${GFLAGS_LIB_NAME})
@@ -135,7 +134,6 @@
target_link_libraries(quickstep_queryexecution_ForemanSingleNode
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
quickstep_threading_ThreadUtil
- quickstep_utility_EqualsAnyConstant
quickstep_utility_Macros
tmb
${GFLAGS_LIB_NAME})
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8d7284de/query_execution/ForemanDistributed.cpp
--
diff --git a/query_execution/ForemanDistributed.cpp
b/query_execution/ForemanDistributed.cpp
index 942f383..82cc624 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -48,7 +48,6 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageManager.hpp"
#include "threading/ThreadUtil.hpp"
-#include "utility/EqualsAnyConstant.hpp"
#include "glog/logging.h"
@@ -233,9 +232,7 @@ void ForemanDistributed::run() {
}
bool ForemanDistributed::canCollectNewMessages(const tmb::message_type_id
message_type) {
- return !QUICKSTEP_EQUALS_ANY_CONSTANT(message_type,
-kCatalogRelationNewBlockMessage,
-kWorkOrderFeedbackMessage);
+ return message_type != kCatalogRelationNewBlockMessage;
}
bool ForemanDistributed::isAggregationRelatedWorkOrder(const
S::WorkOrderMessage &proto,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8d7284de/query_execution/ForemanSingleNode.cpp
--
diff --git a/query_execution/ForemanSingleNode.cpp
b/query_execution/ForemanSingleNode.cpp
index 1501408..d66f1f5 100644
--- a/query_execution/ForemanSingleNode.cpp
+++ b/query_execution/ForemanSingleNode.cpp
@@ -33,7 +33,6 @@
#include "query_execution/WorkerDirectory.hpp"
#include "query_execution/WorkerMessage.hpp"
#include "threading/ThreadUtil.hpp"
-#include "utility/EqualsAnyConstant.hpp"
#include "utility/Macros.hpp"
#include "gflags/gflags.h"
@@ -179,18 +178,13 @@ void ForemanSingleNode::run() {
}
bool ForemanSingleNode::canCollectNewMessages(const tmb::message_type_id
message_type) {
- if (QUICKSTEP_EQUALS_ANY_CONSTANT(message_type,
-kCatalogRelationNewBlockMessage,
-kWorkOrderFeedbackMessage)) {
-return false;
- } else if (worker_directory_->getLeastLoadedWorker().second <=
- FLAGS_min_load_per_worker) {
-// If the least loaded
incubator-quickstep git commit: Fix a bug in LineReader for recognizing command
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 71aa8d265 -> 77960a42d
Fix a bug in LineReader for recognizing command
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/77960a42
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/77960a42
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/77960a42
Branch: refs/heads/master
Commit: 77960a42dcfb3d27de5601548a04d81a6be79375
Parents: 71aa8d2
Author: Jianqiao Zhu
Authored: Tue Sep 19 22:02:02 2017 -0500
Committer: Jianqiao Zhu
Committed: Tue Sep 19 23:00:20 2017 -0500
--
cli/CMakeLists.txt | 2 ++
cli/LineReader.cpp | 11 +--
cli/tests/command_executor/D.test | 23 +++
cli/tests/command_executor/Dt.test | 4
4 files changed, 38 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/77960a42/cli/CMakeLists.txt
--
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 33d10e3..03c5408 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -189,10 +189,12 @@ if(QUICKSTEP_HAVE_LIBNUMA)
endif()
if(USE_LINENOISE)
target_link_libraries(quickstep_cli_LineReader
+glog
linenoise
quickstep_utility_Macros)
else()
target_link_libraries(quickstep_cli_LineReader
+glog
quickstep_utility_Macros)
endif()
target_link_libraries(quickstep_cli_LineReaderBuffered
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/77960a42/cli/LineReader.cpp
--
diff --git a/cli/LineReader.cpp b/cli/LineReader.cpp
index 002727d..1a23dd3 100644
--- a/cli/LineReader.cpp
+++ b/cli/LineReader.cpp
@@ -23,6 +23,8 @@
#include
#include
+#include "glog/logging.h"
+
using std::ispunct;
using std::size_t;
using std::string;
@@ -171,7 +173,7 @@ std::string LineReader::getNextCommand() {
case '.':
case '\\': // Fall Through.
// If the dot or forward slash begins the line, begin a command
search.
- if (scan_position == 0) {
+ if (special_char_location ==
multiline_buffer.find_first_not_of(" \t\r\n")) {
line_state = kCommand;
} else {
// This is a regular character, so skip over it.
@@ -217,7 +219,12 @@ std::string LineReader::getNextCommand() {
if (std::all_of(leftover_.begin(), leftover_.end(), ::isspace)) {
leftover_.clear();
}
-return multiline_buffer.substr(0, special_char_location + 1);
+// Skip all the whitespaces before the command.
+const std::size_t start_position =
+multiline_buffer.find_first_not_of(" \t\r\n");
+DCHECK_LT(start_position, special_char_location + 1);
+return multiline_buffer.substr(start_position,
+ special_char_location + 1 -
start_position);
}
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/77960a42/cli/tests/command_executor/D.test
--
diff --git a/cli/tests/command_executor/D.test
b/cli/tests/command_executor/D.test
index 36e9a92..c3564a6 100644
--- a/cli/tests/command_executor/D.test
+++ b/cli/tests/command_executor/D.test
@@ -58,6 +58,8 @@ INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
==
+
+
\d foo
--
Table "foo"
@@ -69,6 +71,7 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
col4 | Float
col5 | Char(5)
==
+
\d foo2
--
Table "foo2"
@@ -80,6 +83,7 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
col4 | Float
averyverylongcolumnnamefortest | Char(5)
==
+
\d foo3
--
Table "foo3"
@@ -93,6 +97,7 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
Indexes
"foo3_index_1" CSB_TREE (col1)
==
+
\d foo4
--
Table "foo4"
@@ -107,6 +112,7 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
"foo4_index_2" CSB_TREE (col3, col4)
"foo4_index_1" CSB_TREE (col1, col2)
==
+
\d foo_hash_part
--
Table "foo_hash_part"
@@ -118,6 +124,7 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
PARTITION BY HASH ( col1 ) PARTITIONS 4
| 1 | 1 | 1 | 1 |
==
+
\d
--
List of relations
@@ -132,6 +139,22 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
averylongtablenamethatseemstoneverend | table | 1
==
+
+\d
+--
+ List of relations
+
+ Name
incubator-quickstep git commit: Printed out the partition info in QueryPlan.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master d85f7a9a3 -> 475704ec9
Printed out the partition info in QueryPlan.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/475704ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/475704ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/475704ec
Branch: refs/heads/master
Commit: 475704ec9510793a70e149b938d02569611c6177
Parents: d85f7a9
Author: Zuyu Zhang
Authored: Mon Sep 11 11:57:10 2017 -0500
Committer: Zuyu Zhang
Committed: Thu Sep 14 13:15:10 2017 -0500
--
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/OptimizerTree.hpp | 39
query_optimizer/logical/CreateTable.cpp | 5 +++
query_optimizer/logical/CreateTable.hpp | 20 ++
query_optimizer/physical/CreateTable.cpp| 5 +++
query_optimizer/physical/CreateTable.hpp| 20 ++
.../tests/logical_generator/Create.test | 15 +++-
.../tests/physical_generator/Create.test| 26 +
query_optimizer/tests/resolver/Create.test | 17 +++--
9 files changed, 128 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 4ea21b2..5e0db44 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -212,6 +212,7 @@
target_link_libraries(quickstep_queryoptimizer_OptimizerContext
quickstep_queryoptimizer_expressions_ExprId
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_OptimizerTree
+ quickstep_catalog_Catalog_proto
quickstep_storage_StorageBlockLayout_proto
quickstep_utility_Macros
quickstep_utility_TreeStringSerializable)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/OptimizerTree.hpp
--
diff --git a/query_optimizer/OptimizerTree.hpp
b/query_optimizer/OptimizerTree.hpp
index 62df66d..c54ce20 100644
--- a/query_optimizer/OptimizerTree.hpp
+++ b/query_optimizer/OptimizerTree.hpp
@@ -25,6 +25,7 @@
#include
#include
+#include "catalog/Catalog.pb.h"
#include "storage/StorageBlockLayout.pb.h"
#include "utility/Macros.hpp"
#include "utility/TreeStringSerializable.hpp"
@@ -283,6 +284,44 @@ OptimizerProtoRepresentation*
getOptimizerRepresentationForProto(
return node.release();
}
+template
+OptimizerProtoRepresentation* getOptimizerRepresentationForProto(
+const serialization::PartitionSchemeHeader *partition_header) {
+ if (partition_header == nullptr) {
+return nullptr;
+ }
+
+ auto node = std::make_unique>();
+
+ // Add properties based on the partition type.
+ switch (partition_header->partition_type()) {
+case serialization::PartitionSchemeHeader::HASH: {
+ node->addProperty("partition_type", "hash");
+ break;
+}
+case serialization::PartitionSchemeHeader::RANDOM: {
+ node->addProperty("partition_type", "random");
+ break;
+}
+case serialization::PartitionSchemeHeader::RANGE: {
+ node->addProperty("partition_type", "range");
+ // TODO(quickstep-team): display the range boundaries.
+ node->addProperty("range_boundaries", "TODO");
+ break;
+}
+default:
+ LOG(FATAL) << "Unrecognized partition type in protobuf message.";
+ }
+ // Every case will specify a partition number and a partition attributes.
+ node->addProperty("num_partitions", partition_header->num_partitions());
+
+ for (int i = 0; i < partition_header->partition_attribute_ids_size(); ++i) {
+node->addProperty("partition_attr_id",
partition_header->partition_attribute_ids(i));
+ }
+
+ return node.release();
+}
+
/** @} */
} // namespace optimizer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/logical/CreateTable.cpp
--
diff --git a/query_optimizer/logical/CreateTable.cpp
b/query_optimizer/logical/CreateTable.cpp
index 111d04b..9977a0f 100644
--- a/query_optimizer/logical/CreateTable.cpp
+++ b/query_optimizer/logical/CreateTable.cpp
@@ -49,6 +49,11 @@ void CreateTable::getFieldStringItems(
non_container_child_field_names->push_back("block_properties");
non_container_child_fields->push_back(block_properties_representation_);
}
+
+ if (partition_scheme_header_proto_repr
incubator-quickstep git commit: Crash if block not found in the local storage in the single node.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master c4f7614dc -> b1c3356a1
Crash if block not found in the local storage in the single node.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b1c3356a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b1c3356a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b1c3356a
Branch: refs/heads/master
Commit: b1c3356a19efef985ea87ae5e127f4ed2dd40ad7
Parents: c4f7614
Author: Zuyu Zhang
Authored: Tue Sep 12 20:29:39 2017 -0500
Committer: Zuyu Zhang
Committed: Tue Sep 12 20:29:39 2017 -0500
--
storage/StorageManager.cpp | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b1c3356a/storage/StorageManager.cpp
--
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 7eb132c..5924607 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -683,8 +683,6 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
loaded_handle.block_memory = block_buffer;
loaded_handle.block_memory_size = num_slots;
} else {
-bool pull_succeeded = false;
-
#ifdef QUICKSTEP_DISTRIBUTED
const string domain_network_address =
getPeerDomainNetworkAddress(BlockIdUtil::Domain(block));
DLOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from "
<< domain_network_address;
@@ -693,11 +691,9 @@ StorageManager::BlockHandle
StorageManager::loadBlockOrBlob(
while (!client.Pull(block, numa_node, &loaded_handle)) {
LOG(INFO) << "Retry pulling Block " << BlockIdUtil::ToString(block) << "
from " << domain_network_address;
}
-
-pull_succeeded = true;
+#else
+LOG(FATAL) << "Block not found from persistent storage: " << block;
#endif
-
-CHECK(pull_succeeded) << "Failed to pull Block " <<
BlockIdUtil::ToString(block) << " from remote peers.";
}
#ifdef QUICKSTEP_DISTRIBUTED
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/redirect-stream [deleted] b815b388d
incubator-quickstep git commit: Redirect stdout and stderr in network mode.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 4578c63f0 -> b815b388d
Redirect stdout and stderr in network mode.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b815b388
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b815b388
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b815b388
Branch: refs/heads/master
Commit: b815b388d4fb501c4454a1b4e9560a92e4fbb469
Parents: 4578c63
Author: Jianqiao Zhu
Authored: Fri Sep 1 12:07:13 2017 -0500
Committer: Jianqiao Zhu
Committed: Mon Sep 11 16:15:38 2017 -0500
--
CMakeLists.txt | 1 +
cli/Flags.cpp | 3 +
cli/Flags.hpp | 2 +
cli/IOInterface.hpp | 11 +-
cli/NetworkCliClientMain.cpp| 1 -
cli/QuickstepCli.cpp| 15 +-
.../tests/ExecutionGeneratorTestRunner.cpp | 4 +-
utility/CMakeLists.txt | 13 ++
utility/ScopedReassignment.hpp | 81
utility/tests/ScopedReassignment_unittest.cpp | 183 +++
10 files changed, 304 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ed80fab..e0d020b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -751,6 +751,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Macros
quickstep_utility_PtrVector
+ quickstep_utility_ScopedReassignment
quickstep_utility_SqlError
quickstep_utility_StringUtil)
if (ENABLE_HDFS)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/cli/Flags.cpp
--
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
index 362eac3..279c503 100644
--- a/cli/Flags.cpp
+++ b/cli/Flags.cpp
@@ -36,6 +36,9 @@ DEFINE_bool(print_query, false,
"Print each input query statement. This is useful when running a "
"large number of queries in a batch.");
+DEFINE_bool(display_timing, true,
+"Whether to display execution time of each statement.");
+
DEFINE_bool(initialize_db, false, "If true, initialize a database.");
static bool ValidateNumWorkers(const char *flagname, int value) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/cli/Flags.hpp
--
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
index 1ae37c4..614cca5 100644
--- a/cli/Flags.hpp
+++ b/cli/Flags.hpp
@@ -45,6 +45,8 @@ DECLARE_string(storage_path);
DECLARE_bool(preload_buffer_pool);
+DECLARE_bool(display_timing);
+
/** @} */
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/cli/IOInterface.hpp
--
diff --git a/cli/IOInterface.hpp b/cli/IOInterface.hpp
index 815596e..dc0d5b2 100644
--- a/cli/IOInterface.hpp
+++ b/cli/IOInterface.hpp
@@ -28,7 +28,7 @@
namespace quickstep {
/**
- * An individual IO interaction with Quickstep.
+ * @brief An individual IO interaction with Quickstep.
*/
class IOHandle {
public:
@@ -56,18 +56,21 @@ class IOHandle {
};
/**
- * Virtual base defines a generic, file-based interface around IO. One IO
interaction (eg a SQL query) will be assigned
- * an IOHandle. On destruction of the IOHandle, the IO interaction has
finished.
+ * @brief Virtual base defines a generic, file-based interface around IO.
+ *One IO interaction (eg a SQL query) will be assigned an IOHandle.
+ *On destruction of the IOHandle, the IO interaction has finished.
*/
class IOInterface {
public:
/**
- * @note Destructing the IOInterface should close any outstanding IO state
(eg an open port).
+ * @note Destructing the IOInterface should close any outstanding IO state
+ * (e.g. an open port).
*/
virtual ~IOInterface() {}
/**
* @brief Retrieves the next IOHandle. Blocks if no IO ready.
+ *
* @return An IOHandle.
*/
virtual IOHandle* getNextIOHandle() = 0;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/cli/NetworkCliClientMain.cpp
--
diff --git a/cli/NetworkCliClientMain.cpp b/cli/NetworkCliClientMain.cpp
index 862941c..c55819b 100644
--- a/cli/Ne
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/copy-to [deleted] 298165115
[5/6] incubator-quickstep git commit: Add "COPY TO" operator for exporting data from Quickstep.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29816511/parser/preprocessed/SqlLexer_gen.cpp -- diff --git a/parser/preprocessed/SqlLexer_gen.cpp b/parser/preprocessed/SqlLexer_gen.cpp index bd77ee1..4800cde 100644 --- a/parser/preprocessed/SqlLexer_gen.cpp +++ b/parser/preprocessed/SqlLexer_gen.cpp @@ -9,88 +9,244 @@ #define FLEX_SCANNER #define YY_FLEX_MAJOR_VERSION 2 #define YY_FLEX_MINOR_VERSION 6 -#define YY_FLEX_SUBMINOR_VERSION 3 +#define YY_FLEX_SUBMINOR_VERSION 4 #if YY_FLEX_SUBMINOR_VERSION > 0 #define FLEX_BETA #endif -#define yy_create_buffer quickstep_yy_create_buffer +#ifdef yy_create_buffer +#define quickstep_yy_create_buffer_ALREADY_DEFINED +#else +#define yy_create_buffer quickstep_yy_create_buffer +#endif -#define yy_delete_buffer quickstep_yy_delete_buffer +#ifdef yy_delete_buffer +#define quickstep_yy_delete_buffer_ALREADY_DEFINED +#else +#define yy_delete_buffer quickstep_yy_delete_buffer +#endif -#define yy_scan_buffer quickstep_yy_scan_buffer +#ifdef yy_scan_buffer +#define quickstep_yy_scan_buffer_ALREADY_DEFINED +#else +#define yy_scan_buffer quickstep_yy_scan_buffer +#endif -#define yy_scan_string quickstep_yy_scan_string +#ifdef yy_scan_string +#define quickstep_yy_scan_string_ALREADY_DEFINED +#else +#define yy_scan_string quickstep_yy_scan_string +#endif -#define yy_scan_bytes quickstep_yy_scan_bytes +#ifdef yy_scan_bytes +#define quickstep_yy_scan_bytes_ALREADY_DEFINED +#else +#define yy_scan_bytes quickstep_yy_scan_bytes +#endif -#define yy_init_buffer quickstep_yy_init_buffer +#ifdef yy_init_buffer +#define quickstep_yy_init_buffer_ALREADY_DEFINED +#else +#define yy_init_buffer quickstep_yy_init_buffer +#endif -#define yy_flush_buffer quickstep_yy_flush_buffer +#ifdef yy_flush_buffer +#define quickstep_yy_flush_buffer_ALREADY_DEFINED +#else +#define yy_flush_buffer quickstep_yy_flush_buffer +#endif -#define yy_load_buffer_state quickstep_yy_load_buffer_state +#ifdef yy_load_buffer_state +#define quickstep_yy_load_buffer_state_ALREADY_DEFINED +#else +#define yy_load_buffer_state quickstep_yy_load_buffer_state +#endif -#define yy_switch_to_buffer quickstep_yy_switch_to_buffer +#ifdef yy_switch_to_buffer +#define quickstep_yy_switch_to_buffer_ALREADY_DEFINED +#else +#define yy_switch_to_buffer quickstep_yy_switch_to_buffer +#endif -#define yypush_buffer_state quickstep_yypush_buffer_state +#ifdef yypush_buffer_state +#define quickstep_yypush_buffer_state_ALREADY_DEFINED +#else +#define yypush_buffer_state quickstep_yypush_buffer_state +#endif -#define yypop_buffer_state quickstep_yypop_buffer_state +#ifdef yypop_buffer_state +#define quickstep_yypop_buffer_state_ALREADY_DEFINED +#else +#define yypop_buffer_state quickstep_yypop_buffer_state +#endif -#define yyensure_buffer_stack quickstep_yyensure_buffer_stack +#ifdef yyensure_buffer_stack +#define quickstep_yyensure_buffer_stack_ALREADY_DEFINED +#else +#define yyensure_buffer_stack quickstep_yyensure_buffer_stack +#endif -#define yylex quickstep_yylex +#ifdef yylex +#define quickstep_yylex_ALREADY_DEFINED +#else +#define yylex quickstep_yylex +#endif -#define yyrestart quickstep_yyrestart +#ifdef yyrestart +#define quickstep_yyrestart_ALREADY_DEFINED +#else +#define yyrestart quickstep_yyrestart +#endif -#define yylex_init quickstep_yylex_init +#ifdef yylex_init +#define quickstep_yylex_init_ALREADY_DEFINED +#else +#define yylex_init quickstep_yylex_init +#endif -#define yylex_init_extra quickstep_yylex_init_extra +#ifdef yylex_init_extra +#define quickstep_yylex_init_extra_ALREADY_DEFINED +#else +#define yylex_init_extra quickstep_yylex_init_extra +#endif -#define yylex_destroy quickstep_yylex_destroy +#ifdef yylex_destroy +#define quickstep_yylex_destroy_ALREADY_DEFINED +#else +#define yylex_destroy quickstep_yylex_destroy +#endif -#define yyget_debug quickstep_yyget_debug +#ifdef yyget_debug +#define quickstep_yyget_debug_ALREADY_DEFINED +#else +#define yyget_debug quickstep_yyget_debug +#endif -#define yyset_debug quickstep_yyset_debug +#ifdef yyset_debug +#define quickstep_yyset_debug_ALREADY_DEFINED +#else +#define yyset_debug quickstep_yyset_debug +#endif -#define yyget_extra quickstep_yyget_extra +#ifdef yyget_extra +#define quickstep_yyget_extra_ALREADY_DEFINED +#else +#define yyget_extra quickstep_yyget_extra +#endif -#define yyset_extra quickstep_yyset_extra +#ifdef yyset_extra +#define quickstep_yyset_extra_ALREADY_DEFINED +#else +#define yyset_extra quickstep_yyset_extra +#endif -#define yyget_in quickstep_yyget_in +#ifdef yyget_in +#define quickstep_yyget_in_ALREADY_DEFINED +#else +#define yyget_in quickstep_yyget_in +#endif -#define yyset_in quickstep_yyset_in +#ifdef yyset_in +#define quickstep_yyset_in_ALREADY_DEFINED +#else +#define yyset_in quickstep_yyset_in +#endif -#define y
[2/6] incubator-quickstep git commit: Add "COPY TO" operator for exporting data from Quickstep.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29816511/parser/preprocessed/SqlParser_gen.hpp -- diff --git a/parser/preprocessed/SqlParser_gen.hpp b/parser/preprocessed/SqlParser_gen.hpp index a6d12e2..f6b5247 100644 --- a/parser/preprocessed/SqlParser_gen.hpp +++ b/parser/preprocessed/SqlParser_gen.hpp @@ -94,90 +94,91 @@ extern int quickstep_yydebug; TOKEN_DECIMAL = 304, TOKEN_DEFAULT = 305, TOKEN_DELETE = 306, -TOKEN_DELIMITER = 307, -TOKEN_DESC = 308, -TOKEN_DISTINCT = 309, -TOKEN_DOUBLE = 310, -TOKEN_DROP = 311, -TOKEN_ELSE = 312, -TOKEN_END = 313, -TOKEN_ESCAPE_STRINGS = 314, -TOKEN_EXISTS = 315, -TOKEN_EXTRACT = 316, -TOKEN_FALSE = 317, -TOKEN_FIRST = 318, -TOKEN_FLOAT = 319, -TOKEN_FOLLOWING = 320, -TOKEN_FOR = 321, -TOKEN_FOREIGN = 322, -TOKEN_FROM = 323, -TOKEN_FULL = 324, -TOKEN_GROUP = 325, -TOKEN_HASH = 326, -TOKEN_HAVING = 327, -TOKEN_HOUR = 328, -TOKEN_IN = 329, -TOKEN_INDEX = 330, -TOKEN_INNER = 331, -TOKEN_INSERT = 332, -TOKEN_INTEGER = 333, -TOKEN_INTERVAL = 334, -TOKEN_INTO = 335, -TOKEN_JOIN = 336, -TOKEN_KEY = 337, -TOKEN_LAST = 338, -TOKEN_LEFT = 339, -TOKEN_LIMIT = 340, -TOKEN_LONG = 341, -TOKEN_MINUTE = 342, -TOKEN_MONTH = 343, -TOKEN_NULL = 344, -TOKEN_NULLS = 345, -TOKEN_OFF = 346, -TOKEN_ON = 347, -TOKEN_ORDER = 348, -TOKEN_OUTER = 349, -TOKEN_OVER = 350, -TOKEN_PARTITION = 351, -TOKEN_PARTITIONS = 352, -TOKEN_PERCENT = 353, -TOKEN_PRECEDING = 354, -TOKEN_PRIMARY = 355, -TOKEN_PRIORITY = 356, -TOKEN_QUIT = 357, -TOKEN_RANGE = 358, -TOKEN_REAL = 359, -TOKEN_REFERENCES = 360, -TOKEN_RIGHT = 361, -TOKEN_ROW = 362, -TOKEN_ROW_DELIMITER = 363, -TOKEN_ROWS = 364, -TOKEN_SECOND = 365, -TOKEN_SELECT = 366, -TOKEN_SET = 367, -TOKEN_SMA = 368, -TOKEN_SMALLINT = 369, +TOKEN_DESC = 307, +TOKEN_DISTINCT = 308, +TOKEN_DOUBLE = 309, +TOKEN_DROP = 310, +TOKEN_ELSE = 311, +TOKEN_END = 312, +TOKEN_EXISTS = 313, +TOKEN_EXTRACT = 314, +TOKEN_FALSE = 315, +TOKEN_FIRST = 316, +TOKEN_FLOAT = 317, +TOKEN_FOLLOWING = 318, +TOKEN_FOR = 319, +TOKEN_FOREIGN = 320, +TOKEN_FROM = 321, +TOKEN_FULL = 322, +TOKEN_GROUP = 323, +TOKEN_HASH = 324, +TOKEN_HAVING = 325, +TOKEN_HOUR = 326, +TOKEN_IN = 327, +TOKEN_INDEX = 328, +TOKEN_INNER = 329, +TOKEN_INSERT = 330, +TOKEN_INTEGER = 331, +TOKEN_INTERVAL = 332, +TOKEN_INTO = 333, +TOKEN_JOIN = 334, +TOKEN_KEY = 335, +TOKEN_LAST = 336, +TOKEN_LEFT = 337, +TOKEN_LIMIT = 338, +TOKEN_LONG = 339, +TOKEN_MINUTE = 340, +TOKEN_MONTH = 341, +TOKEN_NULL = 342, +TOKEN_NULLS = 343, +TOKEN_OFF = 344, +TOKEN_ON = 345, +TOKEN_ORDER = 346, +TOKEN_OUTER = 347, +TOKEN_OVER = 348, +TOKEN_PARTITION = 349, +TOKEN_PARTITIONS = 350, +TOKEN_PERCENT = 351, +TOKEN_PRECEDING = 352, +TOKEN_PRIMARY = 353, +TOKEN_PRIORITY = 354, +TOKEN_QUIT = 355, +TOKEN_RANGE = 356, +TOKEN_REAL = 357, +TOKEN_REFERENCES = 358, +TOKEN_RIGHT = 359, +TOKEN_ROW = 360, +TOKEN_ROW_DELIMITER = 361, +TOKEN_ROWS = 362, +TOKEN_SECOND = 363, +TOKEN_SELECT = 364, +TOKEN_SET = 365, +TOKEN_SMA = 366, +TOKEN_SMALLINT = 367, +TOKEN_STDERR = 368, +TOKEN_STDOUT = 369, TOKEN_SUBSTRING = 370, TOKEN_TABLE = 371, TOKEN_THEN = 372, TOKEN_TIME = 373, TOKEN_TIMESTAMP = 374, -TOKEN_TRUE = 375, -TOKEN_TUPLESAMPLE = 376, -TOKEN_UNBOUNDED = 377, -TOKEN_UNIQUE = 378, -TOKEN_UPDATE = 379, -TOKEN_USING = 380, -TOKEN_VALUES = 381, -TOKEN_VARCHAR = 382, -TOKEN_WHEN = 383, -TOKEN_WHERE = 384, -TOKEN_WINDOW = 385, -TOKEN_WITH = 386, -TOKEN_YEAR = 387, -TOKEN_YEARMONTH = 388, -TOKEN_EOF = 389, -TOKEN_LEX_ERROR = 390 +TOKEN_TO = 375, +TOKEN_TRUE = 376, +TOKEN_TUPLESAMPLE = 377, +TOKEN_UNBOUNDED = 378, +TOKEN_UNIQUE = 379, +TOKEN_UPDATE = 380, +TOKEN_USING = 381, +TOKEN_VALUES = 382, +TOKEN_VARCHAR = 383, +TOKEN_WHEN = 384, +TOKEN_WHERE = 385, +TOKEN_WINDOW = 386, +TOKEN_WITH = 387, +TOKEN_YEAR = 388, +TOKEN_YEARMONTH = 389, +TOKEN_EOF = 390, +TOKEN_LEX_ERROR = 391 }; #endif @@ -237,8 +238,7 @@ union YYSTYPE quickstep::ParseKeyStringValue *key_string_value_; quickstep::ParseKeyStringList *key_string_list_; quickstep::ParseKeyIntegerValue *key_integer_value_; - - quickstep::ParseCopyFromParams *copy_from_params_; + quickstep::ParseKeyBoolValue *key_bool_value_; quickstep::ParseAssignment *assignment_; quickstep::PtrList *assignment_list_; @@ -251,7 +251,7 @@ union YYSTYPE quicks
[6/6] incubator-quickstep git commit: Add "COPY TO" operator for exporting data from Quickstep.
Add "COPY TO" operator for exporting data from Quickstep.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/29816511
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/29816511
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/29816511
Branch: refs/heads/master
Commit: 2981651159d36de1322a7bb6452d7ffe6029563b
Parents: 877cae0
Author: Jianqiao Zhu
Authored: Fri Aug 4 16:49:45 2017 -0500
Committer: Jianqiao
Committed: Wed Aug 30 17:35:48 2017 -0500
--
parser/ParseKeyValue.hpp| 83 +-
parser/ParseStatement.hpp | 237 +-
parser/SqlLexer.lpp |9 +-
parser/SqlParser.ypp| 93 +-
parser/preprocessed/SqlLexer_gen.cpp| 1889 -
parser/preprocessed/SqlLexer_gen.hpp| 467 ++-
parser/preprocessed/SqlParser_gen.cpp | 3633 +-
parser/preprocessed/SqlParser_gen.hpp | 164 +-
parser/tests/Copy.test | 119 +-
query_optimizer/CMakeLists.txt |2 +
query_optimizer/ExecutionGenerator.cpp | 42 +-
query_optimizer/ExecutionGenerator.hpp |8 +
query_optimizer/logical/CMakeLists.txt | 12 +
query_optimizer/logical/CopyFrom.cpp|6 +-
query_optimizer/logical/CopyFrom.hpp| 43 +-
query_optimizer/logical/CopyTo.cpp | 75 +
query_optimizer/logical/CopyTo.hpp | 141 +
query_optimizer/logical/LogicalType.hpp |3 +-
query_optimizer/physical/CMakeLists.txt | 15 +-
query_optimizer/physical/CopyFrom.cpp |6 +-
query_optimizer/physical/CopyFrom.hpp | 41 +-
query_optimizer/physical/CopyTo.cpp | 75 +
query_optimizer/physical/CopyTo.hpp | 147 +
query_optimizer/physical/PhysicalType.hpp |3 +-
query_optimizer/resolver/CMakeLists.txt |2 +
query_optimizer/resolver/Resolver.cpp | 276 +-
query_optimizer/resolver/Resolver.hpp | 21 +-
query_optimizer/strategy/CMakeLists.txt |2 +
query_optimizer/strategy/OneToOne.cpp | 27 +-
.../tests/ExecutionGeneratorTestRunner.cpp |5 +-
.../tests/execution_generator/CMakeLists.txt|6 +
.../tests/execution_generator/Copy.test | 127 +
.../tests/physical_generator/Copy.test |4 +-
query_optimizer/tests/resolver/Copy.test| 164 +-
relational_operators/CMakeLists.txt | 23 +
relational_operators/RelationalOperator.hpp |1 +
relational_operators/TableExportOperator.cpp| 336 ++
relational_operators/TableExportOperator.hpp| 267 ++
relational_operators/TextScanOperator.cpp | 70 +-
relational_operators/TextScanOperator.hpp | 18 +-
relational_operators/WorkOrder.proto|3 +-
.../tests/TextScanOperator_unittest.cpp |9 +-
utility/BulkIoConfiguration.cpp | 50 +
utility/BulkIoConfiguration.hpp | 198 +
utility/CMakeLists.txt |4 +
utility/ExecutionDAGVisualizer.cpp |9 +-
46 files changed, 5792 insertions(+), 3143 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29816511/parser/ParseKeyValue.hpp
--
diff --git a/parser/ParseKeyValue.hpp b/parser/ParseKeyValue.hpp
index 9d6d511..6134379 100644
--- a/parser/ParseKeyValue.hpp
+++ b/parser/ParseKeyValue.hpp
@@ -37,14 +37,15 @@ namespace quickstep {
*/
class ParseKeyValue : public ParseTreeNode {
public:
- enum class KeyValueType {
+ enum KeyValueType {
+kStringBool = 0,
+kStringInteger,
kStringString,
-kStringStringList,
-kStringInteger
+kStringStringList
};
/**
- * @brief Single value constructor.
+ * @brief Constructor.
*
* @param line_number Line number of the first token of this node in the SQL
statement.
* @param column_number Column number of the first token of this node in the
SQL statement.
@@ -87,7 +88,7 @@ class ParseKeyValue : public ParseTreeNode {
class ParseKeyStringValue : public ParseKeyValue {
public:
/**
- * @brief Single value constructor.
+ * @brief Constructor.
*
* @param line_number Line number of the first token of this node in the SQL
statement.
* @param column_number Column number of the first token of this node in the
SQL statement.
@@ -102,7 +103,7 @@ class ParseKeyStringValue : public ParseKeyValue {
value_(value) { }
KeyValueType getKeyValueType() const override {
-return ParseKeyValue::KeyValueType::kStringString;
[3/6] incubator-quickstep git commit: Add "COPY TO" operator for exporting data from Quickstep.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29816511/parser/preprocessed/SqlParser_gen.cpp -- diff --git a/parser/preprocessed/SqlParser_gen.cpp b/parser/preprocessed/SqlParser_gen.cpp index 5cd7426..72c61dd 100644 --- a/parser/preprocessed/SqlParser_gen.cpp +++ b/parser/preprocessed/SqlParser_gen.cpp @@ -237,90 +237,91 @@ extern int quickstep_yydebug; TOKEN_DECIMAL = 304, TOKEN_DEFAULT = 305, TOKEN_DELETE = 306, -TOKEN_DELIMITER = 307, -TOKEN_DESC = 308, -TOKEN_DISTINCT = 309, -TOKEN_DOUBLE = 310, -TOKEN_DROP = 311, -TOKEN_ELSE = 312, -TOKEN_END = 313, -TOKEN_ESCAPE_STRINGS = 314, -TOKEN_EXISTS = 315, -TOKEN_EXTRACT = 316, -TOKEN_FALSE = 317, -TOKEN_FIRST = 318, -TOKEN_FLOAT = 319, -TOKEN_FOLLOWING = 320, -TOKEN_FOR = 321, -TOKEN_FOREIGN = 322, -TOKEN_FROM = 323, -TOKEN_FULL = 324, -TOKEN_GROUP = 325, -TOKEN_HASH = 326, -TOKEN_HAVING = 327, -TOKEN_HOUR = 328, -TOKEN_IN = 329, -TOKEN_INDEX = 330, -TOKEN_INNER = 331, -TOKEN_INSERT = 332, -TOKEN_INTEGER = 333, -TOKEN_INTERVAL = 334, -TOKEN_INTO = 335, -TOKEN_JOIN = 336, -TOKEN_KEY = 337, -TOKEN_LAST = 338, -TOKEN_LEFT = 339, -TOKEN_LIMIT = 340, -TOKEN_LONG = 341, -TOKEN_MINUTE = 342, -TOKEN_MONTH = 343, -TOKEN_NULL = 344, -TOKEN_NULLS = 345, -TOKEN_OFF = 346, -TOKEN_ON = 347, -TOKEN_ORDER = 348, -TOKEN_OUTER = 349, -TOKEN_OVER = 350, -TOKEN_PARTITION = 351, -TOKEN_PARTITIONS = 352, -TOKEN_PERCENT = 353, -TOKEN_PRECEDING = 354, -TOKEN_PRIMARY = 355, -TOKEN_PRIORITY = 356, -TOKEN_QUIT = 357, -TOKEN_RANGE = 358, -TOKEN_REAL = 359, -TOKEN_REFERENCES = 360, -TOKEN_RIGHT = 361, -TOKEN_ROW = 362, -TOKEN_ROW_DELIMITER = 363, -TOKEN_ROWS = 364, -TOKEN_SECOND = 365, -TOKEN_SELECT = 366, -TOKEN_SET = 367, -TOKEN_SMA = 368, -TOKEN_SMALLINT = 369, +TOKEN_DESC = 307, +TOKEN_DISTINCT = 308, +TOKEN_DOUBLE = 309, +TOKEN_DROP = 310, +TOKEN_ELSE = 311, +TOKEN_END = 312, +TOKEN_EXISTS = 313, +TOKEN_EXTRACT = 314, +TOKEN_FALSE = 315, +TOKEN_FIRST = 316, +TOKEN_FLOAT = 317, +TOKEN_FOLLOWING = 318, +TOKEN_FOR = 319, +TOKEN_FOREIGN = 320, +TOKEN_FROM = 321, +TOKEN_FULL = 322, +TOKEN_GROUP = 323, +TOKEN_HASH = 324, +TOKEN_HAVING = 325, +TOKEN_HOUR = 326, +TOKEN_IN = 327, +TOKEN_INDEX = 328, +TOKEN_INNER = 329, +TOKEN_INSERT = 330, +TOKEN_INTEGER = 331, +TOKEN_INTERVAL = 332, +TOKEN_INTO = 333, +TOKEN_JOIN = 334, +TOKEN_KEY = 335, +TOKEN_LAST = 336, +TOKEN_LEFT = 337, +TOKEN_LIMIT = 338, +TOKEN_LONG = 339, +TOKEN_MINUTE = 340, +TOKEN_MONTH = 341, +TOKEN_NULL = 342, +TOKEN_NULLS = 343, +TOKEN_OFF = 344, +TOKEN_ON = 345, +TOKEN_ORDER = 346, +TOKEN_OUTER = 347, +TOKEN_OVER = 348, +TOKEN_PARTITION = 349, +TOKEN_PARTITIONS = 350, +TOKEN_PERCENT = 351, +TOKEN_PRECEDING = 352, +TOKEN_PRIMARY = 353, +TOKEN_PRIORITY = 354, +TOKEN_QUIT = 355, +TOKEN_RANGE = 356, +TOKEN_REAL = 357, +TOKEN_REFERENCES = 358, +TOKEN_RIGHT = 359, +TOKEN_ROW = 360, +TOKEN_ROW_DELIMITER = 361, +TOKEN_ROWS = 362, +TOKEN_SECOND = 363, +TOKEN_SELECT = 364, +TOKEN_SET = 365, +TOKEN_SMA = 366, +TOKEN_SMALLINT = 367, +TOKEN_STDERR = 368, +TOKEN_STDOUT = 369, TOKEN_SUBSTRING = 370, TOKEN_TABLE = 371, TOKEN_THEN = 372, TOKEN_TIME = 373, TOKEN_TIMESTAMP = 374, -TOKEN_TRUE = 375, -TOKEN_TUPLESAMPLE = 376, -TOKEN_UNBOUNDED = 377, -TOKEN_UNIQUE = 378, -TOKEN_UPDATE = 379, -TOKEN_USING = 380, -TOKEN_VALUES = 381, -TOKEN_VARCHAR = 382, -TOKEN_WHEN = 383, -TOKEN_WHERE = 384, -TOKEN_WINDOW = 385, -TOKEN_WITH = 386, -TOKEN_YEAR = 387, -TOKEN_YEARMONTH = 388, -TOKEN_EOF = 389, -TOKEN_LEX_ERROR = 390 +TOKEN_TO = 375, +TOKEN_TRUE = 376, +TOKEN_TUPLESAMPLE = 377, +TOKEN_UNBOUNDED = 378, +TOKEN_UNIQUE = 379, +TOKEN_UPDATE = 380, +TOKEN_USING = 381, +TOKEN_VALUES = 382, +TOKEN_VARCHAR = 383, +TOKEN_WHEN = 384, +TOKEN_WHERE = 385, +TOKEN_WINDOW = 386, +TOKEN_WITH = 387, +TOKEN_YEAR = 388, +TOKEN_YEARMONTH = 389, +TOKEN_EOF = 390, +TOKEN_LEX_ERROR = 391 }; #endif @@ -380,8 +381,7 @@ union YYSTYPE quickstep::ParseKeyStringValue *key_string_value_; quickstep::ParseKeyStringList *key_string_list_; quickstep::ParseKeyIntegerValue *key_integer_value_; - - quickstep::ParseCopyFromParams *copy_from_params_; + quickstep::ParseKeyBoolValue *key_bool_value_; quickstep::ParseAssignment *assignment_; quickstep::PtrList *assignment_list_; @@ -394,7 +394,7 @@ union YYSTYPE quic
[4/6] incubator-quickstep git commit: Add "COPY TO" operator for exporting data from Quickstep.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29816511/parser/preprocessed/SqlLexer_gen.hpp -- diff --git a/parser/preprocessed/SqlLexer_gen.hpp b/parser/preprocessed/SqlLexer_gen.hpp index 86e36f0..5fafae5 100644 --- a/parser/preprocessed/SqlLexer_gen.hpp +++ b/parser/preprocessed/SqlLexer_gen.hpp @@ -13,88 +13,244 @@ #define FLEX_SCANNER #define YY_FLEX_MAJOR_VERSION 2 #define YY_FLEX_MINOR_VERSION 6 -#define YY_FLEX_SUBMINOR_VERSION 3 +#define YY_FLEX_SUBMINOR_VERSION 4 #if YY_FLEX_SUBMINOR_VERSION > 0 #define FLEX_BETA #endif -#define yy_create_buffer quickstep_yy_create_buffer +#ifdef yy_create_buffer +#define quickstep_yy_create_buffer_ALREADY_DEFINED +#else +#define yy_create_buffer quickstep_yy_create_buffer +#endif -#define yy_delete_buffer quickstep_yy_delete_buffer +#ifdef yy_delete_buffer +#define quickstep_yy_delete_buffer_ALREADY_DEFINED +#else +#define yy_delete_buffer quickstep_yy_delete_buffer +#endif -#define yy_scan_buffer quickstep_yy_scan_buffer +#ifdef yy_scan_buffer +#define quickstep_yy_scan_buffer_ALREADY_DEFINED +#else +#define yy_scan_buffer quickstep_yy_scan_buffer +#endif -#define yy_scan_string quickstep_yy_scan_string +#ifdef yy_scan_string +#define quickstep_yy_scan_string_ALREADY_DEFINED +#else +#define yy_scan_string quickstep_yy_scan_string +#endif -#define yy_scan_bytes quickstep_yy_scan_bytes +#ifdef yy_scan_bytes +#define quickstep_yy_scan_bytes_ALREADY_DEFINED +#else +#define yy_scan_bytes quickstep_yy_scan_bytes +#endif -#define yy_init_buffer quickstep_yy_init_buffer +#ifdef yy_init_buffer +#define quickstep_yy_init_buffer_ALREADY_DEFINED +#else +#define yy_init_buffer quickstep_yy_init_buffer +#endif -#define yy_flush_buffer quickstep_yy_flush_buffer +#ifdef yy_flush_buffer +#define quickstep_yy_flush_buffer_ALREADY_DEFINED +#else +#define yy_flush_buffer quickstep_yy_flush_buffer +#endif -#define yy_load_buffer_state quickstep_yy_load_buffer_state +#ifdef yy_load_buffer_state +#define quickstep_yy_load_buffer_state_ALREADY_DEFINED +#else +#define yy_load_buffer_state quickstep_yy_load_buffer_state +#endif -#define yy_switch_to_buffer quickstep_yy_switch_to_buffer +#ifdef yy_switch_to_buffer +#define quickstep_yy_switch_to_buffer_ALREADY_DEFINED +#else +#define yy_switch_to_buffer quickstep_yy_switch_to_buffer +#endif -#define yypush_buffer_state quickstep_yypush_buffer_state +#ifdef yypush_buffer_state +#define quickstep_yypush_buffer_state_ALREADY_DEFINED +#else +#define yypush_buffer_state quickstep_yypush_buffer_state +#endif -#define yypop_buffer_state quickstep_yypop_buffer_state +#ifdef yypop_buffer_state +#define quickstep_yypop_buffer_state_ALREADY_DEFINED +#else +#define yypop_buffer_state quickstep_yypop_buffer_state +#endif -#define yyensure_buffer_stack quickstep_yyensure_buffer_stack +#ifdef yyensure_buffer_stack +#define quickstep_yyensure_buffer_stack_ALREADY_DEFINED +#else +#define yyensure_buffer_stack quickstep_yyensure_buffer_stack +#endif -#define yylex quickstep_yylex +#ifdef yylex +#define quickstep_yylex_ALREADY_DEFINED +#else +#define yylex quickstep_yylex +#endif -#define yyrestart quickstep_yyrestart +#ifdef yyrestart +#define quickstep_yyrestart_ALREADY_DEFINED +#else +#define yyrestart quickstep_yyrestart +#endif -#define yylex_init quickstep_yylex_init +#ifdef yylex_init +#define quickstep_yylex_init_ALREADY_DEFINED +#else +#define yylex_init quickstep_yylex_init +#endif -#define yylex_init_extra quickstep_yylex_init_extra +#ifdef yylex_init_extra +#define quickstep_yylex_init_extra_ALREADY_DEFINED +#else +#define yylex_init_extra quickstep_yylex_init_extra +#endif -#define yylex_destroy quickstep_yylex_destroy +#ifdef yylex_destroy +#define quickstep_yylex_destroy_ALREADY_DEFINED +#else +#define yylex_destroy quickstep_yylex_destroy +#endif -#define yyget_debug quickstep_yyget_debug +#ifdef yyget_debug +#define quickstep_yyget_debug_ALREADY_DEFINED +#else +#define yyget_debug quickstep_yyget_debug +#endif -#define yyset_debug quickstep_yyset_debug +#ifdef yyset_debug +#define quickstep_yyset_debug_ALREADY_DEFINED +#else +#define yyset_debug quickstep_yyset_debug +#endif -#define yyget_extra quickstep_yyget_extra +#ifdef yyget_extra +#define quickstep_yyget_extra_ALREADY_DEFINED +#else +#define yyget_extra quickstep_yyget_extra +#endif -#define yyset_extra quickstep_yyset_extra +#ifdef yyset_extra +#define quickstep_yyset_extra_ALREADY_DEFINED +#else +#define yyset_extra quickstep_yyset_extra +#endif -#define yyget_in quickstep_yyget_in +#ifdef yyget_in +#define quickstep_yyget_in_ALREADY_DEFINED +#else +#define yyget_in quickstep_yyget_in +#endif -#define yyset_in quickstep_yyset_in +#ifdef yyset_in +#define quickstep_yyset_in_ALREADY_DEFINED +#else +#define yyset_in quickstep_yyset_in +#endif -#define
[1/6] incubator-quickstep git commit: Add "COPY TO" operator for exporting data from Quickstep.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 877cae08b -> 298165115
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29816511/relational_operators/CMakeLists.txt
--
diff --git a/relational_operators/CMakeLists.txt
b/relational_operators/CMakeLists.txt
index 57ba9f9..7b9ed96 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -71,6 +71,7 @@
add_library(quickstep_relationaloperators_SortMergeRunOperatorHelpers SortMergeR
SortMergeRunOperatorHelpers.hpp)
add_library(quickstep_relationaloperators_SortRunGenerationOperator
SortRunGenerationOperator.cpp
SortRunGenerationOperator.hpp)
+add_library(quickstep_relationaloperators_TableExportOperator
TableExportOperator.cpp TableExportOperator.hpp)
add_library(quickstep_relationaloperators_TableGeneratorOperator
TableGeneratorOperator.cpp TableGeneratorOperator.hpp)
add_library(quickstep_relationaloperators_TextScanOperator
TextScanOperator.cpp TextScanOperator.hpp)
add_library(quickstep_relationaloperators_UnionAllOperator
UnionAllOperator.cpp UnionAllOperator.hpp)
@@ -473,6 +474,26 @@
target_link_libraries(quickstep_relationaloperators_SortRunGenerationOperator
quickstep_utility_Macros
quickstep_utility_SortConfiguration
tmb)
+target_link_libraries(quickstep_relationaloperators_TableExportOperator
+ glog
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
+ quickstep_queryexecution_WorkOrdersContainer
+ quickstep_relationaloperators_RelationalOperator
+ quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_ValueAccessor
+ quickstep_threading_SpinMutex
+ quickstep_types_TypedValue
+ quickstep_types_containers_Tuple
+ quickstep_utility_BulkIoConfiguration
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil)
target_link_libraries(quickstep_relationaloperators_TableGeneratorOperator
glog
quickstep_catalog_CatalogRelation
@@ -508,6 +529,7 @@
target_link_libraries(quickstep_relationaloperators_TextScanOperator
quickstep_types_containers_ColumnVector
quickstep_types_containers_ColumnVectorsValueAccessor
quickstep_types_containers_Tuple
+ quickstep_utility_BulkIoConfiguration
quickstep_utility_Glob
quickstep_utility_Macros
tmb)
@@ -637,6 +659,7 @@ target_link_libraries(quickstep_relationaloperators
quickstep_relationaloperators_SortMergeRunOperatorHelpers
quickstep_relationaloperators_SortMergeRunOperator_proto
quickstep_relationaloperators_SortRunGenerationOperator
+ quickstep_relationaloperators_TableExportOperator
quickstep_relationaloperators_TableGeneratorOperator
quickstep_relationaloperators_TextScanOperator
quickstep_relationaloperators_UnionAllOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29816511/relational_operators/RelationalOperator.hpp
--
diff --git a/relational_operators/RelationalOperator.hpp
b/relational_operators/RelationalOperator.hpp
index 8035685..8eb59f0 100644
--- a/relational_operators/RelationalOperator.hpp
+++ b/relational_operators/RelationalOperator.hpp
@@ -86,6 +86,7 @@ class RelationalOperator {
kSelect,
kSortMergeRun,
kSortRunGeneration,
+kTableExport,
kTableGenerator,
kTextScan,
kUnionAll,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29816511/relational_operators/TableExportOperator.cpp
--
diff --git a/relational_operators/TableExportOperator.cpp
b/relational_operators/TableExportOperator.cpp
new file mode 100644
index 000..f6a73bf
--- /dev/null
+++ b/relational_operators/TableExportOperator.cpp
@@ -0,0 +1,336 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE f
incubator-quickstep git commit: Fixed a minor bug in PR 282.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 6c21bdb61 -> 5bfa8e1b7
Fixed a minor bug in PR 282.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5bfa8e1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5bfa8e1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5bfa8e1b
Branch: refs/heads/master
Commit: 5bfa8e1b7752153e6afb499a7a81c24191717e7b
Parents: 6c21bdb
Author: Zuyu Zhang
Authored: Mon Aug 28 16:00:27 2017 -0500
Committer: Zuyu Zhang
Committed: Mon Aug 28 16:00:27 2017 -0500
--
utility/ExecutionDAGVisualizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5bfa8e1b/utility/ExecutionDAGVisualizer.cpp
--
diff --git a/utility/ExecutionDAGVisualizer.cpp
b/utility/ExecutionDAGVisualizer.cpp
index 2dc6976..7c319f5 100644
--- a/utility/ExecutionDAGVisualizer.cpp
+++ b/utility/ExecutionDAGVisualizer.cpp
@@ -154,7 +154,7 @@ ExecutionDAGVisualizer::ExecutionDAGVisualizer(const
QueryPlan &plan) {
std::string input_stored_relation_names;
std::size_t num_input_stored_relations = 0;
for (const auto &input_relation : union_all_op.input_relations()) {
- if (!input_relation->isTemporary()) {
+ if (input_relation->isTemporary()) {
continue;
}
++num_input_stored_relations;
incubator-quickstep git commit: Added output_num_partitions in RelationalOperators.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 5f066fa4b -> 9afad881d
Added output_num_partitions in RelationalOperators.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9afad881
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9afad881
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9afad881
Branch: refs/heads/master
Commit: 9afad881da17dd3a8488a88922ab06b8ee96fbb9
Parents: 5f066fa
Author: Zuyu Zhang
Authored: Wed Aug 23 12:57:38 2017 -0500
Committer: Zuyu Zhang
Committed: Sun Aug 27 01:14:24 2017 -0500
--
relational_operators/DeleteOperator.hpp| 3 ++-
.../FinalizeAggregationOperator.hpp| 6 --
relational_operators/HashJoinOperator.hpp | 3 ++-
relational_operators/InsertOperator.hpp| 2 +-
relational_operators/NestedLoopsJoinOperator.hpp | 2 +-
relational_operators/RelationalOperator.hpp| 17 +++--
relational_operators/SelectOperator.hpp| 8 ++--
relational_operators/SortMergeRunOperator.hpp | 5 -
relational_operators/SortRunGenerationOperator.hpp | 7 +--
relational_operators/TableGeneratorOperator.hpp| 3 ++-
relational_operators/TextScanOperator.hpp | 3 ++-
relational_operators/UpdateOperator.hpp| 3 ++-
relational_operators/WindowAggregationOperator.hpp | 5 -
13 files changed, 50 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/DeleteOperator.hpp
--
diff --git a/relational_operators/DeleteOperator.hpp
b/relational_operators/DeleteOperator.hpp
index b084560..cc6f6cf 100644
--- a/relational_operators/DeleteOperator.hpp
+++ b/relational_operators/DeleteOperator.hpp
@@ -74,7 +74,8 @@ class DeleteOperator : public RelationalOperator {
const CatalogRelation &relation,
const QueryContext::predicate_id predicate_index,
const bool relation_is_stored)
- : RelationalOperator(query_id, relation.getNumPartitions()),
+ : RelationalOperator(query_id, relation.getNumPartitions(), false /*
has_repartition */,
+ relation.getNumPartitions()),
relation_(relation),
predicate_index_(predicate_index),
relation_is_stored_(relation_is_stored),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/FinalizeAggregationOperator.hpp
--
diff --git a/relational_operators/FinalizeAggregationOperator.hpp
b/relational_operators/FinalizeAggregationOperator.hpp
index b344c13..5931ca2 100644
--- a/relational_operators/FinalizeAggregationOperator.hpp
+++ b/relational_operators/FinalizeAggregationOperator.hpp
@@ -74,12 +74,14 @@ class FinalizeAggregationOperator : public
RelationalOperator {
const std::size_t aggr_state_num_partitions,
const CatalogRelation &output_relation,
const QueryContext::insert_destination_id output_destination_index)
- : RelationalOperator(query_id, num_partitions, has_repartition),
+ : RelationalOperator(query_id, num_partitions, has_repartition,
output_relation.getNumPartitions()),
aggr_state_index_(aggr_state_index),
aggr_state_num_partitions_(aggr_state_num_partitions),
output_relation_(output_relation),
output_destination_index_(output_destination_index),
-started_(false) {}
+started_(false) {
+DCHECK(has_repartition || num_partitions == output_num_partitions_);
+ }
~FinalizeAggregationOperator() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/HashJoinOperator.hpp
--
diff --git a/relational_operators/HashJoinOperator.hpp
b/relational_operators/HashJoinOperator.hpp
index a5e6eb4..306875e 100644
--- a/relational_operators/HashJoinOperator.hpp
+++ b/relational_operators/HashJoinOperator.hpp
@@ -139,7 +139,7 @@ class HashJoinOperator : public RelationalOperator {
const QueryContext::scalar_group_id selection_index,
const std::vector *is_selection_on_build = nullptr,
const JoinType join_type = JoinType::kInnerJoin)
- : RelationalOperator(query_id, num_partitions, has_repartition),
+ : RelationalOperator(query_id, num_partitions, has_repartition,
output_relation.getNumPartitions()),
build_relation_(build_relation),
probe_relation_(probe_relation),
probe_relation_is_stored_(probe_relation_is_stored),
@@ -157,6 +157,
incubator-quickstep git commit: Added has_repartition in RelationalOperators.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 942a15bee -> 5f066fa4b
Added has_repartition in RelationalOperators.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5f066fa4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5f066fa4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5f066fa4
Branch: refs/heads/master
Commit: 5f066fa4bd53ec11d82c4a68bd3a120c65c873e8
Parents: 942a15b
Author: Zuyu Zhang
Authored: Wed Aug 9 14:24:03 2017 -0500
Committer: Zuyu Zhang
Committed: Sun Aug 27 00:59:20 2017 -0500
--
query_optimizer/ExecutionGenerator.cpp | 18 ++
query_optimizer/physical/Aggregate.cpp | 3 +
query_optimizer/physical/Aggregate.hpp | 14 +-
query_optimizer/physical/BinaryJoin.cpp | 3 +
query_optimizer/physical/BinaryJoin.hpp | 4 +-
query_optimizer/physical/HashJoin.cpp | 1 +
query_optimizer/physical/HashJoin.hpp | 13 +-
query_optimizer/physical/Join.hpp | 4 +-
query_optimizer/physical/NestedLoopsJoin.hpp| 14 +-
query_optimizer/physical/Physical.hpp | 21 ++-
query_optimizer/physical/Selection.cpp | 9 +-
query_optimizer/physical/Selection.hpp | 13 +-
query_optimizer/physical/TableGenerator.hpp | 9 +-
query_optimizer/physical/TableReference.hpp | 2 +-
query_optimizer/rules/Partition.cpp | 30 +++-
.../physical_generator/CommonSubexpression.test | 23 ++-
.../tests/physical_generator/Join.test | 44 ++---
.../tests/physical_generator/Select.test| 164 +--
.../FinalizeAggregationOperator.hpp | 4 +-
relational_operators/HashJoinOperator.hpp | 4 +-
.../NestedLoopsJoinOperator.hpp | 4 +-
relational_operators/RelationalOperator.hpp | 15 +-
relational_operators/SelectOperator.hpp | 8 +-
.../tests/AggregationOperator_unittest.cpp | 3 +
.../tests/HashJoinOperator_unittest.cpp | 12 ++
25 files changed, 281 insertions(+), 158 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5f066fa4/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp
b/query_optimizer/ExecutionGenerator.cpp
index d43432c..d82a0c7 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -747,6 +747,8 @@ void ExecutionGenerator::convertSelection(
insert_destination_proto);
// Create and add a Select operator.
+ const bool has_repartition = physical_selection->hasRepartition();
+
// Use the "simple" form of the selection operator (a pure projection that
// doesn't require any expression evaluation or intermediate copies) if
// possible.
@@ -755,6 +757,7 @@ void ExecutionGenerator::convertSelection(
convertSimpleProjection(project_expressions_group_index, &attributes)
? new SelectOperator(query_handle_->query_id(),
input_relation,
+ has_repartition,
*output_relation,
insert_destination_index,
execution_predicate_index,
@@ -762,6 +765,7 @@ void ExecutionGenerator::convertSelection(
input_relation_info->isStoredRelation())
: new SelectOperator(query_handle_->query_id(),
input_relation,
+ has_repartition,
*output_relation,
insert_destination_index,
execution_predicate_index,
@@ -1040,6 +1044,7 @@ void ExecutionGenerator::convertHashJoin(const
P::HashJoinPtr &physical_plan) {
probe_attribute_ids,
any_probe_attributes_nullable,
probe_num_partitions,
+ physical_plan->hasRepartition(),
*output_relation,
insert_destination_index,
join_hash_table_index,
@@ -1152,6 +1157,7 @@ void ExecutionGenerator::convertNestedLoopsJoin(
left_relation,
right_relation,
num_partitions,
+ physical_plan->hasRepartition(),
*output_relation,
insert_destination_index,
execution_join_predicate_index,
@@ -1539,6 +1545,15 @@ void ExecutionGenerato
incubator-quickstep git commit: Added num_partitions in RelationalOperator.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 32b5b83f3 -> 942a15bee
Added num_partitions in RelationalOperator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/942a15be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/942a15be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/942a15be
Branch: refs/heads/master
Commit: 942a15bee482ffedef757d9b0d808f154606af3f
Parents: 32b5b83
Author: Zuyu Zhang
Authored: Wed Aug 9 12:59:45 2017 -0500
Committer: Zuyu Zhang
Committed: Sun Aug 27 00:50:53 2017 -0500
--
catalog/CMakeLists.txt | 1 +
catalog/CatalogRelation.cpp | 5 ++
catalog/CatalogRelation.hpp | 11
query_optimizer/ExecutionGenerator.cpp | 68 ++--
relational_operators/AggregationOperator.hpp| 4 +-
.../BuildAggregationExistenceMapOperator.hpp| 4 +-
relational_operators/BuildHashOperator.hpp | 7 +-
relational_operators/BuildLIPFilterOperator.hpp | 13 +---
relational_operators/CMakeLists.txt | 5 +-
relational_operators/CreateIndexOperator.hpp| 2 +-
relational_operators/CreateTableOperator.hpp| 2 +-
relational_operators/DeleteOperator.cpp | 61 +++---
relational_operators/DeleteOperator.hpp | 34 +++---
.../DestroyAggregationStateOperator.hpp | 4 +-
relational_operators/DestroyHashOperator.cpp| 4 +-
relational_operators/DestroyHashOperator.hpp| 8 +--
.../FinalizeAggregationOperator.hpp | 5 +-
relational_operators/HashJoinOperator.hpp | 4 +-
.../InitializeAggregationOperator.hpp | 5 +-
.../NestedLoopsJoinOperator.hpp | 5 +-
relational_operators/RelationalOperator.hpp | 18 +-
relational_operators/SaveBlocksOperator.cpp | 43 +++--
relational_operators/SaveBlocksOperator.hpp | 17 +++--
relational_operators/SelectOperator.hpp | 29 -
relational_operators/UpdateOperator.cpp | 64 ++
relational_operators/UpdateOperator.hpp | 24 +--
relational_operators/WorkOrder.proto| 3 +
relational_operators/WorkOrderFactory.cpp | 31 +++--
28 files changed, 266 insertions(+), 215 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/942a15be/catalog/CMakeLists.txt
--
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index 0ca014e..1aa2b41 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -108,6 +108,7 @@ target_link_libraries(quickstep_catalog_CatalogRelation
quickstep_catalog_Catalog_proto
quickstep_catalog_IndexScheme
quickstep_catalog_PartitionScheme
+ quickstep_catalog_PartitionSchemeHeader
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageBlockLayout
quickstep_storage_StorageBlockLayout_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/942a15be/catalog/CatalogRelation.cpp
--
diff --git a/catalog/CatalogRelation.cpp b/catalog/CatalogRelation.cpp
index 793fc2d..38c86f6 100644
--- a/catalog/CatalogRelation.cpp
+++ b/catalog/CatalogRelation.cpp
@@ -33,6 +33,7 @@
#endif
#include "catalog/PartitionScheme.hpp"
+#include "catalog/PartitionSchemeHeader.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageBlockLayout.hpp"
#include "storage/StorageBlockLayout.pb.h"
@@ -180,6 +181,10 @@ serialization::CatalogRelationSchema
CatalogRelation::getProto() const {
void CatalogRelation::setPartitionScheme(PartitionScheme* partition_scheme) {
DCHECK_EQ(0u, size_blocks());
partition_scheme_.reset(partition_scheme);
+
+ if (partition_scheme_) {
+num_partitions_ =
partition_scheme_->getPartitionSchemeHeader().getNumPartitions();
+ }
}
void CatalogRelation::setDefaultStorageBlockLayout(StorageBlockLayout
*default_layout) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/942a15be/catalog/CatalogRelation.hpp
--
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index af7f273..e40b599 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -81,6 +81,7 @@ class CatalogRelation : public CatalogRelationSchema {
bool temporary = false)
: CatalogRelationSchema(parent, name, id, temporary),
default_layout_(nullptr),
+num_pa
incubator-quickstep git commit: Added partition_id in WorkOrder.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 83c4fe557 -> 32b5b83f3
Added partition_id in WorkOrder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/32b5b83f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/32b5b83f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/32b5b83f
Branch: refs/heads/master
Commit: 32b5b83f3e37033a3959c935aca102f2db2822fc
Parents: 83c4fe5
Author: Zuyu Zhang
Authored: Thu Aug 24 13:26:04 2017 -0500
Committer: Zuyu Zhang
Committed: Sat Aug 26 16:54:29 2017 -0500
--
query_execution/PolicyEnforcerBase.cpp | 4 +-
query_execution/QueryExecutionMessages.proto| 2 +
query_execution/QueryManagerBase.cpp| 6 +-
query_execution/QueryManagerBase.hpp| 8 ++-
query_execution/Worker.cpp | 13 ++--
.../tests/QueryManagerSingleNode_unittest.cpp | 10 ++-
relational_operators/AggregationOperator.cpp| 2 +
relational_operators/AggregationOperator.hpp| 4 +-
.../BuildAggregationExistenceMapOperator.cpp| 2 +
.../BuildAggregationExistenceMapOperator.hpp| 4 +-
relational_operators/BuildHashOperator.hpp | 16 +
relational_operators/BuildLIPFilterOperator.cpp | 2 +
relational_operators/BuildLIPFilterOperator.hpp | 4 +-
relational_operators/CMakeLists.txt | 1 +
.../DestroyAggregationStateOperator.cpp | 2 +-
.../DestroyAggregationStateOperator.hpp | 4 +-
relational_operators/DestroyHashOperator.cpp| 2 +-
relational_operators/DestroyHashOperator.hpp| 4 +-
.../FinalizeAggregationOperator.cpp | 1 -
.../FinalizeAggregationOperator.hpp | 5 +-
relational_operators/HashJoinOperator.cpp | 6 +-
relational_operators/HashJoinOperator.hpp | 66 +++-
.../InitializeAggregationOperator.cpp | 1 +
.../InitializeAggregationOperator.hpp | 4 +-
.../NestedLoopsJoinOperator.cpp | 4 ++
.../NestedLoopsJoinOperator.hpp | 4 +-
relational_operators/RebuildWorkOrder.hpp | 6 +-
relational_operators/SelectOperator.cpp | 2 +-
relational_operators/SelectOperator.hpp | 7 +--
relational_operators/WorkOrder.hpp | 21 ++-
relational_operators/WorkOrderFactory.cpp | 15 -
31 files changed, 114 insertions(+), 118 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32b5b83f/query_execution/PolicyEnforcerBase.cpp
--
diff --git a/query_execution/PolicyEnforcerBase.cpp
b/query_execution/PolicyEnforcerBase.cpp
index 32f29a3..f97f711 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -72,7 +72,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage
&tagged_message) {
DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
op_index = proto.operator_index();
- admitted_queries_[query_id]->processWorkOrderCompleteMessage(op_index);
+ admitted_queries_[query_id]->processWorkOrderCompleteMessage(op_index,
proto.partition_id());
break;
}
case kRebuildWorkOrderCompleteMessage: {
@@ -87,7 +87,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage
&tagged_message) {
DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
op_index = proto.operator_index();
-
admitted_queries_[query_id]->processRebuildWorkOrderCompleteMessage(op_index);
+
admitted_queries_[query_id]->processRebuildWorkOrderCompleteMessage(op_index,
proto.partition_id());
break;
}
case kCatalogRelationNewBlockMessage: {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32b5b83f/query_execution/QueryExecutionMessages.proto
--
diff --git a/query_execution/QueryExecutionMessages.proto
b/query_execution/QueryExecutionMessages.proto
index 6aa8769..abc47cf 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -31,6 +31,7 @@ import "relational_operators/WorkOrder.proto";
// order completion message, we may be interested in adding the compression
// ratio or dictionary size of the rebuilt block.
+// Next tag: 9.
message WorkOrderCompletionMessage {
enum WorkOrderType {
NORMAL = 0;
@@ -42,6 +43,7 @@ message WorkOrderCompletionMessage {
required uint64 operator_index = 2;
required uint64 worker_thread_index = 3;
required uint64 query_id = 4;
+ required uint64 partition_id = 8;
// Epoch time in microseconds.
optional uint64 execution_start
incubator-quickstep git commit: Minor cleanups for the create operators.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 09ccb7ca7 -> 83c4fe557
Minor cleanups for the create operators.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/83c4fe55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/83c4fe55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/83c4fe55
Branch: refs/heads/master
Commit: 83c4fe55707fe8e5934d53b28c60673c8cdf66eb
Parents: 09ccb7c
Author: Zuyu Zhang
Authored: Sat Aug 26 13:46:53 2017 -0500
Committer: Zuyu Zhang
Committed: Sat Aug 26 15:23:51 2017 -0500
--
relational_operators/CMakeLists.txt | 4 +--
relational_operators/CreateIndexOperator.cpp | 40 -
relational_operators/CreateIndexOperator.hpp | 9 +++--
relational_operators/CreateTableOperator.cpp | 43 ---
relational_operators/CreateTableOperator.hpp | 9 +++--
5 files changed, 16 insertions(+), 89 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/83c4fe55/relational_operators/CMakeLists.txt
--
diff --git a/relational_operators/CMakeLists.txt
b/relational_operators/CMakeLists.txt
index 5ad9c3b..6083dd5 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -38,8 +38,8 @@
add_library(quickstep_relationaloperators_BuildAggregationExistenceMapOperator
BuildAggregationExistenceMapOperator.hpp)
add_library(quickstep_relationaloperators_BuildHashOperator
BuildHashOperator.cpp BuildHashOperator.hpp)
add_library(quickstep_relationaloperators_BuildLIPFilterOperator
BuildLIPFilterOperator.cpp BuildLIPFilterOperator.hpp)
-add_library(quickstep_relationaloperators_CreateIndexOperator
CreateIndexOperator.cpp CreateIndexOperator.hpp)
-add_library(quickstep_relationaloperators_CreateTableOperator
CreateTableOperator.cpp CreateTableOperator.hpp)
+add_library(quickstep_relationaloperators_CreateIndexOperator ../empty_src.cpp
CreateIndexOperator.hpp)
+add_library(quickstep_relationaloperators_CreateTableOperator ../empty_src.cpp
CreateTableOperator.hpp)
add_library(quickstep_relationaloperators_DestroyAggregationStateOperator
DestroyAggregationStateOperator.cpp
DestroyAggregationStateOperator.hpp)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/83c4fe55/relational_operators/CreateIndexOperator.cpp
--
diff --git a/relational_operators/CreateIndexOperator.cpp
b/relational_operators/CreateIndexOperator.cpp
deleted file mode 100644
index ab3624c..000
--- a/relational_operators/CreateIndexOperator.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#include "relational_operators/CreateIndexOperator.hpp"
-
-#include
-
-#include "tmb/id_typedefs.h"
-
-namespace quickstep {
-
-bool CreateIndexOperator::getAllWorkOrders(WorkOrdersContainer *container,
- QueryContext *query_context,
- StorageManager *storage_manager,
- const tmb::client_id
scheduler_client_id,
- tmb::MessageBus *bus) {
- return true;
-}
-
-void CreateIndexOperator::updateCatalogOnCompletion() {
- relation_->addIndex(index_name_, std::move(index_description_));
-}
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/83c4fe55/relational_operators/CreateIndexOperator.hpp
--
diff --git a/relational_operators/CreateIndexOperator.hpp
b/relational_operators/CreateIndexOperator.hpp
index c08947f..09d5d81 100644
--- a/relational_operators/CreateIndexOperator.hpp
+++ b/relational_operators/CreateIndexOperator.hpp
@@ -22,6 +22,7 @@
#include
#include
+#include
#include
incubator-quickstep git commit: Minor changes in QueryManager and QueryExecutionState.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master c018882f5 -> 09ccb7ca7
Minor changes in QueryManager and QueryExecutionState.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/09ccb7ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/09ccb7ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/09ccb7ca
Branch: refs/heads/master
Commit: 09ccb7ca75dd914ece283ca6b8dc2ae759b12e48
Parents: c018882
Author: Zuyu Zhang
Authored: Sat Aug 26 13:01:12 2017 -0500
Committer: Zuyu Zhang
Committed: Sat Aug 26 13:13:25 2017 -0500
--
query_execution/QueryExecutionState.hpp| 19 ---
query_execution/QueryManagerBase.cpp | 1 -
query_execution/QueryManagerSingleNode.cpp | 18 +-
query_execution/QueryManagerSingleNode.hpp | 6 --
4 files changed, 21 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/09ccb7ca/query_execution/QueryExecutionState.hpp
--
diff --git a/query_execution/QueryExecutionState.hpp
b/query_execution/QueryExecutionState.hpp
index a4273dc..0f98964 100644
--- a/query_execution/QueryExecutionState.hpp
+++ b/query_execution/QueryExecutionState.hpp
@@ -99,14 +99,10 @@ class QueryExecutionState {
const bool rebuild_initiated) {
DCHECK(operator_index < num_operators_);
auto search_res = rebuild_status_.find(operator_index);
-if (search_res != rebuild_status_.end()) {
- search_res->second.has_initiated = rebuild_initiated;
- search_res->second.num_pending_workorders = num_rebuild_workorders;
-} else {
- RebuildStatus rebuild_status(rebuild_initiated, num_rebuild_workorders);
+DCHECK(search_res != rebuild_status_.end());
- rebuild_status_.emplace(operator_index, std::move(rebuild_status));
-}
+search_res->second.has_initiated = rebuild_initiated;
+search_res->second.num_pending_workorders = num_rebuild_workorders;
}
#ifdef QUICKSTEP_DISTRIBUTED
@@ -272,6 +268,8 @@ class QueryExecutionState {
inline void setRebuildRequired(const std::size_t operator_index) {
DCHECK(operator_index < num_operators_);
rebuild_required_[operator_index] = true;
+
+rebuild_status_.emplace(operator_index, RebuildStatus());
}
/**
@@ -353,10 +351,9 @@ class QueryExecutionState {
std::vector execution_finished_;
struct RebuildStatus {
-RebuildStatus(const bool initiated,
- const std::size_t num_workorders)
-: has_initiated(initiated),
- num_pending_workorders(num_workorders) {}
+RebuildStatus()
+: has_initiated(false),
+ num_pending_workorders(0) {}
// Whether rebuild for operator at index i has been initiated.
bool has_initiated;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/09ccb7ca/query_execution/QueryManagerBase.cpp
--
diff --git a/query_execution/QueryManagerBase.cpp
b/query_execution/QueryManagerBase.cpp
index 14c9ba5..f353b64 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -64,7 +64,6 @@ QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
if (insert_destination_index != QueryContext::kInvalidInsertDestinationId)
{
// Rebuild is necessary whenever InsertDestination is present.
query_exec_state_->setRebuildRequired(node_index);
- query_exec_state_->setRebuildStatus(node_index, 0, false);
}
for (const pair &dependent_link :
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/09ccb7ca/query_execution/QueryManagerSingleNode.cpp
--
diff --git a/query_execution/QueryManagerSingleNode.cpp
b/query_execution/QueryManagerSingleNode.cpp
index 001faa8..82a0de6 100644
--- a/query_execution/QueryManagerSingleNode.cpp
+++ b/query_execution/QueryManagerSingleNode.cpp
@@ -126,22 +126,20 @@ bool QueryManagerSingleNode::initiateRebuild(const
dag_node_index index) {
DCHECK(checkRebuildRequired(index));
DCHECK(!checkRebuildInitiated(index));
- getRebuildWorkOrders(index, workorders_container_.get());
+ const std::size_t num_rebuild_work_orders = getRebuildWorkOrders(index,
workorders_container_.get());
+ DCHECK_EQ(workorders_container_->getNumRebuildWorkOrders(index),
num_rebuild_work_orders);
query_exec_state_->setRebuildStatus(
- index, workorders_container_->getNumRebuildWorkOrders(index), true);
+ index, num_rebuild_work_orders, true);
- return (query_exec_state_->getNumRebuildWorkOrders(index) == 0);
+ re
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/fix-compare-aggregate [deleted] 679257094
incubator-quickstep git commit: Fix the bug with min/max aggregation.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master b1aab0096 -> 679257094
Fix the bug with min/max aggregation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/67925709
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/67925709
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/67925709
Branch: refs/heads/master
Commit: 67925709450be3ceef8a789c8054a446dc693a0a
Parents: b1aab00
Author: Jianqiao Zhu
Authored: Wed Aug 2 15:47:30 2017 -0500
Committer: Jianqiao Zhu
Committed: Fri Aug 4 16:12:15 2017 -0500
--
.../comparisons/LiteralComparators-inl.hpp | 76 +++-
1 file changed, 60 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/67925709/types/operations/comparisons/LiteralComparators-inl.hpp
--
diff --git a/types/operations/comparisons/LiteralComparators-inl.hpp
b/types/operations/comparisons/LiteralComparators-inl.hpp
index fd59e2e..ad54d4e 100644
--- a/types/operations/comparisons/LiteralComparators-inl.hpp
+++ b/types/operations/comparisons/LiteralComparators-inl.hpp
@@ -543,7 +543,11 @@ TypedValue LiteralUncheckedComparator();
+ }
InvokeOnValueAccessorMaybeTupleIdSequenceAdapter(
accessor,
@@ -555,32 +559,72 @@ TypedValue LiteralUncheckedComparator>
column_accessor(accessor->template
getColumnAccessor(value_accessor_id));
DCHECK(column_accessor != nullptr);
- while (accessor->next()) {
-const void *va_value = column_accessor->getUntypedValue();
-if (left_nullable && !va_value) {
- continue;
+
+ // Locate the first non-null value.
+ if (is_null) {
+const void *va_value = nullptr;
+while (accessor->next()) {
+ va_value = column_accessor->getUntypedValue();
+ if (!left_nullable || va_value) {
+break;
+ }
+}
+if (va_value != nullptr) {
+ is_null = false;
+ current_literal = *static_cast(va_value);
}
-if (!current_literal || this->compareDataPtrsHelper(va_value,
current_literal)) {
- current_literal = va_value;
+ }
+
+ // Aggregate on the remaining values.
+ if (!accessor->iterationFinished()) {
+DCHECK(!is_null);
+while (accessor->next()) {
+ const void *va_value = column_accessor->getUntypedValue();
+ if (left_nullable && !va_value) {
+continue;
+ }
+ if (this->compareDataPtrsHelper(va_value, ¤t_literal)) {
+current_literal = *static_cast(va_value);
+ }
}
}
} else {
- while (accessor->next()) {
-const void *va_value = accessor->template
getUntypedValue(value_accessor_id);
-if (left_nullable && !va_value) {
- continue;
+ // Locate the first non-null value.
+ if (is_null) {
+const void *va_value = nullptr;
+while (accessor->next()) {
+ va_value = accessor->template
getUntypedValue(value_accessor_id);
+ if (!left_nullable || va_value) {
+break;
+ }
+}
+if (va_value != nullptr) {
+ is_null = false;
+ current_literal = *static_cast(va_value);
}
-if (!current_literal || this->compareDataPtrsHelper(va_value,
current_literal)) {
- current_literal = va_value;
+ }
+
+ // Aggregate on the remaining values.
+ if (!accessor->iterationFinished()) {
+DCHECK(!is_null);
+while (accessor->next()) {
+ const void *va_value =
+ accessor->template
getUntypedValue(value_accessor_id);
+ if (left_nullable && !va_value) {
+continue;
+ }
+ if (this->compareDataPtrsHelper(va_value, ¤t_literal)) {
+current_literal = *static_cast(va_value);
+ }
}
}
}
});
- if (current_literal) {
-return TypedValue(*static_cast(current_literal));
- } else {
+ if (is_null) {
return TypedValue(current.getTypeID());
+ } else {
+return TypedValue(current_literal);
}
}
incubator-quickstep git commit: Disabled NetworkUtil_unittest.
Repository: incubator-quickstep Updated Branches: refs/heads/master e80e5c828 -> 9dc9dfbfc Disabled NetworkUtil_unittest. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9dc9dfbf Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9dc9dfbf Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9dc9dfbf Branch: refs/heads/master Commit: 9dc9dfbfcbb072b2280394d78abc8f2554881d9a Parents: e80e5c8 Author: Zuyu Zhang Authored: Wed Aug 2 19:46:16 2017 -0500 Committer: Zuyu Zhang Committed: Wed Aug 2 20:46:21 2017 -0500 -- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9dc9dfbf/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index 868ffaf..9820599 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,7 +83,7 @@ script: - ./validate_cmakelists.py - ./cyclic_dependency.py - (cd build && make) - - (cd build && ctest --output-on-failure -j$TEST_JOBS) + - (cd build && ctest --output-on-failure -E NetworkUtil_unittest -j$TEST_JOBS) after_failure: - df -h
incubator-quickstep git commit: Added Partition Rule For NestedLoopsJoin.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master bdf7ede11 -> 174b92c80
Added Partition Rule For NestedLoopsJoin.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/174b92c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/174b92c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/174b92c8
Branch: refs/heads/master
Commit: 174b92c807f2a6736ca56df68931c81dd9c077f0
Parents: bdf7ede
Author: Zuyu Zhang
Authored: Sat Jun 17 19:03:52 2017 -0500
Committer: Zuyu Zhang
Committed: Fri Jul 14 15:40:08 2017 -0500
--
query_optimizer/ExecutionGenerator.cpp | 17 ++--
query_optimizer/physical/NestedLoopsJoin.hpp| 22 -
query_optimizer/rules/Partition.cpp | 97
.../tests/execution_generator/Partition.test| 92 +++
.../NestedLoopsJoinOperator.hpp | 11 ++-
5 files changed, 221 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/174b92c8/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp
b/query_optimizer/ExecutionGenerator.cpp
index 7963d97..4f6f807 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1004,20 +1004,17 @@ void ExecutionGenerator::convertNestedLoopsJoin(
}
const PartitionScheme *left_partition_scheme =
left_relation.getPartitionScheme();
- const PartitionScheme *right_partition_scheme =
right_relation.getPartitionScheme();
- if (left_partition_scheme && right_partition_scheme) {
-
DCHECK_EQ(left_partition_scheme->getPartitionSchemeHeader().getNumPartitions(),
-
right_partition_scheme->getPartitionSchemeHeader().getNumPartitions());
- } else if (left_partition_scheme) {
-LOG(FATAL) << "Left side has partitions, but right does not";
- } else if (right_partition_scheme) {
-LOG(FATAL) << "Right side has partitions, but left does not";
- }
-
const std::size_t num_partitions =
left_partition_scheme ?
left_partition_scheme->getPartitionSchemeHeader().getNumPartitions()
: 1u;
+#ifdef QUICKSTEP_DEBUG
+ const PartitionScheme *right_partition_scheme =
right_relation.getPartitionScheme();
+ if (right_partition_scheme) {
+DCHECK_EQ(num_partitions,
right_partition_scheme->getPartitionSchemeHeader().getNumPartitions());
+ }
+#endif
+
const std::size_t nested_loops_join_index =
query_context_proto_->num_partitions_for_nested_loops_joins_size();
query_context_proto_->add_num_partitions_for_nested_loops_joins(num_partitions);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/174b92c8/query_optimizer/physical/NestedLoopsJoin.hpp
--
diff --git a/query_optimizer/physical/NestedLoopsJoin.hpp
b/query_optimizer/physical/NestedLoopsJoin.hpp
index c547ae8..f4c3a42 100644
--- a/query_optimizer/physical/NestedLoopsJoin.hpp
+++ b/query_optimizer/physical/NestedLoopsJoin.hpp
@@ -48,6 +48,8 @@ namespace physical {
class NestedLoopsJoin;
typedef std::shared_ptr NestedLoopsJoinPtr;
+struct PartitionSchemeHeader;
+
/**
* @brief Physical nested loops join node.
*/
@@ -72,7 +74,8 @@ class NestedLoopsJoin : public BinaryJoin {
return Create(new_children[0],
new_children[1],
join_predicate_,
- project_expressions());
+ project_expressions(),
+ cloneOutputPartitionSchemeHeader());
}
std::vector getReferencedAttributes()
const override;
@@ -81,6 +84,11 @@ class NestedLoopsJoin : public BinaryJoin {
const expressions::UnorderedNamedExpressionSet &referenced_expressions,
PhysicalPtr *output) const override;
+ PhysicalPtr copyWithNewOutputPartitionSchemeHeader(
+ PartitionSchemeHeader *partition_scheme_header) const override {
+return Create(left(), right(), join_predicate_, project_expressions(),
partition_scheme_header);
+ }
+
/**
* @brief Creates a NestedLoopsJoin.
*
@@ -88,15 +96,18 @@ class NestedLoopsJoin : public BinaryJoin {
* @param right The right operand.
* @param join_predicate The join predicate.
* @param project_expressions The project expressions.
+ * @param partition_scheme_header The optional output partition scheme
header.
+ *
* @return An immutable NestedLoopsJoin.
*/
static NestedLoopsJoinPtr Create(
const PhysicalPtr &left,
const PhysicalPtr &right,
const expressions::PredicatePtr &join_predicate,
- const std::vector &project_expressions)
{
+ const s
incubator-quickstep git commit: Added Physical PartitionSchemeHeader in Physical Aggregate.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master d9ce64308 -> b7a70951a
Added Physical PartitionSchemeHeader in Physical Aggregate.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b7a70951
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b7a70951
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b7a70951
Branch: refs/heads/master
Commit: b7a70951ab000c06190d0ab6c366decfc7c1e000
Parents: d9ce643
Author: Zuyu Zhang
Authored: Tue Jul 11 10:25:50 2017 -0500
Committer: Zuyu Zhang
Committed: Tue Jul 11 10:40:00 2017 -0500
--
query_optimizer/physical/Aggregate.cpp | 7 ++
query_optimizer/physical/Aggregate.hpp | 34
query_optimizer/physical/CMakeLists.txt | 1 +
3 files changed, 32 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b7a70951/query_optimizer/physical/Aggregate.cpp
--
diff --git a/query_optimizer/physical/Aggregate.cpp
b/query_optimizer/physical/Aggregate.cpp
index 35476af..f249e95 100644
--- a/query_optimizer/physical/Aggregate.cpp
+++ b/query_optimizer/physical/Aggregate.cpp
@@ -27,6 +27,8 @@
#include "query_optimizer/expressions/ExpressionUtil.hpp"
#include "query_optimizer/expressions/NamedExpression.hpp"
#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/physical/PartitionSchemeHeader.hpp"
+#include "query_optimizer/physical/Physical.hpp"
#include "utility/Cast.hpp"
namespace quickstep {
@@ -94,6 +96,11 @@ void Aggregate::getFieldStringItems(
std::vector *non_container_child_fields,
std::vector *container_child_field_names,
std::vector>
*container_child_fields) const {
+ if (partition_scheme_header_) {
+inline_field_names->push_back("output_partition_scheme_header");
+inline_field_values->push_back(partition_scheme_header_->toString());
+ }
+
non_container_child_field_names->push_back("input");
non_container_child_fields->push_back(input_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b7a70951/query_optimizer/physical/Aggregate.hpp
--
diff --git a/query_optimizer/physical/Aggregate.hpp
b/query_optimizer/physical/Aggregate.hpp
index de36cc3..c19a951 100644
--- a/query_optimizer/physical/Aggregate.hpp
+++ b/query_optimizer/physical/Aggregate.hpp
@@ -47,6 +47,8 @@ namespace physical {
class Aggregate;
typedef std::shared_ptr AggregatePtr;
+struct PartitionSchemeHeader;
+
/**
* @brief Aggregate operator that computes aggregate expressions for each
combination
*of the values of grouping expressions.
@@ -86,13 +88,20 @@ class Aggregate : public Physical {
PhysicalPtr copyWithNewChildren(
const std::vector &new_children) const override {
DCHECK_EQ(getNumChildren(), new_children.size());
-return Create(new_children[0], grouping_expressions_,
aggregate_expressions_, filter_predicate_);
+return Create(new_children[0], grouping_expressions_,
aggregate_expressions_, filter_predicate_,
+ cloneOutputPartitionSchemeHeader());
}
std::vector getOutputAttributes() const
override;
std::vector getReferencedAttributes()
const override;
+ PhysicalPtr copyWithNewOutputPartitionSchemeHeader(
+ PartitionSchemeHeader *partition_scheme_header) const override {
+return Create(input_, grouping_expressions_, aggregate_expressions_,
filter_predicate_,
+ partition_scheme_header);
+ }
+
bool maybeCopyWithPrunedExpressions(
const expressions::UnorderedNamedExpressionSet &referenced_expressions,
PhysicalPtr *output) const override {
@@ -107,15 +116,18 @@ class Aggregate : public Physical {
* @param aggregate_expressions The aggregate expressions.
* @param grouping_expressions The grouping expressions.
* @param filter_predicate The filtering predicate applied before
aggregation. Can be NULL.
+ * @param partition_scheme_header The optional output partition scheme
header.
+ *
* @return An immutable Aggregate node.
*/
static AggregatePtr Create(
PhysicalPtr input,
const std::vector &grouping_expressions,
const std::vector &aggregate_expressions,
- const expressions::PredicatePtr &filter_predicate) {
+ const expressions::PredicatePtr &filter_predicate,
+ PartitionSchemeHeader *partition_scheme_header = nullptr) {
return AggregatePtr(
-new Aggregate(input, grouping_expressions, aggregate_expressions,
filter_predicate));
+new Aggregate(input, grouping_expressions, aggregate_expressions,
filter_predicate, partition_schem
incubator-quickstep git commit: Minor style fixes in Partition rule.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 331ab02a7 -> d9ce64308
Minor style fixes in Partition rule.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d9ce6430
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d9ce6430
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d9ce6430
Branch: refs/heads/master
Commit: d9ce64308f0ebd8c3b7cdc81742ee8c7c4c46a5c
Parents: 331ab02
Author: Zuyu Zhang
Authored: Tue Jul 11 10:32:08 2017 -0500
Committer: Zuyu Zhang
Committed: Tue Jul 11 10:32:08 2017 -0500
--
query_optimizer/rules/Partition.cpp | 15 ---
1 file changed, 8 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d9ce6430/query_optimizer/rules/Partition.cpp
--
diff --git a/query_optimizer/rules/Partition.cpp
b/query_optimizer/rules/Partition.cpp
index 074c4e7..39546c6 100644
--- a/query_optimizer/rules/Partition.cpp
+++ b/query_optimizer/rules/Partition.cpp
@@ -84,14 +84,15 @@ bool needsSelection(const P::PhysicalType physical_type) {
P::PhysicalType::kUnionAll);
}
-P::PhysicalPtr Repartition(const P::PhysicalPtr &node, const
vector &join_attributes,
- const size_t num_partitions) {
+P::PhysicalPtr HashRepartition(const P::PhysicalPtr &node,
+ const vector
&repartition_attributes,
+ const size_t num_repartitions) {
P::PartitionSchemeHeader::PartitionExprIds repartition_expr_ids;
- for (const E::AttributeReferencePtr &attr : join_attributes) {
+ for (const E::AttributeReferencePtr &attr : repartition_attributes) {
repartition_expr_ids.push_back({ attr->id() });
}
auto repartition_scheme_header = make_unique(
- P::PartitionSchemeHeader::PartitionType::kHash, num_partitions,
move(repartition_expr_ids));
+ P::PartitionSchemeHeader::PartitionType::kHash, num_repartitions,
move(repartition_expr_ids));
if (needsSelection(node->getPhysicalType())) {
// Add a Selection node.
@@ -136,13 +137,13 @@ P::PhysicalPtr Partition::applyToNode(const
P::PhysicalPtr &node) {
needsRepartitionForHashJoin(left_partition_scheme_header,
left_join_attributes,
right_partition_scheme_header,
right_join_attributes,
&left_needs_repartition,
&right_needs_repartition, &num_partitions);
- // Repartition.
+ // Hash repartition.
if (left_needs_repartition) {
-left = Repartition(left, left_join_attributes, num_partitions);
+left = HashRepartition(left, left_join_attributes, num_partitions);
}
if (right_needs_repartition) {
-right = Repartition(right, right_join_attributes, num_partitions);
+right = HashRepartition(right, right_join_attributes, num_partitions);
}
unordered_set project_expr_ids;
incubator-quickstep git commit: IWYU for Selection strategy.
Repository: incubator-quickstep Updated Branches: refs/heads/master 85b537451 -> f35bd8f60 IWYU for Selection strategy. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f35bd8f6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f35bd8f6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f35bd8f6 Branch: refs/heads/master Commit: f35bd8f60b850ab079778ba1819e91369d99ba86 Parents: 85b5374 Author: Zuyu Zhang Authored: Tue Jun 20 18:24:44 2017 -0500 Committer: Zuyu Zhang Committed: Tue Jun 20 18:24:44 2017 -0500 -- query_optimizer/strategy/CMakeLists.txt | 2 ++ query_optimizer/strategy/Selection.cpp | 4 2 files changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f35bd8f6/query_optimizer/strategy/CMakeLists.txt -- diff --git a/query_optimizer/strategy/CMakeLists.txt b/query_optimizer/strategy/CMakeLists.txt index e3e6d76..e011126 100644 --- a/query_optimizer/strategy/CMakeLists.txt +++ b/query_optimizer/strategy/CMakeLists.txt @@ -118,6 +118,7 @@ target_link_libraries(quickstep_queryoptimizer_strategy_Selection quickstep_queryoptimizer_LogicalToPhysicalMapper quickstep_queryoptimizer_OptimizerContext quickstep_queryoptimizer_expressions_AttributeReference + quickstep_queryoptimizer_expressions_ExprId quickstep_queryoptimizer_expressions_ExpressionUtil quickstep_queryoptimizer_expressions_NamedExpression quickstep_queryoptimizer_expressions_Predicate @@ -125,6 +126,7 @@ target_link_libraries(quickstep_queryoptimizer_strategy_Selection quickstep_queryoptimizer_logical_Logical quickstep_queryoptimizer_logical_PatternMatcher quickstep_queryoptimizer_logical_Project + quickstep_queryoptimizer_physical_PartitionSchemeHeader quickstep_queryoptimizer_physical_PatternMatcher quickstep_queryoptimizer_physical_Physical quickstep_queryoptimizer_physical_Selection http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f35bd8f6/query_optimizer/strategy/Selection.cpp -- diff --git a/query_optimizer/strategy/Selection.cpp b/query_optimizer/strategy/Selection.cpp index d738921..bc057de 100644 --- a/query_optimizer/strategy/Selection.cpp +++ b/query_optimizer/strategy/Selection.cpp @@ -20,11 +20,14 @@ #include "query_optimizer/strategy/Selection.hpp" #include +#include +#include #include #include "query_optimizer/LogicalToPhysicalMapper.hpp" #include "query_optimizer/OptimizerContext.hpp" #include "query_optimizer/expressions/AttributeReference.hpp" +#include "query_optimizer/expressions/ExprId.hpp" #include "query_optimizer/expressions/ExpressionUtil.hpp" #include "query_optimizer/expressions/NamedExpression.hpp" #include "query_optimizer/expressions/Predicate.hpp" @@ -32,6 +35,7 @@ #include "query_optimizer/logical/Logical.hpp" #include "query_optimizer/logical/PatternMatcher.hpp" #include "query_optimizer/logical/Project.hpp" +#include "query_optimizer/physical/PartitionSchemeHeader.hpp" #include "query_optimizer/physical/PatternMatcher.hpp" #include "query_optimizer/physical/Physical.hpp" #include "query_optimizer/physical/Selection.hpp"
incubator-quickstep git commit: Refactored Create Factory Method for Physical Selection.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 261c955ad -> 85b537451
Refactored Create Factory Method for Physical Selection.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/85b53745
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/85b53745
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/85b53745
Branch: refs/heads/master
Commit: 85b537451d7684befbaebb263ed983e440c1eb30
Parents: 261c955
Author: Zuyu Zhang
Authored: Tue Jun 20 15:42:31 2017 -0500
Committer: Zuyu Zhang
Committed: Tue Jun 20 15:42:31 2017 -0500
--
query_optimizer/physical/Selection.cpp | 25
query_optimizer/physical/Selection.hpp | 5 +++-
query_optimizer/rules/CollapseSelection.cpp | 3 ++-
.../rules/ExtractCommonSubexpression.cpp| 3 ++-
.../PushDownLowCostDisjunctivePredicate.cpp | 3 ++-
query_optimizer/rules/ReorderColumns.cpp| 3 ++-
query_optimizer/strategy/Selection.cpp | 17 -
7 files changed, 28 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/85b53745/query_optimizer/physical/Selection.cpp
--
diff --git a/query_optimizer/physical/Selection.cpp
b/query_optimizer/physical/Selection.cpp
index e2fa69e..3307e9e 100644
--- a/query_optimizer/physical/Selection.cpp
+++ b/query_optimizer/physical/Selection.cpp
@@ -42,31 +42,6 @@ namespace physical {
namespace E = ::quickstep::optimizer::expressions;
-SelectionPtr Selection::Create(
- const PhysicalPtr &input,
- const std::vector &project_expressions,
- const E::PredicatePtr &filter_predicate,
- PartitionSchemeHeader *output_partition_scheme_header) {
- std::unique_ptr
partition_scheme_header(output_partition_scheme_header);
-
- if (!partition_scheme_header) {
-const PartitionSchemeHeader *input_partition_scheme_header =
input->getOutputPartitionSchemeHeader();
-if (input_partition_scheme_header) {
- unordered_set project_expr_ids;
- for (const E::NamedExpressionPtr &project_expression :
project_expressions) {
-project_expr_ids.insert(project_expression->id());
- }
-
- if
(input_partition_scheme_header->reusablePartitionScheme(project_expr_ids)) {
-partition_scheme_header =
std::make_unique(*input_partition_scheme_header);
- }
-}
- }
-
- return SelectionPtr(
- new Selection(input, project_expressions, filter_predicate,
partition_scheme_header.release()));
-}
-
PhysicalPtr Selection::copyWithNewChildren(
const std::vector &new_children) const {
DCHECK_EQ(children().size(), new_children.size());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/85b53745/query_optimizer/physical/Selection.hpp
--
diff --git a/query_optimizer/physical/Selection.hpp
b/query_optimizer/physical/Selection.hpp
index 204eb2f..4cbc856 100644
--- a/query_optimizer/physical/Selection.hpp
+++ b/query_optimizer/physical/Selection.hpp
@@ -109,7 +109,10 @@ class Selection : public Physical {
const PhysicalPtr &input,
const std::vector &project_expressions,
const expressions::PredicatePtr &filter_predicate,
- PartitionSchemeHeader *output_partition_scheme_header = nullptr);
+ PartitionSchemeHeader *output_partition_scheme_header = nullptr) {
+return SelectionPtr(
+new Selection(input, project_expressions, filter_predicate,
output_partition_scheme_header));
+ }
/**
* @brief Creates a conjunctive predicate with \p filter_predicates
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/85b53745/query_optimizer/rules/CollapseSelection.cpp
--
diff --git a/query_optimizer/rules/CollapseSelection.cpp
b/query_optimizer/rules/CollapseSelection.cpp
index f92e1b2..fc45ffd 100644
--- a/query_optimizer/rules/CollapseSelection.cpp
+++ b/query_optimizer/rules/CollapseSelection.cpp
@@ -49,7 +49,8 @@ P::PhysicalPtr CollapseSelection::applyToNode(const
P::PhysicalPtr &input) {
{ &project_expressions } /*
project_expression_lists */);
return P::Selection::Create(child_selection->input(),
project_expressions,
-selection->filter_predicate());
+selection->filter_predicate(),
+
child_selection->input()->cloneOutputPartitionSchemeHeader());
}
return input;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/85b53745/query_optimizer/rules/Extr
incubator-quickstep git commit: Minor refactor for printing FilterJoin in a Physical Plan.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 4b8963493 -> 261c955ad
Minor refactor for printing FilterJoin in a Physical Plan.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/261c955a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/261c955a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/261c955a
Branch: refs/heads/master
Commit: 261c955ad45c3098dd1792bb3763b13643fd9906
Parents: 4b89634
Author: Zuyu Zhang
Authored: Mon Jun 19 16:06:01 2017 -0500
Committer: Zuyu Zhang
Committed: Mon Jun 19 16:06:01 2017 -0500
--
query_optimizer/physical/FilterJoin.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/261c955a/query_optimizer/physical/FilterJoin.cpp
--
diff --git a/query_optimizer/physical/FilterJoin.cpp
b/query_optimizer/physical/FilterJoin.cpp
index ca46791..06f5a16 100644
--- a/query_optimizer/physical/FilterJoin.cpp
+++ b/query_optimizer/physical/FilterJoin.cpp
@@ -99,7 +99,7 @@ void FilterJoin::getFieldStringItems(
container_child_fields);
inline_field_names->push_back("is_anti_join");
- inline_field_values->push_back(std::to_string(is_anti_join_));
+ inline_field_values->push_back(is_anti_join_ ? "true" : "false");
if (build_side_filter_predicate_ != nullptr) {
non_container_child_field_names->emplace_back("build_side_filter_predicate");
incubator-quickstep git commit: Added Partition Rule for HashJoin.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 1c749c911 -> 4b8963493
Added Partition Rule for HashJoin.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4b896349
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4b896349
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4b896349
Branch: refs/heads/master
Commit: 4b8963493b8ebcce38ea1d1cc0a6392dbbae26e2
Parents: 1c749c9
Author: Zuyu Zhang
Authored: Tue Jun 13 21:50:41 2017 -0500
Committer: Zuyu Zhang
Committed: Fri Jun 16 17:17:06 2017 -0500
--
cli/distributed/QuickstepDistributedCli.cpp | 5 +
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/ExecutionGenerator.cpp | 1 -
query_optimizer/PhysicalGenerator.cpp | 11 +
query_optimizer/physical/FilterJoin.cpp | 4 +-
query_optimizer/physical/FilterJoin.hpp | 26 +-
query_optimizer/physical/Physical.hpp | 20 +-
query_optimizer/rules/CMakeLists.txt| 25 ++
query_optimizer/rules/InjectJoinFilters.cpp | 9 +-
query_optimizer/rules/Partition.cpp | 297 +
query_optimizer/rules/Partition.hpp | 106 ++
query_optimizer/rules/tests/CMakeLists.txt | 4 +
.../rules/tests/Partition_unittest.cpp | 326 +++
.../tests/execution_generator/Partition.test| 85 -
14 files changed, 891 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b896349/cli/distributed/QuickstepDistributedCli.cpp
--
diff --git a/cli/distributed/QuickstepDistributedCli.cpp
b/cli/distributed/QuickstepDistributedCli.cpp
index 513bedd..e8fa6a1 100644
--- a/cli/distributed/QuickstepDistributedCli.cpp
+++ b/cli/distributed/QuickstepDistributedCli.cpp
@@ -69,6 +69,11 @@ using quickstep::FLAGS_role;
int main(int argc, char *argv[]) {
google::InitGoogleLogging(argv[0]);
+ // TODO(quickstep-team): Fix JIRA QUICKSTEP-94 for adding LIP filter support
+ // with partitioned inputs in the distributed version.
+ quickstep::optimizer::FLAGS_use_lip_filters = false;
+ quickstep::optimizer::FLAGS_use_filter_joins = false;
+
gflags::ParseCommandLineFlags(&argc, &argv, true);
grpc_init();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b896349/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 04af02c..564c5c8 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -220,6 +220,7 @@
target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_queryoptimizer_rules_ExtractCommonSubexpression
quickstep_queryoptimizer_rules_FuseAggregateJoin
quickstep_queryoptimizer_rules_InjectJoinFilters
+ quickstep_queryoptimizer_rules_Partition
quickstep_queryoptimizer_rules_PruneColumns
quickstep_queryoptimizer_rules_PushDownLowCostDisjunctivePredicate
quickstep_queryoptimizer_rules_ReduceGroupByAttributes
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b896349/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp
b/query_optimizer/ExecutionGenerator.cpp
index a7c7328..7963d97 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -850,7 +850,6 @@ void ExecutionGenerator::convertHashJoin(const
P::HashJoinPtr &physical_plan) {
probe_partition_scheme_header ?
probe_partition_scheme_header->num_partitions : 1u;
hash_table_context_proto->set_num_partitions(probe_num_partitions);
-
S::HashTable *hash_table_proto =
hash_table_context_proto->mutable_join_hash_table();
// SimplifyHashTableImplTypeProto() switches the hash table implementation
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b896349/query_optimizer/PhysicalGenerator.cpp
--
diff --git a/query_optimizer/PhysicalGenerator.cpp
b/query_optimizer/PhysicalGenerator.cpp
index 5f3b1b7..6932b30 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -31,6 +31,7 @@
#include "query_optimizer/rules/ExtractCommonSubexpression.hpp"
#include "query_optimizer/rules/FuseAggregateJoin.hpp"
#include "query_optimizer/rules/InjectJoinFilters.hpp"
+#include "query_optimiz
incubator-quickstep git commit: Added the execution support for LIP with partitions.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 5fbfd2111 -> 1c749c911
Added the execution support for LIP with partitions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1c749c91
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1c749c91
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1c749c91
Branch: refs/heads/master
Commit: 1c749c911814926c48b8c4f7f34519605dc435a2
Parents: 5fbfd21
Author: Zuyu Zhang
Authored: Thu Jun 15 17:38:53 2017 -0500
Committer: Zuyu Zhang
Committed: Thu Jun 15 17:38:53 2017 -0500
--
query_execution/ForemanDistributed.cpp| 5 -
query_execution/PolicyEnforcerDistributed.cpp | 2 ++
query_execution/PolicyEnforcerDistributed.hpp | 2 ++
query_execution/QueryContext.proto| 2 ++
query_execution/QueryManagerDistributed.cpp | 6 ++
query_execution/QueryManagerDistributed.hpp | 24 --
6 files changed, 34 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1c749c91/query_execution/ForemanDistributed.cpp
--
diff --git a/query_execution/ForemanDistributed.cpp
b/query_execution/ForemanDistributed.cpp
index 49f2101..942f383 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -329,6 +329,7 @@ bool ForemanDistributed::isLipRelatedWorkOrder(const
S::WorkOrderMessage &proto,
size_t
*shiftboss_index_for_lip) {
const S::WorkOrder &work_order_proto = proto.work_order();
vector lip_filter_indexes;
+ partition_id part_id;
block_id block = kInvalidBlockId;
switch (work_order_proto.work_order_type()) {
@@ -336,12 +337,14 @@ bool ForemanDistributed::isLipRelatedWorkOrder(const
S::WorkOrderMessage &proto,
for (int i = 0; i <
work_order_proto.ExtensionSize(S::BuildLIPFilterWorkOrder::lip_filter_indexes);
++i) {
lip_filter_indexes.push_back(work_order_proto.GetExtension(S::BuildLIPFilterWorkOrder::lip_filter_indexes,
i));
}
+ part_id =
work_order_proto.GetExtension(S::BuildLIPFilterWorkOrder::partition_id);
block =
work_order_proto.GetExtension(S::BuildLIPFilterWorkOrder::build_block_id);
break;
case S::SELECT:
for (int i = 0; i <
work_order_proto.ExtensionSize(S::SelectWorkOrder::lip_filter_indexes); ++i) {
lip_filter_indexes.push_back(work_order_proto.GetExtension(S::SelectWorkOrder::lip_filter_indexes,
i));
}
+ part_id =
work_order_proto.GetExtension(S::SelectWorkOrder::partition_id);
block = work_order_proto.GetExtension(S::SelectWorkOrder::block_id);
break;
default:
@@ -349,7 +352,7 @@ bool ForemanDistributed::isLipRelatedWorkOrder(const
S::WorkOrderMessage &proto,
}
static_cast(policy_enforcer_.get())->getShiftbossIndexForLip(
- proto.query_id(), lip_filter_indexes, block_locator_, block,
next_shiftboss_index_to_schedule,
+ proto.query_id(), lip_filter_indexes, part_id, block_locator_, block,
next_shiftboss_index_to_schedule,
shiftboss_index_for_lip);
return true;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1c749c91/query_execution/PolicyEnforcerDistributed.cpp
--
diff --git a/query_execution/PolicyEnforcerDistributed.cpp
b/query_execution/PolicyEnforcerDistributed.cpp
index 766c351..2bd25fa 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -232,6 +232,7 @@ void
PolicyEnforcerDistributed::getShiftbossIndexForHashJoin(
void PolicyEnforcerDistributed::getShiftbossIndexForLip(
const std::size_t query_id,
const vector &lip_filter_indexes,
+const partition_id part_id,
const BlockLocator &block_locator,
const block_id block,
const std::size_t next_shiftboss_index_to_schedule,
@@ -239,6 +240,7 @@ void PolicyEnforcerDistributed::getShiftbossIndexForLip(
DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
QueryManagerDistributed *query_manager =
static_cast(admitted_queries_[query_id].get());
query_manager->getShiftbossIndexForLip(lip_filter_indexes,
+ part_id,
block_locator,
block,
next_shiftboss_index_to_schedule,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1c749c91/query_execution/PolicyEnforcerDistributed.hpp
--
diff --git a/qu
incubator-quickstep git commit: Using PartitionSchemeHeader in Physical Plan node.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 13c16b9cb -> 5fbfd2111
Using PartitionSchemeHeader in Physical Plan node.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5fbfd211
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5fbfd211
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5fbfd211
Branch: refs/heads/master
Commit: 5fbfd21110091cfaf262daa8d9d2620a4e3f03bb
Parents: 13c16b9
Author: Zuyu Zhang
Authored: Tue Jun 13 21:50:41 2017 -0500
Committer: Zuyu Zhang
Committed: Wed Jun 14 20:42:37 2017 -0500
--
query_optimizer/CMakeLists.txt | 3 +-
query_optimizer/ExecutionGenerator.cpp | 100 ---
query_optimizer/physical/BinaryJoin.cpp | 7 ++
query_optimizer/physical/BinaryJoin.hpp | 8 +-
query_optimizer/physical/CMakeLists.txt | 8 ++
query_optimizer/physical/HashJoin.cpp | 4 +-
query_optimizer/physical/HashJoin.hpp | 24 --
query_optimizer/physical/Join.hpp | 9 +-
query_optimizer/physical/Physical.hpp | 52 +++-
query_optimizer/physical/Selection.cpp | 40 -
query_optimizer/physical/Selection.hpp | 27 --
query_optimizer/physical/TableGenerator.hpp | 18 +++-
query_optimizer/physical/TableReference.cpp | 55 +
query_optimizer/physical/TableReference.hpp | 21 ++---
query_optimizer/rules/CMakeLists.txt| 4 +-
relational_operators/HashJoinOperator.cpp | 6 ++
relational_operators/HashJoinOperator.hpp | 4 +-
relational_operators/SelectOperator.cpp | 13 +--
relational_operators/SelectOperator.hpp | 10 ++-
relational_operators/WorkOrder.proto| 2 +
relational_operators/WorkOrderFactory.cpp | 9 +-
storage/CMakeLists.txt | 1 +
storage/InsertDestination.cpp | 34 ++--
storage/InsertDestination.hpp | 34 +++-
utility/CMakeLists.txt | 1 +
utility/PlanVisualizer.cpp | 6 ++
26 files changed, 394 insertions(+), 106 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fbfd211/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index c969f16..04af02c 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -103,6 +103,7 @@
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_queryoptimizer_physical_InsertTuple
quickstep_queryoptimizer_physical_LIPFilterConfiguration
quickstep_queryoptimizer_physical_NestedLoopsJoin
+ quickstep_queryoptimizer_physical_PartitionSchemeHeader
quickstep_queryoptimizer_physical_PatternMatcher
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
@@ -211,6 +212,7 @@ target_link_libraries(quickstep_queryoptimizer_OptimizerTree
target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
${GFLAGS_LIB_NAME}
quickstep_queryoptimizer_LogicalToPhysicalMapper
+ quickstep_queryoptimizer_Validator
quickstep_queryoptimizer_logical_Logical
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_rules_AttachLIPFilters
@@ -230,7 +232,6 @@
target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_queryoptimizer_strategy_OneToOne
quickstep_queryoptimizer_strategy_Selection
quickstep_queryoptimizer_strategy_Strategy
- quickstep_queryoptimizer_Validator
quickstep_utility_Macros
quickstep_utility_PlanVisualizer)
target_link_libraries(quickstep_queryoptimizer_QueryHandle
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fbfd211/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp
b/query_optimizer/ExecutionGenerator.cpp
index 3b2fe08..a7c7328 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -84,6 +84,7 @@
#include "query_optimizer/physical/InsertTuple.hpp"
#include "query_optimizer/physical/LIPFilterConfiguration.hpp"
#include "query_optimizer/physical/NestedLoopsJoin.hpp"
+#include "query_optimizer/physical/PartitionSchemeHeader.hpp"
#inclu
incubator-quickstep git commit: Added PartitionSchemeHeader in Physical Plan node.
Repository: incubator-quickstep Updated Branches: refs/heads/master e6ac59d5a -> 13c16b9cb Added PartitionSchemeHeader in Physical Plan node. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/13c16b9c Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/13c16b9c Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/13c16b9c Branch: refs/heads/master Commit: 13c16b9cbee8a380ad1ab98ef307475376e9e661 Parents: e6ac59d Author: Zuyu Zhang Authored: Wed Jun 14 20:34:57 2017 -0500 Committer: Zuyu Zhang Committed: Wed Jun 14 20:34:57 2017 -0500 -- query_optimizer/physical/CMakeLists.txt | 6 + .../physical/PartitionSchemeHeader.cpp | 113 .../physical/PartitionSchemeHeader.hpp | 132 +++ 3 files changed, 251 insertions(+) -- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13c16b9c/query_optimizer/physical/CMakeLists.txt -- diff --git a/query_optimizer/physical/CMakeLists.txt b/query_optimizer/physical/CMakeLists.txt index 2751c6e..1e777b1 100644 --- a/query_optimizer/physical/CMakeLists.txt +++ b/query_optimizer/physical/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(quickstep_queryoptimizer_physical_InsertTuple InsertTuple.cpp Insert add_library(quickstep_queryoptimizer_physical_Join ../../empty_src.cpp Join.hpp) add_library(quickstep_queryoptimizer_physical_LIPFilterConfiguration ../../empty_src.cpp LIPFilterConfiguration.hpp) add_library(quickstep_queryoptimizer_physical_NestedLoopsJoin NestedLoopsJoin.cpp NestedLoopsJoin.hpp) +add_library(quickstep_queryoptimizer_physical_PartitionSchemeHeader PartitionSchemeHeader.cpp PartitionSchemeHeader.hpp) add_library(quickstep_queryoptimizer_physical_PatternMatcher ../../empty_src.cpp PatternMatcher.hpp) add_library(quickstep_queryoptimizer_physical_Physical ../../empty_src.cpp Physical.hpp) add_library(quickstep_queryoptimizer_physical_PhysicalType ../../empty_src.cpp PhysicalType.hpp) @@ -194,6 +195,10 @@ target_link_libraries(quickstep_queryoptimizer_physical_NestedLoopsJoin quickstep_queryoptimizer_physical_Physical quickstep_queryoptimizer_physical_PhysicalType quickstep_utility_Macros) +target_link_libraries(quickstep_queryoptimizer_physical_PartitionSchemeHeader + glog + quickstep_queryoptimizer_expressions_ExprId + quickstep_utility_Macros) target_link_libraries(quickstep_queryoptimizer_physical_PatternMatcher quickstep_queryoptimizer_physical_PhysicalType) target_link_libraries(quickstep_queryoptimizer_physical_Physical @@ -325,6 +330,7 @@ target_link_libraries(quickstep_queryoptimizer_physical quickstep_queryoptimizer_physical_Join quickstep_queryoptimizer_physical_LIPFilterConfiguration quickstep_queryoptimizer_physical_NestedLoopsJoin + quickstep_queryoptimizer_physical_PartitionSchemeHeader quickstep_queryoptimizer_physical_PatternMatcher quickstep_queryoptimizer_physical_Physical quickstep_queryoptimizer_physical_PhysicalType http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13c16b9c/query_optimizer/physical/PartitionSchemeHeader.cpp -- diff --git a/query_optimizer/physical/PartitionSchemeHeader.cpp b/query_optimizer/physical/PartitionSchemeHeader.cpp new file mode 100644 index 000..829b52b --- /dev/null +++ b/query_optimizer/physical/PartitionSchemeHeader.cpp @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +#include "query_optimizer/physical/PartitionSchemeHeader.hpp" + +#include +#include +#include + +#include "query_optimizer/expressions/Ex
incubator-quickstep git commit: Added the support for partitioned (but not parallel) aggregations.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 361a65fa6 -> e6ac59d5a
Added the support for partitioned (but not parallel) aggregations.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e6ac59d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e6ac59d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e6ac59d5
Branch: refs/heads/master
Commit: e6ac59d5a966f23a4fad3d907be7cc9b4ba53820
Parents: 361a65f
Author: Zuyu Zhang
Authored: Fri May 26 02:08:57 2017 -0700
Committer: Zuyu Zhang
Committed: Wed Jun 14 19:00:28 2017 -0500
--
query_execution/BlockLocator.hpp| 2 +
query_execution/ForemanDistributed.cpp | 14 +++-
query_execution/PolicyEnforcerDistributed.cpp | 2 +
query_execution/PolicyEnforcerDistributed.hpp | 2 +
query_execution/QueryContext.cpp| 24 +--
query_execution/QueryContext.hpp| 26 +---
query_execution/QueryContext.proto | 7 +-
query_execution/QueryManagerDistributed.cpp | 5 +-
query_execution/QueryManagerDistributed.hpp | 16 +++--
query_optimizer/ExecutionGenerator.cpp | 57
relational_operators/AggregationOperator.cpp| 62 ++---
relational_operators/AggregationOperator.hpp| 38 ---
.../BuildAggregationExistenceMapOperator.cpp| 64 +++---
.../BuildAggregationExistenceMapOperator.hpp| 37 ---
relational_operators/CMakeLists.txt | 4 ++
.../DestroyAggregationStateOperator.cpp | 24 ---
.../DestroyAggregationStateOperator.hpp | 12 +++-
.../FinalizeAggregationOperator.cpp | 58 ++--
.../FinalizeAggregationOperator.hpp | 18 +++--
.../InitializeAggregationOperator.cpp | 44
.../InitializeAggregationOperator.hpp | 15 +++--
relational_operators/WorkOrder.proto| 19 +-
relational_operators/WorkOrderFactory.cpp | 70
.../tests/AggregationOperator_unittest.cpp | 17 +++--
24 files changed, 457 insertions(+), 180 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e6ac59d5/query_execution/BlockLocator.hpp
--
diff --git a/query_execution/BlockLocator.hpp b/query_execution/BlockLocator.hpp
index 82c28ae..01492f9 100644
--- a/query_execution/BlockLocator.hpp
+++ b/query_execution/BlockLocator.hpp
@@ -123,6 +123,8 @@ class BlockLocator : public Thread {
* @return Whether the block locality info has found.
**/
bool getBlockLocalityInfo(const block_id block, std::size_t
*shiftboss_index_for_block) const {
+if (block == kInvalidBlockId) { return false; }
+
const std::unordered_set block_domains =
getBlockDomains(block);
if (!block_domains.empty()) {
// NOTE(zuyu): This lock is held for the rest duration of this call, as
the
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e6ac59d5/query_execution/ForemanDistributed.cpp
--
diff --git a/query_execution/ForemanDistributed.cpp
b/query_execution/ForemanDistributed.cpp
index 34b5b76..49f2101 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -243,12 +243,14 @@ bool
ForemanDistributed::isAggregationRelatedWorkOrder(const S::WorkOrderMessage
size_t
*shiftboss_index_for_aggregation) {
const S::WorkOrder &work_order_proto = proto.work_order();
QueryContext::aggregation_state_id aggr_state_index;
+ partition_id part_id;
vector lip_filter_indexes;
block_id block = kInvalidBlockId;
switch (work_order_proto.work_order_type()) {
case S::AGGREGATION:
aggr_state_index =
work_order_proto.GetExtension(S::AggregationWorkOrder::aggr_state_index);
+ part_id =
work_order_proto.GetExtension(S::AggregationWorkOrder::partition_id);
for (int i = 0; i <
work_order_proto.ExtensionSize(S::AggregationWorkOrder::lip_filter_indexes);
++i) {
lip_filter_indexes.push_back(work_order_proto.GetExtension(S::AggregationWorkOrder::lip_filter_indexes,
i));
@@ -256,18 +258,28 @@ bool
ForemanDistributed::isAggregationRelatedWorkOrder(const S::WorkOrderMessage
block = work_order_proto.GetExtension(S::AggregationWorkOrder::block_id);
break;
+case S::BUILD_AGGREGATION_EXISTENCE_MAP:
+ aggr_state_index =
work_order_proto.GetExtension(S::BuildAggregationExistenceMapWorkOrder::aggr_state_index);
+ part_id =
work_order_proto.GetExtension(S::BuildAggregationExistenceMapWorkO
incubator-quickstep git commit: Added operator and execution support for partitioned nested loops join.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master cbb84b4d6 -> 361a65fa6
Added operator and execution support for partitioned nested loops join.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/361a65fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/361a65fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/361a65fa
Branch: refs/heads/master
Commit: 361a65fa615fcdcfd3f35869b2690d19e8156e86
Parents: cbb84b4
Author: Zuyu Zhang
Authored: Sat Jun 3 18:59:14 2017 -0700
Committer: Zuyu Zhang
Committed: Wed Jun 14 18:57:15 2017 -0500
--
query_execution/ForemanDistributed.cpp | 25 +-
query_execution/PolicyEnforcerDistributed.cpp | 20 ++
query_execution/PolicyEnforcerDistributed.hpp | 27 ++
query_execution/QueryContext.proto | 2 +
query_execution/QueryManagerDistributed.cpp | 5 +
query_execution/QueryManagerDistributed.hpp | 41 +++
query_optimizer/ExecutionGenerator.cpp | 29 +-
relational_operators/CMakeLists.txt | 2 +
.../NestedLoopsJoinOperator.cpp | 340 ++-
.../NestedLoopsJoinOperator.hpp | 83 +++--
relational_operators/WorkOrder.proto| 3 +
11 files changed, 378 insertions(+), 199 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/361a65fa/query_execution/ForemanDistributed.cpp
--
diff --git a/query_execution/ForemanDistributed.cpp
b/query_execution/ForemanDistributed.cpp
index fbac18e..34b5b76 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -347,22 +347,6 @@ namespace {
constexpr size_t kDefaultShiftbossIndex = 0u;
-bool isNestedLoopsJoinWorkOrder(const serialization::WorkOrder
&work_order_proto,
-const BlockLocator &block_locator,
-std::size_t *shiftboss_index_for_join) {
- if (work_order_proto.work_order_type() != S::NESTED_LOOP_JOIN) {
-return false;
- }
-
- const block_id left_block =
work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::left_block_id);
- if (block_locator.getBlockLocalityInfo(left_block,
shiftboss_index_for_join)) {
-return true;
- }
-
- const block_id right_block =
work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::right_block_id);
- return block_locator.getBlockLocalityInfo(right_block,
shiftboss_index_for_join);
-}
-
bool hasBlockLocalityInfo(const serialization::WorkOrder &work_order_proto,
const BlockLocator &block_locator,
std::size_t *shiftboss_index_for_block) {
@@ -401,10 +385,15 @@ void ForemanDistributed::dispatchWorkOrderMessages(const
vector(policy_enforcer_.get())->getShiftbossIndexForNestedLoopsJoin(
+ proto.query_id(),
work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::nested_loops_join_index),
+
work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::partition_id),
block_locator_,
+
work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::left_block_id),
+
work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::right_block_id),
+ shiftboss_index, &shiftboss_index_for_particular_work_order_type);
} else if (hasBlockLocalityInfo(work_order_proto, block_locator_,
&shiftboss_index_for_particular_work_order_type)) {
-} else if (isNestedLoopsJoinWorkOrder(work_order_proto, block_locator_,
-
&shiftboss_index_for_particular_work_order_type)) {
} else {
shiftboss_index_for_particular_work_order_type = shiftboss_index;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/361a65fa/query_execution/PolicyEnforcerDistributed.cpp
--
diff --git a/query_execution/PolicyEnforcerDistributed.cpp
b/query_execution/PolicyEnforcerDistributed.cpp
index 0a4fd30..7e9a81d 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -243,6 +243,26 @@ void PolicyEnforcerDistributed::getShiftbossIndexForLip(
shiftboss_index);
}
+void PolicyEnforcerDistributed::getShiftbossIndexForNestedLoopsJoin(
+const std::size_t query_id,
+const std::size_t nested_loops_join_index,
+const partition_id part_id,
+const BlockLocator &block_locator,
+const block_id left_block,
+const block_id right_block,
+const std::size_t next_shiftboss_index_to_schedule,
+std::size_t *shiftboss_index_for_nested_lo
incubator-quickstep git commit: Added RandomPartitionSchemeHeader.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 7c5d76223 -> cbb84b4d6
Added RandomPartitionSchemeHeader.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/cbb84b4d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/cbb84b4d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/cbb84b4d
Branch: refs/heads/master
Commit: cbb84b4d619540f8ca7c586695fece0e46cfeaf7
Parents: 7c5d762
Author: Zuyu Zhang
Authored: Sat Jun 3 09:39:48 2017 -0700
Committer: Zuyu Zhang
Committed: Wed Jun 14 18:53:43 2017 -0500
--
catalog/CMakeLists.txt | 2 ++
catalog/Catalog.proto | 2 ++
catalog/PartitionSchemeHeader.cpp | 7
catalog/PartitionSchemeHeader.hpp | 43 -
catalog/tests/PartitionScheme_unittest.cpp | 41 +++
5 files changed, 94 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cbb84b4d/catalog/CMakeLists.txt
--
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index 9684cfe..0ca014e 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -168,6 +168,8 @@
target_link_libraries(quickstep_catalog_PartitionSchemeHeader
glog
quickstep_catalog_CatalogTypedefs
quickstep_catalog_Catalog_proto
+ quickstep_storage_StorageConstants
+ quickstep_threading_SpinMutex
quickstep_types_Type
quickstep_types_TypeFactory
quickstep_types_Type_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cbb84b4d/catalog/Catalog.proto
--
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index 4e2fafe..6c6accf 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -31,8 +31,10 @@ message CatalogAttribute {
// TODO(zuyu): Move PartitionScheme to a dedicate proto file.
message PartitionSchemeHeader {
+ // Next tag: 3.
enum PartitionType {
HASH = 0;
+RANDOM = 2;
RANGE = 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cbb84b4d/catalog/PartitionSchemeHeader.cpp
--
diff --git a/catalog/PartitionSchemeHeader.cpp
b/catalog/PartitionSchemeHeader.cpp
index 30fa58d..f7a0971 100644
--- a/catalog/PartitionSchemeHeader.cpp
+++ b/catalog/PartitionSchemeHeader.cpp
@@ -74,6 +74,7 @@ bool PartitionSchemeHeader::ProtoIsValid(
// Check that the proto has a valid partition type.
switch (proto.partition_type()) {
case serialization::PartitionSchemeHeader::HASH:
+case serialization::PartitionSchemeHeader::RANDOM:
return true;
case serialization::PartitionSchemeHeader::RANGE: {
return
proto.HasExtension(serialization::RangePartitionSchemeHeader::partition_range_boundaries)
&&
@@ -104,6 +105,9 @@ PartitionSchemeHeader*
PartitionSchemeHeader::ReconstructFromProto(
case serialization::PartitionSchemeHeader::HASH: {
return new HashPartitionSchemeHeader(proto.num_partitions(),
move(partition_attribute_ids));
}
+case serialization::PartitionSchemeHeader::RANDOM: {
+ return new RandomPartitionSchemeHeader(proto.num_partitions());
+}
case serialization::PartitionSchemeHeader::RANGE: {
std::vector attr_types;
for (int i = 0; i <
proto.ExtensionSize(serialization::RangePartitionSchemeHeader::partition_attr_types);
++i) {
@@ -142,6 +146,9 @@ serialization::PartitionSchemeHeader
PartitionSchemeHeader::getProto() const {
case PartitionType::kHash:
proto.set_partition_type(serialization::PartitionSchemeHeader::HASH);
break;
+case PartitionType::kRandom:
+ proto.set_partition_type(serialization::PartitionSchemeHeader::RANDOM);
+ break;
case PartitionType::kRange:
proto.set_partition_type(serialization::PartitionSchemeHeader::RANGE);
break;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cbb84b4d/catalog/PartitionSchemeHeader.hpp
--
diff --git a/catalog/PartitionSchemeHeader.hpp
b/catalog/PartitionSchemeHeader.hpp
index a03b0e2..9bbbc0f 100644
--- a/catalog/PartitionSchemeHeader.hpp
+++ b/catalog/PartitionSchemeHeader.hpp
@@ -22,11 +22,14 @@
#include
#include
+#include
#include
#include
#include "catalog/Catalog.pb.h"
#include "catalog/CatalogTypedefs.hpp"
+#include "storage/StorageConstants.hpp"
+#include "threading/Sp
incubator-quickstep git commit: Added operator support for BuildLipFilter with partitioned inputs.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 0f549acb5 -> 7c5d76223
Added operator support for BuildLipFilter with partitioned inputs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7c5d7622
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7c5d7622
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7c5d7622
Branch: refs/heads/master
Commit: 7c5d7622332c6684948daa2bc8de7f38488521f7
Parents: 0f549ac
Author: Zuyu Zhang
Authored: Wed Jun 14 15:22:13 2017 -0500
Committer: Zuyu Zhang
Committed: Wed Jun 14 16:10:20 2017 -0500
--
query_optimizer/ExecutionGenerator.cpp | 11 +++-
relational_operators/BuildLIPFilterOperator.cpp | 64
relational_operators/BuildLIPFilterOperator.hpp | 37 ---
relational_operators/CMakeLists.txt | 2 +
relational_operators/WorkOrder.proto| 2 +
5 files changed, 82 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7c5d7622/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp
b/query_optimizer/ExecutionGenerator.cpp
index acc2bf1..d3870df 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -663,6 +663,14 @@ void ExecutionGenerator::convertFilterJoin(const
P::FilterJoinPtr &physical_plan
const CatalogRelationInfo *build_relation_info =
findRelationInfoOutputByPhysical(build_physical);
+ const CatalogRelation &build_relation = *build_relation_info->relation;
+ const PartitionScheme *build_partition_scheme =
build_relation.getPartitionScheme();
+
+ const std::size_t build_num_partitions =
+ build_partition_scheme
+ ?
build_partition_scheme->getPartitionSchemeHeader().getNumPartitions()
+ : 1u;
+
// Create a BuildLIPFilterOperator for the FilterJoin. This operator builds
// LIP filters that are applied properly in downstream operators to achieve
// the filter-join semantics.
@@ -670,7 +678,8 @@ void ExecutionGenerator::convertFilterJoin(const
P::FilterJoinPtr &physical_plan
execution_plan_->addRelationalOperator(
new BuildLIPFilterOperator(
query_handle_->query_id(),
- *build_relation_info->relation,
+ build_relation,
+ build_num_partitions,
build_side_predicate_index,
build_relation_info->isStoredRelation()));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7c5d7622/relational_operators/BuildLIPFilterOperator.cpp
--
diff --git a/relational_operators/BuildLIPFilterOperator.cpp
b/relational_operators/BuildLIPFilterOperator.cpp
index e7e549c..925dfb5 100644
--- a/relational_operators/BuildLIPFilterOperator.cpp
+++ b/relational_operators/BuildLIPFilterOperator.cpp
@@ -55,8 +55,12 @@ bool BuildLIPFilterOperator::getAllWorkOrders(
query_context->getPredicate(build_side_predicate_index_);
if (input_relation_is_stored_) {
-if (!started_) {
- for (const block_id input_block_id : input_relation_block_ids_) {
+if (started_) {
+ return true;
+}
+
+for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
+ for (const block_id input_block_id : input_relation_block_ids_[part_id])
{
container->addNormalWorkOrder(
new BuildLIPFilterWorkOrder(
query_id_,
@@ -68,22 +72,24 @@ bool BuildLIPFilterOperator::getAllWorkOrders(
CreateLIPFilterBuilderHelper(lip_deployment_index_,
query_context)),
op_index_);
}
- started_ = true;
}
+started_ = true;
return true;
} else {
-while (num_workorders_generated_ < input_relation_block_ids_.size()) {
- container->addNormalWorkOrder(
- new BuildLIPFilterWorkOrder(
- query_id_,
- input_relation_,
- input_relation_block_ids_[num_workorders_generated_],
- build_side_predicate,
- storage_manager,
- CreateLIPFilterAdaptiveProberHelper(lip_deployment_index_,
query_context),
- CreateLIPFilterBuilderHelper(lip_deployment_index_,
query_context)),
- op_index_);
- ++num_workorders_generated_;
+for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
+ while (num_workorders_generated_[part_id] <
input_relation_block_ids_[part_id].size()) {
+container->addNormalWorkOrder(
+new BuildLIPFilterWorkOrder(
+query_id_,
+input_relation_,
incubator-quickstep git commit: Minor refactors in WorkOrderFactory.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master d0c55320f -> 0f549acb5
Minor refactors in WorkOrderFactory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0f549acb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0f549acb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0f549acb
Branch: refs/heads/master
Commit: 0f549acb54c56337a3d7e805e125e78aea48e013
Parents: d0c5532
Author: Zuyu Zhang
Authored: Wed Jun 14 15:59:48 2017 -0500
Committer: Zuyu Zhang
Committed: Wed Jun 14 15:59:48 2017 -0500
--
relational_operators/WorkOrderFactory.cpp | 103 +
1 file changed, 53 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0f549acb/relational_operators/WorkOrderFactory.cpp
--
diff --git a/relational_operators/WorkOrderFactory.cpp
b/relational_operators/WorkOrderFactory.cpp
index bb71ec4..4c1ffa9 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -61,6 +61,7 @@
#include "tmb/id_typedefs.h"
using std::move;
+using std::size_t;
using std::vector;
namespace quickstep {
@@ -83,12 +84,14 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
<< "Attempted to create WorkOrder from an invalid proto description:\n"
<< proto.DebugString();
+ const size_t query_id = proto.query_id();
+
switch (proto.work_order_type()) {
case serialization::AGGREGATION: {
- LOG(INFO) << "Creating AggregationWorkOrder for Query " <<
proto.query_id()
+ LOG(INFO) << "Creating AggregationWorkOrder for Query " << query_id
<< " in Shiftboss " << shiftboss_index;
return new AggregationWorkOrder(
- proto.query_id(),
+ query_id,
proto.GetExtension(serialization::AggregationWorkOrder::block_id),
query_context->getAggregationState(
proto.GetExtension(serialization::AggregationWorkOrder::aggr_state_index)),
@@ -96,11 +99,11 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
proto.GetExtension(serialization::AggregationWorkOrder::lip_deployment_index),
query_context));
}
case serialization::BUILD_AGGREGATION_EXISTENCE_MAP: {
- LOG(INFO) << "Creating BuildAggregationExistenceMapWorkOrder for Query "
<< proto.query_id()
+ LOG(INFO) << "Creating BuildAggregationExistenceMapWorkOrder for Query "
<< query_id
<< " in Shiftboss " << shiftboss_index;
return new BuildAggregationExistenceMapWorkOrder(
- proto.query_id(),
+ query_id,
catalog_database->getRelationSchemaById(
proto.GetExtension(serialization::BuildAggregationExistenceMapWorkOrder::relation_id)),
proto.GetExtension(serialization::BuildAggregationExistenceMapWorkOrder::build_block_id),
@@ -110,14 +113,14 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
storage_manager);
}
case serialization::BUILD_LIP_FILTER: {
- LOG(INFO) << "Creating BuildLIPFilterWorkOrder for Query " <<
proto.query_id()
+ LOG(INFO) << "Creating BuildLIPFilterWorkOrder for Query " << query_id
<< " in Shiftboss " << shiftboss_index;
const QueryContext::lip_deployment_id lip_deployment_index =
proto.GetExtension(serialization::BuildLIPFilterWorkOrder::lip_deployment_index);
return new BuildLIPFilterWorkOrder(
- proto.query_id(),
+ query_id,
catalog_database->getRelationSchemaById(
proto.GetExtension(serialization::BuildLIPFilterWorkOrder::relation_id)),
proto.GetExtension(serialization::BuildLIPFilterWorkOrder::build_block_id),
@@ -131,7 +134,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
const partition_id part_id =
proto.GetExtension(serialization::BuildHashWorkOrder::partition_id);
- LOG(INFO) << "Creating BuildHashWorkOrder (Partition " << part_id << ")
for Query " << proto.query_id()
+ LOG(INFO) << "Creating BuildHashWorkOrder (Partition " << part_id << ")
for Query " << query_id
<< " in Shiftboss " << shiftboss_index;
vector join_key_attributes;
for (int i = 0; i <
proto.ExtensionSize(serialization::BuildHashWorkOrder::join_key_attributes);
++i) {
@@ -140,7 +143,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
}
return new BuildHashWorkOrder(
- proto.query_id(),
+ query_id,
cata
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/storage-fix [deleted] d894e43ce
incubator-quickstep git commit: Avoided crashes using PartitionAwareInsertDest in Inner HashJoins.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 45e691e67 -> f8181b526
Avoided crashes using PartitionAwareInsertDest in Inner HashJoins.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f8181b52
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f8181b52
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f8181b52
Branch: refs/heads/master
Commit: f8181b52607df16833f7cb89b6e4271732cc52f6
Parents: 45e691e
Author: Zuyu Zhang
Authored: Thu May 25 18:38:15 2017 -0700
Committer: Zuyu Zhang
Committed: Thu May 25 18:38:15 2017 -0700
--
relational_operators/HashJoinOperator.cpp | 4 +++-
storage/InsertDestination.cpp | 13 +++-
storage/InsertDestination.hpp | 29 --
3 files changed, 34 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8181b52/relational_operators/HashJoinOperator.cpp
--
diff --git a/relational_operators/HashJoinOperator.cpp
b/relational_operators/HashJoinOperator.cpp
index ea90828..f8acd74 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -471,7 +471,9 @@ void HashInnerJoinWorkOrder::execute() {
base_accessor->createSharedTupleIdSequenceAdapterVirtual(*existence_map));
}
- if (probe_accessor->getImplementationType() ==
ValueAccessor::Implementation::kSplitRowStore) {
+ if (probe_accessor->getImplementationType() ==
ValueAccessor::Implementation::kSplitRowStore &&
+ output_destination_->getInsertDestinationType() ==
+
InsertDestination::InsertDestinationType::kBlockPoolInsertDestination) {
executeWithCopyElision(probe_accessor.get());
} else {
executeWithoutCopyElision(probe_accessor.get());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8181b52/storage/InsertDestination.cpp
--
diff --git a/storage/InsertDestination.cpp b/storage/InsertDestination.cpp
index 891b5a1..aeef08a 100644
--- a/storage/InsertDestination.cpp
+++ b/storage/InsertDestination.cpp
@@ -57,14 +57,16 @@ using std::vector;
namespace quickstep {
-InsertDestination::InsertDestination(const CatalogRelationSchema &relation,
+InsertDestination::InsertDestination(const InsertDestinationType
insert_dest_type,
+ const CatalogRelationSchema &relation,
const StorageBlockLayout *layout,
StorageManager *storage_manager,
const std::size_t relational_op_index,
const std::size_t query_id,
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus)
-: thread_id_map_(*ClientIDMap::Instance()),
+: insert_dest_type_(insert_dest_type),
+ thread_id_map_(*ClientIDMap::Instance()),
storage_manager_(storage_manager),
relation_(relation),
layout_(layout),
@@ -289,7 +291,7 @@ void InsertDestination::bulkInsertTuplesFromValueAccessors(
// same tuples from the other ValueAccessors.
for (auto &p : reduced_accessor_attribute_map) {
ValueAccessor *accessor = p.first;
- std::vector attribute_map = p.second;
+ const std::vector &attribute_map = p.second;
InvokeOnAnyValueAccessor(
accessor,
@@ -308,7 +310,7 @@ void InsertDestination::bulkInsertTuplesFromValueAccessors(
// Since the bulk insertion of the first ValueAccessor should already
// have reserved the space for all the other ValueAccessors' columns,
// we must have been able to insert all the tuples we asked to insert.
-DCHECK(num_tuples_inserted == num_tuples_to_insert);
+DCHECK_EQ(num_tuples_inserted, num_tuples_to_insert);
}
}
@@ -491,7 +493,8 @@
PartitionAwareInsertDestination::PartitionAwareInsertDestination(
const std::size_t query_id,
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus)
-: InsertDestination(relation,
+:
InsertDestination(InsertDestinationType::kPartitionAwareInsertDestination,
+relation,
layout,
storage_manager,
relational_op_index,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8181b52/storage/InsertDestination.hpp
--
diff --git a/storage/InsertDestination.hpp b/storage/InsertDestination.hpp
index 66c67d
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/improve-insert-dest [deleted] f8181b526
incubator-quickstep git commit: Avoided crashes using PartitionAwareInsertDest in Inner HashJoins.
Repository: incubator-quickstep
Updated Branches:
refs/heads/improve-insert-dest [created] f8181b526
Avoided crashes using PartitionAwareInsertDest in Inner HashJoins.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f8181b52
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f8181b52
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f8181b52
Branch: refs/heads/improve-insert-dest
Commit: f8181b52607df16833f7cb89b6e4271732cc52f6
Parents: 45e691e
Author: Zuyu Zhang
Authored: Thu May 25 18:38:15 2017 -0700
Committer: Zuyu Zhang
Committed: Thu May 25 18:38:15 2017 -0700
--
relational_operators/HashJoinOperator.cpp | 4 +++-
storage/InsertDestination.cpp | 13 +++-
storage/InsertDestination.hpp | 29 --
3 files changed, 34 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8181b52/relational_operators/HashJoinOperator.cpp
--
diff --git a/relational_operators/HashJoinOperator.cpp
b/relational_operators/HashJoinOperator.cpp
index ea90828..f8acd74 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -471,7 +471,9 @@ void HashInnerJoinWorkOrder::execute() {
base_accessor->createSharedTupleIdSequenceAdapterVirtual(*existence_map));
}
- if (probe_accessor->getImplementationType() ==
ValueAccessor::Implementation::kSplitRowStore) {
+ if (probe_accessor->getImplementationType() ==
ValueAccessor::Implementation::kSplitRowStore &&
+ output_destination_->getInsertDestinationType() ==
+
InsertDestination::InsertDestinationType::kBlockPoolInsertDestination) {
executeWithCopyElision(probe_accessor.get());
} else {
executeWithoutCopyElision(probe_accessor.get());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8181b52/storage/InsertDestination.cpp
--
diff --git a/storage/InsertDestination.cpp b/storage/InsertDestination.cpp
index 891b5a1..aeef08a 100644
--- a/storage/InsertDestination.cpp
+++ b/storage/InsertDestination.cpp
@@ -57,14 +57,16 @@ using std::vector;
namespace quickstep {
-InsertDestination::InsertDestination(const CatalogRelationSchema &relation,
+InsertDestination::InsertDestination(const InsertDestinationType
insert_dest_type,
+ const CatalogRelationSchema &relation,
const StorageBlockLayout *layout,
StorageManager *storage_manager,
const std::size_t relational_op_index,
const std::size_t query_id,
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus)
-: thread_id_map_(*ClientIDMap::Instance()),
+: insert_dest_type_(insert_dest_type),
+ thread_id_map_(*ClientIDMap::Instance()),
storage_manager_(storage_manager),
relation_(relation),
layout_(layout),
@@ -289,7 +291,7 @@ void InsertDestination::bulkInsertTuplesFromValueAccessors(
// same tuples from the other ValueAccessors.
for (auto &p : reduced_accessor_attribute_map) {
ValueAccessor *accessor = p.first;
- std::vector attribute_map = p.second;
+ const std::vector &attribute_map = p.second;
InvokeOnAnyValueAccessor(
accessor,
@@ -308,7 +310,7 @@ void InsertDestination::bulkInsertTuplesFromValueAccessors(
// Since the bulk insertion of the first ValueAccessor should already
// have reserved the space for all the other ValueAccessors' columns,
// we must have been able to insert all the tuples we asked to insert.
-DCHECK(num_tuples_inserted == num_tuples_to_insert);
+DCHECK_EQ(num_tuples_inserted, num_tuples_to_insert);
}
}
@@ -491,7 +493,8 @@
PartitionAwareInsertDestination::PartitionAwareInsertDestination(
const std::size_t query_id,
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus)
-: InsertDestination(relation,
+:
InsertDestination(InsertDestinationType::kPartitionAwareInsertDestination,
+relation,
layout,
storage_manager,
relational_op_index,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f8181b52/storage/InsertDestination.hpp
--
diff --git a/storage/InsertDestination.hpp b/storage/InsertDesti
incubator-quickstep git commit: Removed namespace definitions in a hpp file.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master a0b97342d -> 45e691e67
Removed namespace definitions in a hpp file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/45e691e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/45e691e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/45e691e6
Branch: refs/heads/master
Commit: 45e691e6787effaf1da4400919deb93d195d8285
Parents: a0b9734
Author: Zuyu Zhang
Authored: Sun May 21 15:01:36 2017 -0700
Committer: Zuyu Zhang
Committed: Sun May 21 15:01:36 2017 -0700
--
query_optimizer/rules/SwapProbeBuild.cpp | 4
query_optimizer/rules/SwapProbeBuild.hpp | 12
2 files changed, 8 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/45e691e6/query_optimizer/rules/SwapProbeBuild.cpp
--
diff --git a/query_optimizer/rules/SwapProbeBuild.cpp
b/query_optimizer/rules/SwapProbeBuild.cpp
index cc3f1e2..d707e01 100644
--- a/query_optimizer/rules/SwapProbeBuild.cpp
+++ b/query_optimizer/rules/SwapProbeBuild.cpp
@@ -34,6 +34,10 @@
namespace quickstep {
namespace optimizer {
+namespace C = cost;
+namespace E = expressions;
+namespace P = physical;
+
P::PhysicalPtr SwapProbeBuild::applyToNode(const P::PhysicalPtr &input) {
P::HashJoinPtr hash_join;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/45e691e6/query_optimizer/rules/SwapProbeBuild.hpp
--
diff --git a/query_optimizer/rules/SwapProbeBuild.hpp
b/query_optimizer/rules/SwapProbeBuild.hpp
index ad756fb..9e0dae3 100644
--- a/query_optimizer/rules/SwapProbeBuild.hpp
+++ b/query_optimizer/rules/SwapProbeBuild.hpp
@@ -36,15 +36,11 @@ namespace optimizer {
* @{
*/
-namespace P = ::quickstep::optimizer::physical;
-namespace E = ::quickstep::optimizer::expressions;
-namespace C = ::quickstep::optimizer::cost;
-
/**
* @brief Rule that applies to a physical plan to arrange probe and
*build side based on the cardinalities.
*/
-class SwapProbeBuild : public BottomUpRule {
+class SwapProbeBuild : public BottomUpRule {
public:
SwapProbeBuild() {
}
@@ -52,11 +48,11 @@ class SwapProbeBuild : public BottomUpRule {
std::string getName() const override { return "SwapProbeBuild"; }
protected:
- P::PhysicalPtr applyToNode(const P::PhysicalPtr &input) override;
- void init(const P::PhysicalPtr &input) override;
+ physical::PhysicalPtr applyToNode(const physical::PhysicalPtr &input)
override;
+ void init(const physical::PhysicalPtr &input) override;
private:
- std::unique_ptr cost_model_;
+ std::unique_ptr cost_model_;
DISALLOW_COPY_AND_ASSIGN(SwapProbeBuild);
};
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/namespace-hpp [deleted] 45e691e67
incubator-quickstep git commit: Removed namespace definitions in a hpp file.
Repository: incubator-quickstep
Updated Branches:
refs/heads/namespace-hpp [created] 45e691e67
Removed namespace definitions in a hpp file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/45e691e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/45e691e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/45e691e6
Branch: refs/heads/namespace-hpp
Commit: 45e691e6787effaf1da4400919deb93d195d8285
Parents: a0b9734
Author: Zuyu Zhang
Authored: Sun May 21 15:01:36 2017 -0700
Committer: Zuyu Zhang
Committed: Sun May 21 15:01:36 2017 -0700
--
query_optimizer/rules/SwapProbeBuild.cpp | 4
query_optimizer/rules/SwapProbeBuild.hpp | 12
2 files changed, 8 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/45e691e6/query_optimizer/rules/SwapProbeBuild.cpp
--
diff --git a/query_optimizer/rules/SwapProbeBuild.cpp
b/query_optimizer/rules/SwapProbeBuild.cpp
index cc3f1e2..d707e01 100644
--- a/query_optimizer/rules/SwapProbeBuild.cpp
+++ b/query_optimizer/rules/SwapProbeBuild.cpp
@@ -34,6 +34,10 @@
namespace quickstep {
namespace optimizer {
+namespace C = cost;
+namespace E = expressions;
+namespace P = physical;
+
P::PhysicalPtr SwapProbeBuild::applyToNode(const P::PhysicalPtr &input) {
P::HashJoinPtr hash_join;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/45e691e6/query_optimizer/rules/SwapProbeBuild.hpp
--
diff --git a/query_optimizer/rules/SwapProbeBuild.hpp
b/query_optimizer/rules/SwapProbeBuild.hpp
index ad756fb..9e0dae3 100644
--- a/query_optimizer/rules/SwapProbeBuild.hpp
+++ b/query_optimizer/rules/SwapProbeBuild.hpp
@@ -36,15 +36,11 @@ namespace optimizer {
* @{
*/
-namespace P = ::quickstep::optimizer::physical;
-namespace E = ::quickstep::optimizer::expressions;
-namespace C = ::quickstep::optimizer::cost;
-
/**
* @brief Rule that applies to a physical plan to arrange probe and
*build side based on the cardinalities.
*/
-class SwapProbeBuild : public BottomUpRule {
+class SwapProbeBuild : public BottomUpRule {
public:
SwapProbeBuild() {
}
@@ -52,11 +48,11 @@ class SwapProbeBuild : public BottomUpRule {
std::string getName() const override { return "SwapProbeBuild"; }
protected:
- P::PhysicalPtr applyToNode(const P::PhysicalPtr &input) override;
- void init(const P::PhysicalPtr &input) override;
+ physical::PhysicalPtr applyToNode(const physical::PhysicalPtr &input)
override;
+ void init(const physical::PhysicalPtr &input) override;
private:
- std::unique_ptr cost_model_;
+ std::unique_ptr cost_model_;
DISALLOW_COPY_AND_ASSIGN(SwapProbeBuild);
};
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/union-test [deleted] a0b97342d
incubator-quickstep git commit: Keep the Union execution test stand-alone.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master e4dd66ea2 -> a0b97342d
Keep the Union execution test stand-alone.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a0b97342
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a0b97342
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a0b97342
Branch: refs/heads/master
Commit: a0b97342d7186f391a5faa614e58409aa454b563
Parents: e4dd66e
Author: Zuyu Zhang
Authored: Wed May 17 01:03:20 2017 -0700
Committer: Zuyu Zhang
Committed: Wed May 17 01:03:20 2017 -0700
--
.../tests/execution_generator/CMakeLists.txt| 7 ++
.../tests/execution_generator/Select.test | 86
.../tests/execution_generator/Union.test| 100 +++
3 files changed, 107 insertions(+), 86 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0b97342/query_optimizer/tests/execution_generator/CMakeLists.txt
--
diff --git a/query_optimizer/tests/execution_generator/CMakeLists.txt
b/query_optimizer/tests/execution_generator/CMakeLists.txt
index 595d09d..09a7647 100644
--- a/query_optimizer/tests/execution_generator/CMakeLists.txt
+++ b/query_optimizer/tests/execution_generator/CMakeLists.txt
@@ -75,6 +75,11 @@
add_test(quickstep_queryoptimizer_tests_executiongenerator_tablegenerator
"${CMAKE_CURRENT_SOURCE_DIR}/TableGenerator.test"
"${CMAKE_CURRENT_BINARY_DIR}/TableGenerator.test"
"${CMAKE_CURRENT_BINARY_DIR}/TableGenerator/")
+add_test(quickstep_queryoptimizer_tests_executiongenerator_union
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Union.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Union.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Union/")
add_test(quickstep_queryoptimizer_tests_executiongenerator_update
"../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
"${CMAKE_CURRENT_SOURCE_DIR}/Update.test"
@@ -142,6 +147,7 @@ if (ENABLE_DISTRIBUTED)
"${CMAKE_CURRENT_SOURCE_DIR}/TableGenerator.test"
"${CMAKE_CURRENT_BINARY_DIR}/TableGeneratorDistributed.test"
"${CMAKE_CURRENT_BINARY_DIR}/TableGeneratorDistributed/")
+ # No union test for the distributed version due to its nondeterminism.
add_test(quickstep_queryoptimizer_tests_executiongenerator_update_distributed
"../quickstep_queryoptimizer_tests_DistributedExecutionGeneratorTest"
"${CMAKE_CURRENT_SOURCE_DIR}/Update.test"
@@ -163,6 +169,7 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Partition)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Select)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/StringPatternMatching)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/TableGenerator)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Union)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Update)
if (ENABLE_DISTRIBUTED)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0b97342/query_optimizer/tests/execution_generator/Select.test
--
diff --git a/query_optimizer/tests/execution_generator/Select.test
b/query_optimizer/tests/execution_generator/Select.test
index b39..494e759 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -1072,89 +1072,3 @@ WINDOW w AS
++
| -18|
++
-==
-
-SELECT int_col AS result FROM test
-WHERE int_col < 5
-UNION
-SELECT int_col + 3 AS result FROM test
-WHERE int_col < 5;
---
-+---+
-|result |
-+---+
-| -1|
-| 2|
-| -3|
-| 4|
-| -5|
-| -7|
-| -9|
-|-11|
-|-13|
-|-15|
-|-17|
-|-19|
-|-21|
-|-23|
-| 5|
-| 0|
-| 7|
-| -2|
-| -4|
-| -6|
-| -8|
-|-10|
-|-12|
-|-14|
-|-16|
-|-18|
-|-20|
-+---+
-==
-
-SELECT i + 1 AS result
-FROM generate_series(1, 5) AS gs(i)
-INTERSECT
-SELECT i * 2 AS result
-FROM generate_series(1, 5) AS gs(i)
---
-+---+
-|result |
-+---+
-| 2|
-| 4|
-| 6|
-+---+
-==
-
-SELECT float_col FROM test
-WHERE int_col < 5
-ORDER BY int_col DESC
-LIMIT 5
-UNION ALL
-SELECT float_col FROM test
-WHERE int_col < 5
-ORDER BY int_col DESC
-LIMIT 10
---
-+---+
-|float_col |
-+---+
-| 2|
-| 1.41421354|
-| 1|
-|
incubator-quickstep git commit: Keep the Union execution test stand-alone.
Repository: incubator-quickstep
Updated Branches:
refs/heads/union-test [created] a0b97342d
Keep the Union execution test stand-alone.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a0b97342
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a0b97342
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a0b97342
Branch: refs/heads/union-test
Commit: a0b97342d7186f391a5faa614e58409aa454b563
Parents: e4dd66e
Author: Zuyu Zhang
Authored: Wed May 17 01:03:20 2017 -0700
Committer: Zuyu Zhang
Committed: Wed May 17 01:03:20 2017 -0700
--
.../tests/execution_generator/CMakeLists.txt| 7 ++
.../tests/execution_generator/Select.test | 86
.../tests/execution_generator/Union.test| 100 +++
3 files changed, 107 insertions(+), 86 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0b97342/query_optimizer/tests/execution_generator/CMakeLists.txt
--
diff --git a/query_optimizer/tests/execution_generator/CMakeLists.txt
b/query_optimizer/tests/execution_generator/CMakeLists.txt
index 595d09d..09a7647 100644
--- a/query_optimizer/tests/execution_generator/CMakeLists.txt
+++ b/query_optimizer/tests/execution_generator/CMakeLists.txt
@@ -75,6 +75,11 @@
add_test(quickstep_queryoptimizer_tests_executiongenerator_tablegenerator
"${CMAKE_CURRENT_SOURCE_DIR}/TableGenerator.test"
"${CMAKE_CURRENT_BINARY_DIR}/TableGenerator.test"
"${CMAKE_CURRENT_BINARY_DIR}/TableGenerator/")
+add_test(quickstep_queryoptimizer_tests_executiongenerator_union
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Union.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Union.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Union/")
add_test(quickstep_queryoptimizer_tests_executiongenerator_update
"../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
"${CMAKE_CURRENT_SOURCE_DIR}/Update.test"
@@ -142,6 +147,7 @@ if (ENABLE_DISTRIBUTED)
"${CMAKE_CURRENT_SOURCE_DIR}/TableGenerator.test"
"${CMAKE_CURRENT_BINARY_DIR}/TableGeneratorDistributed.test"
"${CMAKE_CURRENT_BINARY_DIR}/TableGeneratorDistributed/")
+ # No union test for the distributed version due to its nondeterminism.
add_test(quickstep_queryoptimizer_tests_executiongenerator_update_distributed
"../quickstep_queryoptimizer_tests_DistributedExecutionGeneratorTest"
"${CMAKE_CURRENT_SOURCE_DIR}/Update.test"
@@ -163,6 +169,7 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Partition)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Select)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/StringPatternMatching)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/TableGenerator)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Union)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Update)
if (ENABLE_DISTRIBUTED)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0b97342/query_optimizer/tests/execution_generator/Select.test
--
diff --git a/query_optimizer/tests/execution_generator/Select.test
b/query_optimizer/tests/execution_generator/Select.test
index b39..494e759 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -1072,89 +1072,3 @@ WINDOW w AS
++
| -18|
++
-==
-
-SELECT int_col AS result FROM test
-WHERE int_col < 5
-UNION
-SELECT int_col + 3 AS result FROM test
-WHERE int_col < 5;
---
-+---+
-|result |
-+---+
-| -1|
-| 2|
-| -3|
-| 4|
-| -5|
-| -7|
-| -9|
-|-11|
-|-13|
-|-15|
-|-17|
-|-19|
-|-21|
-|-23|
-| 5|
-| 0|
-| 7|
-| -2|
-| -4|
-| -6|
-| -8|
-|-10|
-|-12|
-|-14|
-|-16|
-|-18|
-|-20|
-+---+
-==
-
-SELECT i + 1 AS result
-FROM generate_series(1, 5) AS gs(i)
-INTERSECT
-SELECT i * 2 AS result
-FROM generate_series(1, 5) AS gs(i)
---
-+---+
-|result |
-+---+
-| 2|
-| 4|
-| 6|
-+---+
-==
-
-SELECT float_col FROM test
-WHERE int_col < 5
-ORDER BY int_col DESC
-LIMIT 5
-UNION ALL
-SELECT float_col FROM test
-WHERE int_col < 5
-ORDER BY int_col DESC
-LIMIT 10
---
-+---+
-|float_col |
-+---+
-| 2|
-| 1.41421354|
-| 1|
-
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/batched-aggr-wo [deleted] e56527dd5
incubator-quickstep git commit: Print the partition id in WorkOrder Reconstruction.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 97650fb85 -> e4dd66ea2
Print the partition id in WorkOrder Reconstruction.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e4dd66ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e4dd66ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e4dd66ea
Branch: refs/heads/master
Commit: e4dd66ea2501c475aeea07a551a4c5c11800bc3c
Parents: 97650fb
Author: Zuyu Zhang
Authored: Sat May 13 22:45:15 2017 -0700
Committer: Zuyu Zhang
Committed: Tue May 16 22:46:59 2017 -0700
--
relational_operators/WorkOrderFactory.cpp | 25 ++---
1 file changed, 14 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e4dd66ea/relational_operators/WorkOrderFactory.cpp
--
diff --git a/relational_operators/WorkOrderFactory.cpp
b/relational_operators/WorkOrderFactory.cpp
index d63bb62..bb71ec4 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -128,16 +128,17 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
CreateLIPFilterBuilderHelper(lip_deployment_index, query_context));
}
case serialization::BUILD_HASH: {
- LOG(INFO) << "Creating BuildHashWorkOrder for Query " <<
proto.query_id() << " in Shiftboss " << shiftboss_index;
+ const partition_id part_id =
+ proto.GetExtension(serialization::BuildHashWorkOrder::partition_id);
+
+ LOG(INFO) << "Creating BuildHashWorkOrder (Partition " << part_id << ")
for Query " << proto.query_id()
+<< " in Shiftboss " << shiftboss_index;
vector join_key_attributes;
for (int i = 0; i <
proto.ExtensionSize(serialization::BuildHashWorkOrder::join_key_attributes);
++i) {
join_key_attributes.push_back(
proto.GetExtension(serialization::BuildHashWorkOrder::join_key_attributes, i));
}
- const partition_id part_id =
- proto.GetExtension(serialization::BuildHashWorkOrder::partition_id);
-
return new BuildHashWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -176,14 +177,16 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
query_context);
}
case serialization::DESTROY_HASH: {
- LOG(INFO) << "Creating DestroyHashWorkOrder for Query " <<
proto.query_id()
+ const partition_id part_id =
+
proto.GetExtension(serialization::DestroyHashWorkOrder::partition_id);
+
+ LOG(INFO) << "Creating DestroyHashWorkOrder (Partition " << part_id <<
") for Query " << proto.query_id()
<< " in Shiftboss " << shiftboss_index;
return new DestroyHashWorkOrder(
proto.query_id(),
proto.GetExtension(
serialization::DestroyHashWorkOrder::join_hash_table_index),
- proto.GetExtension(
- serialization::DestroyHashWorkOrder::partition_id),
+ part_id,
query_context);
}
case serialization::DROP_TABLE: {
@@ -265,7 +268,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
switch (hash_join_work_order_type) {
case serialization::HashJoinWorkOrder::HASH_ANTI_JOIN: {
- LOG(INFO) << "Creating HashAntiJoinWorkOrder for Query " <<
proto.query_id()
+ LOG(INFO) << "Creating HashAntiJoinWorkOrder (Partition " << part_id
<< ") for Query " << proto.query_id()
<< " in Shiftboss " << shiftboss_index;
return new HashAntiJoinWorkOrder(
proto.query_id(),
@@ -283,7 +286,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
lip_filter_adaptive_prober);
}
case serialization::HashJoinWorkOrder::HASH_INNER_JOIN: {
- LOG(INFO) << "Creating HashInnerJoinWorkOrder for Query " <<
proto.query_id()
+ LOG(INFO) << "Creating HashInnerJoinWorkOrder (Partition " <<
part_id << ") for Query " << proto.query_id()
<< " in Shiftboss " << shiftboss_index;
return new HashInnerJoinWorkOrder(
proto.query_id(),
@@ -309,7 +312,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const
serialization::WorkOrder
proto.GetExtension(serialization::HashJoinWorkOrder::is_selection_on_build, i));
}
- LOG(INFO) << "Creating HashOuterJoinWorkOrder for Query " <<
proto.query_id()
+ LOG(INFO) << "Creating HashOuterJoinWorkOrder (Partition " <<
part_id << ") for Query " <<
incubator-quickstep git commit: Minor refactor scheduler in the distributed version.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master ed72e2477 -> 97650fb85
Minor refactor scheduler in the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/97650fb8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/97650fb8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/97650fb8
Branch: refs/heads/master
Commit: 97650fb85cee9c93aadc37adf6cbcc0f6d61244c
Parents: ed72e24
Author: Zuyu Zhang
Authored: Tue May 16 21:29:15 2017 -0700
Committer: Zuyu Zhang
Committed: Tue May 16 21:29:15 2017 -0700
--
query_execution/PolicyEnforcerDistributed.cpp | 8
query_execution/QueryManagerDistributed.cpp | 10 +++---
query_execution/QueryManagerDistributed.hpp | 6 +-
3 files changed, 8 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/97650fb8/query_execution/PolicyEnforcerDistributed.cpp
--
diff --git a/query_execution/PolicyEnforcerDistributed.cpp
b/query_execution/PolicyEnforcerDistributed.cpp
index 36becf2..b410152 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -112,11 +112,11 @@ void PolicyEnforcerDistributed::getWorkOrderProtoMessages(
DCHECK(curr_query_manager != nullptr);
std::size_t messages_collected_curr_query = 0;
while (messages_collected_curr_query < per_query_share) {
- S::WorkOrderMessage *next_work_order_message =
-
static_cast(curr_query_manager)->getNextWorkOrderMessage(0);
- if (next_work_order_message != nullptr) {
+ unique_ptr next_work_order_message(
+
static_cast(curr_query_manager)->getNextWorkOrderMessage());
+ if (next_work_order_message) {
++messages_collected_curr_query;
-
work_order_proto_messages->push_back(unique_ptr(next_work_order_message));
+work_order_proto_messages->push_back(move(next_work_order_message));
} else {
// No more work ordes from the current query at this time.
// Check if the query's execution is over.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/97650fb8/query_execution/QueryManagerDistributed.cpp
--
diff --git a/query_execution/QueryManagerDistributed.cpp
b/query_execution/QueryManagerDistributed.cpp
index 92645b6..b304f9f 100644
--- a/query_execution/QueryManagerDistributed.cpp
+++ b/query_execution/QueryManagerDistributed.cpp
@@ -79,19 +79,15 @@
QueryManagerDistributed::QueryManagerDistributed(QueryHandle *query_handle,
}
}
-serialization::WorkOrderMessage*
QueryManagerDistributed::getNextWorkOrderMessage(
-const dag_node_index start_operator_index) {
+serialization::WorkOrderMessage*
QueryManagerDistributed::getNextWorkOrderMessage() {
// Default policy: Operator with lowest index first.
- size_t num_operators_checked = 0;
- for (dag_node_index index = start_operator_index;
- num_operators_checked < num_operators_in_dag_;
- index = (index + 1) % num_operators_in_dag_, ++num_operators_checked) {
+ for (dag_node_index index = 0u; index < num_operators_in_dag_; ++index) {
if (query_exec_state_->hasExecutionFinished(index)) {
continue;
}
unique_ptr work_order_proto(
normal_workorder_protos_container_->getWorkOrderProto(index));
-if (work_order_proto != nullptr) {
+if (work_order_proto) {
query_exec_state_->incrementNumQueuedWorkOrders(index);
unique_ptr message_proto(new
serialization::WorkOrderMessage);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/97650fb8/query_execution/QueryManagerDistributed.hpp
--
diff --git a/query_execution/QueryManagerDistributed.hpp
b/query_execution/QueryManagerDistributed.hpp
index 6a47ce8..ab4479c 100644
--- a/query_execution/QueryManagerDistributed.hpp
+++ b/query_execution/QueryManagerDistributed.hpp
@@ -91,14 +91,10 @@ class QueryManagerDistributed final : public
QueryManagerBase {
* @brief Get the next normal workorder to be excuted, wrapped in a
*WorkOrderMessage proto.
*
- * @param start_operator_index Begin the search for the schedulable WorkOrder
- *with the operator at this index.
- *
* @return A pointer to the WorkOrderMessage proto. If there is no WorkOrder
* to be executed, return NULL.
**/
- serialization::WorkOrderMessage* getNextWorkOrderMessage(
- const dag_node_index start_operator_index);
+ serialization::WorkOrderMessage* getNextWorkOrderMessage();
/**
* @brief Get t
incubator-quickstep git commit: Batched Aggr WorkOrders in the distributed version.
Repository: incubator-quickstep
Updated Branches:
refs/heads/batched-aggr-wo [created] e56527dd5
Batched Aggr WorkOrders in the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e56527dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e56527dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e56527dd
Branch: refs/heads/batched-aggr-wo
Commit: e56527dd5d74d60ab9dd20771e01d0ed308e0665
Parents: ed72e24
Author: Zuyu Zhang
Authored: Mon May 15 19:29:43 2017 -0700
Committer: Zuyu Zhang
Committed: Mon May 15 19:29:43 2017 -0700
--
query_execution/ForemanDistributed.cpp | 2 +-
query_execution/Shiftboss.cpp| 38 +++---
relational_operators/AggregationOperator.cpp | 13 +-
relational_operators/WorkOrder.proto | 2 +-
relational_operators/WorkOrderFactory.cpp| 149 ++
relational_operators/WorkOrderFactory.hpp| 4 +-
6 files changed, 129 insertions(+), 79 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e56527dd/query_execution/ForemanDistributed.cpp
--
diff --git a/query_execution/ForemanDistributed.cpp
b/query_execution/ForemanDistributed.cpp
index e5e0eee..ca80a79 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -248,7 +248,7 @@ bool
ForemanDistributed::isAggregationRelatedWorkOrder(const S::WorkOrderMessage
switch (work_order_proto.work_order_type()) {
case S::AGGREGATION:
aggr_state_index =
work_order_proto.GetExtension(S::AggregationWorkOrder::aggr_state_index);
- block = work_order_proto.GetExtension(S::AggregationWorkOrder::block_id);
+ block = work_order_proto.GetExtension(S::AggregationWorkOrder::block_id,
0);
break;
case S::FINALIZE_AGGREGATION:
aggr_state_index =
work_order_proto.GetExtension(S::FinalizeAggregationWorkOrder::aggr_state_index);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e56527dd/query_execution/Shiftboss.cpp
--
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index 21e7858..ce3eab6 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -185,28 +185,30 @@ void Shiftboss::run() {
const std::size_t query_id = proto.query_id();
DCHECK_EQ(1u, query_contexts_.count(query_id));
- unique_ptr work_order(
+ vector> work_orders(
WorkOrderFactory::ReconstructFromProto(proto.work_order(),
shiftboss_index_, &database_cache_,
query_contexts_[query_id].get(), storage_manager_,
shiftboss_client_id_local_, bus_local_, hdfs_));
- unique_ptr worker_message(
- WorkerMessage::WorkOrderMessage(work_order.release(),
proto.operator_index()));
-
- TaggedMessage worker_tagged_message(worker_message.get(),
- sizeof(*worker_message),
- kWorkOrderMessage);
-
- const size_t worker_index = getSchedulableWorker();
- DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " with Client " <<
shiftboss_client_id_local_
- << " forwarded WorkOrderMessage from Foreman to Worker "
<< worker_index;
-
- const MessageBus::SendStatus send_status =
- QueryExecutionUtil::SendTMBMessage(bus_local_,
- shiftboss_client_id_local_,
-
workers_->getClientID(worker_index),
- move(worker_tagged_message));
- CHECK(send_status == MessageBus::SendStatus::kOK);
+ for (int i = 0; i < work_orders.size(); ++i) {
+unique_ptr worker_message(
+WorkerMessage::WorkOrderMessage(work_orders[i].release(),
proto.operator_index()));
+
+TaggedMessage worker_tagged_message(worker_message.get(),
+sizeof(*worker_message),
+kWorkOrderMessage);
+
+const size_t worker_index = getSchedulableWorker();
+DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " with Client "
<< shiftboss_client_id_local_
+ << " forwarded WorkOrderMessage from Foreman to Worker
" << worker_index;
+
+const MessageBus::SendStatus send_status =
+QueryExecutionUtil::SendTMBMessage(bu
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/improve-pushdown [deleted] ed72e2477
incubator-quickstep git commit: Improve disjunctive predicate pushdown to optimize more scenarios.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 8f662921b -> ed72e2477
Improve disjunctive predicate pushdown to optimize more scenarios.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ed72e247
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ed72e247
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ed72e247
Branch: refs/heads/master
Commit: ed72e2477033438e922195aa2632003ab037ab7b
Parents: 8f66292
Author: Jianqiao Zhu
Authored: Tue May 2 23:55:52 2017 -0500
Committer: Jianqiao Zhu
Committed: Thu May 4 14:40:06 2017 -0500
--
.../PushDownLowCostDisjunctivePredicate.cpp | 34 ++--
1 file changed, 24 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ed72e247/query_optimizer/rules/PushDownLowCostDisjunctivePredicate.cpp
--
diff --git a/query_optimizer/rules/PushDownLowCostDisjunctivePredicate.cpp
b/query_optimizer/rules/PushDownLowCostDisjunctivePredicate.cpp
index e39f155..1c71485 100644
--- a/query_optimizer/rules/PushDownLowCostDisjunctivePredicate.cpp
+++ b/query_optimizer/rules/PushDownLowCostDisjunctivePredicate.cpp
@@ -51,6 +51,11 @@
DEFINE_uint64(push_down_disjunctive_predicate_cardinality_threshold, 100u,
"PushDownLowCostDisjunctivePredicate optimization rule to push "
"down a disjunctive predicate to pre-filter that relation.");
+DEFINE_double(push_down_disjunctive_predicate_selectivity_threshold, 0.2,
+ "The estimated selectivity threshold below which the "
+ "PushDownLowCostDisjunctivePredicate optimization rule will push
"
+ "down a disjunctive predicate to pre-filter a stored relation.");
+
namespace E = ::quickstep::optimizer::expressions;
namespace P = ::quickstep::optimizer::physical;
@@ -77,11 +82,7 @@ void
PushDownLowCostDisjunctivePredicate::collectApplicablePredicates(
const physical::PhysicalPtr &input) {
P::TableReferencePtr table_reference;
if (P::SomeTableReference::MatchesWithConditionalCast(input,
&table_reference)) {
-// Consider only stored relations with small cardinality as targets.
-if (cost_model_->estimateCardinality(input) <=
-FLAGS_push_down_disjunctive_predicate_cardinality_threshold) {
- applicable_nodes_.emplace_back(input,
&table_reference->attribute_list());
-}
+applicable_nodes_.emplace_back(input, &table_reference->attribute_list());
return;
}
@@ -191,11 +192,24 @@ P::PhysicalPtr
PushDownLowCostDisjunctivePredicate::attachPredicates(
const auto &node_it = applicable_predicates_.find(input);
if (node_it != applicable_predicates_.end()) {
-const E::PredicatePtr filter_predicate =
-CreateConjunctive(node_it->second.predicates);
-return P::Selection::Create(output,
-
E::ToNamedExpressions(output->getOutputAttributes()),
-filter_predicate);
+const P::PhysicalPtr selection =
+P::Selection::Create(output,
+
E::ToNamedExpressions(output->getOutputAttributes()),
+ CreateConjunctive(node_it->second.predicates));
+
+// Applicable case 1: The stored relation has small cardinality.
+const bool is_small_cardinality_relation =
+cost_model_->estimateCardinality(input) <=
+FLAGS_push_down_disjunctive_predicate_cardinality_threshold;
+
+// Applicable case 2: The filter predicate has low selectivity.
+const bool is_selective_predicate =
+cost_model_->estimateSelectivityForFilterPredicate(selection) <=
+FLAGS_push_down_disjunctive_predicate_selectivity_threshold;
+
+if (is_small_cardinality_relation || is_selective_predicate) {
+ return selection;
+}
}
return output;
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/fix-copy-group [deleted] 8f662921b
incubator-quickstep git commit: Fix a problem in CopyGroupList + minor style fixes.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master ece7e424b -> 8f662921b
Fix a problem in CopyGroupList + minor style fixes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8f662921
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8f662921
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8f662921
Branch: refs/heads/master
Commit: 8f662921bf8307aad1bddad74c5e9e037420246e
Parents: ece7e42
Author: Jianqiao Zhu
Authored: Tue May 2 23:55:52 2017 -0500
Committer: Jianqiao Zhu
Committed: Thu May 4 00:13:25 2017 -0500
--
storage/SplitRowStoreTupleStorageSubBlock.cpp | 169 ++-
storage/SplitRowStoreTupleStorageSubBlock.hpp | 133 ---
...litRowStoreTupleStorageSubBlock_unittest.cpp | 80 ++---
3 files changed, 209 insertions(+), 173 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f662921/storage/SplitRowStoreTupleStorageSubBlock.cpp
--
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.cpp
b/storage/SplitRowStoreTupleStorageSubBlock.cpp
index ad583eb..0e5cfe6 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.cpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.cpp
@@ -244,14 +244,12 @@ tuple_id
SplitRowStoreTupleStorageSubBlock::bulkInsertDispatcher(
CopyGroupList copy_groups;
getCopyGroupsForAttributeMap(attribute_map, ©_groups);
- auto impl = accessor->getImplementationType();
- const bool is_rowstore_source = impl ==
ValueAccessor::Implementation::kSplitRowStore;
- if (is_rowstore_source) {
-copy_groups.merge_contiguous();
+ if (accessor->getImplementationType() ==
ValueAccessor::Implementation::kSplitRowStore) {
+copy_groups.mergeContiguous();
}
- const bool copy_nulls = copy_groups.nullable_attrs_.size() > 0;
- const bool copy_varlen = copy_groups.varlen_attrs_.size() > 0;
+ const bool copy_nulls = copy_groups.nullable_attrs.size() > 0;
+ const bool copy_varlen = copy_groups.varlen_attrs.size() > 0;
if (fill_to_capacity) {
if (relation_.hasNullableAttributes()) {
@@ -311,96 +309,101 @@ tuple_id
SplitRowStoreTupleStorageSubBlock::bulkInsertPartialTuplesImpl(
std::size_t num_tuples_inserted = 0;
// We only append to the end of the block to cut down on complexity.
- char *tuple_slot = static_cast(tuple_storage_) +
header_->num_tuples * tuple_slot_bytes_;
+ char *tuple_slot = static_cast(tuple_storage_) + header_->num_tuples
* tuple_slot_bytes_;
std::uint32_t varlen_heap_offset = tuple_storage_bytes_ -
header_->variable_length_bytes_allocated;
std::uint32_t varlen_heap_offset_orig = varlen_heap_offset;
- BitVector tuple_null_bitmap(tuple_slot, num_null_attrs_);
- char *fixed_len_cursor = tuple_slot +
BitVector::BytesNeeded(num_null_attrs_);
-
-
-
std::size_t storage_available = tuple_storage_bytes_ -
(header_->variable_length_bytes_allocated +
header_->num_tuples * tuple_slot_bytes_);
+ const std::vector &contiguous_attrs =
copy_groups.contiguous_attrs;
+ const std::vector &varlen_attrs = copy_groups.varlen_attrs;
+ const std::vector &nullable_attrs = copy_groups.nullable_attrs;
+
// The number of bytes that must be reserved per tuple inserted due to gaps.
std::size_t varlen_reserve = relation_.getMaximumVariableByteLength();
if (fill_to_capacity) {
-for (std::size_t vattr_idx = 0; vattr_idx <
copy_groups.varlen_attrs_.size(); vattr_idx++) {
+for (std::size_t vattr_idx = 0; vattr_idx < varlen_attrs.size();
vattr_idx++) {
varlen_reserve -= relation_.getAttributeById(
-
copy_groups.varlen_attrs_[vattr_idx].dst_attr_id_)->getType().maximumByteLength();
+ varlen_attrs[vattr_idx].dst_attr_id)->getType().maximumByteLength();
}
DCHECK_GE(relation_.getMaximumVariableByteLength(), varlen_reserve);
}
InvokeOnAnyValueAccessor(
-accessor,
-[&](auto *accessor) -> void { // NOLINT(build/c++11
- do {
-const std::size_t num_c_attr = copy_groups.contiguous_attrs_.size();
-const std::size_t num_n_attr = copy_groups.nullable_attrs_.size();
-const std::size_t num_v_attr = copy_groups.varlen_attrs_.size();
-
-const std::size_t nullmap_size =
BitVector::BytesNeeded(num_null_attrs_);
-
-while (num_tuples_inserted < max_num_tuples_to_insert &&
accessor->next()) {
- for (std::size_t cattr_idx = 0; cattr_idx < num_c_attr; cattr_idx++)
{
-const ContiguousAttrs &cattr =
copy_groups.contiguous_attrs_[cattr_idx];
-fixed_len_cursor += cattr.bytes_to_advance_;
-const void *attr_value = a
incubator-quickstep git commit: Dispatch BuildHashJoin and Aggregation based on block locality.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 0572f4007 -> ece7e424b
Dispatch BuildHashJoin and Aggregation based on block locality.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ece7e424
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ece7e424
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ece7e424
Branch: refs/heads/master
Commit: ece7e424b5c43e91dbe6e52b1ca95312696e57d2
Parents: 0572f40
Author: Zuyu Zhang
Authored: Tue Apr 11 00:26:40 2017 -0700
Committer: Zuyu Zhang
Committed: Wed May 3 12:50:03 2017 -0700
--
query_execution/CMakeLists.txt| 2 ++
query_execution/ForemanDistributed.cpp| 9 +++--
query_execution/PolicyEnforcerDistributed.cpp | 8
query_execution/PolicyEnforcerDistributed.hpp | 15 --
query_execution/QueryManagerDistributed.hpp | 23 ++
5 files changed, 49 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ece7e424/query_execution/CMakeLists.txt
--
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index eeed791..c74fa36 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -267,6 +267,7 @@
target_link_libraries(quickstep_queryexecution_QueryManagerBase
if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_QueryManagerDistributed
quickstep_catalog_CatalogTypedefs
+quickstep_queryexecution_BlockLocator
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
quickstep_queryexecution_QueryExecutionMessages_proto
@@ -277,6 +278,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryexecution_WorkOrderProtosContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder_proto
+quickstep_storage_StorageBlockInfo
quickstep_utility_DAG
quickstep_utility_Macros
tmb)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ece7e424/query_execution/ForemanDistributed.cpp
--
diff --git a/query_execution/ForemanDistributed.cpp
b/query_execution/ForemanDistributed.cpp
index 81b5ec1..e5e0eee 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -243,10 +243,12 @@ bool
ForemanDistributed::isAggregationRelatedWorkOrder(const S::WorkOrderMessage
size_t
*shiftboss_index_for_aggregation) {
const S::WorkOrder &work_order_proto = proto.work_order();
QueryContext::aggregation_state_id aggr_state_index;
+ block_id block = kInvalidBlockId;
switch (work_order_proto.work_order_type()) {
case S::AGGREGATION:
aggr_state_index =
work_order_proto.GetExtension(S::AggregationWorkOrder::aggr_state_index);
+ block = work_order_proto.GetExtension(S::AggregationWorkOrder::block_id);
break;
case S::FINALIZE_AGGREGATION:
aggr_state_index =
work_order_proto.GetExtension(S::FinalizeAggregationWorkOrder::aggr_state_index);
@@ -259,7 +261,8 @@ bool
ForemanDistributed::isAggregationRelatedWorkOrder(const S::WorkOrderMessage
}
static_cast(policy_enforcer_.get())->getShiftbossIndexForAggregation(
- proto.query_id(), aggr_state_index, next_shiftboss_index_to_schedule,
shiftboss_index_for_aggregation);
+ proto.query_id(), aggr_state_index, block_locator_, block,
next_shiftboss_index_to_schedule,
+ shiftboss_index_for_aggregation);
return true;
}
@@ -270,11 +273,13 @@ bool ForemanDistributed::isHashJoinRelatedWorkOrder(const
S::WorkOrderMessage &p
const S::WorkOrder &work_order_proto = proto.work_order();
QueryContext::join_hash_table_id join_hash_table_index;
partition_id part_id;
+ block_id block = kInvalidBlockId;
switch (work_order_proto.work_order_type()) {
case S::BUILD_HASH:
join_hash_table_index =
work_order_proto.GetExtension(S::BuildHashWorkOrder::join_hash_table_index);
part_id =
work_order_proto.GetExtension(S::BuildHashWorkOrder::partition_id);
+ block = work_order_proto.GetExtension(S::BuildHashWorkOrder::block_id);
break;
case S::HASH_JOIN:
join_hash_table_index =
work_order_proto.GetExtension(S::HashJoinWorkOrder::join_hash_table_index);
@@ -289,7 +294,7 @@ bool ForemanDistributed::isHas
incubator-quickstep git commit: Partition-aware CopyFrom.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 678e2c395 -> 0572f4007
Partition-aware CopyFrom.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0572f400
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0572f400
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0572f400
Branch: refs/heads/master
Commit: 0572f40075534db25a090ee28d31da8ab7a5ee76
Parents: 678e2c3
Author: Zuyu Zhang
Authored: Sat Apr 29 16:32:14 2017 -0700
Committer: Zuyu Zhang
Committed: Tue May 2 17:16:43 2017 -0700
--
query_optimizer/ExecutionGenerator.cpp | 19 +--
1 file changed, 13 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0572f400/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp
b/query_optimizer/ExecutionGenerator.cpp
index 9625a91..acc2bf1 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1011,16 +1011,23 @@ void ExecutionGenerator::convertCopyFrom(
query_context_proto_->insert_destinations_size();
S::InsertDestination *insert_destination_proto =
query_context_proto_->add_insert_destinations();
-
insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
+ if (output_relation->hasPartitionScheme()) {
+
insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::PARTITION_AWARE);
+
insert_destination_proto->MutableExtension(S::PartitionAwareInsertDestination::partition_scheme)
+->MergeFrom(output_relation->getPartitionScheme()->getProto());
+ } else {
+
insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
+
+const vector blocks(output_relation->getBlocksSnapshot());
+for (const block_id block : blocks) {
+
insert_destination_proto->AddExtension(S::BlockPoolInsertDestination::blocks,
block);
+}
+ }
+
insert_destination_proto->set_relation_id(output_rel_id);
insert_destination_proto->mutable_layout()->MergeFrom(
output_relation->getDefaultStorageBlockLayout().getDescription());
- const vector
blocks(physical_plan->catalog_relation()->getBlocksSnapshot());
- for (const block_id block : blocks) {
-
insert_destination_proto->AddExtension(S::BlockPoolInsertDestination::blocks,
block);
- }
-
const QueryPlan::DAGNodeIndex scan_operator_index =
execution_plan_->addRelationalOperator(
new TextScanOperator(
incubator-quickstep git commit: Assigned partition_id to RebuildWorkOrder.
Repository: incubator-quickstep
Updated Branches:
refs/heads/master 30021acf8 -> 5ee3f11a5
Assigned partition_id to RebuildWorkOrder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5ee3f11a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5ee3f11a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5ee3f11a
Branch: refs/heads/master
Commit: 5ee3f11a5f36d56b27df0e7df3a270852911d7a0
Parents: 30021ac
Author: Zuyu Zhang
Authored: Sat Apr 22 21:09:19 2017 -0700
Committer: Zuyu Zhang
Committed: Mon Apr 24 17:57:41 2017 -0700
--
query_execution/QueryManagerSingleNode.cpp | 4 +-
query_execution/Shiftboss.cpp | 4 +-
relational_operators/RebuildWorkOrder.hpp | 5 +++
storage/InsertDestination.cpp | 7 ++-
storage/InsertDestination.hpp | 58 +++--
5 files changed, 51 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ee3f11a/query_execution/QueryManagerSingleNode.cpp
--
diff --git a/query_execution/QueryManagerSingleNode.cpp
b/query_execution/QueryManagerSingleNode.cpp
index e3f349f..79c4026 100644
--- a/query_execution/QueryManagerSingleNode.cpp
+++ b/query_execution/QueryManagerSingleNode.cpp
@@ -174,12 +174,13 @@ void QueryManagerSingleNode::getRebuildWorkOrders(const
dag_node_index index,
}
std::vector partially_filled_block_refs;
+ std::vector part_ids;
DCHECK(query_context_ != nullptr);
InsertDestination *insert_destination =
query_context_->getInsertDestination(insert_destination_index);
DCHECK(insert_destination != nullptr);
- insert_destination->getPartiallyFilledBlocks(&partially_filled_block_refs);
+ insert_destination->getPartiallyFilledBlocks(&partially_filled_block_refs,
&part_ids);
for (std::vector::size_type i = 0;
i < partially_filled_block_refs.size();
@@ -189,6 +190,7 @@ void QueryManagerSingleNode::getRebuildWorkOrders(const
dag_node_index index,
std::move(partially_filled_block_refs[i]),
index,
op.getOutputRelationID(),
+ part_ids[i],
foreman_client_id_,
bus_),
index);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ee3f11a/query_execution/Shiftboss.cpp
--
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index 72d8913..21e7858 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -405,7 +405,8 @@ void Shiftboss::processInitiateRebuildMessage(const
std::size_t query_id,
DCHECK(insert_destination != nullptr);
vector partially_filled_block_refs;
- insert_destination->getPartiallyFilledBlocks(&partially_filled_block_refs);
+ vector part_ids;
+ insert_destination->getPartiallyFilledBlocks(&partially_filled_block_refs,
&part_ids);
serialization::InitiateRebuildResponseMessage proto;
proto.set_query_id(query_id);
@@ -439,6 +440,7 @@ void Shiftboss::processInitiateRebuildMessage(const
std::size_t query_id,
move(partially_filled_block_refs[i]),
op_index,
rel_id,
+ part_ids[i],
shiftboss_client_id_local_,
bus_local_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ee3f11a/relational_operators/RebuildWorkOrder.hpp
--
diff --git a/relational_operators/RebuildWorkOrder.hpp
b/relational_operators/RebuildWorkOrder.hpp
index 7f0f7fc..8615d74 100644
--- a/relational_operators/RebuildWorkOrder.hpp
+++ b/relational_operators/RebuildWorkOrder.hpp
@@ -57,6 +57,7 @@ class RebuildWorkOrder : public WorkOrder {
*query plan DAG that produced the output block.
* @param input_relation_id The ID of the CatalogRelation to which the given
*storage block belongs to.
+ * @param part_id The partition_id of the block, if any.
* @param scheduler_client_id The TMB client ID of the scheduler thread.
* @param bus A pointer to the TMB.
**/
@@ -65,12 +66,14 @@ class RebuildWorkOrder : public WorkOrder {
MutableBlockReference &&block_ref, // NOLINT(whitespace/operators)
const std::size_t input_operator_index,
const relation_id input_relation_id,
+ const partition_id part_id,
const client_id scheduler_client_id,
MessageBus *bus)
: WorkOrder(
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/fix-common-subexpression [deleted] 30021acf8
[1/2] incubator-quickstep git commit: Add ThreadPrivateCompactKeyHashTable as a fast path data structure for aggregation. [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/fix-common-subexpression c7333e02f -> 30021acf8 (forced update)
Add ThreadPrivateCompactKeyHashTable as a fast path data structure for
aggregation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d6a01e7c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d6a01e7c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d6a01e7c
Branch: refs/heads/fix-common-subexpression
Commit: d6a01e7c867354ca05545595644a62b03de56b81
Parents: 8169306
Author: Jianqiao Zhu
Authored: Fri Apr 21 23:23:13 2017 -0500
Committer: Jianqiao Zhu
Committed: Mon Apr 24 13:40:17 2017 -0500
--
query_optimizer/ExecutionGenerator.cpp | 14 +-
.../cost_model/StarSchemaSimpleCostModel.cpp| 71 +++-
.../cost_model/StarSchemaSimpleCostModel.hpp| 12 +
storage/AggregationOperationState.cpp | 68 ++-
storage/AggregationOperationState.hpp | 6 +-
storage/CMakeLists.txt | 24 ++
storage/CollisionFreeVectorTable.hpp| 4 +
storage/HashTable.proto | 1 +
storage/HashTableBase.hpp | 18 +-
storage/HashTableFactory.hpp| 13 +-
storage/HashTablePool.hpp | 9 +
storage/PackedPayloadHashTable.hpp | 4 +
storage/ThreadPrivateCompactKeyHashTable.cpp| 421 +++
storage/ThreadPrivateCompactKeyHashTable.hpp| 230 ++
14 files changed, 870 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d6a01e7c/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp
b/query_optimizer/ExecutionGenerator.cpp
index 3e0f647..9625a91 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1580,14 +1580,22 @@ void ExecutionGenerator::convertAggregate(
->canUseCollisionFreeAggregation(physical_plan,
estimated_num_groups,
&max_num_groups)) {
+ // First option: use array-based aggregation if applicable.
aggr_state_proto->set_hash_table_impl_type(
serialization::HashTableImplType::COLLISION_FREE_VECTOR);
aggr_state_proto->set_estimated_num_entries(max_num_groups);
use_parallel_initialization = true;
} else {
- // Otherwise, use SeparateChaining.
- aggr_state_proto->set_hash_table_impl_type(
- serialization::HashTableImplType::SEPARATE_CHAINING);
+ if (cost_model_for_aggregation_->canUseTwoPhaseCompactKeyAggregation(
+ physical_plan, estimated_num_groups)) {
+// Second option: use thread-private compact-key aggregation if
applicable.
+aggr_state_proto->set_hash_table_impl_type(
+serialization::HashTableImplType::THREAD_PRIVATE_COMPACT_KEY);
+ } else {
+// Otherwise, use SeparateChaining.
+aggr_state_proto->set_hash_table_impl_type(
+serialization::HashTableImplType::SEPARATE_CHAINING);
+ }
aggr_state_proto->set_estimated_num_entries(std::max(16uL,
estimated_num_groups));
}
} else {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d6a01e7c/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
--
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index b17fac0..e0e3dff 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -71,8 +71,8 @@ namespace optimizer {
namespace cost {
DEFINE_int64(collision_free_vector_table_max_size, 10,
- "The maximum allowed key range (number of entries) for using a "
- "CollisionFreeVectorTable.");
+ "The maximum allowed key range (number of entries) for using a "
+ "CollisionFreeVectorTable.");
namespace E = ::quickstep::optimizer::expressions;
namespace P = ::quickstep::optimizer::physical;
@@ -700,6 +700,73 @@ bool
StarSchemaSimpleCostModel::canUseCollisionFreeAggregation(
return true;
}
+bool StarSchemaSimpleCostModel::canUseTwoPhaseCompactKeyAggregation(
+const physical::AggregatePtr &aggregate,
+const std::size_t estimated_num_groups) {
+ // Require estimated number of groups to be below the specified threshold.
+ //
+ // TODO(jianqiao): It is good to have the threshold to be the same as
+ // FLAGS_partition_aggr
incubator-quickstep git commit: Fix the issues with the common subexpression feature
Repository: incubator-quickstep
Updated Branches:
refs/heads/master d6a01e7c8 -> 30021acf8
Fix the issues with the common subexpression feature
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/30021acf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/30021acf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/30021acf
Branch: refs/heads/master
Commit: 30021acf8bdf6d33f6e940f3343760384d971830
Parents: d6a01e7
Author: Jianqiao Zhu
Authored: Mon Apr 24 02:33:39 2017 -0500
Committer: Zuyu Zhang
Committed: Mon Apr 24 17:55:22 2017 -0700
--
expressions/predicate/Predicate.hpp | 6 +--
expressions/scalar/Scalar.cpp | 2 +-
expressions/scalar/Scalar.hpp | 5 +-
expressions/scalar/ScalarSharedExpression.cpp | 51 +---
expressions/scalar/ScalarSharedExpression.hpp | 13 +++--
.../tests/ScalarCaseExpression_unittest.cpp | 26 +++---
.../expressions/CommonSubexpression.hpp | 4 +-
query_optimizer/expressions/SimpleCase.cpp | 6 +--
query_optimizer/rules/CollapseSelection.cpp | 2 +-
query_optimizer/rules/CollapseSelection.hpp | 2 +
.../rules/ExtractCommonSubexpression.cpp| 2 +-
.../rules/ExtractCommonSubexpression.hpp| 2 +-
.../rules/ReuseAggregateExpressions.cpp | 7 +--
.../rules/ReuseAggregateExpressions.hpp | 2 +
14 files changed, 68 insertions(+), 62 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30021acf/expressions/predicate/Predicate.hpp
--
diff --git a/expressions/predicate/Predicate.hpp
b/expressions/predicate/Predicate.hpp
index 6a2ba6d..df04644 100644
--- a/expressions/predicate/Predicate.hpp
+++ b/expressions/predicate/Predicate.hpp
@@ -65,11 +65,7 @@ class Predicate : public Expression {
**/
static const char *kPredicateTypeNames[kNumPredicateTypes];
- /**
- * @brief Virtual destructor.
- *
- **/
- virtual ~Predicate() {
+ ~Predicate() override {
}
std::string getName() const override {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30021acf/expressions/scalar/Scalar.cpp
--
diff --git a/expressions/scalar/Scalar.cpp b/expressions/scalar/Scalar.cpp
index c552d8b..da0fc1b 100644
--- a/expressions/scalar/Scalar.cpp
+++ b/expressions/scalar/Scalar.cpp
@@ -31,8 +31,8 @@ const char *Scalar::kScalarDataSourceNames[] = {
"Attribute",
"UnaryExpression",
"BinaryExpression",
- "SharedExpression",
"SimpleCase"
+ "SharedExpression",
};
const TypedValue& Scalar::getStaticValue() const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30021acf/expressions/scalar/Scalar.hpp
--
diff --git a/expressions/scalar/Scalar.hpp b/expressions/scalar/Scalar.hpp
index 472b71c..6e482c2 100644
--- a/expressions/scalar/Scalar.hpp
+++ b/expressions/scalar/Scalar.hpp
@@ -69,10 +69,7 @@ class Scalar : public Expression {
**/
static const char *kScalarDataSourceNames[kNumScalarDataSources];
- /**
- * @brief Virtual destructor.
- **/
- virtual ~Scalar() {
+ ~Scalar() override {
}
std::string getName() const override {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30021acf/expressions/scalar/ScalarSharedExpression.cpp
--
diff --git a/expressions/scalar/ScalarSharedExpression.cpp
b/expressions/scalar/ScalarSharedExpression.cpp
index f97c60b..e301116 100644
--- a/expressions/scalar/ScalarSharedExpression.cpp
+++ b/expressions/scalar/ScalarSharedExpression.cpp
@@ -34,19 +34,12 @@ namespace quickstep {
struct SubBlocksReference;
-ScalarSharedExpression::ScalarSharedExpression(const int share_id,
- Scalar *operand)
-: Scalar(operand->getType()),
- share_id_(share_id),
- operand_(operand) {
-}
-
serialization::Scalar ScalarSharedExpression::getProto() const {
serialization::Scalar proto;
proto.set_data_source(serialization::Scalar::SHARED_EXPRESSION);
proto.SetExtension(serialization::ScalarSharedExpression::share_id,
share_id_);
proto.MutableExtension(serialization::ScalarSharedExpression::operand)
- ->CopyFrom(operand_->getProto());
+ ->MergeFrom(operand_->getProto());
return proto;
}
@@ -81,16 +74,16 @@ ColumnVectorPtr ScalarSharedExpression::getAllValues(
ColumnVectorCache *cv_cache) const {
if (cv_cache == nullptr) {
return operand_->getAllValues(accessor, sub_blocks_ref, cv_cache);
+ }
+
+
[2/2] incubator-quickstep git commit: Fix the issues with the common subexpression feature
Fix the issues with the common subexpression feature
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/30021acf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/30021acf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/30021acf
Branch: refs/heads/fix-common-subexpression
Commit: 30021acf8bdf6d33f6e940f3343760384d971830
Parents: d6a01e7
Author: Jianqiao Zhu
Authored: Mon Apr 24 02:33:39 2017 -0500
Committer: Zuyu Zhang
Committed: Mon Apr 24 17:55:22 2017 -0700
--
expressions/predicate/Predicate.hpp | 6 +--
expressions/scalar/Scalar.cpp | 2 +-
expressions/scalar/Scalar.hpp | 5 +-
expressions/scalar/ScalarSharedExpression.cpp | 51 +---
expressions/scalar/ScalarSharedExpression.hpp | 13 +++--
.../tests/ScalarCaseExpression_unittest.cpp | 26 +++---
.../expressions/CommonSubexpression.hpp | 4 +-
query_optimizer/expressions/SimpleCase.cpp | 6 +--
query_optimizer/rules/CollapseSelection.cpp | 2 +-
query_optimizer/rules/CollapseSelection.hpp | 2 +
.../rules/ExtractCommonSubexpression.cpp| 2 +-
.../rules/ExtractCommonSubexpression.hpp| 2 +-
.../rules/ReuseAggregateExpressions.cpp | 7 +--
.../rules/ReuseAggregateExpressions.hpp | 2 +
14 files changed, 68 insertions(+), 62 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30021acf/expressions/predicate/Predicate.hpp
--
diff --git a/expressions/predicate/Predicate.hpp
b/expressions/predicate/Predicate.hpp
index 6a2ba6d..df04644 100644
--- a/expressions/predicate/Predicate.hpp
+++ b/expressions/predicate/Predicate.hpp
@@ -65,11 +65,7 @@ class Predicate : public Expression {
**/
static const char *kPredicateTypeNames[kNumPredicateTypes];
- /**
- * @brief Virtual destructor.
- *
- **/
- virtual ~Predicate() {
+ ~Predicate() override {
}
std::string getName() const override {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30021acf/expressions/scalar/Scalar.cpp
--
diff --git a/expressions/scalar/Scalar.cpp b/expressions/scalar/Scalar.cpp
index c552d8b..da0fc1b 100644
--- a/expressions/scalar/Scalar.cpp
+++ b/expressions/scalar/Scalar.cpp
@@ -31,8 +31,8 @@ const char *Scalar::kScalarDataSourceNames[] = {
"Attribute",
"UnaryExpression",
"BinaryExpression",
- "SharedExpression",
"SimpleCase"
+ "SharedExpression",
};
const TypedValue& Scalar::getStaticValue() const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30021acf/expressions/scalar/Scalar.hpp
--
diff --git a/expressions/scalar/Scalar.hpp b/expressions/scalar/Scalar.hpp
index 472b71c..6e482c2 100644
--- a/expressions/scalar/Scalar.hpp
+++ b/expressions/scalar/Scalar.hpp
@@ -69,10 +69,7 @@ class Scalar : public Expression {
**/
static const char *kScalarDataSourceNames[kNumScalarDataSources];
- /**
- * @brief Virtual destructor.
- **/
- virtual ~Scalar() {
+ ~Scalar() override {
}
std::string getName() const override {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30021acf/expressions/scalar/ScalarSharedExpression.cpp
--
diff --git a/expressions/scalar/ScalarSharedExpression.cpp
b/expressions/scalar/ScalarSharedExpression.cpp
index f97c60b..e301116 100644
--- a/expressions/scalar/ScalarSharedExpression.cpp
+++ b/expressions/scalar/ScalarSharedExpression.cpp
@@ -34,19 +34,12 @@ namespace quickstep {
struct SubBlocksReference;
-ScalarSharedExpression::ScalarSharedExpression(const int share_id,
- Scalar *operand)
-: Scalar(operand->getType()),
- share_id_(share_id),
- operand_(operand) {
-}
-
serialization::Scalar ScalarSharedExpression::getProto() const {
serialization::Scalar proto;
proto.set_data_source(serialization::Scalar::SHARED_EXPRESSION);
proto.SetExtension(serialization::ScalarSharedExpression::share_id,
share_id_);
proto.MutableExtension(serialization::ScalarSharedExpression::operand)
- ->CopyFrom(operand_->getProto());
+ ->MergeFrom(operand_->getProto());
return proto;
}
@@ -81,16 +74,16 @@ ColumnVectorPtr ScalarSharedExpression::getAllValues(
ColumnVectorCache *cv_cache) const {
if (cv_cache == nullptr) {
return operand_->getAllValues(accessor, sub_blocks_ref, cv_cache);
+ }
+
+ ColumnVectorPtr result;
+ if (cv_cache->contains(share_id_)) {
+result
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/common-subexpression [deleted] 8169306c2
[incubator-quickstep] Git Push Summary
Repository: incubator-quickstep Updated Branches: refs/heads/reorder-partitioned-hash-join [deleted] 238b2d66a
incubator-quickstep git commit: Implement optimizer and execution layers for UNION and INTERSECT.
Repository: incubator-quickstep Updated Branches: refs/heads/master 5b7b5cb84 -> 563abe043 Implement optimizer and execution layers for UNION and INTERSECT. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/563abe04 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/563abe04 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/563abe04 Branch: refs/heads/master Commit: 563abe0430af0e26f571cf07ba03079232c59fd3 Parents: 5b7b5cb Author: Tianrun Authored: Mon Apr 10 10:21:11 2017 -0600 Committer: Tianrun Committed: Tue Apr 11 23:52:18 2017 -0600 -- query_optimizer/CMakeLists.txt | 2 + query_optimizer/ExecutionGenerator.cpp | 68 + query_optimizer/ExecutionGenerator.hpp | 8 + query_optimizer/cost_model/CMakeLists.txt | 2 + query_optimizer/cost_model/SimpleCostModel.cpp | 13 + query_optimizer/cost_model/SimpleCostModel.hpp | 6 + .../cost_model/StarSchemaSimpleCostModel.cpp| 12 + .../cost_model/StarSchemaSimpleCostModel.hpp| 6 +- query_optimizer/physical/CMakeLists.txt | 10 + query_optimizer/physical/PhysicalType.hpp | 1 + query_optimizer/physical/UnionAll.hpp | 188 + query_optimizer/resolver/Resolver.cpp | 6 +- query_optimizer/strategy/CMakeLists.txt | 5 + query_optimizer/strategy/Join.cpp | 42 +++ query_optimizer/strategy/OneToOne.cpp | 30 ++ .../tests/execution_generator/Select.test | 86 ++ .../tests/physical_generator/Select.test| 280 +++ query_optimizer/tests/resolver/CMakeLists.txt | 4 + .../tests/resolver/SetOperation.test| 185 relational_operators/CMakeLists.txt | 16 ++ relational_operators/RelationalOperator.hpp | 1 + relational_operators/UnionAllOperator.cpp | 144 ++ relational_operators/UnionAllOperator.hpp | 228 +++ 23 files changed, 1339 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/563abe04/query_optimizer/CMakeLists.txt -- diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt index 9bdb753..08b6467 100644 --- a/query_optimizer/CMakeLists.txt +++ b/query_optimizer/CMakeLists.txt @@ -113,6 +113,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator quickstep_queryoptimizer_physical_TableGenerator quickstep_queryoptimizer_physical_TableReference quickstep_queryoptimizer_physical_TopLevelPlan + quickstep_queryoptimizer_physical_UnionAll quickstep_queryoptimizer_physical_UpdateTable quickstep_queryoptimizer_physical_WindowAggregate quickstep_relationaloperators_AggregationOperator @@ -138,6 +139,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator quickstep_relationaloperators_SortRunGenerationOperator quickstep_relationaloperators_TableGeneratorOperator quickstep_relationaloperators_TextScanOperator + quickstep_relationaloperators_UnionAllOperator quickstep_relationaloperators_UpdateOperator quickstep_relationaloperators_WindowAggregationOperator quickstep_storage_AggregationOperationState_proto http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/563abe04/query_optimizer/ExecutionGenerator.cpp -- diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp index 6fec85b..3e0f647 100644 --- a/query_optimizer/ExecutionGenerator.cpp +++ b/query_optimizer/ExecutionGenerator.cpp @@ -94,6 +94,7 @@ #include "query_optimizer/physical/TableGenerator.hpp" #include "query_optimizer/physical/TableReference.hpp" #include "query_optimizer/physical/TopLevelPlan.hpp" +#include "query_optimizer/physical/UnionAll.hpp" #include "query_optimizer/physical/UpdateTable.hpp" #include "query_optimizer/physical/WindowAggregate.hpp" #include "relational_operators/AggregationOperator.hpp" @@ -119,6 +120,7 @@ #include "relational_operators/SortRunGenerationOperator.hpp" #include "relational_operators/TableGeneratorOperator.hpp" #include "relational_operators/TextScanOperator.hpp" +#include "relational_operators/UnionAllOperator.hpp" #include "relational_operators/UpdateOperator.hpp" #include "relational_operators/WindowAggregationOperator.hpp" #include "st
