incubator-quickstep git commit: Update ntohll/htonll for protability

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/trace-dev 29024a3e9 -> 9005d9935


Update ntohll/htonll for protability


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9005d993
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9005d993
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9005d993

Branch: refs/heads/trace-dev
Commit: 9005d99359b8d7cae8fc2c31539e70bd5e45761f
Parents: 29024a3
Author: Jianqiao Zhu 
Authored: Fri Nov 9 22:23:04 2018 -0600
Committer: Jianqiao Zhu 
Committed: Fri Nov 9 22:30:39 2018 -0600

--
 cli/simple_socket/SimpleSocketConnection.hpp | 25 ++-
 1 file changed, 20 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9005d993/cli/simple_socket/SimpleSocketConnection.hpp
--
diff --git a/cli/simple_socket/SimpleSocketConnection.hpp 
b/cli/simple_socket/SimpleSocketConnection.hpp
index 111db6a..0bd7774 100644
--- a/cli/simple_socket/SimpleSocketConnection.hpp
+++ b/cli/simple_socket/SimpleSocketConnection.hpp
@@ -20,6 +20,7 @@
 #ifndef QUICKSTEP_CLI_SIMPLE_SOCKET_SIMPLE_SOCKET_CONNECTION_HPP_
 #define QUICKSTEP_CLI_SIMPLE_SOCKET_SIMPLE_SOCKET_CONNECTION_HPP_
 
+#include 
 #include 
 
 #include 
@@ -72,12 +73,12 @@ class SimpleSocketConnection {
 // Decode request data.
 const std::uint64_t *size_ptr =
 static_cast(request_data_);
-const std::uint64_t num_fields = ntohll(*size_ptr++);
+const std::uint64_t num_fields = Ntohll(*size_ptr++);
 
 std::vector> field_sizes;
 for (std::size_t i = 0; i < num_fields; ++i) {
-  const std::uint64_t key_size = ntohll(*size_ptr++);
-  const std::uint64_t value_size = ntohll(*size_ptr++);
+  const std::uint64_t key_size = Ntohll(*size_ptr++);
+  const std::uint64_t value_size = Ntohll(*size_ptr++);
   field_sizes.emplace_back(key_size, value_size);
 }
 
@@ -108,11 +109,11 @@ class SimpleSocketConnection {
   inline std::uint64_t receiveUInt64() const {
 std::uint64_t code;
 receiveData(, sizeof(std::uint64_t));
-return ntohll(code);
+return Ntohll(code);
   }
 
   inline void writeUInt64(const std::uint64_t value) const {
-const uint64_t code = htonll(value);
+const uint64_t code = Htonll(value);
 write(socket_fd_, , sizeof(std::uint64_t));
   }
 
@@ -146,6 +147,20 @@ class SimpleSocketConnection {
 }
   }
 
+  inline static std::uint64_t Ntohll(const std::uint64_t code) {
+const std::uint32_t lo32 = static_cast(code);
+const std::uint32_t hi32 = static_cast(code >> 32);
+return (static_cast(ntohl(lo32)) << 32)
+   | static_cast(ntohl(hi32));
+  }
+
+  inline static std::uint64_t Htonll(const std::uint64_t code) {
+const std::uint32_t lo32 = static_cast(code);
+const std::uint32_t hi32 = static_cast(code >> 32);
+return (static_cast(htonl(lo32)) << 32)
+   | static_cast(htonl(hi32));
+  }
+
   const int socket_fd_;
   std::uint64_t request_data_length_;
   void *request_data_;

[2/2] incubator-quickstep git commit: Add socket cli support

[jianqiao]

Add socket cli support


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/29024a3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/29024a3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/29024a3e

Branch: refs/heads/trace-dev
Commit: 29024a3e9e092010f70c067de73f0c4cef1dd6c0
Parents: f477c5d
Author: Jianqiao Zhu 
Authored: Mon Oct 29 23:32:47 2018 -0500
Committer: Jianqiao Zhu 
Committed: Mon Nov 5 01:20:39 2018 -0600

--
 CMakeLists.txt  |1 +
 cli/CMakeLists.txt  |   25 +-
 cli/IOInterface.hpp |6 +-
 cli/LocalIO.hpp |9 +-
 cli/NetworkIO.hpp   |   11 +-
 cli/QuickstepCli.cpp|  268 ++--
 cli/SocketIO.cpp|   36 +
 cli/SocketIO.hpp|  121 ++
 cli/quickstep/NetworkCliOuterClass.java | 1388 --
 cli/simple_socket/CMakeLists.txt|   35 +
 cli/simple_socket/SimpleSocketConnection.hpp|  159 ++
 cli/simple_socket/SimpleSocketContent.hpp   |   75 +
 cli/simple_socket/SimpleSocketServer.hpp|  115 ++
 query_optimizer/CMakeLists.txt  |2 +
 query_optimizer/QueryHandle.hpp |9 +-
 relational_operators/CMakeLists.txt |3 +
 relational_operators/TextScanOperator.cpp   |   12 +-
 relational_operators/TextScanOperator.hpp   |   11 +-
 relational_operators/WorkOrderFactory.cpp   |3 +-
 .../tests/TextScanOperator_unittest.cpp |3 +-
 utility/StringUtil.hpp  |4 +
 utility/ThreadSafeQueue.hpp |2 +-
 22 files changed, 748 insertions(+), 1550 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29024a3e/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index cb8e9f4..54a69bf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -734,6 +734,7 @@ target_link_libraries(quickstep_cli_shell
   quickstep_cli_LineReader
   quickstep_cli_LocalIO
   quickstep_cli_PrintToScreen
+  quickstep_cli_SocketIO
   quickstep_parser_ParseStatement
   quickstep_parser_SqlParserWrapper
   quickstep_queryexecution_ForemanSingleNode

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29024a3e/cli/CMakeLists.txt
--
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 03c5408..d90de95 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -65,6 +65,9 @@ if (ENABLE_NETWORK_CLI)
 NetworkCli.proto)
 endif()
 
+# Sub-directories:
+add_subdirectory(simple_socket)
+
 # Declare micro-libs and link dependencies:
 add_library(quickstep_cli_CommandExecutor CommandExecutor.cpp 
CommandExecutor.hpp)
 add_library(quickstep_cli_CommandExecutorUtil CommandExecutorUtil.cpp 
CommandExecutorUtil.hpp)
@@ -74,6 +77,7 @@ add_library(quickstep_cli_DropRelation DropRelation.cpp 
DropRelation.hpp)
 add_library(quickstep_cli_Flags Flags.cpp Flags.hpp)
 add_library(quickstep_cli_IOInterface ../empty_src.cpp IOInterface.hpp)
 add_library(quickstep_cli_InputParserUtil InputParserUtil.cpp 
InputParserUtil.hpp)
+add_library(quickstep_cli_SocketIO SocketIO.cpp SocketIO.hpp)
 
 if(USE_LINENOISE)
   add_library(quickstep_cli_LineReader
@@ -178,7 +182,8 @@ target_link_libraries(quickstep_cli_Flags
   quickstep_storage_StorageConstants
   ${GFLAGS_LIB_NAME})
 target_link_libraries(quickstep_cli_IOInterface
-  quickstep_utility_Macros)
+  quickstep_utility_Macros
+  quickstep_utility_StringUtil)
 target_link_libraries(quickstep_cli_InputParserUtil
   glog
   quickstep_utility_Macros
@@ -203,7 +208,8 @@ target_link_libraries(quickstep_cli_LineReaderBuffered
 target_link_libraries(quickstep_cli_LocalIO
   quickstep_cli_LineReader
   quickstep_cli_IOInterface
-  quickstep_utility_Macros)
+  quickstep_utility_Macros
+  quickstep_utility_StringUtil)
 if (ENABLE_NETWORK_CLI)
   target_link_libraries(quickstep_cli_NetworkCli_proto
 ${GRPCPLUSPLUS_LIBRARIES}
@@ -226,6 +232,7 @@ if (ENABLE_NETWORK_CLI)
 quickstep_threading_SpinSharedMutex

[1/2] incubator-quickstep git commit: Add socket cli support

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/trace-dev [created] 29024a3e9


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29024a3e/cli/simple_socket/SimpleSocketServer.hpp
--
diff --git a/cli/simple_socket/SimpleSocketServer.hpp 
b/cli/simple_socket/SimpleSocketServer.hpp
new file mode 100644
index 000..d2f5dfe
--- /dev/null
+++ b/cli/simple_socket/SimpleSocketServer.hpp
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_SIMPLE_SOCKET_SIMPLE_SOCKET_SERVER_HPP_
+#define QUICKSTEP_CLI_SIMPLE_SOCKET_SIMPLE_SOCKET_SERVER_HPP_
+
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+#include "cli/simple_socket/SimpleSocketConnection.hpp"
+#include "utility/Macros.hpp"
+#include "utility/ThreadSafeQueue.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class SimpleSocketServer {
+ public:
+  SimpleSocketServer(const int port)
+  : port_(port) {}
+
+  ~SimpleSocketServer() {
+stop();
+  }
+
+  void start() {
+CHECK(main_loop_thread_ == nullptr);
+CHECK(connections_.empty());
+main_loop_thread_ = std::make_unique([this] {
+  this->mainLoop(this->port_);
+});
+  }
+
+  void stop() {
+if (main_loop_thread_ != nullptr) {
+  main_loop_thread_ = nullptr;
+}
+  }
+
+  SimpleSocketConnection* waitForConnection() {
+return connections_.popOne().release();
+  }
+
+ private:
+  void mainLoop(const int port) {
+main_socket_fd_ = socket(AF_INET, SOCK_STREAM, 0);
+int option = 1;
+setsockopt(main_socket_fd_,
+   SOL_SOCKET, SO_REUSEADDR,
+   ,
+   sizeof(option));
+CHECK(main_socket_fd_ >= 0) << "Error opening socket";
+
+constexpr socklen_t sockaddr_size = sizeof(sockaddr_in);
+sockaddr_in server_address;
+std::memset(_address, 0, sockaddr_size);
+
+server_address.sin_family = AF_INET;
+server_address.sin_addr.s_addr = INADDR_ANY;
+server_address.sin_port = htons(port);
+
+const int bind_retval =
+bind(main_socket_fd_,
+ reinterpret_cast(_address),
+ sockaddr_size);
+CHECK(bind_retval >= 0) << "Error binding socket";
+
+const int listen_retval = listen(main_socket_fd_, 32);
+CHECK(listen_retval >= 0) << "Error listening to socket connection";
+
+while (true) {
+  socklen_t client_addr_len = sockaddr_size;
+  sockaddr_in client_address;
+  const int client_socket_fd =
+  accept(main_socket_fd_,
+ reinterpret_cast(_address),
+ _addr_len);
+  CHECK(client_socket_fd >= 0) << "Error accepting socket connection";
+  
connections_.push(std::make_unique(client_socket_fd));
+}
+  }
+
+  const int port_;
+  std::unique_ptr main_loop_thread_;
+  int main_socket_fd_;
+  ThreadSafeQueue> connections_;
+
+  DISALLOW_COPY_AND_ASSIGN(SimpleSocketServer);
+};
+
+}  // namespace quickstep
+
+#endif  // QUICKSTEP_CLI_SIMPLE_SOCKET_SIMPLE_SOCKET_SERVER_HPP_

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29024a3e/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 5e0db44..1c5d669 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -148,6 +148,7 @@ 
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
   quickstep_relationaloperators_UpdateOperator
   quickstep_relationaloperators_WindowAggregationOperator
   quickstep_storage_AggregationOperationState_proto
+  quickstep_storage_Flags
   quickstep_storage_HashTableFactory
   quickstep_storage_HashTable_proto
   quickstep_storage_InsertDestination_proto
@@ -248,6 +249,7 @@ target_link_libraries(quickstep_queryoptimizer_QueryHandle
   quickstep_queryexecution_QueryContext_proto
   quickstep_queryoptimizer_QueryPlan
   

incubator-quickstep git commit: Fix the inclusion guard of ForemanSingleNode.hpp file

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master e36fc7497 -> 5cbaa7ef8


Fix the inclusion guard of ForemanSingleNode.hpp file


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5cbaa7ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5cbaa7ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5cbaa7ef

Branch: refs/heads/master
Commit: 5cbaa7ef877f3ab4a1d89f3e463f3a281709ec05
Parents: e36fc74
Author: Harshad Deshmukh 
Authored: Thu Jun 21 21:31:07 2018 -0500
Committer: Harshad Deshmukh 
Committed: Thu Jun 21 21:31:07 2018 -0500

--
 query_execution/ForemanSingleNode.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5cbaa7ef/query_execution/ForemanSingleNode.hpp
--
diff --git a/query_execution/ForemanSingleNode.hpp 
b/query_execution/ForemanSingleNode.hpp
index 9fa3b9a..2e4f23b 100644
--- a/query_execution/ForemanSingleNode.hpp
+++ b/query_execution/ForemanSingleNode.hpp
@@ -17,8 +17,8 @@
  * under the License.
  **/
 
-#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
-#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
+#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_SINGLE_NODE_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_SINGLE_NODE_HPP_
 
 #include 
 #include 
@@ -120,4 +120,4 @@ class ForemanSingleNode final : public ForemanBase {
 
 }  // namespace quickstep
 
-#endif  // QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
+#endif  // QUICKSTEP_QUERY_EXECUTION_FOREMAN_SINGLE_NODE_HPP_

incubator-quickstep git commit: Fixed the build issues regarding tmb benchmark.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 97bef178f -> e36fc7497


Fixed the build issues regarding tmb benchmark.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e36fc749
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e36fc749
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e36fc749

Branch: refs/heads/master
Commit: e36fc7497fb63cadff7bcb3e30f42a3c94a087e2
Parents: 97bef17
Author: Zuyu Zhang 
Authored: Mon Jun 18 20:18:26 2018 -0500
Committer: Zuyu Zhang 
Committed: Mon Jun 18 20:20:22 2018 -0500

--
 third_party/src/tmb/benchmarks/CMakeLists.txt   | 16 
 third_party/src/tmb/benchmarks/src/bus_setup.cc |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e36fc749/third_party/src/tmb/benchmarks/CMakeLists.txt
--
diff --git a/third_party/src/tmb/benchmarks/CMakeLists.txt 
b/third_party/src/tmb/benchmarks/CMakeLists.txt
index cec8fd8..d215489 100644
--- a/third_party/src/tmb/benchmarks/CMakeLists.txt
+++ b/third_party/src/tmb/benchmarks/CMakeLists.txt
@@ -109,32 +109,32 @@ target_link_libraries(tmbbench
 add_executable(oneway_throughput src/oneway_throughput.cc)
 target_link_libraries(oneway_throughput
   gflags_nothreads-static
-  tcmalloc_minimal
   tmb
-  tmbbench)
+  tmbbench
+  ${LIBS})
 
 add_executable(oneway_throughput_numa src/oneway_throughput_numa.cc)
 target_link_libraries(oneway_throughput_numa
   gflags_nothreads-static
-  tcmalloc_minimal
   tmb
-  tmbbench)
+  tmbbench
+  ${LIBS})
 
 add_executable(oneway_throughput_distributed
src/oneway_throughput_distributed.cc)
 target_link_libraries(oneway_throughput_distributed
   gflags_nothreads-static
-  tcmalloc_minimal
   tmb
-  tmbbench)
+  tmbbench
+  ${LIBS})
 
 add_executable(oneway_throughput_distributed_coordinator
src/oneway_throughput_distributed_coordinator.cc)
 target_link_libraries(oneway_throughput_distributed_coordinator
   gflags_nothreads-static
-  tcmalloc_minimal
   tmb
-  tmbbench)
+  tmbbench
+  ${LIBS})
 
 add_executable(reset_bus src/reset_bus.cc)
 target_link_libraries(reset_bus

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e36fc749/third_party/src/tmb/benchmarks/src/bus_setup.cc
--
diff --git a/third_party/src/tmb/benchmarks/src/bus_setup.cc 
b/third_party/src/tmb/benchmarks/src/bus_setup.cc
index 8e4c223..205f8e3 100644
--- a/third_party/src/tmb/benchmarks/src/bus_setup.cc
+++ b/third_party/src/tmb/benchmarks/src/bus_setup.cc
@@ -96,7 +96,6 @@ DEFINE_string(servers, "",
   "specifies a comma-separated list of servers to connect to, "
   "each of which is given in HOSTNAME:PORT or IP:PORT format. "
   "Has no effect for other TMB implementations.");
-#endif
 
 namespace {
 
@@ -153,6 +152,7 @@ bool ParseServersString(const std::string _string,
 }
 
 }  // namespace
+#endif
 
 tmb::MessageBus* CreateBus(const bool delete_messages_immediately) {
 #ifdef TMB_MEMORYMIRROR_ENABLED

incubator-quickstep git commit: Data Provider thread added.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 835281ae8 -> dfefe6293


Data Provider thread added.

- Thread used for receiving block requests from clients and sending the
contents of a block back to the client.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/dfefe629
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/dfefe629
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/dfefe629

Branch: refs/heads/master
Commit: dfefe62934c5c890650606782b50714cb2eb0366
Parents: 835281a
Author: Harshad Deshmukh 
Authored: Fri Apr 6 16:00:54 2018 -0500
Committer: Harshad Deshmukh 
Committed: Mon Jun 4 16:45:31 2018 -0500

--
 CMakeLists.txt|   6 +-
 query_execution/QueryExecutionTypedefs.hpp|   4 +
 storage/BlockWire.proto   |  34 +
 storage/CMakeLists.txt|  69 -
 storage/DataProviderThread.cpp|  92 +++
 storage/DataProviderThread.hpp| 103 +
 storage/StorageManager.cpp|  23 +++
 storage/StorageManager.hpp|  17 ++-
 storage/tests/DataProviderThread_unittest.cpp | 169 +
 9 files changed, 512 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dfefe629/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3f1c6cd..434019e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -702,7 +702,9 @@ set(ENABLE_MEMORYMIRROR OFF CACHE BOOL "Enable MemoryMirror 
TMB")
 set(ENABLE_NATIVELOG OFF CACHE BOOL "Enable NativeLog TMB")
 
 # The distributed version requires to use the NativeNet implementation.
-if (NOT ENABLE_DISTRIBUTED)
+if (ENABLE_DISTRIBUTED OR ENABLE_NETWORK_CLI)
+  set(ENABLE_NATIVENET ON CACHE BOOL "Enable NativeNet TMB")
+else()
   set(ENABLE_NATIVENET OFF CACHE BOOL "Enable NativeNet TMB")
 endif()
 
@@ -713,7 +715,7 @@ set(ENABLE_ZOOKEEPER OFF CACHE BOOL "Enable Zookeeper TMB")
 add_subdirectory("${THIRD_PARTY_SOURCE_DIR}/tmb" 
"${CMAKE_CURRENT_BINARY_DIR}/third_party/tmb")
 include_directories(${TMB_INCLUDE_DIRS})
 
-if (ENABLE_DISTRIBUTED)
+if (ENABLE_DISTRIBUTED OR ENABLE_NETWORK_CLI)
   include_directories(${CMAKE_CURRENT_BINARY_DIR}/third_party/tmb/include)
 endif()
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dfefe629/query_execution/QueryExecutionTypedefs.hpp
--
diff --git a/query_execution/QueryExecutionTypedefs.hpp 
b/query_execution/QueryExecutionTypedefs.hpp
index 80da7c5..ea4f8d2 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -81,6 +81,10 @@ enum QueryExecutionMessageType : message_type_id {
   kRebuildWorkOrderMessage,  // From Foreman to Worker.
   kRebuildWorkOrderCompleteMessage,  // From Worker to Foreman.
   kWorkloadCompletionMessage,  // From Foreman to main thread.
+  kBlockRequestMessage,  // To DataProviderThread.
+  kBlockResponseMessage,  // From DataProviderThread.
+  kFinishReadingRelationMessage,  // To DataProviderThread.
+
   kPoisonMessage,  // From the main thread to Foreman and Workers.
 
 #ifdef QUICKSTEP_DISTRIBUTED

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dfefe629/storage/BlockWire.proto
--
diff --git a/storage/BlockWire.proto b/storage/BlockWire.proto
new file mode 100644
index 000..c1a1763
--- /dev/null
+++ b/storage/BlockWire.proto
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax = "proto2";
+
+package quickstep;
+
+message BlockResponse {
+  required bool is_valid = 1;
+  required bytes block = 2;
+}
+
+message BlockRequest {
+  required uint64 block_id = 1;
+  required int32 relation_id = 2;
+}
+
+message 

incubator-quickstep git commit: Fixed the command execution bug in the distributed version.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 8b7c38010 -> 835281ae8


Fixed the command execution bug in the distributed version.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/835281ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/835281ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/835281ae

Branch: refs/heads/master
Commit: 835281ae894eadfdd121e444313bbf001e9b884b
Parents: 8b7c380
Author: Zuyu Zhang 
Authored: Mon May 21 16:39:42 2018 -0500
Committer: Zuyu Zhang 
Committed: Mon May 21 16:47:52 2018 -0500

--
 cli/tests/CMakeLists.txt|  8 +-
 cli/tests/CommandExecutorTestRunner.cpp |  2 +-
 cli/tests/DistributedCommandExecutorTest.cpp|  2 -
 .../DistributedCommandExecutorTestRunner.cpp| 87 +++-
 .../DistributedCommandExecutorTestRunner.hpp| 15 +---
 cli/tests/command_executor/CMakeLists.txt   |  6 ++
 cli/tests/command_executor/D.test   | 25 ++
 cli/tests/command_executor/Dt.test  |  8 ++
 8 files changed, 60 insertions(+), 93 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/CMakeLists.txt
--
diff --git a/cli/tests/CMakeLists.txt b/cli/tests/CMakeLists.txt
index c450e4d..18d321d 100644
--- a/cli/tests/CMakeLists.txt
+++ b/cli/tests/CMakeLists.txt
@@ -76,8 +76,9 @@ if (ENABLE_DISTRIBUTED)
 glog
 gtest
 quickstep_catalog_CatalogTypedefs
-quickstep_cli_CommandExecutorUtil
+quickstep_cli_CommandExecutor
 quickstep_cli_Constants
+quickstep_cli_DefaultsConfigurator
 quickstep_cli_DropRelation
 quickstep_cli_PrintToScreen
 quickstep_parser_ParseStatement
@@ -90,11 +91,10 @@ if (ENABLE_DISTRIBUTED)
 quickstep_queryexecution_Shiftboss
 quickstep_queryexecution_Worker
 quickstep_queryexecution_WorkerDirectory
-quickstep_queryoptimizer_Optimizer
-quickstep_queryoptimizer_OptimizerContext
 quickstep_queryoptimizer_QueryHandle
-quickstep_queryoptimizer_tests_TestDatabaseLoader
+quickstep_queryoptimizer_QueryProcessor
 quickstep_storage_DataExchangerAsync
+quickstep_storage_StorageConstants
 quickstep_storage_StorageManager
 quickstep_utility_Macros
 quickstep_utility_MemStream

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/CommandExecutorTestRunner.cpp
--
diff --git a/cli/tests/CommandExecutorTestRunner.cpp 
b/cli/tests/CommandExecutorTestRunner.cpp
index 4ab94d8..a8307db 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -69,7 +69,7 @@ void CommandExecutorTestRunner::runTestCase(
   std::printf("%s\n", parse_statement.toString().c_str());
   try {
 if (parse_statement.getStatementType() == ParseStatement::kCommand) {
-  quickstep::cli::executeCommand(
+  cli::executeCommand(
   *result.parsed_statement,
   *query_processor_->getDefaultDatabase(),
   main_thread_client_id_,

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/DistributedCommandExecutorTest.cpp
--
diff --git a/cli/tests/DistributedCommandExecutorTest.cpp 
b/cli/tests/DistributedCommandExecutorTest.cpp
index b41a70f..c8574b9 100644
--- a/cli/tests/DistributedCommandExecutorTest.cpp
+++ b/cli/tests/DistributedCommandExecutorTest.cpp
@@ -49,8 +49,6 @@ int main(int argc, char** argv) {
 
   auto test_runner = 
make_unique(argv[3]);
   test_driver = make_unique(_file, 
test_runner.get());
-  test_driver->registerOption(
-  quickstep::DistributedCommandExecutorTestRunner::kResetOption);
 
   ::testing::InitGoogleTest(, argv);
   const int success = RUN_ALL_TESTS();

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/835281ae/cli/tests/DistributedCommandExecutorTestRunner.cpp
--
diff --git a/cli/tests/DistributedCommandExecutorTestRunner.cpp 
b/cli/tests/DistributedCommandExecutorTestRunner.cpp
index 

incubator-quickstep git commit: Fix the problem that Quickstep fails on CPUs with no NUMA support (but libnuma exists)

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/trace bfe32cd70 -> e4997d921


Fix the problem that Quickstep fails on CPUs with no NUMA support (but libnuma 
exists)


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e4997d92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e4997d92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e4997d92

Branch: refs/heads/trace
Commit: e4997d921987496be7b6266ab19988e1319ae44a
Parents: bfe32cd
Author: Jianqiao Zhu 
Authored: Tue May 29 18:03:08 2018 -0500
Committer: Jianqiao Zhu 
Committed: Tue May 29 18:03:08 2018 -0500

--
 cli/InputParserUtil.cpp | 53 ++--
 1 file changed, 27 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e4997d92/cli/InputParserUtil.cpp
--
diff --git a/cli/InputParserUtil.cpp b/cli/InputParserUtil.cpp
index b092dfa..b8bd5a7 100644
--- a/cli/InputParserUtil.cpp
+++ b/cli/InputParserUtil.cpp
@@ -77,36 +77,37 @@ std::vector InputParserUtil::ParseWorkerAffinities(
 // This code is inspired from the print_node_cpus() function of numactl.
 // WARNING - If some NUMA sockets are disabled, we can't detect it.
 const int num_sockets = numa_num_configured_nodes();
-CHECK_GT(num_sockets, 0);
-// A vector V where V[i] denotes a vector of CPU cores that belong to the
-// socket i.
-std::vector> cpus_from_sockets;
-cpus_from_sockets.resize(num_sockets);
-for (int curr_socket = 0; curr_socket < num_sockets; ++curr_socket) {
-  std::unique_ptr cpus(numa_allocate_cpumask());
-  const int err = numa_node_to_cpus(curr_socket, cpus.get());
-  if (err >= 0) {
-for (int i = 0; i < static_cast(cpus->size); i++) {
-  if (numa_bitmask_isbitset(cpus.get(), i)) {
-// The current CPU belongs to curr_socket.
-cpus_from_sockets[curr_socket].push_back(i);
+if (num_sockets > 0) {
+  // A vector V where V[i] denotes a vector of CPU cores that belong to the
+  // socket i.
+  std::vector> cpus_from_sockets;
+  cpus_from_sockets.resize(num_sockets);
+  for (int curr_socket = 0; curr_socket < num_sockets; ++curr_socket) {
+std::unique_ptr cpus(numa_allocate_cpumask());
+const int err = numa_node_to_cpus(curr_socket, cpus.get());
+if (err >= 0) {
+  for (int i = 0; i < static_cast(cpus->size); i++) {
+if (numa_bitmask_isbitset(cpus.get(), i)) {
+  // The current CPU belongs to curr_socket.
+  cpus_from_sockets[curr_socket].push_back(i);
+}
   }
 }
   }
-}
-// Now assign affinity to each worker, picking one CPU from each socket in 
a
-// round robin manner.
-int curr_socket = 0;
-std::size_t iteration = 0;
-for (int curr_worker = 0; curr_worker < num_workers; ++curr_worker) {
-  if (iteration < cpus_from_sockets[curr_socket].size()) {
-const int curr_worker_affinity =
-cpus_from_sockets[curr_socket][iteration];
-affinities.push_back(curr_worker_affinity);
+  // Now assign affinity to each worker, picking one CPU from each socket 
in a
+  // round robin manner.
+  int curr_socket = 0;
+  std::size_t iteration = 0;
+  for (int curr_worker = 0; curr_worker < num_workers; ++curr_worker) {
+if (iteration < cpus_from_sockets[curr_socket].size()) {
+  const int curr_worker_affinity =
+  cpus_from_sockets[curr_socket][iteration];
+  affinities.push_back(curr_worker_affinity);
+}
+// Increase iteration number only when we are at the last socket.
+iteration = iteration + ((curr_socket + 1) / num_sockets);
+curr_socket = (curr_socket + 1) % num_sockets;
   }
-  // Increase iteration number only when we are at the last socket.
-  iteration = iteration + ((curr_socket + 1) / num_sockets);
-  curr_socket = (curr_socket + 1) % num_sockets;
 }
 #endif
   }

[1/2] incubator-quickstep git commit: Fixes to bulk load

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/trace f477c5d31 -> bfe32cd70


Fixes to bulk load


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d4355009
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d4355009
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d4355009

Branch: refs/heads/trace
Commit: d43550098c550fab3d9fba25ef8337bf7baa026c
Parents: f477c5d
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Sat May 26 21:23:53 2018 -0500
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Sat May 26 21:27:02 2018 -0500

--
 relational_operators/TextScanOperator.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d4355009/relational_operators/TextScanOperator.hpp
--
diff --git a/relational_operators/TextScanOperator.hpp 
b/relational_operators/TextScanOperator.hpp
index dfedb2b..3b5b2d2 100644
--- a/relational_operators/TextScanOperator.hpp
+++ b/relational_operators/TextScanOperator.hpp
@@ -138,7 +138,7 @@ class TextScanOperator : public RelationalOperator {
 options_(options),
 output_relation_(output_relation),
 output_destination_index_(output_destination_index),
-serial_bulk_insert_(mem_data != nullptr),
+serial_bulk_insert_(mem_data != nullptr && file_pattern == "$stdin"),
 num_remaining_chunks_(0),
 serial_worker_ready_(true),
 work_generated_(false) {}

[2/2] incubator-quickstep git commit: Client exit with -1 on RPC failure

[jianqiao]

Client exit with -1 on RPC failure


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/bfe32cd7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/bfe32cd7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/bfe32cd7

Branch: refs/heads/trace
Commit: bfe32cd70f8593f319295613fec1465a999c49aa
Parents: d435500
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Sun May 27 20:14:43 2018 -0500
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Sun May 27 20:14:43 2018 -0500

--
 cli/NetworkCliClient.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bfe32cd7/cli/NetworkCliClient.hpp
--
diff --git a/cli/NetworkCliClient.hpp b/cli/NetworkCliClient.hpp
index 5c11cb2..f3693f4 100644
--- a/cli/NetworkCliClient.hpp
+++ b/cli/NetworkCliClient.hpp
@@ -66,7 +66,7 @@ class NetworkCliClient {
 } else {
   std::cout << "RPC call failed with code " << status.error_code()
 << " and message: " << status.error_message() << "\n";
-  return "RPC failed";
+  std::exit(-1);
 }
   }
 

[2/3] incubator-quickstep git commit: Updates for grail

[jianqiao]

Updates for grail


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a0024a2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a0024a2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a0024a2e

Branch: refs/heads/trace
Commit: a0024a2e9ec76d14013fcdee0e8d714d8776c1a3
Parents: 56e0a4e
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Tue May 8 17:53:57 2018 -0500
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Tue May 8 17:53:57 2018 -0500

--
 .../aggregation/AggregationHandleMax.hpp|  2 +
 .../aggregation/AggregationHandleMin.hpp|  2 +
 query_execution/WorkOrdersContainer.hpp |  8 ++-
 query_optimizer/ExecutionGenerator.cpp  |  6 +--
 .../cost_model/StarSchemaSimpleCostModel.cpp|  6 +--
 relational_operators/UnionAllOperator.cpp   |  6 ++-
 storage/AggregationOperationState.cpp   |  2 +-
 storage/PackedPayloadHashTable.cpp  |  8 +--
 storage/StorageManager.cpp  | 42 ++--
 utility/ShardedLockManager.hpp  | 52 +---
 10 files changed, 29 insertions(+), 105 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0024a2e/expressions/aggregation/AggregationHandleMax.hpp
--
diff --git a/expressions/aggregation/AggregationHandleMax.hpp 
b/expressions/aggregation/AggregationHandleMax.hpp
index 8f8c0d8..d6fdd12 100644
--- a/expressions/aggregation/AggregationHandleMax.hpp
+++ b/expressions/aggregation/AggregationHandleMax.hpp
@@ -191,6 +191,7 @@ class AggregationHandleMax : public 
AggregationConcreteHandle {
 if (state->max_.isNull() ||
 fast_comparator_->compareTypedValues(value, state->max_)) {
   state->max_ = value;
+  state->max_.ensureNotReference();
 }
   }
 
@@ -200,6 +201,7 @@ class AggregationHandleMax : public 
AggregationConcreteHandle {
 if (max_ptr->isNull() ||
 fast_comparator_->compareTypedValues(value, *max_ptr)) {
   *max_ptr = value;
+  max_ptr->ensureNotReference();
 }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0024a2e/expressions/aggregation/AggregationHandleMin.hpp
--
diff --git a/expressions/aggregation/AggregationHandleMin.hpp 
b/expressions/aggregation/AggregationHandleMin.hpp
index 0e62be5..441af52 100644
--- a/expressions/aggregation/AggregationHandleMin.hpp
+++ b/expressions/aggregation/AggregationHandleMin.hpp
@@ -191,6 +191,7 @@ class AggregationHandleMin : public 
AggregationConcreteHandle {
 if (state->min_.isNull() ||
 fast_comparator_->compareTypedValues(value, state->min_)) {
   state->min_ = value;
+  state->min_.ensureNotReference();
 }
   }
 
@@ -200,6 +201,7 @@ class AggregationHandleMin : public 
AggregationConcreteHandle {
 if (min_ptr->isNull() ||
 fast_comparator_->compareTypedValues(value, *min_ptr)) {
   *min_ptr = value;
+  min_ptr->ensureNotReference();
 }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0024a2e/query_execution/WorkOrdersContainer.hpp
--
diff --git a/query_execution/WorkOrdersContainer.hpp 
b/query_execution/WorkOrdersContainer.hpp
index 431a270..55ec185 100644
--- a/query_execution/WorkOrdersContainer.hpp
+++ b/query_execution/WorkOrdersContainer.hpp
@@ -362,7 +362,13 @@ class WorkOrdersContainer {
 return nullptr;
   }
 
-  std::cerr << "# work orders: " << workorders_.size() << "\r";
+  /*
+  const std::size_t num_remaining_workorders = workorders_.size();
+  if (num_remaining_workorders % 1000 == 0) {
+std::cerr << "# work orders: " << num_remaining_workorders
+  << "\r";
+  }
+  */
 
   WorkOrder *work_order = workorders_.front().release();
   workorders_.pop();

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a0024a2e/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index 8ca367b..75f15ac 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -190,7 +190,7 @@ static const volatile bool num_aggregation_partitions_dummy
 = gflags::RegisterFlagValidator(_num_aggregation_partitions, 
);
 
 DEFINE_uint64(partition_aggregation_num_groups

[3/3] incubator-quickstep git commit: Updates for aggregation result type

[jianqiao]

Updates for aggregation result type


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f477c5d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f477c5d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f477c5d3

Branch: refs/heads/trace
Commit: f477c5d319e8dc6e501c9ce1a366d3552ec907e0
Parents: a0024a2
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Wed May 16 14:42:54 2018 -0500
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Wed May 16 14:42:54 2018 -0500

--
 expressions/aggregation/AggregateFunction.hpp   |  3 ++-
 .../aggregation/AggregateFunctionAvg.cpp| 19 ---
 .../aggregation/AggregateFunctionAvg.hpp|  3 ++-
 .../aggregation/AggregateFunctionCount.cpp  |  3 ++-
 .../aggregation/AggregateFunctionCount.hpp  |  3 ++-
 .../aggregation/AggregateFunctionMax.cpp| 10 ++
 .../aggregation/AggregateFunctionMax.hpp|  3 ++-
 .../aggregation/AggregateFunctionMin.cpp| 10 ++
 .../aggregation/AggregateFunctionMin.hpp|  3 ++-
 .../aggregation/AggregateFunctionSum.cpp| 20 
 .../aggregation/AggregateFunctionSum.hpp|  3 ++-
 .../tests/AggregationHandleAvg_unittest.cpp |  4 ++--
 .../tests/AggregationHandleCount_unittest.cpp   |  4 ++--
 .../tests/AggregationHandleMax_unittest.cpp |  4 ++--
 .../tests/AggregationHandleMin_unittest.cpp |  4 ++--
 .../tests/AggregationHandleSum_unittest.cpp |  4 ++--
 .../expressions/AggregateFunction.cpp   |  6 --
 types/Type.cpp  |  4 
 types/Type.hpp  |  2 ++
 19 files changed, 70 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f477c5d3/expressions/aggregation/AggregateFunction.hpp
--
diff --git a/expressions/aggregation/AggregateFunction.hpp 
b/expressions/aggregation/AggregateFunction.hpp
index 699..5f290d1 100644
--- a/expressions/aggregation/AggregateFunction.hpp
+++ b/expressions/aggregation/AggregateFunction.hpp
@@ -112,7 +112,8 @@ class AggregateFunction {
* applicable to the specified Type(s).
**/
   virtual const Type* resultTypeForArgumentTypes(
-  const std::vector _types) const = 0;
+  const std::vector _types,
+  const bool is_vector_aggregate) const = 0;
 
   /**
* @brief Create an AggregationHandle to compute aggregates.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f477c5d3/expressions/aggregation/AggregateFunctionAvg.cpp
--
diff --git a/expressions/aggregation/AggregateFunctionAvg.cpp 
b/expressions/aggregation/AggregateFunctionAvg.cpp
index 040d7d9..e966d4a 100644
--- a/expressions/aggregation/AggregateFunctionAvg.cpp
+++ b/expressions/aggregation/AggregateFunctionAvg.cpp
@@ -48,22 +48,27 @@ bool AggregateFunctionAvg::canApplyToTypes(
 }
 
 const Type* AggregateFunctionAvg::resultTypeForArgumentTypes(
-const std::vector _types) const {
+const std::vector _types,
+const bool is_vector_aggregate) const {
   if (!canApplyToTypes(argument_types)) {
 return nullptr;
   }
 
-  // The type used to sum values is nullable, and we automatically widen int to
-  // long and float to double to have more headroom when adding up many values.
-  const Type *sum_type = &(argument_types.front()->getNullableVersion());
-  switch (sum_type->getTypeID()) {
+  // We automatically widen int to long and float to double to have more
+  // headroom when adding up many values.
+  const Type *argument_type = argument_types.front();
+  const bool nullable = argument_type->isNullable() || !is_vector_aggregate;
+
+  const Type *sum_type;
+  switch (argument_type->getTypeID()) {
 case kInt:
-  sum_type = ::GetType(kLong, true);
+  sum_type = ::GetType(kLong, nullable);
   break;
 case kFloat:
-  sum_type = ::GetType(kDouble, true);
+  sum_type = ::GetType(kDouble, nullable);
   break;
 default:
+  sum_type = ::GetType(argument_type->getTypeID(), nullable);
   break;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f477c5d3/expressions/aggregation/AggregateFunctionAvg.hpp
--
diff --git a/expressions/aggregation/AggregateFunctionAvg.hpp 
b/expressions/aggregation/AggregateFunctionAvg.hpp
index 9d08d1c..d040607 100644
--- a/expressions/aggregation/AggregateFunctionAvg.hpp
+++ b/expressions/aggregation/AggregateFunctionAvg.hpp
@@ -54,7 +54,8 @@ class AggregateFunctionAvg : public A

[1/3] incubator-quickstep git commit: Some updates for large data

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/trace ba16c5aef -> f477c5d31


Some updates for large data


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/56e0a4ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/56e0a4ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/56e0a4ed

Branch: refs/heads/trace
Commit: 56e0a4ed70ce0fdefae18fb96bc04e830aface93
Parents: ba16c5a
Author: jianqiao 
<jianq...@node-2.jianqiao-qv36188.quickstep-pg0.wisc.cloudlab.us>
Authored: Fri Apr 27 13:00:00 2018 -0500
Committer: jianqiao 
<jianq...@node-2.jianqiao-qv36188.quickstep-pg0.wisc.cloudlab.us>
Committed: Fri Apr 27 13:00:00 2018 -0500

--
 query_execution/WorkOrdersContainer.hpp | 3 +++
 query_optimizer/ExecutionGenerator.cpp  | 2 ++
 2 files changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56e0a4ed/query_execution/WorkOrdersContainer.hpp
--
diff --git a/query_execution/WorkOrdersContainer.hpp 
b/query_execution/WorkOrdersContainer.hpp
index 3c2d9bf..431a270 100644
--- a/query_execution/WorkOrdersContainer.hpp
+++ b/query_execution/WorkOrdersContainer.hpp
@@ -21,6 +21,7 @@
 #define QUICKSTEP_QUERY_EXECUTION_WORKORDERS_CONTAINER_HPP_
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -361,6 +362,8 @@ class WorkOrdersContainer {
 return nullptr;
   }
 
+  std::cerr << "# work orders: " << workorders_.size() << "\r";
+
   WorkOrder *work_order = workorders_.front().release();
   workorders_.pop();
   return work_order;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56e0a4ed/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index dede95d..8ca367b 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1484,6 +1484,7 @@ void ExecutionGenerator::convertInsertSelection(
 
   // FIXME(qzeng): A better way is using a traits struct to look up whether a 
storage
   //   block supports ad-hoc insertion instead of hard-coding the 
block types.
+/*
   const StorageBlockLayout _block_layout =
   destination_relation.getDefaultStorageBlockLayout();
   if 
(storage_block_layout.getDescription().tuple_store_description().sub_block_type()
 ==
@@ -1494,6 +1495,7 @@ void ExecutionGenerator::convertInsertSelection(
   << destination_relation.getName()
   << ", because its storage blocks do not support ad-hoc 
insertion";
   }
+*/
 
   // Create InsertDestination proto.
   const QueryContext::insert_destination_id insert_destination_index =

[2/2] incubator-quickstep git commit: QUICKSTEP-121: Added the self-join support.

[jianqiao]

QUICKSTEP-121: Added the self-join support.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8b7c3801
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8b7c3801
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8b7c3801

Branch: refs/heads/master
Commit: 8b7c38010e1505ef03e0b6de8c6ac16cceb04695
Parents: 42588d4
Author: Zuyu Zhang 
Authored: Tue May 1 00:48:34 2018 -0500
Committer: Zuyu Zhang 
Committed: Wed May 9 16:34:47 2018 -0500

--
 expressions/ExpressionFactories.cpp | 17 +++-
 expressions/Expressions.proto   |  7 ++
 expressions/predicate/ComparisonPredicate.cpp   | 26 +++---
 expressions/predicate/ComparisonPredicate.hpp   |  2 -
 expressions/predicate/ConjunctionPredicate.cpp  |  4 -
 expressions/predicate/ConjunctionPredicate.hpp  |  2 -
 expressions/predicate/DisjunctionPredicate.cpp  |  4 -
 expressions/predicate/DisjunctionPredicate.hpp  |  2 -
 expressions/predicate/NegationPredicate.cpp |  4 -
 expressions/predicate/NegationPredicate.hpp |  2 -
 expressions/predicate/Predicate.hpp |  6 --
 expressions/predicate/TrivialPredicates.hpp |  4 -
 expressions/scalar/Scalar.hpp   | 50 +--
 expressions/scalar/ScalarAttribute.cpp  | 58 +
 expressions/scalar/ScalarAttribute.hpp  | 10 +--
 expressions/scalar/ScalarBinaryExpression.cpp   | 80 +
 expressions/scalar/ScalarBinaryExpression.hpp   |  4 -
 expressions/scalar/ScalarCaseExpression.cpp | 62 +++---
 expressions/scalar/ScalarCaseExpression.hpp |  4 -
 expressions/scalar/ScalarLiteral.cpp|  2 -
 expressions/scalar/ScalarLiteral.hpp|  4 -
 expressions/scalar/ScalarSharedExpression.cpp   | 14 +--
 expressions/scalar/ScalarSharedExpression.hpp   |  4 -
 expressions/scalar/ScalarUnaryExpression.cpp| 18 +---
 expressions/scalar/ScalarUnaryExpression.hpp|  4 -
 .../tests/ScalarCaseExpression_unittest.cpp | 58 ++---
 query_optimizer/ExecutionGenerator.cpp  | 88 ---
 query_optimizer/ExecutionGenerator.hpp  | 21 +++--
 query_optimizer/expressions/Alias.cpp   |  5 +-
 query_optimizer/expressions/Alias.hpp   |  5 +-
 .../expressions/AttributeReference.cpp  | 19 -
 .../expressions/AttributeReference.hpp  |  5 +-
 .../expressions/BinaryExpression.cpp|  9 +-
 .../expressions/BinaryExpression.hpp|  5 +-
 query_optimizer/expressions/CMakeLists.txt  |  1 +
 query_optimizer/expressions/Cast.cpp| 10 ++-
 query_optimizer/expressions/Cast.hpp|  5 +-
 .../expressions/CommonSubexpression.cpp |  7 +-
 .../expressions/CommonSubexpression.hpp |  5 +-
 .../expressions/ComparisonExpression.cpp|  9 +-
 .../expressions/ComparisonExpression.hpp|  5 +-
 query_optimizer/expressions/Exists.cpp  |  5 +-
 query_optimizer/expressions/Exists.hpp  |  5 +-
 query_optimizer/expressions/InTableQuery.cpp|  5 +-
 query_optimizer/expressions/InTableQuery.hpp|  5 +-
 query_optimizer/expressions/InValueList.cpp |  7 +-
 query_optimizer/expressions/InValueList.hpp |  5 +-
 query_optimizer/expressions/LogicalAnd.cpp  |  9 +-
 query_optimizer/expressions/LogicalAnd.hpp  |  5 +-
 query_optimizer/expressions/LogicalNot.cpp  |  7 +-
 query_optimizer/expressions/LogicalNot.hpp  |  5 +-
 query_optimizer/expressions/LogicalOr.cpp   |  9 +-
 query_optimizer/expressions/LogicalOr.hpp   |  5 +-
 query_optimizer/expressions/Predicate.hpp   |  7 +-
 .../expressions/PredicateLiteral.cpp|  5 +-
 .../expressions/PredicateLiteral.hpp|  5 +-
 query_optimizer/expressions/Scalar.hpp  |  8 +-
 query_optimizer/expressions/ScalarLiteral.cpp   |  5 +-
 query_optimizer/expressions/ScalarLiteral.hpp   |  5 +-
 query_optimizer/expressions/SearchedCase.cpp| 17 ++--
 query_optimizer/expressions/SearchedCase.hpp|  5 +-
 query_optimizer/expressions/SimpleCase.cpp  |  7 +-
 query_optimizer/expressions/SimpleCase.hpp  |  5 +-
 .../expressions/SubqueryExpression.cpp  |  5 +-
 .../expressions/SubqueryExpression.hpp  |  5 +-
 query_optimizer/expressions/UnaryExpression.cpp |  7 +-
 query_optimizer/expressions/UnaryExpression.hpp |  5 +-
 .../tests/execution_generator/Select.test   | 56 +++-
 relational_operators/HashJoinOperator.cpp   | 90 +++-
 .../NestedLoopsJoinOperator.cpp |  9 +-
 .../tests/HashJoinOperator_unittest.cpp | 34 
 71 files changed, 536 insertions(+), 467 deletions(-)
--

[1/2] incubator-quickstep git commit: QUICKSTEP-121: Added the self-join support.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 42588d433 -> 8b7c38010


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8b7c3801/query_optimizer/expressions/BinaryExpression.cpp
--
diff --git a/query_optimizer/expressions/BinaryExpression.cpp 
b/query_optimizer/expressions/BinaryExpression.cpp
index 07eb5ff..af4929c 100644
--- a/query_optimizer/expressions/BinaryExpression.cpp
+++ b/query_optimizer/expressions/BinaryExpression.cpp
@@ -23,6 +23,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 
@@ -101,11 +102,13 @@ std::vector 
BinaryExpression::getReferencedAttributes() c
 }
 
 ::quickstep::Scalar *BinaryExpression::concretize(
-const std::unordered_map 
_map) const {
+const std::unordered_map 
_map,
+const std::unordered_set _expr_ids,
+const std::unordered_set _expr_ids) const {
   return new ::quickstep::ScalarBinaryExpression(
   operation_,
-  left_->concretize(substitution_map),
-  right_->concretize(substitution_map));
+  left_->concretize(substitution_map, left_expr_ids, right_expr_ids),
+  right_->concretize(substitution_map, left_expr_ids, right_expr_ids));
 }
 
 std::size_t BinaryExpression::computeHash() const {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8b7c3801/query_optimizer/expressions/BinaryExpression.hpp
--
diff --git a/query_optimizer/expressions/BinaryExpression.hpp 
b/query_optimizer/expressions/BinaryExpression.hpp
index df7454c..92419f6 100644
--- a/query_optimizer/expressions/BinaryExpression.hpp
+++ b/query_optimizer/expressions/BinaryExpression.hpp
@@ -23,6 +23,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "query_optimizer/OptimizerTree.hpp"
@@ -88,7 +89,9 @@ class BinaryExpression : public Scalar {
   std::vector getReferencedAttributes() const override;
 
   ::quickstep::Scalar* concretize(
-  const std::unordered_map 
_map) const override;
+  const std::unordered_map 
_map,
+  const std::unordered_set _expr_ids = 
std::unordered_set(),
+  const std::unordered_set _expr_ids = 
std::unordered_set()) const override;
 
   bool equals(const ScalarPtr ) const override;
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8b7c3801/query_optimizer/expressions/CMakeLists.txt
--
diff --git a/query_optimizer/expressions/CMakeLists.txt 
b/query_optimizer/expressions/CMakeLists.txt
index 7032f05..03a8247 100644
--- a/query_optimizer/expressions/CMakeLists.txt
+++ b/query_optimizer/expressions/CMakeLists.txt
@@ -79,6 +79,7 @@ 
target_link_libraries(quickstep_queryoptimizer_expressions_Alias
 target_link_libraries(quickstep_queryoptimizer_expressions_AttributeReference
   glog
   quickstep_catalog_CatalogTypedefs
+  quickstep_expressions_scalar_Scalar
   quickstep_expressions_scalar_ScalarAttribute
   quickstep_queryoptimizer_expressions_ExprId
   quickstep_queryoptimizer_expressions_Expression

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8b7c3801/query_optimizer/expressions/Cast.cpp
--
diff --git a/query_optimizer/expressions/Cast.cpp 
b/query_optimizer/expressions/Cast.cpp
index e6eb1bd..c5d8f94 100644
--- a/query_optimizer/expressions/Cast.cpp
+++ b/query_optimizer/expressions/Cast.cpp
@@ -22,6 +22,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "expressions/scalar/Scalar.hpp"
@@ -52,9 +53,12 @@ ExpressionPtr Cast::copyWithNewChildren(
 }
 
 ::quickstep::Scalar *Cast::concretize(
-const std::unordered_map 
_map) const {
-  return new 
::quickstep::ScalarUnaryExpression(::quickstep::NumericCastOperation::Instance(target_type_),
-
operand_->concretize(substitution_map));
+const std::unordered_map 
_map,
+const std::unordered_set _expr_ids,
+const std::unordered_set _expr_ids) const {
+  return new ::quickstep::ScalarUnaryExpression(
+  ::quickstep::NumericCastOperation::Instance(target_type_),
+  operand_->concretize(substitution_map, left_expr_ids, right_expr_ids));
 }
 
 std::size_t Cast::computeHash() const {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8b7c3801/query_optimizer/expressions/Cast.hpp
--
diff --git a/query_optimizer/expressions/Cast.hpp 
b/query_optimizer/expressions/Cast.hpp
index fa40242..2e7ea96 100644
--- 

[3/3] incubator-quickstep git commit: Minor bug fixes and refactors.

[jianqiao]

Minor bug fixes and refactors.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/42588d43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/42588d43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/42588d43

Branch: refs/heads/master
Commit: 42588d43326b4889fa5696c4103315f490e9808a
Parents: 06982e9
Author: Zuyu Zhang 
Authored: Mon May 7 16:08:35 2018 -0500
Committer: Zuyu Zhang 
Committed: Wed May 9 14:55:00 2018 -0500

--
 parser/SqlLexer.lpp |4 +-
 parser/SqlParser.ypp|8 +-
 parser/preprocessed/SqlLexer_gen.cpp|   34 +-
 parser/preprocessed/SqlParser_gen.cpp   | 1802 +-
 parser/preprocessed/SqlParser_gen.hpp   |  176 +-
 .../cost_model/StarSchemaSimpleCostModel.cpp|3 +-
 query_optimizer/resolver/Resolver.cpp   |   37 +-
 query_optimizer/rules/InjectJoinFilters.cpp |   47 +-
 query_optimizer/rules/ReorderColumns.cpp|4 +-
 .../BarrieredReadWriteConcurrentBitVector.hpp   |2 +-
 utility/ExecutionDAGVisualizer.cpp  |   20 +-
 11 files changed, 1092 insertions(+), 1045 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/42588d43/parser/SqlLexer.lpp
--
diff --git a/parser/SqlLexer.lpp b/parser/SqlLexer.lpp
index d818d0b..6f75424 100644
--- a/parser/SqlLexer.lpp
+++ b/parser/SqlLexer.lpp
@@ -183,9 +183,8 @@ unsigned_numeric_literal 
{exact_numeric_literal}|{approximate_numeric_literal}
   "blockproperties"  return TOKEN_BLOCKPROPERTIES;
   "blocksample"  return TOKEN_BLOCKSAMPLE;
   "bloomfilter"  return TOKEN_BLOOM_FILTER;
-  "case" return TOKEN_CASE;
-  "csbtree"  return TOKEN_CSB_TREE;
   "by"   return TOKEN_BY;
+  "case" return TOKEN_CASE;
   "char" return TOKEN_CHARACTER;
   "character"return TOKEN_CHARACTER;
   "check"return TOKEN_CHECK;
@@ -193,6 +192,7 @@ unsigned_numeric_literal 
{exact_numeric_literal}|{approximate_numeric_literal}
   "constraint"   return TOKEN_CONSTRAINT;
   "copy" return TOKEN_COPY;
   "create"   return TOKEN_CREATE;
+  "csbtree"  return TOKEN_CSB_TREE;
   "current"  return TOKEN_CURRENT;
   "date" return TOKEN_DATE;
   "datetime" return TOKEN_DATETIME;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/42588d43/parser/SqlParser.ypp
--
diff --git a/parser/SqlParser.ypp b/parser/SqlParser.ypp
index ba69b3d..12a6b29 100644
--- a/parser/SqlParser.ypp
+++ b/parser/SqlParser.ypp
@@ -259,7 +259,6 @@ void NotSupported(const YYLTYPE *location, yyscan_t 
yyscanner, const std::string
 %token TOKEN_BLOCKPROPERTIES;
 %token TOKEN_BLOCKSAMPLE;
 %token TOKEN_BLOOM_FILTER;
-%token TOKEN_CSB_TREE;
 %token TOKEN_BY;
 %token TOKEN_CASE;
 %token TOKEN_CHARACTER;
@@ -268,6 +267,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t 
yyscanner, const std::string
 %token TOKEN_CONSTRAINT;
 %token TOKEN_COPY;
 %token TOKEN_CREATE;
+%token TOKEN_CSB_TREE;
 %token TOKEN_CURRENT;
 %token TOKEN_DATE;
 %token TOKEN_DATETIME;
@@ -281,6 +281,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t 
yyscanner, const std::string
 %token TOKEN_DROP;
 %token TOKEN_ELSE;
 %token TOKEN_END;
+%token TOKEN_EOF;
 %token TOKEN_EXISTS;
 %token TOKEN_EXTRACT;
 %token TOKEN_FALSE;
@@ -307,6 +308,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t 
yyscanner, const std::string
 %token TOKEN_KEY;
 %token TOKEN_LAST;
 %token TOKEN_LEFT;
+%token TOKEN_LEX_ERROR;
 %token TOKEN_LIMIT;
 %token TOKEN_LONG;
 %token TOKEN_MINUTE;
@@ -333,8 +335,8 @@ void NotSupported(const YYLTYPE *location, yyscan_t 
yyscanner, const std::string
 %token TOKEN_REGEXP;
 %token TOKEN_RIGHT;
 %token TOKEN_ROW;
-%token TOKEN_ROW_DELIMITER;
 %token TOKEN_ROWS;
+%token TOKEN_ROW_DELIMITER;
 %token TOKEN_SECOND;
 %token TOKEN_SELECT;
 %token TOKEN_SET;
@@ -363,8 +365,6 @@ void NotSupported(const YYLTYPE *location, yyscan_t 
yyscanner, const std::string
 %token TOKEN_WITH;
 %token TOKEN_YEAR;
 %token TOKEN_YEARMONTH;
-%token TOKEN_EOF;
-%token TOKEN_LEX_ERROR;
 
 %type 
   any_name

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/42588d43/parser/preprocessed/SqlLexer_gen.cpp
--
diff --git a/parser/preprocessed/SqlLexer_gen.cpp 
b/parser/preprocessed/SqlLexer_gen.cpp
index 4800cde..ff483a4 100644
--- a/parser/preprocessed/SqlLexer_gen.cpp
+++ b/parser/preprocessed/SqlLexer_gen.cpp
@@ -611,7 

incubator-quickstep git commit: QUICKSTEP-125: Fixed the non-determinism in JoinReordering due to pointer cmp.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 3a7b9c796 -> 185c17ca8


QUICKSTEP-125: Fixed the non-determinism in JoinReordering due to pointer cmp.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/185c17ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/185c17ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/185c17ca

Branch: refs/heads/master
Commit: 185c17ca8ac9a44413b678924b9e63ebdac664df
Parents: 3a7b9c7
Author: Zuyu Zhang 
Authored: Sat May 5 14:21:53 2018 -0500
Committer: Zuyu Zhang 
Committed: Sat May 5 14:29:42 2018 -0500

--
 .../StarSchemaHashJoinOrderOptimization.cpp | 30 +++--
 .../StarSchemaHashJoinOrderOptimization.hpp | 45 +---
 2 files changed, 48 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/185c17ca/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
--
diff --git a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp 
b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
index 1007dd5..78c0caf 100644
--- a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
+++ b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
@@ -30,6 +30,7 @@
 
 #include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
 #include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
 #include "query_optimizer/expressions/NamedExpression.hpp"
 #include "query_optimizer/expressions/PatternMatcher.hpp"
 #include "query_optimizer/physical/Aggregate.hpp"
@@ -68,7 +69,7 @@ P::PhysicalPtr 
StarSchemaHashJoinOrderOptimization::applyInternal(const P::Physi
 bool is_valid_cascading_hash_join = false;
 if (hash_join->residual_predicate() == nullptr) {
   is_valid_cascading_hash_join = true;
-  for (const E::NamedExpressionPtr expr : 
hash_join->project_expressions()) {
+  for (const E::NamedExpressionPtr  : 
hash_join->project_expressions()) {
 if (!E::SomeAttributeReference::Matches(expr)) {
   is_valid_cascading_hash_join = false;
   break;
@@ -95,8 +96,8 @@ P::PhysicalPtr 
StarSchemaHashJoinOrderOptimization::applyInternal(const P::Physi
 
 // Gather join attribute pairs.
 for (std::size_t i = 0; i < hash_join->left_join_attributes().size(); ++i) 
{
-  const std::size_t left_attr_id = 
hash_join->left_join_attributes()[i]->id();
-  const std::size_t right_attr_id = 
hash_join->right_join_attributes()[i]->id();
+  const E::ExprId left_attr_id = 
hash_join->left_join_attributes()[i]->id();
+  const E::ExprId right_attr_id = 
hash_join->right_join_attributes()[i]->id();
 
   join_group->join_attribute_pairs.emplace_back(left_attr_id, 
right_attr_id);
 }
@@ -150,7 +151,7 @@ physical::PhysicalPtr 
StarSchemaHashJoinOrderOptimization::generatePlan(
 
   std::vector table_info_storage;
   const std::vector  = join_group.tables;
-  for (std::size_t i = 0; i < join_group.tables.size(); ++i) {
+  for (std::size_t i = 0; i < tables.size(); ++i) {
 table_info_storage.emplace_back(
 i,
 tables[i],
@@ -279,16 +280,8 @@ physical::PhysicalPtr 
StarSchemaHashJoinOrderOptimization::generatePlan(
 remaining_tables.erase(selected_probe_table_info);
 remaining_tables.erase(selected_build_table_info);
 
-// Figure out the output attributes.
 const P::PhysicalPtr _child = selected_probe_table_info->table;
 const P::PhysicalPtr _child = selected_build_table_info->table;
-std::vector output_attributes;
-for (const E::AttributeReferencePtr _attr : 
probe_child->getOutputAttributes()) {
-  output_attributes.emplace_back(probe_attr);
-}
-for (const E::AttributeReferencePtr _attr : 
build_child->getOutputAttributes()) {
-  output_attributes.emplace_back(build_attr);
-}
 
 // Figure out the join attributes.
 std::vector probe_attributes;
@@ -311,6 +304,19 @@ physical::PhysicalPtr 
StarSchemaHashJoinOrderOptimization::generatePlan(
 // the table pool. Return the last table in the table pool if there is only
 // one table left.
 if (remaining_tables.size() > 0) {
+  const auto probe_output_attributes = probe_child->getOutputAttributes();
+  const auto build_output_attributes = build_child->getOutputAttributes();
+
+  // Figure out the output attributes.
+  std::vector output_attributes;
+  output_attributes.reserve(probe_output_attributes.size() + 
build_output_attributes.size());
+  for (const E::AttributeReferencePtr _attr : 
probe_output_attributes) {
+

incubator-quickstep git commit: Fixed the bug regarding EliminateEmptyNode on an aggregate.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 73d796dee -> 876f12ba5


Fixed the bug regarding EliminateEmptyNode on an aggregate.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/876f12ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/876f12ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/876f12ba

Branch: refs/heads/master
Commit: 876f12ba587eed7b8cf7f2b3f7d45affc885e1ac
Parents: 73d796d
Author: Zuyu Zhang 
Authored: Thu May 3 17:34:51 2018 -0500
Committer: Zuyu Zhang 
Committed: Thu May 3 18:08:43 2018 -0500

--
 cli/tests/command_executor/Analyze.test  | 27 
 query_optimizer/ExecutionGenerator.cpp   |  6 ++
 query_optimizer/rules/CMakeLists.txt |  1 +
 query_optimizer/rules/EliminateEmptyNode.cpp | 78 +++
 4 files changed, 99 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/876f12ba/cli/tests/command_executor/Analyze.test
--
diff --git a/cli/tests/command_executor/Analyze.test 
b/cli/tests/command_executor/Analyze.test
index 57b8312..b0bf090 100644
--- a/cli/tests/command_executor/Analyze.test
+++ b/cli/tests/command_executor/Analyze.test
@@ -27,6 +27,15 @@ INSERT INTO t VALUES(0, 0);
 --
 ==
 
+SELECT COUNT(*) FROM r;
+--
+++
+|COUNT(*)|
+++
+|   0|
+++
+==
+
 \analyze
 --
 Analyzing r ... done
@@ -34,6 +43,24 @@ Analyzing s ... done
 Analyzing t ... done
 ==
 
+SELECT COUNT(*) FROM r;
+--
+++
+|COUNT(*)|
+++
+|   0|
+++
+==
+
+SELECT MIN(src) FROM r;
+--
++---+
+|MIN(src)   |
++---+
+|   NULL|
++---+
+==
+
 SELECT r.src, r.dst FROM r;
 --
 +---+---+

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/876f12ba/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index f8f7c9d..cc1319c 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1956,6 +1956,9 @@ void ExecutionGenerator::convertAggregate(
 execution_plan_->addDirectDependency(aggregation_operator_index,
  
input_relation_info->producer_operator_index,
  false /* is_pipeline_breaker */);
+  } else if (input_relation.isTemporary()) {
+// NOTE(zuyu): drop the temp relation created by EliminateEmptyNode rule.
+temporary_relation_info_vec_.emplace_back(aggregation_operator_index, 
_relation);
   }
 
   if (use_parallel_initialization) {
@@ -2451,6 +2454,9 @@ void ExecutionGenerator::convertWindowAggregate(
 execution_plan_->addDirectDependency(window_aggregation_operator_index,
  
input_relation_info->producer_operator_index,
  true /* is_pipeline_breaker */);
+  } else if (input_relation.isTemporary()) {
+// NOTE(zuyu): drop the temp relation created by EliminateEmptyNode rule.
+
temporary_relation_info_vec_.emplace_back(window_aggregation_operator_index, 
_relation);
   }
 
   
insert_destination_proto->set_relational_op_index(window_aggregation_operator_index);

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/876f12ba/query_optimizer/rules/CMakeLists.txt
--
diff --git a/query_optimizer/rules/CMakeLists.txt 
b/query_optimizer/rules/CMakeLists.txt
index 014a99b..8465c89 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -111,6 +111,7 @@ 
target_link_libraries(quickstep_queryoptimizer_rules_EliminateEmptyNode
   quickstep_queryoptimizer_OptimizerContext
   quickstep_queryoptimizer_expressions_Alias
   quickstep_queryoptimizer_expressions_AttributeReference
+  quickstep_queryoptimizer_expressions_ExprId
   quickstep_queryoptimizer_expressions_ExpressionUtil
   quickstep_queryoptimizer_expressions_NamedExpression
   quickstep_queryoptimizer_expressions_PatternMatcher

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/876f12ba/query_optimizer/rules/EliminateEmptyNode.cpp
--
diff --git 

incubator-quickstep git commit: Added the an optimization rule

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 612e103cb -> cc9ab901f


Added the an optimization rule

that eliminates a Physical node with an empty TableReference into
a Selection w/ an temp TableReference.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/cc9ab901
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/cc9ab901
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/cc9ab901

Branch: refs/heads/master
Commit: cc9ab901f92880f4dd33c46797381e081241c281
Parents: 612e103
Author: Zuyu Zhang 
Authored: Wed Apr 18 17:15:34 2018 -0500
Committer: Zuyu Zhang 
Committed: Thu Apr 26 00:24:29 2018 -0500

--
 cli/tests/CMakeLists.txt|   6 +-
 cli/tests/CommandExecutorTest.cpp   |   2 -
 cli/tests/CommandExecutorTestRunner.cpp |  31 +-
 cli/tests/CommandExecutorTestRunner.hpp |  35 +-
 cli/tests/command_executor/Analyze.test | 136 +++
 cli/tests/command_executor/CMakeLists.txt   |   6 +
 cli/tests/command_executor/D.test   |   1 -
 cli/tests/command_executor/Dt.test  |   1 -
 query_optimizer/CMakeLists.txt  |   1 +
 query_optimizer/ExecutionGenerator.cpp  |  18 +
 query_optimizer/Optimizer.cpp   |   3 +-
 query_optimizer/PhysicalGenerator.cpp   |  15 +-
 query_optimizer/PhysicalGenerator.hpp   |  10 +-
 query_optimizer/physical/Aggregate.cpp  |  20 ++
 query_optimizer/physical/Aggregate.hpp  |   3 +
 query_optimizer/physical/CMakeLists.txt |   2 +
 query_optimizer/physical/HashJoin.cpp   |  10 +
 query_optimizer/physical/HashJoin.hpp   |   3 +
 query_optimizer/physical/NestedLoopsJoin.cpp|   9 +
 query_optimizer/physical/NestedLoopsJoin.hpp|   3 +
 query_optimizer/physical/PatternMatcher.hpp |   2 +
 query_optimizer/physical/Physical.hpp   |   6 +
 query_optimizer/physical/Selection.cpp  |   7 +
 query_optimizer/physical/Selection.hpp  |   3 +
 query_optimizer/physical/UnionAll.hpp   |   4 +
 query_optimizer/rules/CMakeLists.txt|  24 ++
 query_optimizer/rules/EliminateEmptyNode.cpp| 358 +++
 query_optimizer/rules/EliminateEmptyNode.hpp|  70 
 .../tests/OptimizerTextTestRunner.cpp   |   3 +-
 query_optimizer/tests/TestDatabaseLoader.cpp|   2 +-
 30 files changed, 741 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cc9ab901/cli/tests/CMakeLists.txt
--
diff --git a/cli/tests/CMakeLists.txt b/cli/tests/CMakeLists.txt
index f402ac0..a59491e 100644
--- a/cli/tests/CMakeLists.txt
+++ b/cli/tests/CMakeLists.txt
@@ -49,6 +49,7 @@ target_link_libraries(quickstep_cli_tests_CommandExecutorTest
   gtest
   quickstep_catalog_CatalogDatabase
   quickstep_cli_CommandExecutor
+  quickstep_cli_DefaultsConfigurator
   quickstep_cli_DropRelation
   quickstep_cli_PrintToScreen
   quickstep_parser_ParseStatement
@@ -59,10 +60,9 @@ target_link_libraries(quickstep_cli_tests_CommandExecutorTest
   quickstep_queryexecution_QueryExecutionUtil
   quickstep_queryexecution_Worker
   quickstep_queryexecution_WorkerDirectory
-  quickstep_queryoptimizer_Optimizer
-  quickstep_queryoptimizer_OptimizerContext
   quickstep_queryoptimizer_QueryHandle
-  quickstep_queryoptimizer_tests_TestDatabaseLoader
+  quickstep_queryoptimizer_QueryProcessor
+  quickstep_storage_StorageConstants
   quickstep_utility_Macros
   quickstep_utility_MemStream
   quickstep_utility_SqlError

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cc9ab901/cli/tests/CommandExecutorTest.cpp
--
diff --git a/cli/tests/CommandExecutorTest.cpp 
b/cli/tests/CommandExecutorTest.cpp
index 6ad2183..a9c64dd 100644
--- a/cli/tests/CommandExecutorTest.cpp
+++ b/cli/tests/CommandExecutorTest.cpp
@@ -45,8 +45,6 @@ int main(int argc, char** argv) {
   new quickstep::CommandExecutorTestRunner(argv[3]));
   test_driver.reset(
   new quickstep::TextBasedTestDriver(_file, test_runner.get()));
-  test_driver->registerOption(
-  quickstep::CommandExecutorTestRunner::kResetOption);
 
   

incubator-quickstep git commit: Upgrade cmake version.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 7cc928429 -> 502960d06


Upgrade cmake version.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/502960d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/502960d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/502960d0

Branch: refs/heads/master
Commit: 502960d06faa730a02755c316b28c93fbf779a7c
Parents: 7cc9284
Author: Harshad Deshmukh 
Authored: Mon Apr 2 11:34:14 2018 -0500
Committer: Harshad Deshmukh 
Committed: Mon Apr 2 11:37:22 2018 -0500

--
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/502960d0/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3965851..1343a18 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-cmake_minimum_required (VERSION 2.8.6)
+cmake_minimum_required (VERSION 3.0)
 project (QUICKSTEP)
 
 set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")

[38/46] incubator-quickstep git commit: Upgrade cpplint

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d886ddb0/third_party/src/cpplint/cpplint.py
--
diff --git a/third_party/src/cpplint/cpplint.py 
b/third_party/src/cpplint/cpplint.py
index 469283f..431c112 100755
--- a/third_party/src/cpplint/cpplint.py
+++ b/third_party/src/cpplint/cpplint.py
@@ -1,15 +1,4 @@
-#!/usr/bin/env python2
-#
-# Original version: svn revision 141
-#
-# This file modified for quickstep as follows:
-#   - Allow no copyright message at the top of the file.
-#   - Allow line length up to 120 characters by default.
-#   - Recognize .hpp files as C++ source.
-#   - Allow use of C++11  header.
-#   - Supress IWYU warnings for std::tuple (since we use "tuple" as a variable
-# name in many places).
-#   - Allow C++11 rvalue references anywhere.
+#!/usr/bin/env python
 #
 # Copyright (c) 2009 Google Inc. All rights reserved.
 #
@@ -52,6 +41,13 @@ We do a small hack, which is to ignore //'s with "'s after 
them on the
 same line, but it is far from perfect (in either direction).
 """
 
+"""
+Additional notes from Quickstep developers:
+commit ID: e8ffd7ce60e5c0e98ba37745749fed0222982668
+Repo: https://github.com/google/styleguide
+Changes: Remove chrono from the banned C++11 headers' list as it appears to be
+just a matter of preference for the Google chromium team.
+"""
 import codecs
 import copy
 import getopt
@@ -67,11 +63,12 @@ import unicodedata
 _USAGE = """
 Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...]
[--counting=total|toplevel|detailed] [--root=subdir]
-   [--linelength=digits]
+   [--linelength=digits] [--headers=x,y,...]
+   [--quiet]
  [file] ...
 
   The style guidelines this tries to follow are those in
-http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
+https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
 
   Every problem is given a confidence score from 1-5, with 5 meaning we are
   certain of the problem, and 1 meaning it could be a legitimate construct.
@@ -94,6 +91,9 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] 
[--filter=-x,+y,...]
 verbose=#
   Specify a number 0-5 to restrict errors to certain verbosity levels.
 
+quiet
+  Don't print anything if no errors are found.
+
 filter=-x,+y,...
   Specify a comma-separated list of category-filters to apply: only
   error messages whose category names pass the filters will be printed.
@@ -125,12 +125,13 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] 
[--filter=-x,+y,...]
   ignored.
 
   Examples:
-Assuming that src/.git exists, the header guard CPP variables for
-src/chrome/browser/ui/browser.h are:
+Assuming that top/src/.git exists (and cwd=top/src), the header guard
+CPP variables for top/src/chrome/browser/ui/browser.h are:
 
 No flag => CHROME_BROWSER_UI_BROWSER_H_
 --root=chrome => BROWSER_UI_BROWSER_H_
 --root=chrome/browser => UI_BROWSER_H_
+--root=.. => SRC_CHROME_BROWSER_UI_BROWSER_H_
 
 linelength=digits
   This is the allowed line length for the project. The default value is
@@ -145,6 +146,14 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] 
[--filter=-x,+y,...]
   Examples:
 --extensions=hpp,cpp
 
+headers=x,y,...
+  The header extensions that cpplint will treat as .h in checks. Values are
+  automatically added to --extensions list.
+
+  Examples:
+--headers=hpp,hxx
+--headers=hpp
+
 cpplint.py supports per-directory configurations specified in CPPLINT.cfg
 files. CPPLINT.cfg file can contain a number of key=value pairs.
 Currently the following options are supported:
@@ -153,6 +162,8 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] 
[--filter=-x,+y,...]
   filter=+filter1,-filter2,...
   exclude_files=regex
   linelength=80
+  root=subdir
+  headers=x,y,...
 
 "set noparent" option prevents cpplint from traversing directory tree
 upwards looking for more .cfg files in parent directories. This option
@@ -168,6 +179,12 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] 
[--filter=-x,+y,...]
 
 "linelength" allows to specify the allowed line length for the project.
 
+The "root" option is similar in function to the --root flag (see example
+above). Paths are relative to the directory of the CPPLINT.cfg.
+
+The "headers" option is similar in function to the --headers flag
+(see example above).
+
 CPPLINT.cfg has an effect on files in the same directory and all
 sub-directories, unless overridden by a nested configuration file.
 
@@ -188,6 +205,8 @@ Syntax: cpplint.py [--verbose=#] [--output=vs7] 
[--filter=-x,+y,...]
 _ERROR_CATEGORIES = [
 'build/class',
 'build/c++11',
+'build/c++14',
+'build/c++tr1',
 'build/deprecated',
 

[24/46] incubator-quickstep git commit: Fix number of work orders generated for insert multiple tuples. (Also added unit tests)

[jianqiao]

Fix number of work orders generated for insert multiple tuples. (Also added 
unit tests)


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3595bc1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3595bc1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3595bc1f

Branch: refs/heads/fix-iwyu
Commit: 3595bc1fdd55bf3979b5a7e98e2263f5bf420406
Parents: b237969
Author: Robert Claus 
Authored: Fri Nov 3 15:11:11 2017 -0500
Committer: Robert Claus 
Committed: Mon Nov 20 14:30:17 2017 -0600

--
 query_execution/QueryContext.hpp|  16 +++
 query_optimizer/ExecutionGenerator.cpp  | 105 ++-
 .../tests/execution_generator/Insert.test   |  15 +++
 query_optimizer/tests/resolver/Insert.test  |  29 +
 relational_operators/InsertOperator.cpp |  18 +++-
 relational_operators/InsertOperator.hpp |  13 +--
 relational_operators/WorkOrder.proto|   2 +-
 relational_operators/WorkOrderFactory.cpp   |  28 -
 8 files changed, 160 insertions(+), 66 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3595bc1f/query_execution/QueryContext.hpp
--
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 7876821..e65f096 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -489,6 +489,22 @@ class QueryContext {
   }
 
   /**
+   * @brief Whether the given vector of Tuple ids is valid.
+   *
+   * @param ids The vector of Tuple ids.
+   *
+   * @return True if valid, otherwise false.
+   **/
+  bool areValidTupleIds(const std::vector ) const {
+for (const tuple_id id : ids) {
+  if (id >= tuples_.size()) {
+return false;
+  }
+}
+return true;
+  }
+
+  /**
* @brief Release the ownership of the Tuple referenced by the id.
*
* @note Each id should use only once.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3595bc1f/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index 14d8949..b0d3c48 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1461,72 +1461,75 @@ void ExecutionGenerator::convertInsertTuple(
   *catalog_database_->getRelationById(
   input_relation_info->relation->getID());
 
+
+  // Construct the tuple proto to be inserted.
+  std::vector tuple_indexes;
+
   for (const std::vector  : 
physical_plan->column_values()) {
-// Construct the tuple proto to be inserted.
 const QueryContext::tuple_id tuple_index = 
query_context_proto_->tuples_size();
-
 S::Tuple *tuple_proto = query_context_proto_->add_tuples();
 for (const E::ScalarLiteralPtr  : tuple) {
   
tuple_proto->add_attribute_values()->CopyFrom(literal->value().getProto());
 }
+tuple_indexes.push_back(tuple_index);
+  }
 
-// FIXME(qzeng): A better way is using a traits struct to look up whether 
a storage
-//   block supports ad-hoc insertion instead of hard-coding 
the block types.
-const StorageBlockLayout _block_layout =
-input_relation.getDefaultStorageBlockLayout();
-if 
(storage_block_layout.getDescription().tuple_store_description().sub_block_type()
 ==
-TupleStorageSubBlockDescription::COMPRESSED_COLUMN_STORE ||
-
storage_block_layout.getDescription().tuple_store_description().sub_block_type()
 ==
-  TupleStorageSubBlockDescription::COMPRESSED_PACKED_ROW_STORE) {
-  THROW_SQL_ERROR() << "INSERT statement is not supported for the relation 
"
-<< input_relation.getName()
-<< ", because its storage blocks do not support ad-hoc 
insertion";
-}
+  // FIXME(qzeng): A better way is using a traits struct to look up whether a 
storage
+  //   block supports ad-hoc insertion instead of hard-coding the 
block types.
+  const StorageBlockLayout _block_layout =
+  input_relation.getDefaultStorageBlockLayout();
+  if 
(storage_block_layout.getDescription().tuple_store_description().sub_block_type()
 ==
+  TupleStorageSubBlockDescription::COMPRESSED_COLUMN_STORE ||
+  
storage_block_layout.getDescription().tuple_store_description().sub_block_type()
 ==
+TupleStorageSubBlockDescription::COMPRESSED_PACKED_ROW_STORE) {
+THROW_SQL_ERROR() << "INSERT statement is not supported for the relation "
+  << input_relation.getName()
+  

[21/46] incubator-quickstep git commit: Support Multiple Tuple Inserts

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fe838df/parser/preprocessed/SqlParser_gen.cpp
--
diff --git a/parser/preprocessed/SqlParser_gen.cpp 
b/parser/preprocessed/SqlParser_gen.cpp
index 72c61dd..9b77875 100644
--- a/parser/preprocessed/SqlParser_gen.cpp
+++ b/parser/preprocessed/SqlParser_gen.cpp
@@ -341,6 +341,7 @@ union YYSTYPE
   quickstep::NumericParseLiteralValue *numeric_literal_value_;
   quickstep::ParseLiteralValue *literal_value_;
   quickstep::PtrList *literal_value_list_;
+  quickstep::PtrList 
*literal_value_list_multiple_;
 
   quickstep::ParseExpression *expression_;
 
@@ -431,7 +432,7 @@ union YYSTYPE
 
   quickstep::ParsePriority *opt_priority_clause_;
 
-#line 435 "SqlParser_gen.cpp" /* yacc.c:355  */
+#line 436 "SqlParser_gen.cpp" /* yacc.c:355  */
 };
 
 typedef union YYSTYPE YYSTYPE;
@@ -460,13 +461,13 @@ int quickstep_yyparse (yyscan_t yyscanner, 
quickstep::ParseStatement **parsedSta
 #endif /* !YY_QUICKSTEP_YY_SQLPARSER_GEN_HPP_INCLUDED  */
 
 /* Copy the second part of user declarations.  */
-#line 222 "../SqlParser.ypp" /* yacc.c:358  */
+#line 223 "../SqlParser.ypp" /* yacc.c:358  */
 
 /* This header needs YYSTYPE, which is defined by the %union directive above */
 #include "SqlLexer_gen.hpp"
 void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const 
std::string );
 
-#line 470 "SqlParser_gen.cpp" /* yacc.c:358  */
+#line 471 "SqlParser_gen.cpp" /* yacc.c:358  */
 
 #ifdef short
 # undef short
@@ -710,16 +711,16 @@ union yyalloc
 /* YYFINAL -- State number of the termination state.  */
 #define YYFINAL  50
 /* YYLAST -- Last index in YYTABLE.  */
-#define YYLAST   1391
+#define YYLAST   1327
 
 /* YYNTOKENS -- Number of terminals.  */
 #define YYNTOKENS  148
 /* YYNNTS -- Number of nonterminals.  */
-#define YYNNTS  110
+#define YYNNTS  111
 /* YYNRULES -- Number of rules.  */
-#define YYNRULES  298
+#define YYNRULES  300
 /* YYNSTATES -- Number of states.  */
-#define YYNSTATES  550
+#define YYNSTATES  555
 
 /* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned
by yylex, with out-of-bounds checking.  */
@@ -779,36 +780,37 @@ static const yytype_uint8 yytranslate[] =
   /* YYRLINE[YYN] -- Source line where rule number YYN was defined.  */
 static const yytype_uint16 yyrline[] =
 {
-   0,   640,   640,   644,   648,   652,   656,   659,   666,   669,
- 672,   675,   678,   681,   684,   687,   690,   693,   699,   705,
- 712,   718,   725,   734,   739,   748,   753,   758,   762,   768,
- 773,   776,   779,   784,   787,   790,   793,   796,   799,   802,
- 805,   808,   811,   823,   826,   829,   847,   867,   870,   873,
- 878,   883,   889,   895,   904,   908,   914,   917,   922,   927,
- 932,   939,   946,   950,   956,   959,   964,   967,   972,   975,
- 980,   983,  1002,  1005,  1010,  1014,  1020,  1023,  1026,  1029,
-1034,  1037,  1040,  1047,  1052,  1063,  1068,  1073,  1077,  1081,
-1087,  1090,  1096,  1104,  1107,  1110,  1116,  1121,  1126,  1130,
-1136,  1140,  1143,  1148,  1151,  1156,  1161,  1166,  1170,  1176,
-1185,  1188,  1193,  1196,  1215,  1220,  1224,  1230,  1236,  1245,
-1250,  1258,  1264,  1270,  1273,  1276,  1281,  1284,  1289,  1293,
-1299,  1302,  1305,  1310,  1315,  1320,  1323,  1326,  1331,  1334,
-1337,  1340,  1343,  1346,  1349,  1352,  1357,  1360,  1365,  1369,
-1373,  1376,  1380,  1383,  1388,  1391,  1396,  1399,  1404,  1408,
-1414,  1417,  1422,  1425,  1430,  1433,  1438,  1441,  1460,  1463,
-1468,  1472,  1478,  1484,  1489,  1492,  1497,  1500,  1505,  1508,
-1513,  1516,  1521,  1522,  1525,  1530,  1531,  1534,  1539,  1543,
-1549,  1556,  1559,  1562,  1567,  1570,  1573,  1579,  1582,  1587,
-1592,  1601,  1606,  1615,  1620,  1623,  1628,  1631,  1636,  1642,
-1648,  1651,  1654,  1657,  1660,  1663,  1669,  1678,  1681,  1686,
-1689,  1694,  1697,  1702,  1705,  1708,  1711,  1715,  1719,  1722,
-1725,  1728,  1731,  1736,  1740,  1744,  1747,  1752,  1757,  1761,
-1767,  1770,  1775,  1779,  1785,  1790,  1794,  1800,  1805,  1808,
-1813,  1817,  1823,  1826,  1829,  1832,  1844,  1848,  1867,  1880,
-1895,  1898,  1901,  1904,  1907,  1910,  1915,  1919,  1925,  1928,
-1933,  1937,  1944,  1947,  1950,  1953,  1956,  1959,  1962,  1965,
-1968,  1971,  1976,  1987,  1990,  1995,  1998,  2001,  2007,  2011,
-2017,  2020,  2028,  2031,  2034,  2037,  2043,  2048,  2053
+   0,   644,   644,   648,   652,   656,   660,   663,   670,   673,
+ 676,   679,   682,   685,   688,   691,   694,   697,   703,   709,
+ 716,   722,   729,   738,   743,   752,   757,   762,   766,   772,
+ 777,   780,   783,   788,   791,   794,   797,   800,   803,   806,
+ 809,   812,   815,   827,   830,   833,   851,   871,   874,   877,
+ 882,   887,   

[27/46] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/windows/raw_logging.cc
--
diff --git a/third_party/src/glog/src/windows/raw_logging.cc 
b/third_party/src/glog/src/windows/raw_logging.cc
deleted file mode 100644
index 7a7409b..000
--- a/third_party/src/glog/src/windows/raw_logging.cc
+++ /dev/null
@@ -1,172 +0,0 @@
-// Copyright (c) 2006, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Maxim Lifantsev
-//
-// logging_unittest.cc covers the functionality herein
-
-#include "utilities.h"
-
-#include 
-#include 
-#include 
-#ifdef HAVE_UNISTD_H
-# include// for close() and write()
-#endif
-#include  // for open()
-#include 
-#include "config.h"
-#include "glog/logging.h"  // To pick up flag settings etc.
-#include "glog/raw_logging.h"
-#include "base/commandlineflags.h"
-
-#ifdef HAVE_STACKTRACE
-# include "stacktrace.h"
-#endif
-
-#if defined(HAVE_SYSCALL_H)
-#include  // for syscall()
-#elif defined(HAVE_SYS_SYSCALL_H)
-#include  // for syscall()
-#endif
-#ifdef HAVE_UNISTD_H
-# include 
-#endif
-
-#if defined(HAVE_SYSCALL_H) || defined(HAVE_SYS_SYSCALL_H)
-# define safe_write(fd, s, len)  syscall(SYS_write, fd, s, len)
-#else
-  // Not so safe, but what can you do?
-# define safe_write(fd, s, len)  write(fd, s, len)
-#endif
-
-_START_GOOGLE_NAMESPACE_
-
-// Data for RawLog__ below. We simply pick up the latest
-// time data created by a normal log message to avoid calling
-// localtime_r which can allocate memory.
-static struct ::tm last_tm_time_for_raw_log;
-static int last_usecs_for_raw_log;
-
-void RawLog__SetLastTime(const struct ::tm& t, int usecs) {
-  memcpy(_tm_time_for_raw_log, , sizeof(last_tm_time_for_raw_log));
-  last_usecs_for_raw_log = usecs;
-}
-
-// CAVEAT: vsnprintf called from *DoRawLog below has some (exotic) code paths
-// that invoke malloc() and getenv() that might acquire some locks.
-// If this becomes a problem we should reimplement a subset of vsnprintf
-// that does not need locks and malloc.
-
-// Helper for RawLog__ below.
-// *DoRawLog writes to *buf of *size and move them past the written portion.
-// It returns true iff there was no overflow or error.
-static bool DoRawLog(char** buf, int* size, const char* format, ...) {
-  va_list ap;
-  va_start(ap, format);
-  int n = vsnprintf(*buf, *size, format, ap);
-  va_end(ap);
-  if (n < 0 || n > *size) return false;
-  *size -= n;
-  *buf += n;
-  return true;
-}
-
-// Helper for RawLog__ below.
-inline static bool VADoRawLog(char** buf, int* size,
-  const char* format, va_list ap) {
-  int n = vsnprintf(*buf, *size, format, ap);
-  if (n < 0 || n > *size) return false;
-  *size -= n;
-  *buf += n;
-  return true;
-}
-
-static const int kLogBufSize = 3000;
-static bool crashed = false;
-static CrashReason crash_reason;
-static char crash_buf[kLogBufSize + 1] = { 0 };  // Will end in '\0'
-
-void RawLog__(LogSeverity severity, const char* file, int line,
-  const char* format, ...) {
-  if (!(FLAGS_logtostderr || severity >= FLAGS_stderrthreshold ||
-FLAGS_alsologtostderr || !IsGoogleLoggingInitialized())) {
-return;  // this stderr log message is suppressed
-  }
-  // can't call localtime_r here: it can allocate
-  struct ::tm& t = last_tm_time_for_raw_log;
-  char 

[29/46] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/windows/glog/logging.h
--
diff --git a/third_party/src/glog/src/windows/glog/logging.h 
b/third_party/src/glog/src/windows/glog/logging.h
deleted file mode 100644
index 1d91b12..000
--- a/third_party/src/glog/src/windows/glog/logging.h
+++ /dev/null
@@ -1,1603 +0,0 @@
-// This file is automatically generated from src/glog/logging.h.in
-// using src/windows/preprocess.sh.
-// DO NOT EDIT!
-
-// Copyright (c) 1999, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Ray Sidney
-//
-// This file contains #include information about logging-related stuff.
-// Pretty much everybody needs to #include this file so that they can
-// log various happenings.
-//
-#ifndef _LOGGING_H_
-#define _LOGGING_H_
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#if 0
-# include 
-#endif
-#include 
-
-// Annoying stuff for windows -- makes sure clients can import these functions
-#ifndef GOOGLE_GLOG_DLL_DECL
-# if defined(_WIN32) && !defined(__CYGWIN__)
-#   define GOOGLE_GLOG_DLL_DECL  __declspec(dllimport)
-# else
-#   define GOOGLE_GLOG_DLL_DECL
-# endif
-#endif
-#if defined(_MSC_VER)
-#define GLOG_MSVC_PUSH_DISABLE_WARNING(n) __pragma(warning(push)) \
- __pragma(warning(disable:n))
-#define GLOG_MSVC_POP_WARNING() __pragma(warning(pop))
-#else
-#define GLOG_MSVC_PUSH_DISABLE_WARNING(n)
-#define GLOG_MSVC_POP_WARNING()
-#endif
-
-// We care a lot about number of bits things take up.  Unfortunately,
-// systems define their bit-specific ints in a lot of different ways.
-// We use our own way, and have a typedef to get there.
-// Note: these commands below may look like "#if 1" or "#if 0", but
-// that's because they were constructed that way at ./configure time.
-// Look at logging.h.in to see how they're calculated (based on your config).
-#if 0
-#include  // the normal place uint16_t is defined
-#endif
-#if 0
-#include   // the normal place u_int16_t is defined
-#endif
-#if 0
-#include// a third place for uint16_t or u_int16_t
-#endif
-
-#if 0
-#include 
-#endif
-
-namespace google {
-
-#if 0  // the C99 format
-typedef int32_t int32;
-typedef uint32_t uint32;
-typedef int64_t int64;
-typedef uint64_t uint64;
-#elif 0   // the BSD format
-typedef int32_t int32;
-typedef u_int32_t uint32;
-typedef int64_t int64;
-typedef u_int64_t uint64;
-#elif 1// the windows (vc7) format
-typedef __int32 int32;
-typedef unsigned __int32 uint32;
-typedef __int64 int64;
-typedef unsigned __int64 uint64;
-#else
-#error Do not know how to define a 32-bit integer quantity on your system
-#endif
-
-}
-
-// The global value of GOOGLE_STRIP_LOG. All the messages logged to
-// LOG(XXX) with severity less than GOOGLE_STRIP_LOG will not be displayed.
-// If it can be determined at compile time that the message will not be
-// printed, the statement will be compiled out.
-//
-// Example: to strip out all INFO and WARNING messages, use the value
-// of 2 below. To make an exception for WARNING messages from a single
-// file, add "#define GOOGLE_STRIP_LOG 1" to that file _before_ including
-// base/logging.h
-#ifndef GOOGLE_STRIP_LOG
-#define GOOGLE_STRIP_LOG 0
-#endif
-
-// GCC can be told that a certain branch is not likely to be taken (for
-// instance, a CHECK failure), and 

[36/46] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/base/mutex.h
--
diff --git a/third_party/src/glog/src/base/mutex.h 
b/third_party/src/glog/src/base/mutex.h
deleted file mode 100644
index 37527d5..000
--- a/third_party/src/glog/src/base/mutex.h
+++ /dev/null
@@ -1,331 +0,0 @@
-// Copyright (c) 2007, Google Inc.
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-// 
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-// ---
-// Author: Craig Silverstein.
-//
-// A simple mutex wrapper, supporting locks and read-write locks.
-// You should assume the locks are *not* re-entrant.
-//
-// To use: you should define the following macros in your configure.ac:
-//   ACX_PTHREAD
-//   AC_RWLOCK
-// The latter is defined in ../autoconf.
-//
-// This class is meant to be internal-only and should be wrapped by an
-// internal namespace.  Before you use this module, please give the
-// name of your internal namespace for this module.  Or, if you want
-// to expose it, you'll want to move it to the Google namespace.  We
-// cannot put this class in global namespace because there can be some
-// problems when we have multiple versions of Mutex in each shared object.
-//
-// NOTE: by default, we have #ifdef'ed out the TryLock() method.
-//   This is for two reasons:
-// 1) TryLock() under Windows is a bit annoying (it requires a
-//#define to be defined very early).
-// 2) TryLock() is broken for NO_THREADS mode, at least in NDEBUG
-//mode.
-// If you need TryLock(), and either these two caveats are not a
-// problem for you, or you're willing to work around them, then
-// feel free to #define GMUTEX_TRYLOCK, or to remove the #ifdefs
-// in the code below.
-//
-// CYGWIN NOTE: Cygwin support for rwlock seems to be buggy:
-//http://www.cygwin.com/ml/cygwin/2008-12/msg00017.html
-// Because of that, we might as well use windows locks for
-// cygwin.  They seem to be more reliable than the cygwin pthreads layer.
-//
-// TRICKY IMPLEMENTATION NOTE:
-// This class is designed to be safe to use during
-// dynamic-initialization -- that is, by global constructors that are
-// run before main() starts.  The issue in this case is that
-// dynamic-initialization happens in an unpredictable order, and it
-// could be that someone else's dynamic initializer could call a
-// function that tries to acquire this mutex -- but that all happens
-// before this mutex's constructor has run.  (This can happen even if
-// the mutex and the function that uses the mutex are in the same .cc
-// file.)  Basically, because Mutex does non-trivial work in its
-// constructor, it's not, in the naive implementation, safe to use
-// before dynamic initialization has run on it.
-//
-// The solution used here is to pair the actual mutex primitive with a
-// bool that is set to true when the mutex is dynamically initialized.
-// (Before that it's false.)  Then we modify all mutex routines to
-// look at the bool, and not try to lock/unlock until the bool makes
-// it to true (which happens after the Mutex constructor has run.)
-//
-// This works because before main() starts -- particularly, during
-// dynamic initialization -- there are no threads, so a) it's ok that
-// the mutex operations are a no-op, since we don't need locking then
-// anyway; and b) we can be quite confident our bool won't change
-// state between a call to Lock() and a call to Unlock() (that would
-// require a global 

[31/46] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/signalhandler.cc
--
diff --git a/third_party/src/glog/src/signalhandler.cc 
b/third_party/src/glog/src/signalhandler.cc
deleted file mode 100644
index d6c203b..000
--- a/third_party/src/glog/src/signalhandler.cc
+++ /dev/null
@@ -1,350 +0,0 @@
-// Copyright (c) 2008, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Satoru Takabayashi
-//
-// Implementation of InstallFailureSignalHandler().
-
-#include "utilities.h"
-#include "stacktrace.h"
-#include "symbolize.h"
-#include "glog/logging.h"
-
-#include 
-#include 
-#ifdef HAVE_UCONTEXT_H
-# include 
-#endif
-#ifdef HAVE_SYS_UCONTEXT_H
-# include 
-#endif
-#include 
-
-_START_GOOGLE_NAMESPACE_
-
-namespace {
-
-// We'll install the failure signal handler for these signals.  We could
-// use strsignal() to get signal names, but we don't use it to avoid
-// introducing yet another #ifdef complication.
-//
-// The list should be synced with the comment in signalhandler.h.
-const struct {
-  int number;
-  const char *name;
-} kFailureSignals[] = {
-  { SIGSEGV, "SIGSEGV" },
-  { SIGILL, "SIGILL" },
-  { SIGFPE, "SIGFPE" },
-  { SIGABRT, "SIGABRT" },
-  { SIGBUS, "SIGBUS" },
-  { SIGTERM, "SIGTERM" },
-};
-
-// Returns the program counter from signal context, NULL if unknown.
-void* GetPC(void* ucontext_in_void) {
-#if (defined(HAVE_UCONTEXT_H) || defined(HAVE_SYS_UCONTEXT_H)) && 
defined(PC_FROM_UCONTEXT)
-  if (ucontext_in_void != NULL) {
-ucontext_t *context = reinterpret_cast(ucontext_in_void);
-return (void*)context->PC_FROM_UCONTEXT;
-  }
-#endif
-  return NULL;
-}
-
-// The class is used for formatting error messages.  We don't use printf()
-// as it's not async signal safe.
-class MinimalFormatter {
- public:
-  MinimalFormatter(char *buffer, int size)
-  : buffer_(buffer),
-cursor_(buffer),
-end_(buffer + size) {
-  }
-
-  // Returns the number of bytes written in the buffer.
-  int num_bytes_written() const { return cursor_ - buffer_; }
-
-  // Appends string from "str" and updates the internal cursor.
-  void AppendString(const char* str) {
-int i = 0;
-while (str[i] != '\0' && cursor_ + i < end_) {
-  cursor_[i] = str[i];
-  ++i;
-}
-cursor_ += i;
-  }
-
-  // Formats "number" in "radix" and updates the internal cursor.
-  // Lowercase letters are used for 'a' - 'z'.
-  void AppendUint64(uint64 number, int radix) {
-int i = 0;
-while (cursor_ + i < end_) {
-  const int tmp = number % radix;
-  number /= radix;
-  cursor_[i] = (tmp < 10 ? '0' + tmp : 'a' + tmp - 10);
-  ++i;
-  if (number == 0) {
-break;
-  }
-}
-// Reverse the bytes written.
-std::reverse(cursor_, cursor_ + i);
-cursor_ += i;
-  }
-
-  // Formats "number" as hexadecimal number, and updates the internal
-  // cursor.  Padding will be added in front if needed.
-  void AppendHexWithPadding(uint64 number, int width) {
-char* start = cursor_;
-AppendString("0x");
-AppendUint64(number, 16);
-// Move to right and add padding in front if needed.
-if (cursor_ < start + width) {
-  const int64 delta = start + width - cursor_;
-  std::copy(start, cursor_, start + delta);
-  std::fill(start, start + delta, ' ');
-  cursor_ = start + width;
-}
-  }
-
- private:
-  char *buffer_;
-  

[25/46] incubator-quickstep git commit: Hash-Join-Fuse: Feature added and tests modified.

[jianqiao]

mp;& physical_plan->build_predicate() == 
nullptr) {
 THROW_SQL_ERROR() << "Self-join is not supported";
   }
 
@@ -1006,7 +1017,8 @@ void ExecutionGenerator::convertHashJoin(const 
P::HashJoinPtr _plan) {
   build_attribute_ids,
   any_build_attributes_nullable,
   probe_num_partitions,
-  join_hash_table_index));
+  join_hash_table_index,
+  build_predicate_index));
 
   // Create InsertDestination proto.
   const CatalogRelation *output_relation = nullptr;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/39c6214a/query_optimizer/PhysicalGenerator.cpp
--
diff --git a/query_optimizer/PhysicalGenerator.cpp 
b/query_optimizer/PhysicalGenerator.cpp
index 865cd11..0d15a66 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -30,6 +30,7 @@
 #include "query_optimizer/rules/CollapseSelection.hpp"
 #include "query_optimizer/rules/ExtractCommonSubexpression.hpp"
 #include "query_optimizer/rules/FuseAggregateJoin.hpp"
+#include "query_optimizer/rules/FuseHashSelect.hpp"
 #include "query_optimizer/rules/InjectJoinFilters.hpp"
 #include "query_optimizer/rules/Partition.hpp"
 #include "query_optimizer/rules/PruneColumns.hpp"
@@ -67,6 +68,10 @@ DEFINE_bool(use_partition_rule, true,
 "If true, apply an optimization to support partitioned inputs. The 
"
 "optimization may add additional Selection for repartitioning.");
 
+DEFINE_bool(use_fuse_hash_select, true,
+"If true, apply an optimization that moves build-side Selection 
nodes"
+"into the hash join operator instead.");
+
 DEFINE_bool(use_filter_joins, true,
 "If true, apply an optimization that strength-reduces HashJoins to 
"
 "FilterJoins (implemented as LIPFilters attached to some anchoring 
"
@@ -176,6 +181,10 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
 rules.push_back(std::make_unique());
   }
 
+  if (FLAGS_use_fuse_hash_select) {
+rules.emplace_back(new FuseHashSelect());
+  }
+
   // NOTE(jianqiao): Adding rules after InjectJoinFilters (or AttachLIPFilters)
   // requires extra handling of LIPFilterConfiguration for transformed nodes.
   // So currently it is suggested that all the new rules be placed before this

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/39c6214a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
--
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp 
b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index e0e3dff..5aec4b4 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -179,8 +179,9 @@ std::size_t 
StarSchemaSimpleCostModel::estimateCardinalityForHashJoin(
   std::size_t right_cardinality = estimateCardinality(physical_plan->right());
   double left_selectivity = estimateSelectivity(physical_plan->left());
   double right_selectivity = estimateSelectivity(physical_plan->right());
-  return std::max(static_cast(left_cardinality * 
right_selectivity + 0.5),
-  static_cast(right_cardinality * 
left_selectivity + 0.5));
+  double build_selectivity = 
estimateSelectivityForPredicate(physical_plan->build_predicate(), 
physical_plan);
+  return std::max(static_cast(left_cardinality * 
right_selectivity * build_selectivity + 0.5),
+  static_cast(right_cardinality * 
left_selectivity * build_selectivity + 0.5));
 }
 
 std::size_t StarSchemaSimpleCostModel::estimateCardinalityForNestedLoopsJoin(
@@ -295,16 +296,20 @@ std::size_t 
StarSchemaSimpleCostModel::estimateNumDistinctValues(
 estimateNumDistinctValues(attribute_id, hash_join->left());
 double right_child_selectivity =
 estimateSelectivity(hash_join->right());
+double build_selectivity =
+estimateSelectivityForPredicate(hash_join->build_predicate(), 
hash_join);
 return static_cast(
-left_child_num_distinct_values * right_child_selectivity * 
filter_selectivity + 0.5);
+left_child_num_distinct_values * right_child_selectivity * 
filter_selectivity * build_selectivity + 0.5);
   }
   if (E::ContainsExprId(hash_join->right()->getOutputAttributes(), 
attribute_id)) {
 std::size_t right_child_num_distinct_values =
 estimateNumDistinctValues(attribute_id, hash_join->right());
 double left_child_selectivity =
 estimateSelectivity(hash_join->left());
+double build_selectivity =
+estimateSelectivityForPredicate(hash_join->build_predicat

[20/46] incubator-quickstep git commit: Support Multiple Tuple Inserts

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fe838df/parser/preprocessed/SqlParser_gen.hpp
--
diff --git a/parser/preprocessed/SqlParser_gen.hpp 
b/parser/preprocessed/SqlParser_gen.hpp
index f6b5247..142059d 100644
--- a/parser/preprocessed/SqlParser_gen.hpp
+++ b/parser/preprocessed/SqlParser_gen.hpp
@@ -198,6 +198,7 @@ union YYSTYPE
   quickstep::NumericParseLiteralValue *numeric_literal_value_;
   quickstep::ParseLiteralValue *literal_value_;
   quickstep::PtrList *literal_value_list_;
+  quickstep::PtrList 
*literal_value_list_multiple_;
 
   quickstep::ParseExpression *expression_;
 
@@ -288,7 +289,7 @@ union YYSTYPE
 
   quickstep::ParsePriority *opt_priority_clause_;
 
-#line 292 "SqlParser_gen.hpp" /* yacc.c:1915  */
+#line 293 "SqlParser_gen.hpp" /* yacc.c:1915  */
 };
 
 typedef union YYSTYPE YYSTYPE;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fe838df/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index 372d576..14d8949 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1461,70 +1461,72 @@ void ExecutionGenerator::convertInsertTuple(
   *catalog_database_->getRelationById(
   input_relation_info->relation->getID());
 
-  // Construct the tuple proto to be inserted.
-  const QueryContext::tuple_id tuple_index = 
query_context_proto_->tuples_size();
+  for (const std::vector  : 
physical_plan->column_values()) {
+// Construct the tuple proto to be inserted.
+const QueryContext::tuple_id tuple_index = 
query_context_proto_->tuples_size();
 
-  S::Tuple *tuple_proto = query_context_proto_->add_tuples();
-  for (const E::ScalarLiteralPtr  : physical_plan->column_values()) {
-tuple_proto->add_attribute_values()->CopyFrom(literal->value().getProto());
-  }
+S::Tuple *tuple_proto = query_context_proto_->add_tuples();
+for (const E::ScalarLiteralPtr  : tuple) {
+  
tuple_proto->add_attribute_values()->CopyFrom(literal->value().getProto());
+}
 
-  // FIXME(qzeng): A better way is using a traits struct to look up whether a 
storage
-  //   block supports ad-hoc insertion instead of hard-coding the 
block types.
-  const StorageBlockLayout _block_layout =
-  input_relation.getDefaultStorageBlockLayout();
-  if 
(storage_block_layout.getDescription().tuple_store_description().sub_block_type()
 ==
-  TupleStorageSubBlockDescription::COMPRESSED_COLUMN_STORE ||
-  
storage_block_layout.getDescription().tuple_store_description().sub_block_type()
 ==
-TupleStorageSubBlockDescription::COMPRESSED_PACKED_ROW_STORE) {
-THROW_SQL_ERROR() << "INSERT statement is not supported for the relation "
-  << input_relation.getName()
-  << ", because its storage blocks do not support ad-hoc 
insertion";
-  }
+// FIXME(qzeng): A better way is using a traits struct to look up whether 
a storage
+//   block supports ad-hoc insertion instead of hard-coding 
the block types.
+const StorageBlockLayout _block_layout =
+input_relation.getDefaultStorageBlockLayout();
+if 
(storage_block_layout.getDescription().tuple_store_description().sub_block_type()
 ==
+TupleStorageSubBlockDescription::COMPRESSED_COLUMN_STORE ||
+
storage_block_layout.getDescription().tuple_store_description().sub_block_type()
 ==
+  TupleStorageSubBlockDescription::COMPRESSED_PACKED_ROW_STORE) {
+  THROW_SQL_ERROR() << "INSERT statement is not supported for the relation 
"
+<< input_relation.getName()
+<< ", because its storage blocks do not support ad-hoc 
insertion";
+}
 
-  // Create InsertDestination proto.
-  const QueryContext::insert_destination_id insert_destination_index =
-  query_context_proto_->insert_destinations_size();
-  S::InsertDestination *insert_destination_proto = 
query_context_proto_->add_insert_destinations();
+// Create InsertDestination proto.
+const QueryContext::insert_destination_id insert_destination_index =
+query_context_proto_->insert_destinations_size();
+S::InsertDestination *insert_destination_proto = 
query_context_proto_->add_insert_destinations();
 
-  insert_destination_proto->set_relation_id(input_relation.getID());
-  insert_destination_proto->mutable_layout()->MergeFrom(
-  input_relation.getDefaultStorageBlockLayout().getDescription());
+insert_destination_proto->set_relation_id(input_relation.getID());
+insert_destination_proto->mutable_layout()->MergeFrom(
+input_relation.getDefaultStorageBlockLayout().getDescription());
 
-  if (input_relation.hasPartitionScheme()) {
-

[35/46] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/glog/log_severity.h
--
diff --git a/third_party/src/glog/src/glog/log_severity.h 
b/third_party/src/glog/src/glog/log_severity.h
deleted file mode 100644
index 99945a4..000
--- a/third_party/src/glog/src/glog/log_severity.h
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright (c) 2007, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef BASE_LOG_SEVERITY_H__
-#define BASE_LOG_SEVERITY_H__
-
-// Annoying stuff for windows -- makes sure clients can import these functions
-#ifndef GOOGLE_GLOG_DLL_DECL
-# if defined(_WIN32) && !defined(__CYGWIN__)
-#   define GOOGLE_GLOG_DLL_DECL  __declspec(dllimport)
-# else
-#   define GOOGLE_GLOG_DLL_DECL
-# endif
-#endif
-
-// Variables of type LogSeverity are widely taken to lie in the range
-// [0, NUM_SEVERITIES-1].  Be careful to preserve this assumption if
-// you ever need to change their values or add a new severity.
-typedef int LogSeverity;
-
-const int GLOG_INFO = 0, GLOG_WARNING = 1, GLOG_ERROR = 2, GLOG_FATAL = 3,
-  NUM_SEVERITIES = 4;
-#ifndef GLOG_NO_ABBREVIATED_SEVERITIES
-# ifdef ERROR
-#  error ERROR macro is defined. Define GLOG_NO_ABBREVIATED_SEVERITIES before 
including logging.h. See the document for detail.
-# endif
-const int INFO = GLOG_INFO, WARNING = GLOG_WARNING,
-  ERROR = GLOG_ERROR, FATAL = GLOG_FATAL;
-#endif
-
-// DFATAL is FATAL in debug mode, ERROR in normal mode
-#ifdef NDEBUG
-#define DFATAL_LEVEL ERROR
-#else
-#define DFATAL_LEVEL FATAL
-#endif
-
-extern GOOGLE_GLOG_DLL_DECL const char* const LogSeverityNames[NUM_SEVERITIES];
-
-// NDEBUG usage helpers related to (RAW_)DCHECK:
-//
-// DEBUG_MODE is for small !NDEBUG uses like
-//   if (DEBUG_MODE) foo.CheckThatFoo();
-// instead of substantially more verbose
-//   #ifndef NDEBUG
-// foo.CheckThatFoo();
-//   #endif
-//
-// IF_DEBUG_MODE is for small !NDEBUG uses like
-//   IF_DEBUG_MODE( string error; )
-//   DCHECK(Foo()) << error;
-// instead of substantially more verbose
-//   #ifndef NDEBUG
-// string error;
-// DCHECK(Foo()) << error;
-//   #endif
-//
-#ifdef NDEBUG
-enum { DEBUG_MODE = 0 };
-#define IF_DEBUG_MODE(x)
-#else
-enum { DEBUG_MODE = 1 };
-#define IF_DEBUG_MODE(x) x
-#endif
-
-#endif  // BASE_LOG_SEVERITY_H__

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/glog/logging.h
--
diff --git a/third_party/src/glog/src/glog/logging.h 
b/third_party/src/glog/src/glog/logging.h
deleted file mode 100644
index 8a6dca0..000
--- a/third_party/src/glog/src/glog/logging.h
+++ /dev/null
@@ -1,1619 +0,0 @@
-// Copyright (c) 1999, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote 

[33/46] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/logging.cc
--
diff --git a/third_party/src/glog/src/logging.cc 
b/third_party/src/glog/src/logging.cc
deleted file mode 100644
index ec334a9..000
--- a/third_party/src/glog/src/logging.cc
+++ /dev/null
@@ -1,2049 +0,0 @@
-// Copyright (c) 1999, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#define _GNU_SOURCE 1 // needed for O_NOFOLLOW and pread()/pwrite()
-
-#include "utilities.h"
-
-#include 
-#include 
-#include 
-#ifdef HAVE_UNISTD_H
-# include   // For _exit.
-#endif
-#include 
-#include 
-#include 
-#ifdef HAVE_SYS_UTSNAME_H
-# include   // For uname.
-#endif
-#include 
-#include 
-#include 
-#include 
-#include 
-#ifdef HAVE_PWD_H
-# include 
-#endif
-#ifdef HAVE_SYSLOG_H
-# include 
-#endif
-#include 
-#include// for errno
-#include 
-#include "base/commandlineflags.h"// to get the program name
-#include "glog/logging.h"
-#include "glog/raw_logging.h"
-#include "base/googleinit.h"
-
-#ifdef HAVE_STACKTRACE
-# include "stacktrace.h"
-#endif
-
-using std::string;
-using std::vector;
-using std::setw;
-using std::setfill;
-using std::hex;
-using std::dec;
-using std::min;
-using std::ostream;
-using std::ostringstream;
-
-using std::FILE;
-using std::fwrite;
-using std::fclose;
-using std::fflush;
-using std::fprintf;
-using std::perror;
-
-#ifdef __QNX__
-using std::fdopen;
-#endif
-
-// There is no thread annotation support.
-#define EXCLUSIVE_LOCKS_REQUIRED(mu)
-
-static bool BoolFromEnv(const char *varname, bool defval) {
-  const char* const valstr = getenv(varname);
-  if (!valstr) {
-return defval;
-  }
-  return memchr("tTyY1\0", valstr[0], 6) != NULL;
-}
-
-GLOG_DEFINE_bool(logtostderr, BoolFromEnv("GOOGLE_LOGTOSTDERR", false),
- "log messages go to stderr instead of logfiles");
-GLOG_DEFINE_bool(alsologtostderr, BoolFromEnv("GOOGLE_ALSOLOGTOSTDERR", false),
- "log messages go to stderr in addition to logfiles");
-GLOG_DEFINE_bool(colorlogtostderr, false,
- "color messages logged to stderr (if supported by terminal)");
-#ifdef OS_LINUX
-GLOG_DEFINE_bool(drop_log_memory, true, "Drop in-memory buffers of log 
contents. "
- "Logs can grow very quickly and they are rarely read before 
they "
- "need to be evicted from memory. Instead, drop them from 
memory "
- "as soon as they are flushed to disk.");
-_START_GOOGLE_NAMESPACE_
-namespace logging {
-static const int64 kPageSize = getpagesize();
-}
-_END_GOOGLE_NAMESPACE_
-#endif
-
-// By default, errors (including fatal errors) get logged to stderr as
-// well as the file.
-//
-// The default is ERROR instead of FATAL so that users can see problems
-// when they run a program without having to look in another file.
-DEFINE_int32(stderrthreshold,
- GOOGLE_NAMESPACE::GLOG_ERROR,
- "log messages at or above this level are copied to stderr in "
- "addition to logfiles.  This flag obsoletes --alsologtostderr.");
-
-GLOG_DEFINE_string(alsologtoemail, "",
-   "log messages go to these email addresses "
-   "in addition to logfiles");
-GLOG_DEFINE_bool(log_prefix, true,
- "Prepend the log prefix to the start of each log line");
-GLOG_DEFINE_int32(minloglevel, 0, "Messages logged at a 

[09/46] incubator-quickstep git commit: Fixed the distributed version due to query execution engine simplification.

[jianqiao]

Fixed the distributed version due to query execution engine simplification.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e496cb58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e496cb58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e496cb58

Branch: refs/heads/fix-iwyu
Commit: e496cb58e10d32de9dc83d69ece84df3f5b62747
Parents: 0898a77
Author: Zuyu Zhang 
Authored: Fri Oct 6 22:33:02 2017 -0500
Committer: Zuyu Zhang 
Committed: Fri Oct 6 22:33:02 2017 -0500

--
 query_execution/QueryManagerDistributed.cpp | 24 +---
 relational_operators/WorkOrderFactory.cpp   |  4 ++--
 2 files changed, 11 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e496cb58/query_execution/QueryManagerDistributed.cpp
--
diff --git a/query_execution/QueryManagerDistributed.cpp 
b/query_execution/QueryManagerDistributed.cpp
index 30a1396..97b451f 100644
--- a/query_execution/QueryManagerDistributed.cpp
+++ b/query_execution/QueryManagerDistributed.cpp
@@ -70,8 +70,11 @@ QueryManagerDistributed::QueryManagerDistributed(QueryHandle 
*query_handle,
   // Collect all the workorders from all the non-blocking relational operators 
in the DAG.
   for (const dag_node_index index : non_dependent_operators_) {
 if (!fetchNormalWorkOrders(index)) {
-  DCHECK(!checkRebuildRequired(index) || initiateRebuild(index));
-  markOperatorFinished(index);
+  if (checkRebuildRequired(index)) {
+initiateRebuild(index);
+  } else {
+markOperatorFinished(index);
+  }
 }
   }
 
@@ -201,21 +204,12 @@ void 
QueryManagerDistributed::processInitiateRebuildResponseMessage(const dag_no
 const 
std::size_t shiftboss_index) {
   query_exec_state_->updateRebuildStatus(op_index, num_rebuild_work_orders, 
shiftboss_index);
 
-  if (!query_exec_state_->hasRebuildFinished(op_index, num_shiftbosses_)) {
-// Wait for the rebuild work orders to finish.
-return;
+  if (query_exec_state_->hasRebuildFinished(op_index, num_shiftbosses_)) {
+// No needs for rebuilds, or the rebuild has finished.
+markOperatorFinished(op_index);
   }
 
-  // No needs for rebuilds, or the rebuild has finished.
-  markOperatorFinished(op_index);
-
-  for (const std::pair _link :
-   query_dag_->getDependents(op_index)) {
-const dag_node_index dependent_op_index = dependent_link.first;
-if (checkAllBlockingDependenciesMet(dependent_op_index)) {
-  fetchNormalWorkOrders(dependent_op_index);
-}
-  }
+  // Wait for the rebuild work orders to finish.
 }
 
 bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e496cb58/relational_operators/WorkOrderFactory.cpp
--
diff --git a/relational_operators/WorkOrderFactory.cpp 
b/relational_operators/WorkOrderFactory.cpp
index 5baa21b..25cc81a 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -741,7 +741,7 @@ bool WorkOrderFactory::ProtoIsValid(const 
serialization::WorkOrder ,
  
proto.GetExtension(serialization::DeleteWorkOrder::predicate_index)) &&
  proto.HasExtension(serialization::DeleteWorkOrder::block_id) &&
  
proto.HasExtension(serialization::DeleteWorkOrder::operator_index) &&
- proto.GetExtension(serialization::DeleteWorkOrder::partition_id);
+ proto.HasExtension(serialization::DeleteWorkOrder::partition_id);
 }
 case serialization::DESTROY_AGGREGATION_STATE: {
   return 
proto.HasExtension(serialization::DestroyAggregationStateWorkOrder::aggr_state_index)
 &&
@@ -1033,7 +1033,7 @@ bool WorkOrderFactory::ProtoIsValid(const 
serialization::WorkOrder ,
  
proto.GetExtension(serialization::UpdateWorkOrder::update_group_index)) &&
  
proto.HasExtension(serialization::UpdateWorkOrder::operator_index) &&
  proto.HasExtension(serialization::UpdateWorkOrder::block_id) &&
- proto.GetExtension(serialization::UpdateWorkOrder::partition_id);
+ proto.HasExtension(serialization::UpdateWorkOrder::partition_id);
 }
 case serialization::WINDOW_AGGREGATION: {
   return 
proto.HasExtension(serialization::WindowAggregationWorkOrder::window_aggr_state_index)
 &&

[40/46] incubator-quickstep git commit: Fixed the bug when partition w/ pruned columns.

[jianqiao]

Fixed the bug when partition w/ pruned columns.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d6347724
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d6347724
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d6347724

Branch: refs/heads/fix-iwyu
Commit: d63477247648a45b577be27db57954f9e85454c3
Parents: d886ddb
Author: Zuyu Zhang 
Authored: Tue Oct 24 16:17:50 2017 -0500
Committer: Zuyu Zhang 
Committed: Thu Dec 21 15:28:38 2017 -0600

--
 query_optimizer/ExecutionGenerator.cpp  | 39 ++--
 .../tests/execution_generator/Partition.test| 25 +
 2 files changed, 20 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d6347724/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index 5ef58a9..555118a 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -486,26 +486,8 @@ void ExecutionGenerator::createTemporaryCatalogRelation(
 const P::PhysicalPtr ,
 const CatalogRelation **catalog_relation_output,
 S::InsertDestination *insert_destination_proto) {
-  std::unique_ptr catalog_relation(
-  new CatalogRelation(catalog_database_,
-  getNewRelationName(),
-  -1 /* id */,
-  true /* is_temporary*/));
-  attribute_id aid = 0;
-  for (const E::NamedExpressionPtr _expression :
-   physical->getOutputAttributes()) {
-// The attribute name is simply set to the attribute id to make it 
distinct.
-std::unique_ptr catalog_attribute(
-new CatalogAttribute(catalog_relation.get(),
- std::to_string(aid),
- project_expression->getValueType(),
- aid,
- project_expression->attribute_alias()));
-attribute_substitution_map_[project_expression->id()] =
-catalog_attribute.get();
-catalog_relation->addAttribute(catalog_attribute.release());
-++aid;
-  }
+  auto catalog_relation =
+  make_unique(catalog_database_, getNewRelationName(), -1 
/* id */, true /* is_temporary*/);
 
   const P::PartitionSchemeHeader *partition_scheme_header = 
physical->getOutputPartitionSchemeHeader();
   if (partition_scheme_header) {
@@ -514,6 +496,9 @@ void ExecutionGenerator::createTemporaryCatalogRelation(
   DCHECK(!partition_equivalent_expr_ids.empty());
   const E::ExprId partition_expr_id = 
*partition_equivalent_expr_ids.begin();
   DCHECK(attribute_substitution_map_.find(partition_expr_id) != 
attribute_substitution_map_.end());
+  // Use the attribute id from the input relation.
+  // NOTE(zuyu): The following line should be before changing
+  // 'attribute_substitution_map_' with the output attributes.
   
output_partition_attr_ids.push_back(attribute_substitution_map_[partition_expr_id]->getID());
 }
 
@@ -544,6 +529,20 @@ void ExecutionGenerator::createTemporaryCatalogRelation(
 
insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
   }
 
+  attribute_id aid = 0;
+  for (const E::NamedExpressionPtr _expression :
+   physical->getOutputAttributes()) {
+// The attribute name is simply set to the attribute id to make it 
distinct.
+auto catalog_attribute =
+make_unique(catalog_relation.get(), 
std::to_string(aid),
+  project_expression->getValueType(), aid,
+  project_expression->attribute_alias());
+attribute_substitution_map_[project_expression->id()] =
+catalog_attribute.get();
+catalog_relation->addAttribute(catalog_attribute.release());
+++aid;
+  }
+
   *catalog_relation_output = catalog_relation.get();
   const relation_id output_rel_id = catalog_database_->addRelation(
   catalog_relation.release());

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d6347724/query_optimizer/tests/execution_generator/Partition.test
--
diff --git a/query_optimizer/tests/execution_generator/Partition.test 
b/query_optimizer/tests/execution_generator/Partition.test
index da9b6b8..747b969 100644
--- a/query_optimizer/tests/execution_generator/Partition.test
+++ b/query_optimizer/tests/execution_generator/Partition.test
@@ -297,30 +297,7 @@ SELECT dim_2_hash_partitions.id as dim_id, fact.id as 
fact_id
 FROM dim_2_hash_partitions, fact
 WHERE 

[12/46] incubator-quickstep git commit: Moved InsertDestination::getTouchedBlocks as a private method.

[jianqiao]

Moved InsertDestination::getTouchedBlocks as a private method.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a61b03dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a61b03dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a61b03dc

Branch: refs/heads/fix-iwyu
Commit: a61b03dcc2446a5bd276a0117f493ba56d8a3ffe
Parents: 79710ca
Author: Zuyu Zhang 
Authored: Thu Oct 5 16:41:38 2017 -0500
Committer: Zuyu Zhang 
Committed: Mon Oct 9 12:00:08 2017 -0500

--
 .../tests/HashJoinOperator_unittest.cpp | 24 +++
 storage/InsertDestination.cpp   | 27 +++-
 storage/InsertDestination.hpp   | 70 
 3 files changed, 65 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a61b03dc/relational_operators/tests/HashJoinOperator_unittest.cpp
--
diff --git a/relational_operators/tests/HashJoinOperator_unittest.cpp 
b/relational_operators/tests/HashJoinOperator_unittest.cpp
index 1fc84fc..cfd4314 100644
--- a/relational_operators/tests/HashJoinOperator_unittest.cpp
+++ b/relational_operators/tests/HashJoinOperator_unittest.cpp
@@ -485,7 +485,7 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
   InsertDestination *insert_destination = 
query_context_->getInsertDestination(prober->getInsertDestinationID());
   DCHECK(insert_destination);
 
-  const std::vector _blocks = 
insert_destination->getTouchedBlocks();
+  const std::vector result_blocks = 
insert_destination->getTouchedBlocks();
   for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
 BlockReference result_block = 
storage_manager_->getBlock(result_blocks[bid],
  
insert_destination->getRelation());
@@ -640,7 +640,7 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
   InsertDestination *insert_destination = 
query_context_->getInsertDestination(prober->getInsertDestinationID());
   DCHECK(insert_destination);
 
-  const std::vector _blocks = 
insert_destination->getTouchedBlocks();
+  const std::vector result_blocks = 
insert_destination->getTouchedBlocks();
   for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
 BlockReference result_block = 
storage_manager_->getBlock(result_blocks[bid],
  
insert_destination->getRelation());
@@ -795,7 +795,7 @@ TEST_P(HashJoinOperatorTest, 
CharKeyCartesianProductHashJoinTest) {
   InsertDestination *insert_destination = 
query_context_->getInsertDestination(prober->getInsertDestinationID());
   DCHECK(insert_destination);
 
-  const std::vector _blocks = 
insert_destination->getTouchedBlocks();
+  const std::vector result_blocks = 
insert_destination->getTouchedBlocks();
   for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
 BlockReference result_block = 
storage_manager_->getBlock(result_blocks[bid],
  
insert_destination->getRelation());
@@ -944,7 +944,7 @@ TEST_P(HashJoinOperatorTest, 
VarCharDuplicateKeyHashJoinTest) {
   InsertDestination *insert_destination = 
query_context_->getInsertDestination(prober->getInsertDestinationID());
   DCHECK(insert_destination);
 
-  const std::vector _blocks = 
insert_destination->getTouchedBlocks();
+  const std::vector result_blocks = 
insert_destination->getTouchedBlocks();
   for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
 BlockReference result_block = 
storage_manager_->getBlock(result_blocks[bid],
  
insert_destination->getRelation());
@@ -1123,7 +1123,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
   InsertDestination *insert_destination = 
query_context_->getInsertDestination(prober->getInsertDestinationID());
   DCHECK(insert_destination);
 
-  const std::vector _blocks = 
insert_destination->getTouchedBlocks();
+  const std::vector result_blocks = 
insert_destination->getTouchedBlocks();
   for (std::size_t bid = 0; bid < result_blocks.size(); ++bid) {
 BlockReference result_block = 
storage_manager_->getBlock(result_blocks[bid],
  
insert_destination->getRelation());
@@ -1313,7 +1313,7 @@ TEST_P(HashJoinOperatorTest, 
CompositeKeyHashJoinWithResidualPredicateTest) {
   InsertDestination *insert_destination = 
query_context_->getInsertDestination(prober->getInsertDestinationID());
   DCHECK(insert_destination);
 
-  const std::vector _blocks = 
insert_destination->getTouchedBlocks();
+  const 

[45/46] incubator-quickstep git commit: Small adjust in star schema cost model for # distinct values

[jianqiao]

Small adjust in star schema cost model for # distinct values


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/023c43a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/023c43a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/023c43a4

Branch: refs/heads/fix-iwyu
Commit: 023c43a40b145c225c6deab103655156a034fa90
Parents: d1dbb0d
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Wed Feb 7 15:42:15 2018 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Fri Feb 23 14:59:06 2018 -0600

--
 .../cost_model/StarSchemaSimpleCostModel.cpp|  4 ++--
 .../tests/execution_generator/Partition.test| 16 
 2 files changed, 10 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/023c43a4/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
--
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp 
b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index 5aec4b4..6ab86e5 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -493,7 +493,7 @@ std::size_t StarSchemaSimpleCostModel::getNumDistinctValues(
   return stat.getNumDistinctValues(rel_attr_id);
 }
   }
-  return estimateCardinalityForTableReference(table_reference);
+  return estimateCardinalityForTableReference(table_reference) * 0.1;
 }
 
 bool StarSchemaSimpleCostModel::impliesUniqueAttributes(
@@ -527,7 +527,7 @@ bool StarSchemaSimpleCostModel::impliesUniqueAttributes(
   std::static_pointer_cast(physical_plan);
   const CatalogRelationStatistics  =
   table_reference->relation()->getStatistics();
-  if (stat.hasNumTuples()) {
+  if (stat.isExact() && stat.hasNumTuples()) {
 const std::size_t num_tuples = stat.getNumTuples();
 for (const auto  : attributes) {
   const attribute_id rel_attr_id =

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/023c43a4/query_optimizer/tests/execution_generator/Partition.test
--
diff --git a/query_optimizer/tests/execution_generator/Partition.test 
b/query_optimizer/tests/execution_generator/Partition.test
index 747b969..4b11a04 100644
--- a/query_optimizer/tests/execution_generator/Partition.test
+++ b/query_optimizer/tests/execution_generator/Partition.test
@@ -116,16 +116,16 @@ WHERE dim_2_hash_partitions.id = fact.id
 +---++
 |id |char_col|
 +---++
-|  2|  2 1.414214|
 |  4|  4 2.00|
-|  6|  6 2.449490|
 |  8|  8 2.828427|
 | 12| 12 3.464102|
-| 14| 14 3.741657|
 | 16| 16 4.00|
+| 24| 24 4.898979|
+|  2|  2 1.414214|
+|  6|  6 2.449490|
+| 14| 14 3.741657|
 | 18| 18 4.242641|
 | 22| 22 4.690416|
-| 24| 24 4.898979|
 +---++
 ==
 
@@ -193,15 +193,15 @@ GROUP BY fact.score;
 +++
 |score   |COUNT(*)|
 +++
+|   8|   1|
 |  41.569219381653056|   1|
-|  76.367532368147124|   1|
 |  64|   1|
 |   52.38320341483518|   1|
-|   8|   1|
-|  2.8284271247461903|   1|
-|  14.696938456699067|   1|
+|  76.367532368147124|   1|
 |  22.627416997969522|   1|
 |  117.57550765359254|   1|
+|  2.8284271247461903|   1|
+|  14.696938456699067|   1|
 |  103.18914671611546|   1|
 +++
 ==

[14/46] incubator-quickstep git commit: Relax the sort requirement in columnstore.

[jianqiao]

Relax the sort requirement in columnstore.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ffb8e055
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ffb8e055
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ffb8e055

Branch: refs/heads/fix-iwyu
Commit: ffb8e055a890a9002235a8516e5f2dece2d6228a
Parents: 69fd94b
Author: Zuyu Zhang 
Authored: Mon Oct 9 11:23:08 2017 -0500
Committer: Zuyu Zhang 
Committed: Tue Oct 10 14:10:03 2017 -0500

--
 parser/CMakeLists.txt  |  1 +
 parser/ParseBlockProperties.hpp| 10 --
 parser/tests/Create.test   | 45 +
 query_optimizer/OptimizerTree.hpp  |  9 +++--
 query_optimizer/resolver/Resolver.cpp  | 15 +
 query_optimizer/tests/resolver/Create.test | 38 +
 6 files changed, 99 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ffb8e055/parser/CMakeLists.txt
--
diff --git a/parser/CMakeLists.txt b/parser/CMakeLists.txt
index b3ddf30..d4aaab4 100644
--- a/parser/CMakeLists.txt
+++ b/parser/CMakeLists.txt
@@ -150,6 +150,7 @@ target_link_libraries(quickstep_parser_ParseBlockProperties
   quickstep_parser_ParseTreeNode
   quickstep_utility_Macros
   quickstep_utility_PtrList
+  quickstep_utility_SqlError
   quickstep_utility_StringUtil)
 target_link_libraries(quickstep_parser_ParseCaseExpressions
   quickstep_parser_ParseExpression

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ffb8e055/parser/ParseBlockProperties.hpp
--
diff --git a/parser/ParseBlockProperties.hpp b/parser/ParseBlockProperties.hpp
index ce0cd92..fa176b1 100644
--- a/parser/ParseBlockProperties.hpp
+++ b/parser/ParseBlockProperties.hpp
@@ -31,6 +31,7 @@
 #include "parser/ParseTreeNode.hpp"
 #include "utility/Macros.hpp"
 #include "utility/PtrList.hpp"
+#include "utility/SqlError.hpp"
 #include "utility/StringUtil.hpp"
 
 #include "glog/logging.h"
@@ -143,10 +144,13 @@ class ParseBlockProperties : public ParseTreeNode {
 if (sort_key_value == nullptr) {
   return nullptr;
 }
-if (sort_key_value->getKeyValueType() !=
-ParseKeyValue::KeyValueType::kStringString) {
-  return nullptr;
+if (sort_key_value->getKeyValueType() ==
+ParseKeyValue::KeyValueType::kStringStringList) {
+  THROW_SQL_ERROR_AT(sort_key_value)
+  << "The SORT property must be a string, not a string list.";
 }
+
+DCHECK(sort_key_value->getKeyValueType() == 
ParseKeyValue::KeyValueType::kStringString);
 return static_cast(sort_key_value)->value();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ffb8e055/parser/tests/Create.test
--
diff --git a/parser/tests/Create.test b/parser/tests/Create.test
index 8c11054..3923c13 100644
--- a/parser/tests/Create.test
+++ b/parser/tests/Create.test
@@ -259,6 +259,51 @@ CreateTableStatement[relation_name=test]
   +-value=String[value=rowstore]
 ==
 
+CREATE TABLE test (attr INT, attr2 INT) WITH BLOCKPROPERTIES
+(TYPE columnstore)
+--
+CreateTableStatement[relation_name=test]
++-attribute_list=
+| +-AttributeDefinition[name=attr,type=Int]
+| +-AttributeDefinition[name=attr2,type=Int]
++-block_properties=
+  +-BlockProperties
++-block_property=KeyStringValue[key=TYPE]
+  +-value=String[value=columnstore]
+==
+
+CREATE TABLE test (attr INT, attr2 INT) WITH BLOCKPROPERTIES
+(TYPE columnstore, SORT attr)
+--
+CreateTableStatement[relation_name=test]
++-attribute_list=
+| +-AttributeDefinition[name=attr,type=Int]
+| +-AttributeDefinition[name=attr2,type=Int]
++-block_properties=
+  +-BlockProperties
++-block_property=KeyStringValue[key=TYPE]
+| +-value=String[value=columnstore]
++-block_property=KeyStringValue[key=SORT]
+  +-value=String[value=attr]
+==
+
+CREATE TABLE test (attr INT, attr2 INT) WITH BLOCKPROPERTIES
+(TYPE columnstore, SORT (attr, attr2))
+--
+CreateTableStatement[relation_name=test]
++-attribute_list=
+| +-AttributeDefinition[name=attr,type=Int]
+| +-AttributeDefinition[name=attr2,type=Int]
++-block_properties=
+  +-BlockProperties
++-block_property=KeyStringValue[key=TYPE]
+| +-value=String[value=columnstore]
++-block_property=KeyStringList[key=SORT]
+  +-value_list=
++-String[value=attr]
++-String[value=attr2]
+==
+
 CREATE TABLE test (attr INT) WITH 

[39/46] incubator-quickstep git commit: Upgrade cpplint

[jianqiao]

Upgrade cpplint

- Added several missing headers.
- Modified the lint_everything file to add the command line flags.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d886ddb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d886ddb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d886ddb0

Branch: refs/heads/fix-iwyu
Commit: d886ddb095d3eaafde00bdfb5f85f97aabe5e7e2
Parents: c43107d
Author: Harshad Deshmukh 
Authored: Tue Dec 19 12:56:36 2017 -0600
Committer: Harshad Deshmukh 
Committed: Thu Dec 21 14:30:47 2017 -0600

--
 .travis.yml |2 +-
 catalog/CatalogRelation.hpp |1 +
 catalog/IndexScheme.hpp |1 +
 catalog/tests/Catalog_unittest.cpp  |1 +
 cli/tests/NetworkIO_unittest.cpp|6 +-
 .../aggregation/AggregationHandleSum.cpp|1 +
 expressions/scalar/ScalarAttribute.cpp  |1 +
 .../WindowAggregateFunctionAvg.cpp  |1 +
 .../WindowAggregateFunctionAvg.hpp  |1 +
 .../WindowAggregateFunctionCount.cpp|1 +
 .../WindowAggregateFunctionCount.hpp|1 +
 .../WindowAggregateFunctionMax.cpp  |1 +
 .../WindowAggregateFunctionMax.hpp  |1 +
 .../WindowAggregateFunctionMin.cpp  |1 +
 .../WindowAggregateFunctionMin.hpp  |1 +
 .../WindowAggregateFunctionSum.cpp  |1 +
 .../WindowAggregateFunctionSum.hpp  |1 +
 lint_everything.py  |   47 +
 parser/ParseBlockProperties.cpp |8 +-
 parser/ParseBlockProperties.hpp |8 +-
 parser/ParseString.hpp  |2 +-
 query_optimizer/LIPFilterGenerator.cpp  |1 +
 .../expressions/BinaryExpression.cpp|1 +
 query_optimizer/expressions/Exists.cpp  |4 +-
 query_optimizer/expressions/InTableQuery.cpp|1 +
 query_optimizer/expressions/InValueList.cpp |2 +
 query_optimizer/expressions/LogicalAnd.cpp  |1 +
 query_optimizer/expressions/LogicalOr.cpp   |1 +
 query_optimizer/expressions/SearchedCase.cpp|1 +
 query_optimizer/expressions/SimpleCase.cpp  |2 +
 .../expressions/SubqueryExpression.cpp  |1 +
 query_optimizer/logical/SetOperation.hpp|1 +
 query_optimizer/logical/Sort.hpp|1 +
 .../physical/SharedSubplanReference.hpp |1 +
 query_optimizer/physical/Sort.hpp   |1 +
 query_optimizer/physical/TableReference.hpp |1 +
 query_optimizer/resolver/NameResolver.cpp   |1 +
 query_optimizer/resolver/NameResolver.hpp   |1 +
 query_optimizer/rules/AttachLIPFilters.cpp  |5 +-
 .../StarSchemaHashJoinOrderOptimization.cpp |2 +
 query_optimizer/rules/UnnestSubqueries.cpp  |1 +
 .../rules/tests/PushDownFilter_unittest.cpp |2 +
 .../rules/tests/UpdateExpression_unittest.cpp   |2 +
 .../strategy/tests/Aggregate_unittest.cpp   |1 +
 .../strategy/tests/Join_unittest.cpp|1 +
 .../strategy/tests/OneToOne_unittest.cpp|2 +
 .../strategy/tests/Selection_unittest.cpp   |2 +
 query_optimizer/strategy/tests/StrategyTest.hpp |2 +
 .../tests/ExecutionGeneratorTestRunner.cpp  |2 +-
 .../tests/ExecutionGeneratorTestRunner.hpp  |2 +-
 relational_operators/AggregationOperator.hpp|1 +
 relational_operators/InsertOperator.cpp |1 +
 relational_operators/InsertOperator.hpp |1 +
 .../WindowAggregationOperator.cpp   |1 +
 .../WindowAggregationOperator.hpp   |1 +
 .../BasicColumnStoreTupleStorageSubBlock.hpp|4 +-
 storage/BasicColumnStoreValueAccessor.hpp   |   13 +-
 storage/CollisionFreeVectorTable.hpp|1 +
 storage/CompressedTupleStorageSubBlock.hpp  |4 +-
 storage/LinearOpenAddressingHashTable.hpp   |1 +
 storage/PackedPayloadHashTable.cpp  |1 +
 storage/SplitRowStoreTupleStorageSubBlock.cpp   |1 +
 storage/SplitRowStoreTupleStorageSubBlock.hpp   |1 +
 storage/StorageManager.cpp  |1 +
 storage/SubBlockTypeRegistry.cpp|1 +
 storage/ThreadPrivateCompactKeyHashTable.cpp|2 +
 .../tests/BloomFilterIndexSubBlock_unittest.cpp |1 +
 storage/tests/FileManager_unittest_common.hpp   |4 -
 storage/tests/StorageBlockSort_unittest.cpp |1 +
 third_party/src/cpplint/cpplint.py  | 1205 +-
 third_party/src/cpplint/lint_everything.py  |   40 -
 transaction/LockManager.cpp |1 +
 

[34/46] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/glog/vlog_is_on.h
--
diff --git a/third_party/src/glog/src/glog/vlog_is_on.h 
b/third_party/src/glog/src/glog/vlog_is_on.h
deleted file mode 100644
index 02b0b86..000
--- a/third_party/src/glog/src/glog/vlog_is_on.h
+++ /dev/null
@@ -1,129 +0,0 @@
-// Copyright (c) 1999, 2007, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Ray Sidney and many others
-//
-// Defines the VLOG_IS_ON macro that controls the variable-verbosity
-// conditional logging.
-//
-// It's used by VLOG and VLOG_IF in logging.h
-// and by RAW_VLOG in raw_logging.h to trigger the logging.
-//
-// It can also be used directly e.g. like this:
-//   if (VLOG_IS_ON(2)) {
-// // do some logging preparation and logging
-// // that can't be accomplished e.g. via just VLOG(2) << ...;
-//   }
-//
-// The truth value that VLOG_IS_ON(level) returns is determined by 
-// the three verbosity level flags:
-//   --v=  Gives the default maximal active V-logging level;
-//0 is the default.
-//Normally positive values are used for V-logging levels.
-//   --vmodule=  Gives the per-module maximal V-logging levels to override
-//the value given by --v.
-//E.g. "my_module=2,foo*=3" would change the logging level
-//for all code in source files "my_module.*" and "foo*.*"
-//("-inl" suffixes are also disregarded for this matching).
-//
-// SetVLOGLevel helper function is provided to do limited dynamic control over
-// V-logging by overriding the per-module settings given via --vmodule flag.
-//
-// CAVEAT: --vmodule functionality is not available in non gcc compilers.
-//
-
-#ifndef BASE_VLOG_IS_ON_H_
-#define BASE_VLOG_IS_ON_H_
-
-#include "glog/log_severity.h"
-
-// Annoying stuff for windows -- makes sure clients can import these functions
-#ifndef GOOGLE_GLOG_DLL_DECL
-# if defined(_WIN32) && !defined(__CYGWIN__)
-#   define GOOGLE_GLOG_DLL_DECL  __declspec(dllimport)
-# else
-#   define GOOGLE_GLOG_DLL_DECL
-# endif
-#endif
-
-#if defined(__GNUC__)
-// We emit an anonymous static int* variable at every VLOG_IS_ON(n) site.
-// (Normally) the first time every VLOG_IS_ON(n) site is hit,
-// we determine what variable will dynamically control logging at this site:
-// it's either FLAGS_v or an appropriate internal variable
-// matching the current source file that represents results of
-// parsing of --vmodule flag and/or SetVLOGLevel calls.
-#define VLOG_IS_ON(verboselevel)\
-  __extension__  \
-  ({ static google::int32* vlocal__ = ::kLogSiteUninitialized;  
 \
- google::int32 verbose_level__ = (verboselevel);\
- (*vlocal__ >= verbose_level__) &&  \
- ((vlocal__ != ::kLogSiteUninitialized) ||   \
-  (google::InitVLOG3__(__, _v, \
-   __FILE__, verbose_level__))); })
-#else
-// GNU extensions not available, so we do not support --vmodule.
-// Dynamic value of FLAGS_v always controls the logging level.
-#define VLOG_IS_ON(verboselevel) (FLAGS_v >= (verboselevel))
-#endif
-
-// Set VLOG(_IS_ON) level for module_pattern to log_level.
-// This lets us dynamically control what is normally set by the --vmodule flag.
-// Returns the 

[03/46] incubator-quickstep git commit: Simplified the work order generation.

[jianqiao]

Simplified the work order generation.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8d7284de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8d7284de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8d7284de

Branch: refs/heads/fix-iwyu
Commit: 8d7284decb7ebf5c0eaac232f39027ddd8bf6144
Parents: 77960a4
Author: Zuyu Zhang 
Authored: Mon Aug 21 19:51:55 2017 -0500
Committer: Zuyu Zhang 
Committed: Fri Sep 22 13:43:08 2017 -0500

--
 query_execution/CMakeLists.txt  |   2 -
 query_execution/ForemanDistributed.cpp  |   5 +-
 query_execution/ForemanSingleNode.cpp   |  16 +--
 query_execution/QueryManagerBase.cpp| 136 ---
 query_execution/QueryManagerBase.hpp|  79 ++-
 query_execution/QueryManagerDistributed.cpp |  54 +++-
 query_execution/QueryManagerDistributed.hpp |   3 +-
 query_execution/QueryManagerSingleNode.cpp  |  58 
 query_execution/QueryManagerSingleNode.hpp  |   7 +-
 query_execution/WorkOrdersContainer.hpp |   1 +
 .../tests/QueryManagerSingleNode_unittest.cpp   |  58 
 11 files changed, 152 insertions(+), 267 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8d7284de/query_execution/CMakeLists.txt
--
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 5c750f0..9394c00 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -119,7 +119,6 @@ if (ENABLE_DISTRIBUTED)
 quickstep_storage_StorageBlockInfo
 quickstep_storage_StorageManager
 quickstep_threading_ThreadUtil
-quickstep_utility_EqualsAnyConstant
 quickstep_utility_Macros
 tmb
 ${GFLAGS_LIB_NAME})
@@ -135,7 +134,6 @@ 
target_link_libraries(quickstep_queryexecution_ForemanSingleNode
   quickstep_queryexecution_WorkerDirectory
   quickstep_queryexecution_WorkerMessage
   quickstep_threading_ThreadUtil
-  quickstep_utility_EqualsAnyConstant
   quickstep_utility_Macros
   tmb
   ${GFLAGS_LIB_NAME})

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8d7284de/query_execution/ForemanDistributed.cpp
--
diff --git a/query_execution/ForemanDistributed.cpp 
b/query_execution/ForemanDistributed.cpp
index 942f383..82cc624 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -48,7 +48,6 @@
 #include "storage/StorageBlockInfo.hpp"
 #include "storage/StorageManager.hpp"
 #include "threading/ThreadUtil.hpp"
-#include "utility/EqualsAnyConstant.hpp"
 
 #include "glog/logging.h"
 
@@ -233,9 +232,7 @@ void ForemanDistributed::run() {
 }
 
 bool ForemanDistributed::canCollectNewMessages(const tmb::message_type_id 
message_type) {
-  return !QUICKSTEP_EQUALS_ANY_CONSTANT(message_type,
-kCatalogRelationNewBlockMessage,
-kWorkOrderFeedbackMessage);
+  return message_type != kCatalogRelationNewBlockMessage;
 }
 
 bool ForemanDistributed::isAggregationRelatedWorkOrder(const 
S::WorkOrderMessage ,

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8d7284de/query_execution/ForemanSingleNode.cpp
--
diff --git a/query_execution/ForemanSingleNode.cpp 
b/query_execution/ForemanSingleNode.cpp
index 1501408..d66f1f5 100644
--- a/query_execution/ForemanSingleNode.cpp
+++ b/query_execution/ForemanSingleNode.cpp
@@ -33,7 +33,6 @@
 #include "query_execution/WorkerDirectory.hpp"
 #include "query_execution/WorkerMessage.hpp"
 #include "threading/ThreadUtil.hpp"
-#include "utility/EqualsAnyConstant.hpp"
 #include "utility/Macros.hpp"
 
 #include "gflags/gflags.h"
@@ -179,18 +178,13 @@ void ForemanSingleNode::run() {
 }
 
 bool ForemanSingleNode::canCollectNewMessages(const tmb::message_type_id 
message_type) {
-  if (QUICKSTEP_EQUALS_ANY_CONSTANT(message_type,
-kCatalogRelationNewBlockMessage,
-kWorkOrderFeedbackMessage)) {
-return false;
-  } else if (worker_directory_->getLeastLoadedWorker().second <=
- FLAGS_min_load_per_worker) {
-// If the least loaded worker has only one pending work order, we should
-// 

[05/46] incubator-quickstep git commit: Bug fix in LockManager loop

[jianqiao]

Bug fix in LockManager loop

- Added a false condition for acquire lock
- Added clarifying comment.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9cbb930b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9cbb930b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9cbb930b

Branch: refs/heads/fix-iwyu
Commit: 9cbb930b5adb589f7a2ba8140d8d59227c9a570e
Parents: bf455e2
Author: Harshad Deshmukh 
Authored: Wed Sep 27 10:02:10 2017 -0500
Committer: Harshad Deshmukh 
Committed: Fri Sep 29 10:42:41 2017 -0500

--
 transaction/LockManager.cpp | 15 ---
 1 file changed, 8 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9cbb930b/transaction/LockManager.cpp
--
diff --git a/transaction/LockManager.cpp b/transaction/LockManager.cpp
index 2a3760f..c917b4b 100644
--- a/transaction/LockManager.cpp
+++ b/transaction/LockManager.cpp
@@ -80,21 +80,22 @@ void LockManager::run() {
 if (request.getRequestType() == RequestType::kReleaseLocks) {
   CHECK(releaseAllLocks(request.getTransactionId()))
   << "Unexpected condition occured.";
-
 } else if (acquireLock(request.getTransactionId(),
request.getResourceId(),
request.getAccessMode())) {
+  // Lock has been acquired.
   LOG(INFO) << "Transaction "
 << std::to_string(request.getTransactionId())
-<< " is waiting " + request.getResourceId().toString();
+<< " acquired " + request.getResourceId().toString();
 
-inner_pending_requests_.push(request);
+  permitted_requests_.push(request);
 } else {
-LOG(INFO) << "Transaction "
-  << std::to_string(request.getTransactionId())
-  << " acquired " + request.getResourceId().toString();
+  // We are unable to acquire lock at this point.
+  LOG(INFO) << "Transaction "
+<< std::to_string(request.getTransactionId())
+<< " is waiting " + request.getResourceId().toString();
 
-permitted_requests_.push(request);
+  inner_pending_requests_.push(request);
 }
   }
 }

[16/46] incubator-quickstep git commit: Added ProbabilityStore class

[jianqiao]

Added ProbabilityStore class

- Used to store probabilities of objects.
- Probabilities are of two kinds: Individual and cumulative.
- All the individual probabilities within the store add up to one.
- Support for finding the object with given cumulative probability.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8f094a1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8f094a1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8f094a1c

Branch: refs/heads/fix-iwyu
Commit: 8f094a1c086445b79d6dba36f81326ac06050209
Parents: f820c45
Author: Harshad Deshmukh 
Authored: Fri Sep 29 15:38:42 2017 -0500
Committer: Harshad Deshmukh 
Committed: Wed Oct 11 10:38:36 2017 -0500

--
 query_execution/CMakeLists.txt  |  13 +
 query_execution/ProbabilityStore.hpp| 263 +++
 .../tests/ProbabilityStore_unittest.cpp | 106 
 3 files changed, 382 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f094a1c/query_execution/CMakeLists.txt
--
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 8f797f7..791434a 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -40,6 +40,7 @@ if (ENABLE_DISTRIBUTED)
   add_library(quickstep_queryexecution_PolicyEnforcerDistributed 
PolicyEnforcerDistributed.cpp PolicyEnforcerDistributed.hpp)
 endif(ENABLE_DISTRIBUTED)
 add_library(quickstep_queryexecution_PolicyEnforcerSingleNode 
PolicyEnforcerSingleNode.cpp PolicyEnforcerSingleNode.hpp)
+add_library(quickstep_queryexecution_ProbabilityStore ../empty_src.cpp 
ProbabilityStore.hpp)
 add_library(quickstep_queryexecution_QueryContext QueryContext.cpp 
QueryContext.hpp)
 add_library(quickstep_queryexecution_QueryContext_proto
 ${queryexecution_QueryContext_proto_srcs}
@@ -201,6 +202,9 @@ 
target_link_libraries(quickstep_queryexecution_PolicyEnforcerSingleNode
   quickstep_utility_Macros
   tmb
   ${GFLAGS_LIB_NAME})
+target_link_libraries(quickstep_queryexecution_ProbabilityStore
+  glog
+  quickstep_utility_Macros)
 target_link_libraries(quickstep_queryexecution_QueryContext
   glog
   quickstep_catalog_CatalogDatabaseLite
@@ -372,6 +376,7 @@ target_link_libraries(quickstep_queryexecution
   quickstep_queryexecution_ForemanSingleNode
   quickstep_queryexecution_PolicyEnforcerBase
   quickstep_queryexecution_PolicyEnforcerSingleNode
+  quickstep_queryexecution_ProbabilityStore
   quickstep_queryexecution_QueryContext
   quickstep_queryexecution_QueryContext_proto
   quickstep_queryexecution_QueryExecutionMessages_proto
@@ -425,6 +430,14 @@ if (ENABLE_DISTRIBUTED)
   add_test(BlockLocator_unittest BlockLocator_unittest)
 endif(ENABLE_DISTRIBUTED)
 
+add_executable(ProbabilityStore_unittest
+"${CMAKE_CURRENT_SOURCE_DIR}/tests/ProbabilityStore_unittest.cpp")
+target_link_libraries(ProbabilityStore_unittest
+  gtest
+  gtest_main
+  quickstep_queryexecution_ProbabilityStore)
+add_test(ProbabilityStore_unittest ProbabilityStore_unittest)
+
 add_executable(QueryManagerSingleNode_unittest
   "${CMAKE_CURRENT_SOURCE_DIR}/tests/QueryManagerSingleNode_unittest.cpp")
 target_link_libraries(QueryManagerSingleNode_unittest

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f094a1c/query_execution/ProbabilityStore.hpp
--
diff --git a/query_execution/ProbabilityStore.hpp 
b/query_execution/ProbabilityStore.hpp
new file mode 100644
index 000..079f60b
--- /dev/null
+++ b/query_execution/ProbabilityStore.hpp
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ 

[41/46] incubator-quickstep git commit: Upgraded benchmark third party library.

[jianqiao]

Upgraded benchmark third party library.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5d7aa5f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5d7aa5f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5d7aa5f0

Branch: refs/heads/fix-iwyu
Commit: 5d7aa5f0d1a898db37c46b772fca986b29c45eaa
Parents: d634772
Author: Harshad Deshmukh 
Authored: Wed Dec 27 13:35:07 2017 -0600
Committer: Harshad Deshmukh 
Committed: Wed Dec 27 13:35:07 2017 -0600

--
 third_party/download_and_patch_prerequisites.sh|  7 +++
 third_party/patches/benchmark/CMakeLists.patch | 11 +++
 third_party/patches/benchmark/benchmarkCMake.patch | 11 ---
 .../patches/benchmark/benchmarkSrcCMakeLists.patch | 13 -
 4 files changed, 14 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5d7aa5f0/third_party/download_and_patch_prerequisites.sh
--
diff --git a/third_party/download_and_patch_prerequisites.sh 
b/third_party/download_and_patch_prerequisites.sh
index 2295525..9d0ff00 100755
--- a/third_party/download_and_patch_prerequisites.sh
+++ b/third_party/download_and_patch_prerequisites.sh
@@ -52,7 +52,7 @@ third_party_dir_names=("benchmark"
"glog"
)
 
-third_party_lib_urls=("https://github.com/google/benchmark/archive/v1.1.0.tar.gz;
+third_party_lib_urls=("https://github.com/google/benchmark/archive/v1.3.0.tar.gz;
   "https://github.com/gflags/gflags/archive/v2.1.2.tar.gz;
   
"https://github.com/google/googletest/archive/release-1.8.0.tar.gz;
   "https://github.com/antirez/linenoise/archive/1.0.tar.gz;
@@ -61,7 +61,7 @@ 
third_party_lib_urls=("https://github.com/google/benchmark/archive/v1.1.0.tar.gz
   "https://github.com/google/glog/archive/v0.3.5.tar.gz;
   )
 
-downloaded_archive_names=("v1.1.0.tar.gz"
+downloaded_archive_names=("v1.3.0.tar.gz"
   "v2.1.2.tar.gz"
   "release-1.8.0.tar.gz"
   "1.0.tar.gz"
@@ -127,8 +127,7 @@ patch ${THIRD_PARTY_SRC_DIR}/gflags/src/gflags_reporting.cc 
${PATCH_DIR}/gflags/
 patch ${THIRD_PARTY_SRC_DIR}/re2/CMakeLists.txt ${PATCH_DIR}/re2/re2CMake.patch
 
 # Apply benchmark patches.
-patch ${THIRD_PARTY_SRC_DIR}/benchmark/CMakeLists.txt 
${PATCH_DIR}/benchmark/benchmarkCMake.patch
-patch ${THIRD_PARTY_SRC_DIR}/benchmark/src/CMakeLists.txt 
${PATCH_DIR}/benchmark/benchmarkSrcCMakeLists.patch
+patch ${THIRD_PARTY_SRC_DIR}/benchmark/CMakeLists.txt 
${PATCH_DIR}/benchmark/CMakeLists.patch
 
 # Apply glog patches.
 patch ${THIRD_PARTY_SRC_DIR}/glog/CMakeLists.txt 
${PATCH_DIR}/glog/glogCMakeLists.txt.patch

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5d7aa5f0/third_party/patches/benchmark/CMakeLists.patch
--
diff --git a/third_party/patches/benchmark/CMakeLists.patch 
b/third_party/patches/benchmark/CMakeLists.patch
new file mode 100644
index 000..4ef6c9c
--- /dev/null
+++ b/third_party/patches/benchmark/CMakeLists.patch
@@ -0,0 +1,11 @@
+--- CMakeLists.txt 2017-12-27 13:11:46.2 -0600
 CMakeLists.txt.new 2017-12-27 13:12:28.2 -0600
+@@ -11,7 +11,7 @@
+   endif()
+ endforeach()
+ 
+-option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON)
++option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." 
OFF)
+ option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the 
benchmark library." ON)
+ option(BENCHMARK_ENABLE_LTO "Enable link time optimisation of the benchmark 
library." OFF)
+ option(BENCHMARK_USE_LIBCXX "Build and test using libc++ as the standard 
library." OFF)

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5d7aa5f0/third_party/patches/benchmark/benchmarkCMake.patch
--
diff --git a/third_party/patches/benchmark/benchmarkCMake.patch 
b/third_party/patches/benchmark/benchmarkCMake.patch
deleted file mode 100644
index 56b54ba..000
--- a/third_party/patches/benchmark/benchmarkCMake.patch
+++ /dev/null
@@ -1,11 +0,0 @@
 CMakeLists.txt 2016-10-28 16:22:22.0 -0500
-+++ CMakeLists.txt.new 2017-01-13 15:46:47.626768358 -0600
-@@ -10,7 +10,7 @@
-   endif()
- endforeach()
- 
--option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON)
-+option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." 
OFF)
- 

[22/46] incubator-quickstep git commit: Support Multiple Tuple Inserts

[jianqiao]

Support Multiple Tuple Inserts

Update Fetch


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0fe838df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0fe838df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0fe838df

Branch: refs/heads/fix-iwyu
Commit: 0fe838dfeac901ff03b8334da46b7b9f364447e3
Parents: 79bfcf9
Author: Robert Claus 
Authored: Tue Oct 24 18:08:57 2017 -0500
Committer: Robert Claus 
Committed: Wed Oct 25 13:24:02 2017 -0500

--
 parser/ParseStatement.hpp|   22 +-
 parser/SqlParser.ypp |   18 +-
 parser/preprocessed/SqlParser_gen.cpp| 2646 +
 parser/preprocessed/SqlParser_gen.hpp|3 +-
 query_optimizer/ExecutionGenerator.cpp   |  110 +-
 query_optimizer/logical/InsertTuple.cpp  |6 +-
 query_optimizer/logical/InsertTuple.hpp  |   10 +-
 query_optimizer/physical/InsertTuple.cpp |6 +-
 query_optimizer/physical/InsertTuple.hpp |8 +-
 query_optimizer/resolver/Resolver.cpp|  122 +-
 10 files changed, 1503 insertions(+), 1448 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fe838df/parser/ParseStatement.hpp
--
diff --git a/parser/ParseStatement.hpp b/parser/ParseStatement.hpp
index cee7221..456bdc2 100644
--- a/parser/ParseStatement.hpp
+++ b/parser/ParseStatement.hpp
@@ -653,9 +653,9 @@ class ParseStatementInsertTuple : public 
ParseStatementInsert {
   ParseStatementInsertTuple(const int line_number,
 const int column_number,
 const ParseString *relation_name,
-PtrList *literal_values)
+PtrList 
*literal_values_list)
   : ParseStatementInsert(line_number, column_number, relation_name),
-literal_values_(literal_values) {
+literal_values_(literal_values_list) {
   }
 
   ~ParseStatementInsertTuple() override {
@@ -666,11 +666,11 @@ class ParseStatementInsertTuple : public 
ParseStatementInsert {
   }
 
   /**
-   * @brief Get the parsed literal attribute values to insert.
+   * @brief Get the list of list of parsed literal attribute values to insert.
*
-   * @return The list of literal values to insert.
+   * @return The list of lists of literal values to insert.
**/
-  const PtrList& getLiteralValues() const {
+  const PtrList& getLiteralValues() const {
 return *literal_values_;
   }
 
@@ -685,15 +685,17 @@ class ParseStatementInsertTuple : public 
ParseStatementInsert {
 inline_field_names->push_back("relation_name");
 inline_field_values->push_back(relation_name()->value());
 
-container_child_field_names->push_back("tuple");
-container_child_fields->emplace_back();
-for (const ParseScalarLiteral& literal_value : *literal_values_) {
-  container_child_fields->back().push_back(_value);
+for (const PtrList& literal_values_single_tuple : 
*literal_values_) {
+  container_child_field_names->push_back("tuple");
+  container_child_fields->emplace_back();
+  for (const ParseScalarLiteral& literal_value : 
literal_values_single_tuple) {
+container_child_fields->back().push_back(_value);
+  }
 }
   }
 
  private:
-  std::unique_ptr literal_values_;
+  std::unique_ptr> literal_values_;
 
   DISALLOW_COPY_AND_ASSIGN(ParseStatementInsertTuple);
 };

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fe838df/parser/SqlParser.ypp
--
diff --git a/parser/SqlParser.ypp b/parser/SqlParser.ypp
index 8fbcdd7..ba69b3d 100644
--- a/parser/SqlParser.ypp
+++ b/parser/SqlParser.ypp
@@ -128,6 +128,7 @@ typedef void* yyscan_t;
   quickstep::NumericParseLiteralValue *numeric_literal_value_;
   quickstep::ParseLiteralValue *literal_value_;
   quickstep::PtrList *literal_value_list_;
+  quickstep::PtrList 
*literal_value_list_multiple_;
 
   quickstep::ParseExpression *expression_;
 
@@ -387,6 +388,9 @@ void NotSupported(const YYLTYPE *location, yyscan_t 
yyscanner, const std::string
 %type 
   literal_value_commalist
 
+%type 
+  literal_value_commalist_multiple
+
 %type 
   expression_base
   unary_expression
@@ -1101,8 +1105,8 @@ insert_statement:
 NotSupported(&@4, yyscanner, "list of column names in INSERT statement");
 YYERROR;
   }
-  | TOKEN_INSERT TOKEN_INTO any_name TOKEN_VALUES '(' literal_value_commalist 
')' {
-$$ = new quickstep::ParseStatementInsertTuple(@1.first_line, 
@1.first_column, $3, $6);
+  | TOKEN_INSERT TOKEN_INTO any_name TOKEN_VALUES 

[46/46] incubator-quickstep git commit: Fix iwyu include path

[jianqiao]

Fix iwyu include path


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c2ed5c69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c2ed5c69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c2ed5c69

Branch: refs/heads/fix-iwyu
Commit: c2ed5c69b6b8dad07d7410beb0c8292ea1a746e0
Parents: 023c43a
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Fri Sep 1 15:07:41 2017 -0500
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Mon Feb 26 13:15:06 2018 -0600

--
 third_party/src/iwyu/iwyu_helper.py | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c2ed5c69/third_party/src/iwyu/iwyu_helper.py
--
diff --git a/third_party/src/iwyu/iwyu_helper.py 
b/third_party/src/iwyu/iwyu_helper.py
index dff4d55..42bf84c 100755
--- a/third_party/src/iwyu/iwyu_helper.py
+++ b/third_party/src/iwyu/iwyu_helper.py
@@ -22,12 +22,12 @@ QUICKSTEP_INCLUDES = [ '.',
'./build/third_party/gflags/include',
'./build/third_party/protobuf/include',
'./build/third_party/tmb/include',
-   './third_party/benchmark/include',
-   './third_party/glog/src',
-   './third_party/googletest/googletest/include',
-   './third_party/protobuf/src',
-   './third_party/re2',
-   './third_party/tmb/include']
+   './third_party/src/benchmark/include',
+   './third_party/src/glog/src',
+   './third_party/src/googletest/googletest/include',
+   './third_party/src/protobuf/src',
+   './third_party/src/re2',
+   './third_party/src/tmb/include']
 QUICKSTEP_DEFINES = [ '-DQUICKSTEP_DEBUG',
   '-DQUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION', ]
 CLANG_FLAGS = [ '-std=c++14', '-x', 'c++', ]

[11/46] incubator-quickstep git commit: Removed the virtual function call in InvokeOnAnyValueAccessor.

[jianqiao]

Removed the virtual function call in InvokeOnAnyValueAccessor.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/79710ca6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/79710ca6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/79710ca6

Branch: refs/heads/fix-iwyu
Commit: 79710ca6c6b75410bf2c26b4646acbfc5d554d7c
Parents: 696a783
Author: Zuyu Zhang 
Authored: Fri Oct 6 14:34:21 2017 -0500
Committer: Zuyu Zhang 
Committed: Mon Oct 9 11:37:16 2017 -0500

--
 storage/SplitRowStoreTupleStorageSubBlock.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/79710ca6/storage/SplitRowStoreTupleStorageSubBlock.cpp
--
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.cpp 
b/storage/SplitRowStoreTupleStorageSubBlock.cpp
index 5060208..9f5a839 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.cpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.cpp
@@ -343,7 +343,7 @@ tuple_id 
SplitRowStoreTupleStorageSubBlock::bulkInsertPartialTuplesImpl(
 
   InvokeOnAnyValueAccessor(
   accessor,
-  [&](auto *accessor) -> void {  // NOLINT(build/c++11
+  [&](auto *accessor) -> void {  // NOLINT(build/c++11)
 BitVector tuple_null_bitmap(tuple_slot, num_null_attrs_);
 const std::size_t nullmap_size = 
BitVector::BytesNeeded(num_null_attrs_);
 
@@ -410,7 +410,7 @@ tuple_id 
SplitRowStoreTupleStorageSubBlock::bulkInsertPartialTuplesImpl(
   max_num_tuples_to_insert += additional_tuples_insert;
 }
   }
-} while (fill_to_capacity && !accessor->iterationFinishedVirtual() &&
+} while (fill_to_capacity && !accessor->iterationFinished() &&
  num_tuples_inserted < max_num_tuples_to_insert);
   });
 

[42/46] incubator-quickstep git commit: IDE Documentation fixes

[jianqiao]

IDE Documentation fixes


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4a945a6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4a945a6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4a945a6b

Branch: refs/heads/fix-iwyu
Commit: 4a945a6b9a64b9735f8cdf91556c6661e4731c43
Parents: 5d7aa5f
Author: Harshad Deshmukh 
Authored: Wed Dec 27 10:47:00 2017 -0600
Committer: Harshad Deshmukh 
Committed: Thu Jan 11 13:52:20 2018 -0600

--
 WORKING_WITH_AN_IDE.md | 24 
 1 file changed, 12 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a945a6b/WORKING_WITH_AN_IDE.md
--
diff --git a/WORKING_WITH_AN_IDE.md b/WORKING_WITH_AN_IDE.md
index 017a174..fca1217 100644
--- a/WORKING_WITH_AN_IDE.md
+++ b/WORKING_WITH_AN_IDE.md
@@ -1,17 +1,17 @@
-#Developing Quickstep with IDEs
+# Developing Quickstep with IDEs
 
-##Who should read this document?
+## Who should read this document?
 Any developer who prefers to work with IDEs instead of a terminal and
 vi, emacs, or the like. In other words, this document aims to make it easier
 for developers of Quickstep to work with IDEs. Over time, there will be
 information about working with other IDEs, but to start out, here are
 instructions for working with XCode on OSX.
 
-##Developing Quickstep with Xcode on OSX
+## Developing Quickstep with Xcode on OSX
 The instructions here were first written and verified on OSX El Capitan,
 v.10.11.2, using Xcode v.7.2.
 
-###1: Install Xcode and command line tools
+### 1: Install Xcode and command line tools
 First, you will need to download and install Xcode and the associated command
 line tools. There are multiple ways to do this, including going to
 https://developer.apple.com/xcode/ and downloading both Xcode and the command
@@ -25,7 +25,7 @@ cc
 This command should trigger a sequence of downloads to get you both Xcode
 and the assocaited command line tools.
 
-###2: Install cmake
+### 2: Install cmake
 Unfortunately, the command line tools do not package `cmake`, which is needed
 to build Quickstep. You can install cmake using brew as follows:
 
@@ -43,7 +43,7 @@ Terminal app by typing:
 brew install cmake
 ```
 
-###3: Build Quicktep
+### 3: Build Quicktep
 Checkout the Quickstep code from git, and also checkout the associated 
submodules.
 If you have not read it already, this would be good time to read the file
 [BUILDING.md](BUILDING.md), but do not run the cmake command mentioned there. 
Instead, go
@@ -103,7 +103,7 @@ these and other command line options by typing:
 ```
 
 
-###4: Debug Quickstep
+### 4: Debug Quickstep
 Now you can debug as you would any normal process in Xcode. Note the
 linenoise option in the cmake command above is important if you are going
 to run quickstep from Xcode (by hitting the "play" button). If you are
@@ -134,7 +134,7 @@ when it starts up. It you had set a breakpoint and the 
program executes that
 code, then Xcode (lldb) will stop at the breakpoint. Or, if there is a crash,
 you can examine the stack in Xcode.
 
-###5: Unit Tests
+### 5: Unit Tests
 Individual unit tests show up as target schemas, so you can simply select them
 and run the unit test of interest.
 
@@ -143,14 +143,14 @@ does not work. So, this is a known limitation at this 
point. You can, however,
 follow the instructions for a [BUILDING.md](command-line build) with cmake and
 then run `ctest` to run the full suite of unit tests.
 
-###6: Other known issues
+### 6: Other known issues
 
-Modifying CMake Files
+ Modifying CMake Files
 If you change any of the cmake files (such as any of the CMakeLists.txt
 files), then you will have to [redo step 3](#3-build-quicktep) above to
 create a new Xcode project file.
 
-Running Python Validation Scripts
+ Running Python Validation Scripts
 Quickstep developers have a few python scripts that are used to mechanically
 check code. These scripts are written in Python 2 and are not compatible with
 Python 3, so they use `python2` as the interpreter in their shebangs. While
@@ -167,7 +167,7 @@ sudo ln -s /usr/bin/python2.7 /usr/local/bin/python2
 2.X version is on your machine.)
 
 After putting the symlink in place, you should be able to run
-`./third_party/cpplint/lint_everything.py` (which applies a modified version of
+`./lint_everything.py` (which applies a modified version of
 Google cpplint to all C++ sources) and `./validate_cmakelists.py` (which checks
 that dependencies in CMakeLists.txt files exactly match included headers in C++
 sources) from the root quickstep source directory to check your code. There is

[13/46] incubator-quickstep git commit: Added a new set API for TupleIdSequence.

[jianqiao]

Added a new set API for TupleIdSequence.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/69fd94b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/69fd94b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/69fd94b8

Branch: refs/heads/fix-iwyu
Commit: 69fd94b8917c53e5a7a3e5899382c6ba12cf1c2b
Parents: a61b03d
Author: Zuyu Zhang 
Authored: Thu Sep 28 19:28:30 2017 -0500
Committer: Zuyu Zhang 
Committed: Mon Oct 9 13:14:07 2017 -0500

--
 expressions/scalar/ScalarCaseExpression.cpp |   2 +-
 relational_operators/HashJoinOperator.cpp   |   2 +-
 storage/AggregationOperationState.cpp   |   3 +-
 storage/BloomFilterIndexSubBlock.cpp|   2 +-
 storage/CMakeLists.txt  |   1 +
 storage/CSBTreeIndexSubBlock.cpp|  22 +-
 ...ompressedColumnStoreTupleStorageSubBlock.cpp |  36 +--
 ...ressedPackedRowStoreTupleStorageSubBlock.cpp |  36 +--
 storage/InsertDestination.hpp   |   4 +-
 storage/SMAIndexSubBlock.cpp|   2 +-
 storage/StorageBlock.cpp|   9 +-
 storage/TupleIdSequence.hpp |  15 ++
 storage/TupleStorageSubBlock.cpp|   2 +-
 utility/BitVector.hpp   |  16 ++
 utility/lip_filter/LIPFilterAdaptiveProber.hpp  |   4 +-
 utility/tests/BitVector_unittest.cpp| 226 +--
 16 files changed, 206 insertions(+), 176 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/69fd94b8/expressions/scalar/ScalarCaseExpression.cpp
--
diff --git a/expressions/scalar/ScalarCaseExpression.cpp 
b/expressions/scalar/ScalarCaseExpression.cpp
index 00a7710..6847425 100644
--- a/expressions/scalar/ScalarCaseExpression.cpp
+++ b/expressions/scalar/ScalarCaseExpression.cpp
@@ -319,7 +319,7 @@ ColumnVectorPtr ScalarCaseExpression::getAllValuesForJoin(
 *right_accessor,
 right_relation_id,
 check_pair.second)) {
-current_case_positions->set(pos, true);
+current_case_positions->set(pos);
 current_case_matches.emplace_back(check_pair);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/69fd94b8/relational_operators/HashJoinOperator.cpp
--
diff --git a/relational_operators/HashJoinOperator.cpp 
b/relational_operators/HashJoinOperator.cpp
index b07e4cb..4083bd3 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -758,7 +758,7 @@ void HashSemiJoinWorkOrder::executeWithResidualPredicate() {
   *probe_accessor,
   probe_relation_id,
   hash_match.second)) {
-filter.set(hash_match.second, true);
+filter.set(hash_match.second);
   }
 }
   }

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/69fd94b8/storage/AggregationOperationState.cpp
--
diff --git a/storage/AggregationOperationState.cpp 
b/storage/AggregationOperationState.cpp
index 0f4795f..73f1983 100644
--- a/storage/AggregationOperationState.cpp
+++ b/storage/AggregationOperationState.cpp
@@ -579,8 +579,7 @@ void 
AggregationOperationState::aggregateBlockHashTableImplPartitioned(
   accessor->getTupleWithAttributes(group_by_key_ids));
   const std::size_t curr_tuple_partition_id =
   curr_tuple->getTupleHash() % num_partitions;
-  partition_membership[curr_tuple_partition_id]->set(
-  accessor->getCurrentPosition(), true);
+  
partition_membership[curr_tuple_partition_id]->set(accessor->getCurrentPosition());
 }
 
 // Aggregate each partition.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/69fd94b8/storage/BloomFilterIndexSubBlock.cpp
--
diff --git a/storage/BloomFilterIndexSubBlock.cpp 
b/storage/BloomFilterIndexSubBlock.cpp
index 4351c05..1af3872 100644
--- a/storage/BloomFilterIndexSubBlock.cpp
+++ b/storage/BloomFilterIndexSubBlock.cpp
@@ -206,7 +206,7 @@ TupleIdSequence* 
BloomFilterIndexSubBlock::getMatchesForPredicate(
 } else {
   for (tuple_id tid = 0; tid <= tuple_store_.getMaxTupleID(); ++tid) {
 if (tuple_store_.hasTupleWithID(tid)) {
-  

[06/46] incubator-quickstep git commit: Created a class to track execution statistics

[jianqiao]

Created a class to track execution statistics

- Stats are maintained for active operators in the query.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1b2698d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1b2698d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1b2698d2

Branch: refs/heads/fix-iwyu
Commit: 1b2698d2225bfab59fb675da5f92a2285dd5650c
Parents: 9cbb930
Author: Harshad Deshmukh 
Authored: Wed Sep 27 14:55:16 2017 -0500
Committer: Harshad Deshmukh 
Committed: Fri Sep 29 12:10:52 2017 -0500

--
 query_execution/CMakeLists.txt |   5 +
 query_execution/ExecutionStats.hpp | 211 
 2 files changed, 216 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b2698d2/query_execution/CMakeLists.txt
--
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 9394c00..8f797f7 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -29,6 +29,7 @@ if (ENABLE_DISTRIBUTED)
   add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp 
BlockLocator.hpp)
   add_library(quickstep_queryexecution_BlockLocatorUtil BlockLocatorUtil.cpp 
BlockLocatorUtil.hpp)
 endif(ENABLE_DISTRIBUTED)
+add_library(quickstep_queryexecution_ExecutionStats ../empty_src.cpp 
ExecutionStats.hpp)
 add_library(quickstep_queryexecution_ForemanBase ../empty_src.cpp 
ForemanBase.hpp)
 if (ENABLE_DISTRIBUTED)
   add_library(quickstep_queryexecution_ForemanDistributed 
ForemanDistributed.cpp ForemanDistributed.hpp)
@@ -123,6 +124,9 @@ if (ENABLE_DISTRIBUTED)
 tmb
 ${GFLAGS_LIB_NAME})
 endif(ENABLE_DISTRIBUTED)
+target_link_libraries(quickstep_queryexecution_ExecutionStats
+  glog
+  quickstep_utility_Macros)
 target_link_libraries(quickstep_queryexecution_ForemanSingleNode
   glog
   quickstep_queryexecution_AdmitRequestMessage
@@ -363,6 +367,7 @@ 
target_link_libraries(quickstep_queryexecution_WorkerSelectionPolicy
 add_library(quickstep_queryexecution ../empty_src.cpp QueryExecutionModule.hpp)
 target_link_libraries(quickstep_queryexecution
   quickstep_queryexecution_AdmitRequestMessage
+  quickstep_queryexecution_ExecutionStats
   quickstep_queryexecution_ForemanBase
   quickstep_queryexecution_ForemanSingleNode
   quickstep_queryexecution_PolicyEnforcerBase

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b2698d2/query_execution/ExecutionStats.hpp
--
diff --git a/query_execution/ExecutionStats.hpp 
b/query_execution/ExecutionStats.hpp
new file mode 100644
index 000..8d19651
--- /dev/null
+++ b/query_execution/ExecutionStats.hpp
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_EXECUTION_STATS_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_EXECUTION_STATS_HPP_
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+/** \addtogroup QueryExecution
+ *  @{
+ */
+
+/**
+ * @brief Record the execution stats of a query.
+ *
+ * @note The time is measured in microseconds.
+ **/
+class ExecutionStats {
+ public:
+  /**
+   * @brief Constructor
+   *
+   * @param max_entries The maximum number of entries we remember for each
+   *operator.
+   **/
+  explicit ExecutionStats(const std::size_t max_entries)
+  : max_entries_(max_entries) {}
+
+  /**
+   * @brief Get the number of active operators in stats.
+   **/
+  std::size_t getNumActiveOperators() const {
+return 

[26/46] incubator-quickstep git commit: Get the list of referenced base relations

[jianqiao]

Get the list of referenced base relations

- Find the base relations that are referenced in the query.
- The referenced relations are stored in the QueryHandle.
- Separate method in QueryProcessor class to just get this list of relations,
  without needing to fully optimize the query.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8a84039e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8a84039e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8a84039e

Branch: refs/heads/fix-iwyu
Commit: 8a84039e140ed80376505b6da950c08a6112470e
Parents: 39c6214
Author: Harshad Deshmukh 
Authored: Tue Nov 28 16:16:59 2017 -0600
Committer: Harshad Deshmukh 
Committed: Fri Dec 1 14:26:19 2017 -0600

--
 query_optimizer/CMakeLists.txt  |  2 +
 query_optimizer/LogicalGenerator.cpp|  1 -
 query_optimizer/Optimizer.cpp   | 11 +++
 query_optimizer/Optimizer.hpp   | 15 
 query_optimizer/QueryHandle.hpp | 21 +
 query_optimizer/QueryProcessor.cpp  |  7 ++
 query_optimizer/QueryProcessor.hpp  | 11 +++
 query_optimizer/resolver/CMakeLists.txt |  1 +
 query_optimizer/resolver/Resolver.cpp   |  7 ++
 query_optimizer/resolver/Resolver.hpp   |  5 ++
 query_optimizer/rules/CMakeLists.txt| 13 
 .../rules/ReferencedBaseRelations.cpp   | 66 
 .../rules/ReferencedBaseRelations.hpp   | 80 
 13 files changed, 239 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 1e4e346..9128c63 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -206,7 +206,9 @@ 
target_link_libraries(quickstep_queryoptimizer_LogicalToPhysicalMapper
 target_link_libraries(quickstep_queryoptimizer_Optimizer
   quickstep_queryoptimizer_ExecutionGenerator
   quickstep_queryoptimizer_LogicalGenerator
+  quickstep_queryoptimizer_OptimizerContext
   quickstep_queryoptimizer_PhysicalGenerator
+  quickstep_queryoptimizer_resolver_Resolver
   quickstep_utility_Macros)
 target_link_libraries(quickstep_queryoptimizer_OptimizerContext
   quickstep_queryoptimizer_expressions_ExprId

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/LogicalGenerator.cpp
--
diff --git a/query_optimizer/LogicalGenerator.cpp 
b/query_optimizer/LogicalGenerator.cpp
index abeca53..111fa1f 100644
--- a/query_optimizer/LogicalGenerator.cpp
+++ b/query_optimizer/LogicalGenerator.cpp
@@ -23,7 +23,6 @@
 #include 
 
 #include "parser/ParseStatement.hpp"
-
 #include "query_optimizer/OptimizerContext.hpp"
 #include "query_optimizer/Validator.hpp"
 #include "query_optimizer/logical/Logical.hpp"

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/Optimizer.cpp
--
diff --git a/query_optimizer/Optimizer.cpp b/query_optimizer/Optimizer.cpp
index 1b91574..d002ca1 100644
--- a/query_optimizer/Optimizer.cpp
+++ b/query_optimizer/Optimizer.cpp
@@ -21,6 +21,8 @@
 
 #include "query_optimizer/ExecutionGenerator.hpp"
 #include "query_optimizer/LogicalGenerator.hpp"
+#include "query_optimizer/OptimizerContext.hpp"
+#include "query_optimizer/resolver/Resolver.hpp"
 
 namespace quickstep {
 namespace optimizer {
@@ -38,5 +40,14 @@ void Optimizer::generateQueryHandle(const ParseStatement 
_statement,
   logical_generator.generatePlan(*catalog_database, parse_statement)));
 }
 
+void Optimizer::findReferencedBaseRelations(const ParseStatement 
_statement,
+CatalogDatabase *catalog_database,
+QueryHandle *query_handle) {
+  OptimizerContext optimizer_context;
+  resolver::Resolver resolver(*catalog_database, _context);
+  resolver.resolve(parse_statement);
+  
query_handle->setReferencedBaseRelations(resolver.getReferencedBaseRelations());
+}
+
 }  // namespace optimizer
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/Optimizer.hpp
--
diff --git a/query_optimizer/Optimizer.hpp 

[43/46] incubator-quickstep git commit: Add a flag to allow disabling of Comparison inline expansion to enable acceleration of Quickstep build.

[jianqiao]

Add a flag to allow disabling of Comparison inline expansion to enable 
acceleration of Quickstep build.

(for development productivity as well as solving the Travis CI timeout problem)


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/539e1ebe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/539e1ebe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/539e1ebe

Branch: refs/heads/fix-iwyu
Commit: 539e1ebe09b5d1a2d86069ed1fdc6e9fb38c5ce7
Parents: 4a945a6
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Fri Feb 2 17:27:59 2018 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Fri Feb 2 17:31:24 2018 -0600

--
 .travis.yml |  3 ++-
 CMakeLists.txt  | 14 ++
 .../comparisons/AsciiStringComparators-inl.hpp  |  2 ++
 .../comparisons/AsciiStringComparators.hpp  |  4 
 types/operations/comparisons/Comparison.cpp | 20 
 .../comparisons/LiteralComparators-inl.hpp  |  2 ++
 .../comparisons/LiteralComparators.hpp  |  4 
 7 files changed, 28 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/.travis.yml
--
diff --git a/.travis.yml b/.travis.yml
index 4e7833f..6517ae8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -76,7 +76,8 @@ before_script:
-D CMAKE_CXX_COMPILER=$CXX
-D CMAKE_LINKER=$CLINKER
-D USE_TCMALLOC=0
-   -D VECTOR_COPY_ELISION_LEVEL=$VECTOR_COPY_ELISION_LEVEL ..)
+   -D VECTOR_COPY_ELISION_LEVEL=$VECTOR_COPY_ELISION_LEVEL
+   -D ENABLE_COMPARISON_INLINE_EXPANSION=OFF ..)
 
 script:
   - ./lint_everything.py

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 88835ac..c777a6a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -145,6 +145,20 @@ if (ENABLE_VECTOR_PREDICATE_SHORT_CIRCUIT)
   )
 endif()
 
+set(COMPARISON_INLINE_EXPANSION_MSG_LIST
+"This option controls whether to enable inlined template expansion "
+"of comparison predicates. WARNING: This option should only be "
+"turned off for development use. Turning off this option will greatly "
+"reduce Quickstep compile time but incur drastic performance degradation.")
+string(REPLACE ";" "" COMPARISON_INLINE_EXPANSION_MSG 
${COMPARISON_INLINE_EXPANSION_MSG_LIST})
+option(ENABLE_COMPARISON_INLINE_EXPANSION ${COMPARISON_INLINE_EXPANSION_MSG} 
ON)
+if (ENABLE_COMPARISON_INLINE_EXPANSION)
+  set_property(
+DIRECTORY
+APPEND PROPERTY COMPILE_DEFINITIONS 
QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
+  )
+endif()
+
 option(ENABLE_NETWORK_CLI "Allows use of the network cli" OFF)
 option(ENABLE_DISTRIBUTED "Use the distributed version of Quickstep" OFF)
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/types/operations/comparisons/AsciiStringComparators-inl.hpp
--
diff --git a/types/operations/comparisons/AsciiStringComparators-inl.hpp 
b/types/operations/comparisons/AsciiStringComparators-inl.hpp
index fd0d17d..8b2c1bf 100644
--- a/types/operations/comparisons/AsciiStringComparators-inl.hpp
+++ b/types/operations/comparisons/AsciiStringComparators-inl.hpp
@@ -46,6 +46,7 @@
 
 namespace quickstep {
 
+#ifdef QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
 template  class ComparisonFunctor,
   bool left_nullable, bool left_null_terminated, bool left_longer,
   bool right_nullable, bool right_null_terminated, bool right_longer>
@@ -586,6 +587,7 @@ TypedValue AsciiStringUncheckedComparator<ComparisonFunctor,
 
   return new_value;
 }
+#endif  // QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
 
 }  // namespace quickstep
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/types/operations/comparisons/AsciiStringComparators.hpp
--
diff --git a/types/operations/comparisons/AsciiStringComparators.hpp 
b/types/operations/comparisons/AsciiStringComparators.hpp
index 936fd1f..2aec8c4 100644
--- a/types/operations/comparisons/AsciiStringComparators.hpp
+++ b/types/operations/comparisons/AsciiStringComparators.hpp
@@ -122,6 +122,7 @@ class AsciiStringUncheckedComparator : public 
UncheckedComparator {
0);
   }
 
+#ifdef QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
   TupleI

[30/46] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/symbolize.h
--
diff --git a/third_party/src/glog/src/symbolize.h 
b/third_party/src/glog/src/symbolize.h
deleted file mode 100644
index 1ebe4dd..000
--- a/third_party/src/glog/src/symbolize.h
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright (c) 2006, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Satoru Takabayashi
-//
-// This library provides Symbolize() function that symbolizes program
-// counters to their corresponding symbol names on linux platforms.
-// This library has a minimal implementation of an ELF symbol table
-// reader (i.e. it doesn't depend on libelf, etc.).
-//
-// The algorithm used in Symbolize() is as follows.
-//
-//   1. Go through a list of maps in /proc/self/maps and find the map
-//   containing the program counter.
-//
-//   2. Open the mapped file and find a regular symbol table inside.
-//   Iterate over symbols in the symbol table and look for the symbol
-//   containing the program counter.  If such a symbol is found,
-//   obtain the symbol name, and demangle the symbol if possible.
-//   If the symbol isn't found in the regular symbol table (binary is
-//   stripped), try the same thing with a dynamic symbol table.
-//
-// Note that Symbolize() is originally implemented to be used in
-// FailureSignalHandler() in base/google.cc.  Hence it doesn't use
-// malloc() and other unsafe operations.  It should be both
-// thread-safe and async-signal-safe.
-
-#ifndef BASE_SYMBOLIZE_H_
-#define BASE_SYMBOLIZE_H_
-
-#include "utilities.h"
-#include "config.h"
-#include "glog/logging.h"
-
-#ifdef HAVE_SYMBOLIZE
-
-#if defined(__ELF__)  // defined by gcc on Linux
-#include 
-#include   // For ElfW() macro.
-
-// If there is no ElfW macro, let's define it by ourself.
-#ifndef ElfW
-# if SIZEOF_VOID_P == 4
-#  define ElfW(type) Elf32_##type
-# elif SIZEOF_VOID_P == 8
-#  define ElfW(type) Elf64_##type
-# else
-#  error "Unknown sizeof(void *)"
-# endif
-#endif
-
-_START_GOOGLE_NAMESPACE_
-
-// Gets the section header for the given name, if it exists. Returns true on
-// success. Otherwise, returns false.
-bool GetSectionHeaderByName(int fd, const char *name, size_t name_len,
-ElfW(Shdr) *out);
-
-_END_GOOGLE_NAMESPACE_
-
-#endif  /* __ELF__ */
-
-_START_GOOGLE_NAMESPACE_
-
-// Installs a callback function, which will be called right before a symbol 
name
-// is printed. The callback is intended to be used for showing a file name and 
a
-// line number preceding a symbol name.
-// "fd" is a file descriptor of the object file containing the program
-// counter "pc". The callback function should write output to "out"
-// and return the size of the output written. On error, the callback
-// function should return -1.
-typedef int (*SymbolizeCallback)(int fd, void *pc, char *out, size_t out_size,
- uint64 relocation);
-void InstallSymbolizeCallback(SymbolizeCallback callback);
-
-_END_GOOGLE_NAMESPACE_
-
-#endif
-
-_START_GOOGLE_NAMESPACE_
-
-// Symbolizes a program counter.  On success, returns true and write the
-// symbol name to "out".  The symbol name is demangled if possible
-// (supports symbols generated by GCC 3.x or newer).  Otherwise,
-// returns false.
-bool Symbolize(void *pc, char *out, int out_size);
-
-_END_GOOGLE_NAMESPACE_
-
-#endif  // 

[15/46] incubator-quickstep git commit: Removed unused argument always_mark_full.

[jianqiao]

Removed unused argument always_mark_full.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f820c45e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f820c45e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f820c45e

Branch: refs/heads/fix-iwyu
Commit: f820c45ee56a9e74671fb1c22c5e6b7e13471b5d
Parents: ffb8e05
Author: Zuyu Zhang 
Authored: Thu Oct 5 17:18:05 2017 -0500
Committer: Zuyu Zhang 
Committed: Tue Oct 10 14:14:29 2017 -0500

--
 storage/InsertDestination.cpp  | 73 ++---
 storage/InsertDestination.hpp  | 21 -
 storage/InsertDestinationInterface.hpp | 12 ++---
 3 files changed, 36 insertions(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f820c45e/storage/InsertDestination.cpp
--
diff --git a/storage/InsertDestination.cpp b/storage/InsertDestination.cpp
index 8821019..c2b44d8 100644
--- a/storage/InsertDestination.cpp
+++ b/storage/InsertDestination.cpp
@@ -199,7 +199,7 @@ void InsertDestination::insertTupleInBatch(const Tuple 
) {
   returnBlock(std::move(output_block), false);
 }
 
-void InsertDestination::bulkInsertTuples(ValueAccessor *accessor, bool 
always_mark_full) {
+void InsertDestination::bulkInsertTuples(ValueAccessor *accessor, const bool 
always_mark_full) {
   InvokeOnAnyValueAccessor(
   accessor,
   [&](auto *accessor) -> void {  // NOLINT(build/c++11)
@@ -207,23 +207,17 @@ void InsertDestination::bulkInsertTuples(ValueAccessor 
*accessor, bool always_ma
 while (!accessor->iterationFinished()) {
   MutableBlockReference output_block = this->getBlockForInsertion();
   // FIXME(chasseur): Deal with TupleTooLargeForBlock exception.
-  if (output_block->bulkInsertTuples(accessor) == 0) {
-// output_block is full.
-this->returnBlock(std::move(output_block), true);
-  } else {
-// Bulk insert into output_block was successful. output_block
-// will be rebuilt when there won't be any more insertions to it.
-this->returnBlock(std::move(output_block),
-  always_mark_full || !accessor->iterationFinished());
-  }
+  const auto num_tuples_inserted = 
output_block->bulkInsertTuples(accessor);
+  this->returnBlock(std::move(output_block),
+num_tuples_inserted == 0 || 
!accessor->iterationFinished() ||
+always_mark_full);
 }
   });
 }
 
 void InsertDestination::bulkInsertTuplesWithRemappedAttributes(
 const std::vector _map,
-ValueAccessor *accessor,
-bool always_mark_full) {
+ValueAccessor *accessor) {
   InvokeOnAnyValueAccessor(
   accessor,
   [&](auto *accessor) -> void {  // NOLINT(build/c++11)
@@ -231,17 +225,10 @@ void 
InsertDestination::bulkInsertTuplesWithRemappedAttributes(
 while (!accessor->iterationFinished()) {
   MutableBlockReference output_block = this->getBlockForInsertion();
   // FIXME(chasseur): Deal with TupleTooLargeForBlock exception.
-  if (output_block->bulkInsertTuplesWithRemappedAttributes(
-  attribute_map,
-  accessor) == 0) {
-// output_block is full.
-this->returnBlock(std::move(output_block), true);
-  } else {
-// Bulk insert into output_block was successful. output_block
-// will be rebuilt when there won't be any more insertions to it.
-this->returnBlock(std::move(output_block),
-  always_mark_full || !accessor->iterationFinished());
-  }
+  const auto num_tuples_inserted =
+  output_block->bulkInsertTuplesWithRemappedAttributes(attribute_map, 
accessor);
+  this->returnBlock(std::move(output_block),
+num_tuples_inserted == 0 || 
!accessor->iterationFinished());
 }
   });
 }
@@ -267,8 +254,7 @@ void removeGapOnlyAccessors(
 }
 
 void InsertDestination::bulkInsertTuplesFromValueAccessors(
-const std::vector> 
_attribute_map,
-bool always_mark_full) {
+const std::vector> 
_attribute_map) {
   // Handle pathological corner case where there are no accessors
   if (accessor_attribute_map.size() == 0)
 return;
@@ -323,9 +309,7 @@ void InsertDestination::bulkInsertTuplesFromValueAccessors(
 
 // Update the header for output_block and then return it.
 output_block->bulkInsertPartialTuplesFinalize(num_tuples_inserted);
-const bool mark_full = always_mark_full
-   || !first_accessor->iterationFinishedVirtual();
-this->returnBlock(std::move(output_block), mark_full);
+

[08/46] incubator-quickstep git commit: Fixed the root path check in the cyclic_dependency.py.

[jianqiao]

Fixed the root path check in the cyclic_dependency.py.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0898a77b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0898a77b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0898a77b

Branch: refs/heads/fix-iwyu
Commit: 0898a77beac5ccb9c97675148bcf853a5490e279
Parents: 7fb7a77
Author: Zuyu Zhang 
Authored: Mon Oct 2 20:47:44 2017 -0500
Committer: Zuyu Zhang 
Committed: Tue Oct 3 10:48:45 2017 -0500

--
 cyclic_dependency.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0898a77b/cyclic_dependency.py
--
diff --git a/cyclic_dependency.py b/cyclic_dependency.py
index adb5fc7..4914e0b 100755
--- a/cyclic_dependency.py
+++ b/cyclic_dependency.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # Script to do analyze the dependencies in Quickstep particularly cycles in the
 # dependency graph. This script can be used to find:
@@ -169,7 +169,7 @@ def find_path(G, nodes_list, nodes_map, source, target):
 print('No path.')
 
 def main():
-if not os.getcwd().endswith("quickstep"):
+if not os.path.isfile("cyclic_dependency.py"):
 print("WARNING: you don't appear to be running in the root quickstep "
   "source directory. Don't blame me if something goes wrong.")
 qs_module_dirs = []

[19/46] incubator-quickstep git commit: Fixed the include path for farmhash.

[jianqiao]

Fixed the include path for farmhash.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/79bfcf9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/79bfcf9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/79bfcf9e

Branch: refs/heads/fix-iwyu
Commit: 79bfcf9ed294477a24823b00bd814df0de54ee5e
Parents: b5130fe
Author: Zuyu Zhang 
Authored: Fri Oct 13 16:07:51 2017 -0500
Committer: Zuyu Zhang 
Committed: Fri Oct 13 16:07:51 2017 -0500

--
 CMakeLists.txt   | 1 +
 types/TypedValue.hpp | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/79bfcf9e/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e0d020b..071f8fc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -635,6 +635,7 @@ endif()
 
 # Add required cmake-controlled third-party libraries (farmhash, gflags, glog, 
and re2).
 add_subdirectory ("${THIRD_PARTY_SOURCE_DIR}/farmhash" 
"${CMAKE_CURRENT_BINARY_DIR}/third_party/farmhash")
+include_directories("${THIRD_PARTY_SOURCE_DIR}")
 
 add_subdirectory ("${THIRD_PARTY_SOURCE_DIR}/gflags" 
"${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags")
 include_directories("${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include")

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/79bfcf9e/types/TypedValue.hpp
--
diff --git a/types/TypedValue.hpp b/types/TypedValue.hpp
index 0ba3d53..3075061 100644
--- a/types/TypedValue.hpp
+++ b/types/TypedValue.hpp
@@ -34,7 +34,7 @@
 #include "utility/HashPair.hpp"
 #include "utility/Macros.hpp"
 
-#include "third_party/src/farmhash/farmhash.h"
+#include "farmhash/farmhash.h"
 
 #include "glog/logging.h"
 

[17/46] incubator-quickstep git commit: Added Vector Aggregation support in the distributed version.

[jianqiao]

Added Vector Aggregation support in the distributed version.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e79b520e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e79b520e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e79b520e

Branch: refs/heads/fix-iwyu
Commit: e79b520ec919fbe101ad72978c02216b6ca6
Parents: 8f094a1
Author: Zuyu Zhang 
Authored: Fri Aug 4 17:03:34 2017 -0500
Committer: Zuyu Zhang 
Committed: Thu Oct 12 11:44:44 2017 -0500

--
 .../FinalizeAggregationOperator.cpp | 31 ++--
 .../InitializeAggregationOperator.cpp   | 23 +++
 relational_operators/WorkOrderFactory.cpp   |  2 --
 3 files changed, 27 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e79b520e/relational_operators/FinalizeAggregationOperator.cpp
--
diff --git a/relational_operators/FinalizeAggregationOperator.cpp 
b/relational_operators/FinalizeAggregationOperator.cpp
index 68d0ef4..92fc7f6 100644
--- a/relational_operators/FinalizeAggregationOperator.cpp
+++ b/relational_operators/FinalizeAggregationOperator.cpp
@@ -67,28 +67,29 @@ bool FinalizeAggregationOperator::getAllWorkOrders(
   return true;
 }
 
-// TODO(quickstep-team) : Think about how the number of partitions could be
-// accessed in this function. Until then, we can't use partitioned aggregation
-// finalization with the distributed version.
 bool 
FinalizeAggregationOperator::getAllWorkOrderProtos(WorkOrderProtosContainer 
*container) {
   if (started_) {
 return true;
   }
 
   for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
-serialization::WorkOrder *proto = new serialization::WorkOrder;
-proto->set_work_order_type(serialization::FINALIZE_AGGREGATION);
-proto->set_query_id(query_id_);
-
proto->SetExtension(serialization::FinalizeAggregationWorkOrder::aggr_state_index,
-aggr_state_index_);
-
proto->SetExtension(serialization::FinalizeAggregationWorkOrder::partition_id,
-part_id);
-
proto->SetExtension(serialization::FinalizeAggregationWorkOrder::state_partition_id,
-0u);
-
proto->SetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index,
-output_destination_index_);
+for (std::size_t state_part_id = 0;
+ state_part_id < aggr_state_num_partitions_;
+ ++state_part_id) {
+  serialization::WorkOrder *proto = new serialization::WorkOrder;
+  proto->set_work_order_type(serialization::FINALIZE_AGGREGATION);
+  proto->set_query_id(query_id_);
+  
proto->SetExtension(serialization::FinalizeAggregationWorkOrder::aggr_state_index,
+  aggr_state_index_);
+  
proto->SetExtension(serialization::FinalizeAggregationWorkOrder::partition_id,
+  part_id);
+  
proto->SetExtension(serialization::FinalizeAggregationWorkOrder::state_partition_id,
+  state_part_id);
+  
proto->SetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index,
+  output_destination_index_);
 
-container->addWorkOrderProto(proto, op_index_);
+  container->addWorkOrderProto(proto, op_index_);
+}
   }
 
   started_ = true;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e79b520e/relational_operators/InitializeAggregationOperator.cpp
--
diff --git a/relational_operators/InitializeAggregationOperator.cpp 
b/relational_operators/InitializeAggregationOperator.cpp
index 39a6fb4..89dfd7e 100644
--- a/relational_operators/InitializeAggregationOperator.cpp
+++ b/relational_operators/InitializeAggregationOperator.cpp
@@ -64,26 +64,25 @@ bool InitializeAggregationOperator::getAllWorkOrders(
   return true;
 }
 
-// TODO(quickstep-team) : Think about how the number of partitions could be
-// accessed in this function. Until then, we can't use partitioned aggregation
-// initialization with the distributed version.
 bool 
InitializeAggregationOperator::getAllWorkOrderProtos(WorkOrderProtosContainer 
*container) {
-  LOG(FATAL) << "Not supported";
-
   if (started_) {
 return true;
   }
 
   for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
-serialization::WorkOrder *proto = new serialization::WorkOrder;
-proto->set_work_order_type(serialization::INITIALIZE_AGGREGATION);
-proto->set_query_id(query_id_);
+for (std::size_t state_part_id = 0;
+ state_part_id < 

[18/46] incubator-quickstep git commit: Fixed gcc compiler warning.

[jianqiao]

Fixed gcc compiler warning.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b5130fea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b5130fea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b5130fea

Branch: refs/heads/fix-iwyu
Commit: b5130feabc571010924cacd1fa6287f4518be8d6
Parents: e79b520
Author: Zuyu Zhang 
Authored: Thu Oct 12 23:01:22 2017 -0500
Committer: Zuyu Zhang 
Committed: Thu Oct 12 23:01:22 2017 -0500

--
 storage/Flags.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b5130fea/storage/Flags.cpp
--
diff --git a/storage/Flags.cpp b/storage/Flags.cpp
index 4672f81..7312cd3 100644
--- a/storage/Flags.cpp
+++ b/storage/Flags.cpp
@@ -40,7 +40,7 @@ static bool ValidateHdfsNameNodePort(const char *flagname,
   }
 }
 DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode.");
-static const bool hdfs_namenode_port_dummy
+static volatile const bool hdfs_namenode_port_dummy
 = gflags::RegisterFlagValidator(_hdfs_namenode_port, 
);
 
 static bool ValidateHdfsNumReplications(const char *flagname,
@@ -55,7 +55,7 @@ static bool ValidateHdfsNumReplications(const char *flagname,
   }
 }
 DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications.");
-static const bool hdfs_num_replications_dummy
+static volatile const bool hdfs_num_replications_dummy
 = gflags::RegisterFlagValidator(_hdfs_num_replications, 
);
 #endif
 

[10/46] incubator-quickstep git commit: Fixed a flaky case in Catalog test.

[jianqiao]

Fixed a flaky case in Catalog test.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/696a783e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/696a783e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/696a783e

Branch: refs/heads/fix-iwyu
Commit: 696a783e5d8adb3ca62ca9044a8d7ccd89f67b3a
Parents: e496cb5
Author: Zuyu Zhang 
Authored: Sun Oct 8 14:20:41 2017 -0500
Committer: Zuyu Zhang 
Committed: Mon Oct 9 11:34:50 2017 -0500

--
 catalog/tests/Catalog_unittest.cpp | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/696a783e/catalog/tests/Catalog_unittest.cpp
--
diff --git a/catalog/tests/Catalog_unittest.cpp 
b/catalog/tests/Catalog_unittest.cpp
index f761026..7b121fa 100644
--- a/catalog/tests/Catalog_unittest.cpp
+++ b/catalog/tests/Catalog_unittest.cpp
@@ -552,13 +552,15 @@ TEST_F(CatalogTest, CatalogIndexTest) {
   IndexSubBlockDescription index_description;
   index_description.set_sub_block_type(IndexSubBlockDescription::CSB_TREE);
   
index_description.add_indexed_attribute_ids(rel->getAttributeByName("attr_idx1")->getID());
+  IndexSubBlockDescription index_description_copy;
+  index_description_copy.MergeFrom(index_description);
 
   EXPECT_TRUE(rel->addIndex("idx1", std::move(index_description)));
   EXPECT_TRUE(rel->hasIndexWithName("idx1"));
   // Adding an index with duplicate name should return false.
-  EXPECT_FALSE(rel->addIndex("idx1", std::move(index_description)));
+  EXPECT_FALSE(rel->addIndex("idx1", IndexSubBlockDescription()));
   // Adding an index of same type with different name on the same attribute 
should return false.
-  EXPECT_FALSE(rel->addIndex("idx2", std::move(index_description)));
+  EXPECT_FALSE(rel->addIndex("idx2", std::move(index_description_copy)));
 
   index_description.Clear();
   index_description.set_sub_block_type(IndexSubBlockDescription::CSB_TREE);

[04/46] incubator-quickstep git commit: Fixed the root path check in the validate_cmakelists script.

[jianqiao]

Fixed the root path check in the validate_cmakelists script.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/bf455e26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/bf455e26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/bf455e26

Branch: refs/heads/fix-iwyu
Commit: bf455e26eb89902731f01928f5eff369a875e5f4
Parents: 8d7284d
Author: Zuyu Zhang 
Authored: Thu Sep 28 19:36:06 2017 -0500
Committer: Zuyu Zhang 
Committed: Thu Sep 28 19:36:06 2017 -0500

--
 validate_cmakelists.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bf455e26/validate_cmakelists.py
--
diff --git a/validate_cmakelists.py b/validate_cmakelists.py
index 0b2e79d..f5f2f89 100755
--- a/validate_cmakelists.py
+++ b/validate_cmakelists.py
@@ -469,7 +469,7 @@ def main(cmakelists_to_process):
 int: The total number of targets that failed validation because of
 missing or superfluous dependencies.
 """
-if not os.getcwd().endswith("quickstep"):
+if not os.path.isfile("validate_cmakelists.py"):
 print("WARNING: you don't appear to be running in the root quickstep "
   "source directory. Don't blame me if something goes wrong.")
 qs_module_dirs = []

[01/46] incubator-quickstep git commit: Prune columns after partition rule. [Forced Update!]

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/fix-iwyu 165bd1fe5 -> c2ed5c69b (forced update)


Prune columns after partition rule.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/71aa8d26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/71aa8d26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/71aa8d26

Branch: refs/heads/fix-iwyu
Commit: 71aa8d265cc2240da0ebf8275a70884002a1ea45
Parents: 475704e
Author: Zuyu Zhang <z...@cs.wisc.edu>
Authored: Tue Sep 19 19:17:29 2017 -0500
Committer: Zuyu Zhang <z...@cs.wisc.edu>
Committed: Tue Sep 19 19:17:29 2017 -0500

--
 query_optimizer/PhysicalGenerator.cpp | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/71aa8d26/query_optimizer/PhysicalGenerator.cpp
--
diff --git a/query_optimizer/PhysicalGenerator.cpp 
b/query_optimizer/PhysicalGenerator.cpp
index 6932b30..865cd11 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -173,6 +173,7 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
   // set output PartitionSchemeHeader in a Physical Plan node, when needed.
   if (FLAGS_use_partition_rule) {
 rules.push_back(std::make_unique(optimizer_context_));
+rules.push_back(std::make_unique());
   }
 
   // NOTE(jianqiao): Adding rules after InjectJoinFilters (or AttachLIPFilters)

incubator-quickstep git commit: Some fixes to cost model

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/transitive-closure 2aefd7bce -> f9e3d35a3


Some fixes to cost model


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f9e3d35a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f9e3d35a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f9e3d35a

Branch: refs/heads/transitive-closure
Commit: f9e3d35a39ad2069d10c6cb604b5f81c71c9e5b1
Parents: 2aefd7b
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Tue Feb 13 16:21:34 2018 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Tue Feb 13 16:21:34 2018 -0600

--
 .../cost_model/StarSchemaSimpleCostModel.cpp|  7 +--
 query_optimizer/rules/InjectJoinFilters.cpp | 47 +++-
 query_optimizer/rules/InjectJoinFilters.hpp |  2 +
 3 files changed, 42 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f9e3d35a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
--
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp 
b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index 875c672..fe4c789 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -464,9 +464,10 @@ double 
StarSchemaSimpleCostModel::estimateSelectivityForPredicate(
E::SomeScalarLiteral::Matches(comparison_expression->left( {
 for (const auto  : physical_plan->children()) {
   if (E::ContainsExprId(child->getOutputAttributes(), attr->id())) {
-const std::size_t child_num_distinct_values = 
estimateNumDistinctValues(attr->id(), child);
+const std::size_t child_num_distinct_values =
+estimateNumDistinctValues(attr->id(), child);
 if (comparison_expression->isEqualityComparisonPredicate()) {
-  return 1.0 / child_num_distinct_values;
+  return 1.0 / std::max(child_num_distinct_values, 
static_cast(1));
 } else {
   return 1.0 / std::max(std::min(child_num_distinct_values / 
100.0, 10.0), 2.0);
 }
@@ -512,7 +513,7 @@ std::size_t StarSchemaSimpleCostModel::getNumDistinctValues(
   return stat.getNumDistinctValues(rel_attr_id);
 }
   }
-  return estimateCardinalityForTableReference(table_reference) * 0.5;
+  return estimateCardinalityForTableReference(table_reference) * 0.1;
 }
 
 bool StarSchemaSimpleCostModel::impliesUniqueAttributes(

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f9e3d35a/query_optimizer/rules/InjectJoinFilters.cpp
--
diff --git a/query_optimizer/rules/InjectJoinFilters.cpp 
b/query_optimizer/rules/InjectJoinFilters.cpp
index 0aa2f5b..a92e433 100644
--- a/query_optimizer/rules/InjectJoinFilters.cpp
+++ b/query_optimizer/rules/InjectJoinFilters.cpp
@@ -62,17 +62,25 @@ P::PhysicalPtr InjectJoinFilters::apply(const 
P::PhysicalPtr ) {
   // Step 1. Transform applicable HashJoin nodes to FilterJoin nodes.
   P::PhysicalPtr output = transformHashJoinToFilters(input);
 
-  // Step 2. Push down FilterJoin nodes to be evaluated early.
+  if (output == input) {
+return input;
+  }
+
+  // Step 2. If the top level plan is a filter join, wrap it with a Selection
+  // to stabilize output columns.
+  output = wrapSelection(output);
+
+  // Step 3. Push down FilterJoin nodes to be evaluated early.
   output = pushDownFilters(output);
 
-  // Step 3. Add Selection nodes for attaching the LIPFilters, if necessary.
+  // Step 4. Add Selection nodes for attaching the LIPFilters, if necessary.
   output = addFilterAnchors(output, false);
 
-  // Step 4. Because of the pushdown of FilterJoin nodes, there are 
optimization
+  // Step 5. Because of the pushdown of FilterJoin nodes, there are 
optimization
   // opportunities for projecting columns early.
   output = PruneColumns().apply(output);
 
-  // Step 5. For each FilterJoin node, attach its corresponding LIPFilter to
+  // Step 6. For each FilterJoin node, attach its corresponding LIPFilter to
   // proper nodes.
   concretizeAsLIPFilters(output, nullptr);
 
@@ -146,13 +154,8 @@ bool InjectJoinFilters::isTransformable(
 P::PhysicalPtr InjectJoinFilters::transformHashJoinToFilters(
 const P::PhysicalPtr ) const {
   std::vector new_children;
-  bool has_changed_children = false;
   for (const P::PhysicalPtr  : input->children()) {
-const P::PhysicalPtr new_child = transformHashJoinToFilters(child);
-if (child != new_child && !has_changed

incubator-quickstep git commit: Fix a bug in InjectJoinFilters [Forced Update!]

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/trace 26c04441c -> ba16c5aef (forced update)


Fix a bug in InjectJoinFilters


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ba16c5ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ba16c5ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ba16c5ae

Branch: refs/heads/trace
Commit: ba16c5aefa25f1d83e0e30e6a2a6a44c5eb984e0
Parents: 0bb163a
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Mon Feb 12 22:01:56 2018 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Mon Feb 12 22:55:34 2018 -0600

--
 query_optimizer/rules/InjectJoinFilters.cpp | 47 ++--
 query_optimizer/rules/InjectJoinFilters.hpp |  2 +
 2 files changed, 38 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ba16c5ae/query_optimizer/rules/InjectJoinFilters.cpp
--
diff --git a/query_optimizer/rules/InjectJoinFilters.cpp 
b/query_optimizer/rules/InjectJoinFilters.cpp
index 0aa2f5b..a92e433 100644
--- a/query_optimizer/rules/InjectJoinFilters.cpp
+++ b/query_optimizer/rules/InjectJoinFilters.cpp
@@ -62,17 +62,25 @@ P::PhysicalPtr InjectJoinFilters::apply(const 
P::PhysicalPtr ) {
   // Step 1. Transform applicable HashJoin nodes to FilterJoin nodes.
   P::PhysicalPtr output = transformHashJoinToFilters(input);
 
-  // Step 2. Push down FilterJoin nodes to be evaluated early.
+  if (output == input) {
+return input;
+  }
+
+  // Step 2. If the top level plan is a filter join, wrap it with a Selection
+  // to stabilize output columns.
+  output = wrapSelection(output);
+
+  // Step 3. Push down FilterJoin nodes to be evaluated early.
   output = pushDownFilters(output);
 
-  // Step 3. Add Selection nodes for attaching the LIPFilters, if necessary.
+  // Step 4. Add Selection nodes for attaching the LIPFilters, if necessary.
   output = addFilterAnchors(output, false);
 
-  // Step 4. Because of the pushdown of FilterJoin nodes, there are 
optimization
+  // Step 5. Because of the pushdown of FilterJoin nodes, there are 
optimization
   // opportunities for projecting columns early.
   output = PruneColumns().apply(output);
 
-  // Step 5. For each FilterJoin node, attach its corresponding LIPFilter to
+  // Step 6. For each FilterJoin node, attach its corresponding LIPFilter to
   // proper nodes.
   concretizeAsLIPFilters(output, nullptr);
 
@@ -146,13 +154,8 @@ bool InjectJoinFilters::isTransformable(
 P::PhysicalPtr InjectJoinFilters::transformHashJoinToFilters(
 const P::PhysicalPtr ) const {
   std::vector new_children;
-  bool has_changed_children = false;
   for (const P::PhysicalPtr  : input->children()) {
-const P::PhysicalPtr new_child = transformHashJoinToFilters(child);
-if (child != new_child && !has_changed_children) {
-  has_changed_children = true;
-}
-new_children.push_back(new_child);
+new_children.emplace_back(transformHashJoinToFilters(child));
   }
 
   P::HashJoinPtr hash_join;
@@ -182,7 +185,7 @@ P::PhysicalPtr 
InjectJoinFilters::transformHashJoinToFilters(
  
hash_join->cloneOutputPartitionSchemeHeader());
   }
 
-  if (has_changed_children) {
+  if (input->children() != new_children) {
 return input->copyWithNewChildren(new_children);
   } else {
 return input;
@@ -437,5 +440,27 @@ bool 
InjectJoinFilters::findExactMinMaxValuesForAttributeHelper(
   }
 }
 
+P::PhysicalPtr InjectJoinFilters::wrapSelection(
+const P::PhysicalPtr ) const {
+  DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+  const P::TopLevelPlanPtr top_level_plan =
+  std::static_pointer_cast(input);
+
+  if (top_level_plan->plan()->getPhysicalType() != 
P::PhysicalType::kFilterJoin) {
+return input;
+  }
+
+  const P::SelectionPtr selection =
+  P::Selection::Create(
+  top_level_plan->plan(),
+  E::ToNamedExpressions(top_level_plan->plan()->getOutputAttributes()),
+  nullptr /* filter_predicate */);
+
+  return P::TopLevelPlan::Create(selection,
+ top_level_plan->shared_subplans(),
+ top_level_plan->uncorrelated_subquery_map());
+}
+
+
 }  // namespace optimizer
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ba16c5ae/query_optimizer/rules/InjectJoinFilters.hpp
--
diff --git a/query_optimizer/rules/InjectJoinFilters.hpp 
b/query_optimizer/rules/InjectJoinFilters.hpp
index c5250b3..71a6c99 100

incubator-quickstep git commit: Fix to a bug in InjectJoinFilters

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/trace 0bb163a7e -> 26c04441c


Fix to a bug in InjectJoinFilters


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/26c04441
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/26c04441
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/26c04441

Branch: refs/heads/trace
Commit: 26c04441c5ed1f54f2a5716312ee6e3cfc3982f3
Parents: 0bb163a
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Mon Feb 12 22:01:56 2018 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Mon Feb 12 22:01:56 2018 -0600

--
 query_optimizer/PhysicalGenerator.cpp   |  2 +-
 query_optimizer/rules/InjectJoinFilters.cpp | 42 +---
 query_optimizer/rules/InjectJoinFilters.hpp |  2 ++
 3 files changed, 34 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/26c04441/query_optimizer/PhysicalGenerator.cpp
--
diff --git a/query_optimizer/PhysicalGenerator.cpp 
b/query_optimizer/PhysicalGenerator.cpp
index 865cd11..b7b0db0 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -194,7 +194,7 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
  << physical_plan_->toString();
   }
 
-  DVLOG(4) << "Optimized physical plan:\n" << physical_plan_->toString();
+  std::cerr << "Optimized physical plan:\n" << physical_plan_->toString();
 
   if (FLAGS_visualize_plan) {
 quickstep::PlanVisualizer plan_visualizer;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/26c04441/query_optimizer/rules/InjectJoinFilters.cpp
--
diff --git a/query_optimizer/rules/InjectJoinFilters.cpp 
b/query_optimizer/rules/InjectJoinFilters.cpp
index 0aa2f5b..b85b8d8 100644
--- a/query_optimizer/rules/InjectJoinFilters.cpp
+++ b/query_optimizer/rules/InjectJoinFilters.cpp
@@ -62,17 +62,24 @@ P::PhysicalPtr InjectJoinFilters::apply(const 
P::PhysicalPtr ) {
   // Step 1. Transform applicable HashJoin nodes to FilterJoin nodes.
   P::PhysicalPtr output = transformHashJoinToFilters(input);
 
-  // Step 2. Push down FilterJoin nodes to be evaluated early.
+  if (output == input) {
+return input;
+  }
+
+  // Step 2. Wrap the top level plan with a Selection to stabilize output 
columns.
+  output = wrapSelection(output);
+
+  // Step 3. Push down FilterJoin nodes to be evaluated early.
   output = pushDownFilters(output);
 
-  // Step 3. Add Selection nodes for attaching the LIPFilters, if necessary.
+  // Step 4. Add Selection nodes for attaching the LIPFilters, if necessary.
   output = addFilterAnchors(output, false);
 
-  // Step 4. Because of the pushdown of FilterJoin nodes, there are 
optimization
+  // Step 5. Because of the pushdown of FilterJoin nodes, there are 
optimization
   // opportunities for projecting columns early.
   output = PruneColumns().apply(output);
 
-  // Step 5. For each FilterJoin node, attach its corresponding LIPFilter to
+  // Step 6. For each FilterJoin node, attach its corresponding LIPFilter to
   // proper nodes.
   concretizeAsLIPFilters(output, nullptr);
 
@@ -146,13 +153,8 @@ bool InjectJoinFilters::isTransformable(
 P::PhysicalPtr InjectJoinFilters::transformHashJoinToFilters(
 const P::PhysicalPtr ) const {
   std::vector new_children;
-  bool has_changed_children = false;
   for (const P::PhysicalPtr  : input->children()) {
-const P::PhysicalPtr new_child = transformHashJoinToFilters(child);
-if (child != new_child && !has_changed_children) {
-  has_changed_children = true;
-}
-new_children.push_back(new_child);
+new_children.emplace_back(transformHashJoinToFilters(child));
   }
 
   P::HashJoinPtr hash_join;
@@ -182,7 +184,7 @@ P::PhysicalPtr 
InjectJoinFilters::transformHashJoinToFilters(
  
hash_join->cloneOutputPartitionSchemeHeader());
   }
 
-  if (has_changed_children) {
+  if (input->children() != new_children) {
 return input->copyWithNewChildren(new_children);
   } else {
 return input;
@@ -437,5 +439,23 @@ bool 
InjectJoinFilters::findExactMinMaxValuesForAttributeHelper(
   }
 }
 
+P::PhysicalPtr InjectJoinFilters::wrapSelection(
+const P::PhysicalPtr ) const {
+  DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+  const P::TopLevelPlanPtr top_level_plan =
+  std::static_pointer_cast(input);
+
+  const P::SelectionPtr selection =
+  P::Selection::Create(
+  top_level_plan->plan(),
+  E::ToNam

incubator-quickstep git commit: Add a flag to allow disabling of Comparison inline expansion to enable acceleration of Quickstep build.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/fix-travis-timeout [created] 539e1ebe0


Add a flag to allow disabling of Comparison inline expansion to enable 
acceleration of Quickstep build.

(for development productivity as well as solving the Travis CI timeout problem)


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/539e1ebe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/539e1ebe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/539e1ebe

Branch: refs/heads/fix-travis-timeout
Commit: 539e1ebe09b5d1a2d86069ed1fdc6e9fb38c5ce7
Parents: 4a945a6
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Fri Feb 2 17:27:59 2018 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Fri Feb 2 17:31:24 2018 -0600

--
 .travis.yml |  3 ++-
 CMakeLists.txt  | 14 ++
 .../comparisons/AsciiStringComparators-inl.hpp  |  2 ++
 .../comparisons/AsciiStringComparators.hpp  |  4 
 types/operations/comparisons/Comparison.cpp | 20 
 .../comparisons/LiteralComparators-inl.hpp  |  2 ++
 .../comparisons/LiteralComparators.hpp  |  4 
 7 files changed, 28 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/.travis.yml
--
diff --git a/.travis.yml b/.travis.yml
index 4e7833f..6517ae8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -76,7 +76,8 @@ before_script:
-D CMAKE_CXX_COMPILER=$CXX
-D CMAKE_LINKER=$CLINKER
-D USE_TCMALLOC=0
-   -D VECTOR_COPY_ELISION_LEVEL=$VECTOR_COPY_ELISION_LEVEL ..)
+   -D VECTOR_COPY_ELISION_LEVEL=$VECTOR_COPY_ELISION_LEVEL
+   -D ENABLE_COMPARISON_INLINE_EXPANSION=OFF ..)
 
 script:
   - ./lint_everything.py

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 88835ac..c777a6a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -145,6 +145,20 @@ if (ENABLE_VECTOR_PREDICATE_SHORT_CIRCUIT)
   )
 endif()
 
+set(COMPARISON_INLINE_EXPANSION_MSG_LIST
+"This option controls whether to enable inlined template expansion "
+"of comparison predicates. WARNING: This option should only be "
+"turned off for development use. Turning off this option will greatly "
+"reduce Quickstep compile time but incur drastic performance degradation.")
+string(REPLACE ";" "" COMPARISON_INLINE_EXPANSION_MSG 
${COMPARISON_INLINE_EXPANSION_MSG_LIST})
+option(ENABLE_COMPARISON_INLINE_EXPANSION ${COMPARISON_INLINE_EXPANSION_MSG} 
ON)
+if (ENABLE_COMPARISON_INLINE_EXPANSION)
+  set_property(
+DIRECTORY
+APPEND PROPERTY COMPILE_DEFINITIONS 
QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
+  )
+endif()
+
 option(ENABLE_NETWORK_CLI "Allows use of the network cli" OFF)
 option(ENABLE_DISTRIBUTED "Use the distributed version of Quickstep" OFF)
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/types/operations/comparisons/AsciiStringComparators-inl.hpp
--
diff --git a/types/operations/comparisons/AsciiStringComparators-inl.hpp 
b/types/operations/comparisons/AsciiStringComparators-inl.hpp
index fd0d17d..8b2c1bf 100644
--- a/types/operations/comparisons/AsciiStringComparators-inl.hpp
+++ b/types/operations/comparisons/AsciiStringComparators-inl.hpp
@@ -46,6 +46,7 @@
 
 namespace quickstep {
 
+#ifdef QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
 template  class ComparisonFunctor,
   bool left_nullable, bool left_null_terminated, bool left_longer,
   bool right_nullable, bool right_null_terminated, bool right_longer>
@@ -586,6 +587,7 @@ TypedValue AsciiStringUncheckedComparator<ComparisonFunctor,
 
   return new_value;
 }
+#endif  // QUICKSTEP_ENABLE_COMPARISON_INLINE_EXPANSION
 
 }  // namespace quickstep
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/539e1ebe/types/operations/comparisons/AsciiStringComparators.hpp
--
diff --git a/types/operations/comparisons/AsciiStringComparators.hpp 
b/types/operations/comparisons/AsciiStringComparators.hpp
index 936fd1f..2aec8c4 100644
--- a/types/operations/comparisons/AsciiStringComparators.hpp
+++ b/types/operations/comparisons/AsciiStringComparators.hpp
@@ -122,6 +122,7 @@ class AsciiStringUnch

incubator-quickstep git commit: IDE Documentation fixes

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 5d7aa5f0d -> 4a945a6b9


IDE Documentation fixes


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4a945a6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4a945a6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4a945a6b

Branch: refs/heads/master
Commit: 4a945a6b9a64b9735f8cdf91556c6661e4731c43
Parents: 5d7aa5f
Author: Harshad Deshmukh 
Authored: Wed Dec 27 10:47:00 2017 -0600
Committer: Harshad Deshmukh 
Committed: Thu Jan 11 13:52:20 2018 -0600

--
 WORKING_WITH_AN_IDE.md | 24 
 1 file changed, 12 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a945a6b/WORKING_WITH_AN_IDE.md
--
diff --git a/WORKING_WITH_AN_IDE.md b/WORKING_WITH_AN_IDE.md
index 017a174..fca1217 100644
--- a/WORKING_WITH_AN_IDE.md
+++ b/WORKING_WITH_AN_IDE.md
@@ -1,17 +1,17 @@
-#Developing Quickstep with IDEs
+# Developing Quickstep with IDEs
 
-##Who should read this document?
+## Who should read this document?
 Any developer who prefers to work with IDEs instead of a terminal and
 vi, emacs, or the like. In other words, this document aims to make it easier
 for developers of Quickstep to work with IDEs. Over time, there will be
 information about working with other IDEs, but to start out, here are
 instructions for working with XCode on OSX.
 
-##Developing Quickstep with Xcode on OSX
+## Developing Quickstep with Xcode on OSX
 The instructions here were first written and verified on OSX El Capitan,
 v.10.11.2, using Xcode v.7.2.
 
-###1: Install Xcode and command line tools
+### 1: Install Xcode and command line tools
 First, you will need to download and install Xcode and the associated command
 line tools. There are multiple ways to do this, including going to
 https://developer.apple.com/xcode/ and downloading both Xcode and the command
@@ -25,7 +25,7 @@ cc
 This command should trigger a sequence of downloads to get you both Xcode
 and the assocaited command line tools.
 
-###2: Install cmake
+### 2: Install cmake
 Unfortunately, the command line tools do not package `cmake`, which is needed
 to build Quickstep. You can install cmake using brew as follows:
 
@@ -43,7 +43,7 @@ Terminal app by typing:
 brew install cmake
 ```
 
-###3: Build Quicktep
+### 3: Build Quicktep
 Checkout the Quickstep code from git, and also checkout the associated 
submodules.
 If you have not read it already, this would be good time to read the file
 [BUILDING.md](BUILDING.md), but do not run the cmake command mentioned there. 
Instead, go
@@ -103,7 +103,7 @@ these and other command line options by typing:
 ```
 
 
-###4: Debug Quickstep
+### 4: Debug Quickstep
 Now you can debug as you would any normal process in Xcode. Note the
 linenoise option in the cmake command above is important if you are going
 to run quickstep from Xcode (by hitting the "play" button). If you are
@@ -134,7 +134,7 @@ when it starts up. It you had set a breakpoint and the 
program executes that
 code, then Xcode (lldb) will stop at the breakpoint. Or, if there is a crash,
 you can examine the stack in Xcode.
 
-###5: Unit Tests
+### 5: Unit Tests
 Individual unit tests show up as target schemas, so you can simply select them
 and run the unit test of interest.
 
@@ -143,14 +143,14 @@ does not work. So, this is a known limitation at this 
point. You can, however,
 follow the instructions for a [BUILDING.md](command-line build) with cmake and
 then run `ctest` to run the full suite of unit tests.
 
-###6: Other known issues
+### 6: Other known issues
 
-Modifying CMake Files
+ Modifying CMake Files
 If you change any of the cmake files (such as any of the CMakeLists.txt
 files), then you will have to [redo step 3](#3-build-quicktep) above to
 create a new Xcode project file.
 
-Running Python Validation Scripts
+ Running Python Validation Scripts
 Quickstep developers have a few python scripts that are used to mechanically
 check code. These scripts are written in Python 2 and are not compatible with
 Python 3, so they use `python2` as the interpreter in their shebangs. While
@@ -167,7 +167,7 @@ sudo ln -s /usr/bin/python2.7 /usr/local/bin/python2
 2.X version is on your machine.)
 
 After putting the symlink in place, you should be able to run
-`./third_party/cpplint/lint_everything.py` (which applies a modified version of
+`./lint_everything.py` (which applies a modified version of
 Google cpplint to all C++ sources) and `./validate_cmakelists.py` (which checks
 that dependencies in CMakeLists.txt files exactly match included 

incubator-quickstep git commit: Upgraded benchmark third party library.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master d63477247 -> 5d7aa5f0d


Upgraded benchmark third party library.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5d7aa5f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5d7aa5f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5d7aa5f0

Branch: refs/heads/master
Commit: 5d7aa5f0d1a898db37c46b772fca986b29c45eaa
Parents: d634772
Author: Harshad Deshmukh 
Authored: Wed Dec 27 13:35:07 2017 -0600
Committer: Harshad Deshmukh 
Committed: Wed Dec 27 13:35:07 2017 -0600

--
 third_party/download_and_patch_prerequisites.sh|  7 +++
 third_party/patches/benchmark/CMakeLists.patch | 11 +++
 third_party/patches/benchmark/benchmarkCMake.patch | 11 ---
 .../patches/benchmark/benchmarkSrcCMakeLists.patch | 13 -
 4 files changed, 14 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5d7aa5f0/third_party/download_and_patch_prerequisites.sh
--
diff --git a/third_party/download_and_patch_prerequisites.sh 
b/third_party/download_and_patch_prerequisites.sh
index 2295525..9d0ff00 100755
--- a/third_party/download_and_patch_prerequisites.sh
+++ b/third_party/download_and_patch_prerequisites.sh
@@ -52,7 +52,7 @@ third_party_dir_names=("benchmark"
"glog"
)
 
-third_party_lib_urls=("https://github.com/google/benchmark/archive/v1.1.0.tar.gz;
+third_party_lib_urls=("https://github.com/google/benchmark/archive/v1.3.0.tar.gz;
   "https://github.com/gflags/gflags/archive/v2.1.2.tar.gz;
   
"https://github.com/google/googletest/archive/release-1.8.0.tar.gz;
   "https://github.com/antirez/linenoise/archive/1.0.tar.gz;
@@ -61,7 +61,7 @@ 
third_party_lib_urls=("https://github.com/google/benchmark/archive/v1.1.0.tar.gz
   "https://github.com/google/glog/archive/v0.3.5.tar.gz;
   )
 
-downloaded_archive_names=("v1.1.0.tar.gz"
+downloaded_archive_names=("v1.3.0.tar.gz"
   "v2.1.2.tar.gz"
   "release-1.8.0.tar.gz"
   "1.0.tar.gz"
@@ -127,8 +127,7 @@ patch ${THIRD_PARTY_SRC_DIR}/gflags/src/gflags_reporting.cc 
${PATCH_DIR}/gflags/
 patch ${THIRD_PARTY_SRC_DIR}/re2/CMakeLists.txt ${PATCH_DIR}/re2/re2CMake.patch
 
 # Apply benchmark patches.
-patch ${THIRD_PARTY_SRC_DIR}/benchmark/CMakeLists.txt 
${PATCH_DIR}/benchmark/benchmarkCMake.patch
-patch ${THIRD_PARTY_SRC_DIR}/benchmark/src/CMakeLists.txt 
${PATCH_DIR}/benchmark/benchmarkSrcCMakeLists.patch
+patch ${THIRD_PARTY_SRC_DIR}/benchmark/CMakeLists.txt 
${PATCH_DIR}/benchmark/CMakeLists.patch
 
 # Apply glog patches.
 patch ${THIRD_PARTY_SRC_DIR}/glog/CMakeLists.txt 
${PATCH_DIR}/glog/glogCMakeLists.txt.patch

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5d7aa5f0/third_party/patches/benchmark/CMakeLists.patch
--
diff --git a/third_party/patches/benchmark/CMakeLists.patch 
b/third_party/patches/benchmark/CMakeLists.patch
new file mode 100644
index 000..4ef6c9c
--- /dev/null
+++ b/third_party/patches/benchmark/CMakeLists.patch
@@ -0,0 +1,11 @@
+--- CMakeLists.txt 2017-12-27 13:11:46.2 -0600
 CMakeLists.txt.new 2017-12-27 13:12:28.2 -0600
+@@ -11,7 +11,7 @@
+   endif()
+ endforeach()
+ 
+-option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON)
++option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." 
OFF)
+ option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the 
benchmark library." ON)
+ option(BENCHMARK_ENABLE_LTO "Enable link time optimisation of the benchmark 
library." OFF)
+ option(BENCHMARK_USE_LIBCXX "Build and test using libc++ as the standard 
library." OFF)

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5d7aa5f0/third_party/patches/benchmark/benchmarkCMake.patch
--
diff --git a/third_party/patches/benchmark/benchmarkCMake.patch 
b/third_party/patches/benchmark/benchmarkCMake.patch
deleted file mode 100644
index 56b54ba..000
--- a/third_party/patches/benchmark/benchmarkCMake.patch
+++ /dev/null
@@ -1,11 +0,0 @@
 CMakeLists.txt 2016-10-28 16:22:22.0 -0500
-+++ CMakeLists.txt.new 2017-01-13 15:46:47.626768358 -0600
-@@ -10,7 +10,7 @@
-   endif()
- endforeach()
- 
--option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." 

[1/7] incubator-quickstep git commit: Quickstep for GRAIL [Forced Update!]

[jianqiao]

;
-  if (stat.hasNumTuples()) {
+  if (stat.isExact() && stat.hasNumTuples()) {
 const std::size_t num_tuples = stat.getNumTuples();
 for (const auto  : attributes) {
   const attribute_id rel_attr_id =

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b75dba28/query_optimizer/resolver/Resolver.cpp
--
diff --git a/query_optimizer/resolver/Resolver.cpp 
b/query_optimizer/resolver/Resolver.cpp
index 0b6dc22..17198c2 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -1019,10 +1019,13 @@ L::LogicalPtr Resolver::resolveInsertSelection(
 if (destination_type.equals(selection_type)) {
   cast_expressions.emplace_back(selection_attributes[aid]);
 } else {
-  // TODO(jianqiao): implement Cast operation for non-numeric types.
+  // TODO(jianqiao): Implement Cast operation for non-numeric types.
+  // TODO(jianqiao): We temporarily disable the safely-coercible check for
+  // tricks that work around "argmin". Will switch it back once the "Cast"
+  // function is supported.
   if (destination_type.getSuperTypeID() == Type::SuperTypeID::kNumeric &&
   selection_type.getSuperTypeID() == Type::SuperTypeID::kNumeric &&
-  destination_type.isSafelyCoercibleFrom(selection_type)) {
+  destination_type.isCoercibleFrom(selection_type)) {
 // Add cast operation
 const E::AttributeReferencePtr attr = selection_attributes[aid];
 const E::ExpressionPtr cast_expr =
@@ -1038,7 +1041,7 @@ L::LogicalPtr Resolver::resolveInsertSelection(
 << insert_statement.relation_name()->value() << "."
 << destination_attributes[aid]->attribute_name() << " has type "
 << selection_attributes[aid]->getValueType().getName()
-<< ", which cannot be safely coerced to the column's type "
+<< ", which cannot be coerced to the column's type "
 << destination_attributes[aid]->getValueType().getName();
  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b75dba28/relational_operators/TableExportOperator.cpp
--
diff --git a/relational_operators/TableExportOperator.cpp 
b/relational_operators/TableExportOperator.cpp
index f6a73bf..cb90d72 100644
--- a/relational_operators/TableExportOperator.cpp
+++ b/relational_operators/TableExportOperator.cpp
@@ -120,6 +120,8 @@ void TableExportOperator::receiveFeedbackMessage(
 } else if (lo_file_name == "$stderr") {
   file_ = stderr;
 } else {
+  DCHECK(!file_name_.empty());
+  DCHECK_EQ('@', file_name_.front());
   file_ = std::fopen(file_name_.substr(1).c_str(), "wb");
   // TODO(quickstep-team): Decent handling of exceptions at query runtime.
   if (file_ == nullptr) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b75dba28/relational_operators/TextScanOperator.cpp
--
diff --git a/relational_operators/TextScanOperator.cpp 
b/relational_operators/TextScanOperator.cpp
index 66137d8..88b7214 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -57,6 +57,7 @@
 #include "types/containers/Tuple.hpp"
 #include "utility/BulkIoConfiguration.hpp"
 #include "utility/Glob.hpp"
+#include "utility/ScopedBuffer.hpp"
 
 #include "gflags/gflags.h"
 #include "glog/logging.h"
@@ -110,19 +111,36 @@ bool TextScanOperator::getAllWorkOrders(
 const tmb::client_id scheduler_client_id,
 tmb::MessageBus *bus) {
   DCHECK(query_context != nullptr);
+  DCHECK(!file_pattern_.empty());
 
-  const std::vector files = 
utility::file::GlobExpand(file_pattern_);
-
-  CHECK_NE(files.size(), 0u)
-  << "No files matched '" << file_pattern_ << "'. Exiting.";
+  if (work_generated_) {
+return true;
+  }
 
   InsertDestination *output_destination =
   query_context->getInsertDestination(output_destination_index_);
 
-  if (work_generated_) {
+  if (file_pattern_ == "$stdin") {
+container->addNormalWorkOrder(
+new TextScanWorkOrder(query_id_,
+  file_pattern_,
+  0,
+  -1 /* text_segment_size */,
+  options_->getDelimiter(),
+  options_->escapeStrings(),
+  output_destination),
+op_index_);
+work_generated_ = true;
 return true;
   }
 
+  DCHECK_EQ('@', file_pattern_.front());
+  const std::v

[7/7] incubator-quickstep git commit: Parallel loading of in memory data

[jianqiao]

Parallel loading of in memory data


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0bb163a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0bb163a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0bb163a7

Branch: refs/heads/trace
Commit: 0bb163a7e4983797fada68ba538ea17147c29ae5
Parents: fff2fab
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Tue Dec 19 17:00:46 2017 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Fri Dec 22 21:50:48 2017 -0600

--
 cli/IOInterface.hpp |   4 +-
 cli/LocalIO.hpp |   4 +-
 cli/NetworkIO.hpp   |  10 +-
 cli/QuickstepCli.cpp|   2 +-
 query_optimizer/ExecutionGenerator.cpp  |   1 +
 query_optimizer/QueryHandle.hpp |  14 +-
 relational_operators/TextScanOperator.cpp   | 294 ++-
 relational_operators/TextScanOperator.hpp   |  64 
 relational_operators/WorkOrderFactory.cpp   |   4 +
 .../tests/TextScanOperator_unittest.cpp |   1 +
 10 files changed, 317 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0bb163a7/cli/IOInterface.hpp
--
diff --git a/cli/IOInterface.hpp b/cli/IOInterface.hpp
index ec125f9..b9b36e1 100644
--- a/cli/IOInterface.hpp
+++ b/cli/IOInterface.hpp
@@ -40,9 +40,9 @@ class IOHandle {
   virtual ~IOHandle() {}
 
   /**
-   * @return A file handle for standard input.
+   * @return Input data.
*/
-  virtual FILE* in() = 0;
+  virtual const std::string* data() const = 0;
 
   /**
* @return A file handle for standard output.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0bb163a7/cli/LocalIO.hpp
--
diff --git a/cli/LocalIO.hpp b/cli/LocalIO.hpp
index 42501d2..06fa014 100644
--- a/cli/LocalIO.hpp
+++ b/cli/LocalIO.hpp
@@ -46,8 +46,8 @@ class LocalIOHandle final : public IOHandle {
 
   ~LocalIOHandle() override {}
 
-  FILE *in() override {
-return stdin;
+  const std::string* data() const override {
+return nullptr;
   }
 
   FILE *out() override {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0bb163a7/cli/NetworkIO.hpp
--
diff --git a/cli/NetworkIO.hpp b/cli/NetworkIO.hpp
index e55c206..f8129b5 100644
--- a/cli/NetworkIO.hpp
+++ b/cli/NetworkIO.hpp
@@ -215,10 +215,6 @@ class NetworkIOHandle final : public IOHandle {
  public:
   explicit NetworkIOHandle(RequestState* state)
   : request_state_(state) {
-const std::string  = request_state_->getRequest().data();
-if (!data.empty()) {
-  std::fwrite(data.c_str(), 1, data.length(), in_stream_.file());
-}
   }
 
   ~NetworkIOHandle() override {
@@ -227,8 +223,8 @@ class NetworkIOHandle final : public IOHandle {
 request_state_->responseReady(out_stream_.str(), err_stream_.str());
   }
 
-  FILE* in() override {
-return in_stream_.file();
+  const std::string* data() const override {
+return _state_->getRequest().data();
   }
 
   FILE* out() override {
@@ -244,7 +240,7 @@ class NetworkIOHandle final : public IOHandle {
   }
 
  private:
-  MemStream in_stream_, out_stream_, err_stream_;
+  MemStream out_stream_, err_stream_;
   RequestState *request_state_;
 
   DISALLOW_COPY_AND_ASSIGN(NetworkIOHandle);

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0bb163a7/cli/QuickstepCli.cpp
--
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index b84b13b..37e366a 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -305,7 +305,6 @@ int main(int argc, char* argv[]) {
   for (;;) {
 string *command_string = new string();
 std::unique_ptr io_handle(io->getNextIOHandle());
-ScopedReassignment<FILE*> reassign_stdin(, io_handle->in());
 ScopedReassignment<FILE*> reassign_stdout(, io_handle->out());
 //ScopedReassignment<FILE*> reassign_stderr(, io_handle->err());
 
@@ -361,6 +360,7 @@ int main(int argc, char* argv[]) {
   auto query_handle = std::make_unique(query_id,
 
main_thread_client_id,
 
statement.getPriority());
+  query_handle->setMemData(io_handle->data());
   query_processor->generateQueryHandle(statement, query_handle.get());
   DCHECK(query_handle->getQueryPlanMutable() != nullptr)

[4/7] incubator-quickstep git commit: Quickstep for GRAIL

[jianqiao]

Quickstep for GRAIL


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b75dba28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b75dba28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b75dba28

Branch: refs/heads/trace
Commit: b75dba2834df330cdc80d0acc2c576954e169048
Parents: 0fe838d
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Sat Sep 2 23:06:37 2017 -0500
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Fri Dec 22 21:50:44 2017 -0600

--
 CMakeLists.txt  |4 +-
 catalog/CatalogRelationStatistics.hpp   |9 +
 cli/CommandExecutor.cpp |  249 +-
 cli/Constants.hpp   |2 +
 cli/IOInterface.hpp |9 +-
 cli/LocalIO.hpp |4 +
 cli/NetworkCli.proto|1 +
 cli/NetworkCliClient.hpp|7 +-
 cli/NetworkCliClientMain.cpp|   30 +-
 cli/NetworkIO.cpp   |5 +
 cli/NetworkIO.hpp   |   33 +-
 cli/QuickstepCli.cpp|3 +-
 cli/quickstep/NetworkCliOuterClass.java | 1388 +++
 cli/tests/NetworkIO_unittest.cpp|2 +-
 .../aggregation/AggregateFunctionMax.cpp|4 +-
 .../aggregation/AggregateFunctionMin.cpp|4 +-
 parser/SqlLexer.lpp |1 +
 parser/SqlParser.ypp|   15 +-
 parser/preprocessed/SqlLexer_gen.cpp|  861 ++---
 parser/preprocessed/SqlLexer_gen.hpp|2 +-
 parser/preprocessed/SqlParser_gen.cpp   | 3436 +-
 parser/preprocessed/SqlParser_gen.hpp   |   49 +-
 query_optimizer/ExecutionGenerator.cpp  |   12 +-
 .../cost_model/StarSchemaSimpleCostModel.cpp|4 +-
 query_optimizer/resolver/Resolver.cpp   |9 +-
 relational_operators/TableExportOperator.cpp|2 +
 relational_operators/TextScanOperator.cpp   |  104 +-
 relational_operators/TextScanOperator.hpp   |2 +
 storage/AggregationOperationState.cpp   |3 +-
 utility/ExecutionDAGVisualizer.cpp  |6 +-
 30 files changed, 4009 insertions(+), 2251 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b75dba28/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 071f8fc..cb8e9f4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -291,7 +291,7 @@ else()
   # Treat warnings as errors, 'cause we hardcore.
   CHECK_CXX_COMPILER_FLAG("-Werror" COMPILER_HAS_WERROR)
   if (COMPILER_HAS_WERROR)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
+# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
   endif()
 
   # Clang reports such warning when using Protoc 3.0 beta.
@@ -585,7 +585,7 @@ if(USE_TCMALLOC)
   CHECK_CXX_COMPILER_FLAG("-Wno-return-type-c-linkage"
   COMPILER_HAS_WNO_RETURN_TYPE_C_LINKAGE)
   if (COMPILER_HAS_WNO_RETURN_TYPE_C_LINKAGE)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type-c-linkage")
+# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type-c-linkage")
   endif()
 endif()
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b75dba28/catalog/CatalogRelationStatistics.hpp
--
diff --git a/catalog/CatalogRelationStatistics.hpp 
b/catalog/CatalogRelationStatistics.hpp
index df95231..55fc747 100644
--- a/catalog/CatalogRelationStatistics.hpp
+++ b/catalog/CatalogRelationStatistics.hpp
@@ -68,6 +68,15 @@ class CatalogRelationStatistics {
   serialization::CatalogRelationStatistics getProto() const;
 
   /**
+   * @brief Clear all statistics.
+   */
+  void clear() {
+num_tuples_ = kNullValue;
+column_stats_.clear();
+is_exact_ = true;
+  }
+
+  /**
* @brief Check whether the statistics are exact for the relation.
*
* return True if the statistics are exact for the relation, false otherwise.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b75dba28/cli/CommandExecutor.cpp
--
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 6a84672..7976d7d 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -201,9 +201,28 @@ void ExecuteAnalyze(const PtrVector 
,
 CatalogRelationStatistics *mutable_stat =
 mutable_relation->getStatisticsMutable();
 
+

[5/7] incubator-quickstep git commit: Improvements and bug fixes

[jianqiao]

 locateBucketInternal(const KeyCode key_code);
+
+  template 
+  inline void constructCompactKeyCodeComponent(const std::size_t num_tuples,
+   const std::size_t offset,
+   const std::size_t key_size,
+   ValueAccessorT *accessor,
+   const attribute_id attr,
+   KeyCode *key_codes);
+
+  static constexpr std::size_t kInitMinPartitionLength = 1024uL * 256uL;
+  static constexpr std::size_t kFinalMinPartitionLength = 1024uL * 4uL;
+
+  struct KeyBucket {
+KeyCode key_code;
+std::atomic next;
+  };
+
+  static constexpr std::size_t kSlotDataSize = 
sizeof(std::atomic);
+  static constexpr std::size_t kKeyBucketDataSize = sizeof(KeyBucket);
+  static constexpr BucketIndex kExclusiveState = 
std::numeric_limits::max();
+
+  const std::vector key_types_;
+  std::vector key_sizes_;
+
+  ScopedArray<std::atomic> slots_;
+  ScopedArray key_buckets_;
+
+  std::size_t num_slots_;
+  std::size_t num_key_buckets_;
+  std::atomic buckets_allocated_;
+
+  std::unique_ptr slots_init_splitter_;
+  std::unique_ptr key_buckets_init_splitter_;
+  mutable std::unique_ptr final_splitter_;
+
+  DISALLOW_COPY_AND_ASSIGN(CompactKeySeparateChainingHashTable);
+};
+
+// 
+// Implementations of class methods follow.
+
+inline CompactKeySeparateChainingHashTable::BucketIndex
+CompactKeySeparateChainingHashTable::locateBucketInternal(const KeyCode 
key_code) {
+  std::atomic *pending_chain = _[key_code % num_slots_];
+
+  for (;;) {
+BucketIndex existing_chain = 0;
+
+// Check if current node is the end of the chain.
+if (pending_chain->compare_exchange_strong(existing_chain,
+   kExclusiveState,
+   std::memory_order_acq_rel)) {
+  const BucketIndex bucket_index =
+  buckets_allocated_.fetch_add(1, std::memory_order_relaxed);
+
+  // TODO(jianqiao): Resize.
+  if (bucket_index > num_key_buckets_) {
+LOG(FATAL) << "Need resize, not handled";
+  }
+
+  // Store key code into key bucket.
+  key_buckets_[bucket_index].key_code = key_code;
+
+  // Update the chaing pointer to point to the new node.
+  pending_chain->store(bucket_index + 1, std::memory_order_release);
+
+  return bucket_index;
+}
+
+// Spin until the pointer is available.
+while (existing_chain == kExclusiveState) {
+  existing_chain = pending_chain->load(std::memory_order_acquire);
+}
+
+if (existing_chain == 0) {
+  // Other thread had to roll back, so try again.
+  continue;
+}
+
+const BucketIndex bucket_index = existing_chain - 1;
+KeyBucket _bucket = key_buckets_[bucket_index];
+if (key_bucket.key_code == key_code) {
+  return bucket_index;
+} else {
+  pending_chain = _bucket.next;
+}
+  }
+}
+
+template 
+inline void CompactKeySeparateChainingHashTable
+::constructCompactKeyCodeComponent(const std::size_t num_tuples,
+   const std::size_t offset,
+   const std::size_t key_size,
+   ValueAccessorT *accessor,
+   const attribute_id attr,
+   KeyCode *key_codes) {
+  accessor->beginIteration();
+  for (std::size_t i = 0; i < num_tuples; ++i) {
+accessor->next();
+std::memcpy(reinterpret_cast<char*>(key_codes + i) + offset,
+accessor->template getUntypedValue(attr),
+key_size);
+  }
+}
+
+}  // namespace quickstep
+
+#endif  // QUICKSTEP_STORAGE_COMPACT_KEY_SEPARATE_CHAINING_HASH_TABLE_HPP_

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fff2fab3/storage/Flags.hpp
--
diff --git a/storage/Flags.hpp b/storage/Flags.hpp
index 1d5527c..87f7da4 100644
--- a/storage/Flags.hpp
+++ b/storage/Flags.hpp
@@ -41,7 +41,6 @@ DECLARE_bool(use_hdfs);
 DECLARE_string(hdfs_namenode_host);
 DECLARE_int32(hdfs_namenode_port);
 DECLARE_int32(hdfs_num_replications);
-
 #endif
 
 /** @} */

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fff2fab3/storage/HashTable.proto
--
diff --git a/storage/HashTable.proto b/storage/HashTable.proto
index d489b9f..40c8e32 100644
--- a/storage/HashTable.proto
+++ b/storage/HashTable.proto
@@ -23,16 +23,11 @@ import "types/Type.proto";
 
 enum HashTableImplType {
   COLLISION_FREE_VECTOR = 0;
-  LINEAR_OPEN_ADDRESSING = 1;
-  SEPARATE_CHAINING = 2;
-  SIMPLE_SCALAR_SEPARATE_CHAINING = 3;
-  THREAD_PRIVATE_COMPACT_KEY

[3/7] incubator-quickstep git commit: Quickstep for GRAIL

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b75dba28/parser/preprocessed/SqlLexer_gen.cpp
--
diff --git a/parser/preprocessed/SqlLexer_gen.cpp 
b/parser/preprocessed/SqlLexer_gen.cpp
index 4800cde..628b0ce 100644
--- a/parser/preprocessed/SqlLexer_gen.cpp
+++ b/parser/preprocessed/SqlLexer_gen.cpp
@@ -592,8 +592,8 @@ static void yynoreturn yy_fatal_error ( const char* msg , 
yyscan_t yyscanner );
yyg->yy_hold_char = *yy_cp; \
*yy_cp = '\0'; \
yyg->yy_c_buf_p = yy_cp;
-#define YY_NUM_RULES 164
-#define YY_END_OF_BUFFER 165
+#define YY_NUM_RULES 165
+#define YY_END_OF_BUFFER 166
 /* This struct is not used in this scanner,
but its presence is necessary. */
 struct yy_trans_info
@@ -601,72 +601,72 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
-static const flex_int16_t yy_accept[589] =
+static const flex_int16_t yy_accept[591] =
 {   0,
 0,0,0,0,0,0,0,0,0,0,
-0,0,  165,2,2,  163,  163,  162,  161,  163,
-  140,  136,  139,  136,  136,  159,  132,  129,  133,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  137,4,5,5,3,  155,
-  155,  152,  156,  156,  150,  157,  157,  154,1,  162,
-  130,  160,  159,  159,  159,0,  134,  131,  135,  158,
-  158,  158,  158,   10,  158,  158,  158,   22,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  138,
-
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,   58,   67,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,   81,   82,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  113,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,4,5,3,  155,  151,  156,
-  149,  149,  141,  143,  144,  145,  146,  147,  148,  149,
-  157,  153,  160,  159,0,  159,6,7,  158,9,
-   11,  158,  158,   15,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,   33,  158,  158,  158,  158,
-
-  158,  158,  158,  158,   43,  158,  158,  158,  158,  158,
-  158,   50,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,   62,  158,   69,  158,  158,  158,  158,  158,  158,
-  158,   77,  158,   80,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,   98,  158,  158,
-  103,  104,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  141,
-  143,  142,  158,  158,  158,  158,  158,  158,  158,   20,
-   23,  158,  158,  158,   28,  158,  158,  158,   31,  158,
-  158,  158,   37,  158,  158,   41,   42,  158,  158,  158,
-
-  158,  158,  158,  158,   52,   53,  158,   55,  158,   57,
-  158,  158,  158,  158,   66,   68,   70,   71,   72,  158,
-   74,  158,  158,   78,  158,  158,   85,  158,  158,  158,
-  158,  158,   92,  158,   94,  158,  158,  158,  100,  158,
-  158,  158,  158,  158,  158,  158,  158,  110,  111,  114,
-  158,  158,  158,  158,  158,  158,  158,  158,  123,  158,
-  158,  126,  127,  141,  142,8,  158,  158,  158,  158,
-  158,  158,  158,   25,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,   46,   47,
-   48,  158,  158,   54,  158,   59,   60,  158,  158,  158,
-
-   73,  158,   76,   79,   83,   84,  158,  158,  158,  158,
-  158,   93,  158,  158,   97,  158,  158,  158,  158,  158,
-  158,  158,  109,  158,  158,  158,  117,  158,  158,  120,
-  158,  158,  124,  158,  158,  158,  158,   14,  158,  158,
-  158,  158,  158,   26,  158,   29,  158,  158,  158,  158,
-  158,   36,  158,  158,   40,   44,  158,  158,  158,   56,
-   61,  158,  158,  158,   75,  158,  158,  158,  158,  158,
-  158,   96,  158,  101,  102,  158,  106,  107,  158,  158,
-  158,  158,  118,  119,  121,  158,  125,  158,  158,   13,
-  158,  158,  158,  158,  158,  158,   21,   30,  158,   34,
-
-   35,  158,  158,   45,  158,   51,   63,  158,  158,  158,
-   88,  158,   90,  158,  158,  158,  158,  158,  158,  158,
-  158,  122,  158,  158,  158,  158,  158,  158,  158,  158,
-   32,  158,   39,  158,  158,   65,  158,  158,   91,  158,
-  158,  105,  158,  158,  158,  158,  158,   12,  158,  158,
-  158,  158,   24,  158,  158,   49,   64,   86,   89,  158,
-  158,  108,  112,  158,  116,  128,   16,  158,  158,  158,
-   27,   38,   87,   95,  158,  158,  158,   18,   19,  158,
-  115,  158,  158,  158,   99,  158,   17,0
+0,0,  166,2,2,  164,  164,  163,  162,  164,
+  141,  137,  140,  

[6/7] incubator-quickstep git commit: Improvements and bug fixes

[jianqiao]

Improvements and bug fixes


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/fff2fab3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/fff2fab3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/fff2fab3

Branch: refs/heads/trace
Commit: fff2fab35d82870b60316781f59a981ee426bc93
Parents: b75dba2
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Sat Dec 16 22:32:05 2017 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Fri Dec 22 21:50:47 2017 -0600

--
 cli/QuickstepCli.cpp|  10 +-
 query_execution/QueryContext.hpp|  16 ++
 query_optimizer/ExecutionGenerator.cpp  | 214 ++---
 .../cost_model/StarSchemaSimpleCostModel.cpp|  39 +++-
 .../cost_model/StarSchemaSimpleCostModel.hpp|   4 +-
 query_optimizer/resolver/Resolver.cpp   |   5 +-
 query_optimizer/rules/FuseAggregateJoin.cpp |   6 -
 query_optimizer/rules/ReorderColumns.cpp|   6 +-
 .../FinalizeAggregationOperator.cpp |  29 +--
 .../FinalizeAggregationOperator.hpp |   3 -
 .../InitializeAggregationOperator.cpp   |  24 +-
 .../InitializeAggregationOperator.hpp   |   7 +-
 relational_operators/InsertOperator.cpp |  18 +-
 relational_operators/InsertOperator.hpp |  13 +-
 relational_operators/WorkOrder.proto|   2 +-
 relational_operators/WorkOrderFactory.cpp   |  28 ++-
 .../tests/AggregationOperator_unittest.cpp  |   2 -
 storage/AggregationOperationState.cpp   | 133 ++-
 storage/AggregationOperationState.hpp   |  38 +--
 storage/AggregationOperationState.proto |   3 -
 storage/CMakeLists.txt  |  24 ++
 storage/CollisionFreeVectorTable.cpp|  58 -
 storage/CollisionFreeVectorTable.hpp|  42 ++--
 storage/CompactKeySeparateChainingHashTable.cpp | 195 
 storage/CompactKeySeparateChainingHashTable.hpp | 234 +++
 storage/Flags.hpp   |   1 -
 storage/HashTable.proto |  15 +-
 storage/HashTableBase.hpp   |   1 +
 storage/HashTableFactory.hpp|  24 +-
 utility/CMakeLists.txt  |  10 +
 utility/ExecutionDAGVisualizer.cpp  |   4 +
 utility/Range.hpp   | 188 +++
 utility/ScopedArray.hpp |  78 +++
 33 files changed, 1113 insertions(+), 361 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fff2fab3/cli/QuickstepCli.cpp
--
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 5db5dfc..b84b13b 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -378,7 +378,15 @@ int main(int argc, char* argv[]) {
   query_handle.release(),
   );
 } catch (const quickstep::SqlError _error) {
-  fprintf(io_handle->err(), "%s", 
sql_error.formatMessage(*command_string).c_str());
+  switch (statement.getStatementType()) {
+case ParseStatement::kDropTable:
+  // Quick hack for QuickGrail for cleaner log information
+  // since we don't have DROP TABLE IF EXISTS yet.
+  break;
+default:
+  fprintf(io_handle->err(), "%s",
+  sql_error.formatMessage(*command_string).c_str());
+  }
   reset_parser = true;
   break;
 }

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fff2fab3/query_execution/QueryContext.hpp
--
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 7876821..e65f096 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -489,6 +489,22 @@ class QueryContext {
   }
 
   /**
+   * @brief Whether the given vector of Tuple ids is valid.
+   *
+   * @param ids The vector of Tuple ids.
+   *
+   * @return True if valid, otherwise false.
+   **/
+  bool areValidTupleIds(const std::vector ) const {
+for (const tuple_id id : ids) {
+  if (id >= tuples_.size()) {
+return false;
+  }
+}
+return true;
+  }
+
+  /**
* @brief Release the ownership of the Tuple referenced by the id.
*
* @note Each id should use only once.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fff2fab3/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/

[2/7] incubator-quickstep git commit: Quickstep for GRAIL

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b75dba28/parser/preprocessed/SqlParser_gen.cpp
--
diff --git a/parser/preprocessed/SqlParser_gen.cpp 
b/parser/preprocessed/SqlParser_gen.cpp
index 9b77875..aca4c87 100644
--- a/parser/preprocessed/SqlParser_gen.cpp
+++ b/parser/preprocessed/SqlParser_gen.cpp
@@ -299,29 +299,30 @@ extern int quickstep_yydebug;
 TOKEN_SMA = 366,
 TOKEN_SMALLINT = 367,
 TOKEN_STDERR = 368,
-TOKEN_STDOUT = 369,
-TOKEN_SUBSTRING = 370,
-TOKEN_TABLE = 371,
-TOKEN_THEN = 372,
-TOKEN_TIME = 373,
-TOKEN_TIMESTAMP = 374,
-TOKEN_TO = 375,
-TOKEN_TRUE = 376,
-TOKEN_TUPLESAMPLE = 377,
-TOKEN_UNBOUNDED = 378,
-TOKEN_UNIQUE = 379,
-TOKEN_UPDATE = 380,
-TOKEN_USING = 381,
-TOKEN_VALUES = 382,
-TOKEN_VARCHAR = 383,
-TOKEN_WHEN = 384,
-TOKEN_WHERE = 385,
-TOKEN_WINDOW = 386,
-TOKEN_WITH = 387,
-TOKEN_YEAR = 388,
-TOKEN_YEARMONTH = 389,
-TOKEN_EOF = 390,
-TOKEN_LEX_ERROR = 391
+TOKEN_STDIN = 369,
+TOKEN_STDOUT = 370,
+TOKEN_SUBSTRING = 371,
+TOKEN_TABLE = 372,
+TOKEN_THEN = 373,
+TOKEN_TIME = 374,
+TOKEN_TIMESTAMP = 375,
+TOKEN_TO = 376,
+TOKEN_TRUE = 377,
+TOKEN_TUPLESAMPLE = 378,
+TOKEN_UNBOUNDED = 379,
+TOKEN_UNIQUE = 380,
+TOKEN_UPDATE = 381,
+TOKEN_USING = 382,
+TOKEN_VALUES = 383,
+TOKEN_VARCHAR = 384,
+TOKEN_WHEN = 385,
+TOKEN_WHERE = 386,
+TOKEN_WINDOW = 387,
+TOKEN_WITH = 388,
+TOKEN_YEAR = 389,
+TOKEN_YEARMONTH = 390,
+TOKEN_EOF = 391,
+TOKEN_LEX_ERROR = 392
   };
 #endif
 
@@ -432,7 +433,7 @@ union YYSTYPE
 
   quickstep::ParsePriority *opt_priority_clause_;
 
-#line 436 "SqlParser_gen.cpp" /* yacc.c:355  */
+#line 437 "SqlParser_gen.cpp" /* yacc.c:355  */
 };
 
 typedef union YYSTYPE YYSTYPE;
@@ -467,7 +468,7 @@ int quickstep_yyparse (yyscan_t yyscanner, 
quickstep::ParseStatement **parsedSta
 #include "SqlLexer_gen.hpp"
 void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const 
std::string );
 
-#line 471 "SqlParser_gen.cpp" /* yacc.c:358  */
+#line 472 "SqlParser_gen.cpp" /* yacc.c:358  */
 
 #ifdef short
 # undef short
@@ -711,21 +712,21 @@ union yyalloc
 /* YYFINAL -- State number of the termination state.  */
 #define YYFINAL  50
 /* YYLAST -- Last index in YYTABLE.  */
-#define YYLAST   1327
+#define YYLAST   1320
 
 /* YYNTOKENS -- Number of terminals.  */
-#define YYNTOKENS  148
+#define YYNTOKENS  149
 /* YYNNTS -- Number of nonterminals.  */
-#define YYNNTS  111
+#define YYNNTS  112
 /* YYNRULES -- Number of rules.  */
-#define YYNRULES  300
+#define YYNRULES  302
 /* YYNSTATES -- Number of states.  */
-#define YYNSTATES  555
+#define YYNSTATES  557
 
 /* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned
by yylex, with out-of-bounds checking.  */
 #define YYUNDEFTOK  2
-#define YYMAXUTOK   391
+#define YYMAXUTOK   392
 
 #define YYTRANSLATE(YYX)\
   ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
@@ -735,11 +736,11 @@ union yyalloc
 static const yytype_uint8 yytranslate[] =
 {
0, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 143, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 144, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-   2, 2, 2, 2, 2, 2, 2,   147, 2, 2,
- 144,   145,23,21,   146,22,27,24, 2, 2,
-   2, 2, 2, 2, 2, 2, 2, 2, 2,   142,
+   2, 2, 2, 2, 2, 2, 2,   148, 2, 2,
+ 145,   146,23,21,   147,22,27,24, 2, 2,
+   2, 2, 2, 2, 2, 2, 2, 2, 2,   143,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
@@ -773,44 +774,44 @@ static const yytype_uint8 yytranslate[] =
  110,   111,   112,   113,   114,   115,   116,   117,   118,   119,
  120,   121,   122,   123,   124,   125,   126,   127,   128,   129,
  130,   131,   132,   133,   134,   135,   136,   137,   138,   139,
- 140,   141
+ 140,   141,   142
 };
 
 #if YYDEBUG
   /* YYRLINE[YYN] -- Source line where rule number YYN was defined.  */
 static const yytype_uint16 yyrline[] =
 {
-   0,   644,   644,   648,   652,   656,   660,   663,   670,   673,
- 676,   679,   682,   685,   688,   691,   694,   697,   703,   709,
- 716,   722,   729,   738,   743,   752,   757,   762,   766,   772,
- 777,   780,   783,   788,   791,   794,   797,   800,   803,   806,
- 809,   812,   815,   

incubator-quickstep git commit: Fixed the bug when partition w/ pruned columns.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master d886ddb09 -> d63477247


Fixed the bug when partition w/ pruned columns.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d6347724
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d6347724
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d6347724

Branch: refs/heads/master
Commit: d63477247648a45b577be27db57954f9e85454c3
Parents: d886ddb
Author: Zuyu Zhang 
Authored: Tue Oct 24 16:17:50 2017 -0500
Committer: Zuyu Zhang 
Committed: Thu Dec 21 15:28:38 2017 -0600

--
 query_optimizer/ExecutionGenerator.cpp  | 39 ++--
 .../tests/execution_generator/Partition.test| 25 +
 2 files changed, 20 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d6347724/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index 5ef58a9..555118a 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -486,26 +486,8 @@ void ExecutionGenerator::createTemporaryCatalogRelation(
 const P::PhysicalPtr ,
 const CatalogRelation **catalog_relation_output,
 S::InsertDestination *insert_destination_proto) {
-  std::unique_ptr catalog_relation(
-  new CatalogRelation(catalog_database_,
-  getNewRelationName(),
-  -1 /* id */,
-  true /* is_temporary*/));
-  attribute_id aid = 0;
-  for (const E::NamedExpressionPtr _expression :
-   physical->getOutputAttributes()) {
-// The attribute name is simply set to the attribute id to make it 
distinct.
-std::unique_ptr catalog_attribute(
-new CatalogAttribute(catalog_relation.get(),
- std::to_string(aid),
- project_expression->getValueType(),
- aid,
- project_expression->attribute_alias()));
-attribute_substitution_map_[project_expression->id()] =
-catalog_attribute.get();
-catalog_relation->addAttribute(catalog_attribute.release());
-++aid;
-  }
+  auto catalog_relation =
+  make_unique(catalog_database_, getNewRelationName(), -1 
/* id */, true /* is_temporary*/);
 
   const P::PartitionSchemeHeader *partition_scheme_header = 
physical->getOutputPartitionSchemeHeader();
   if (partition_scheme_header) {
@@ -514,6 +496,9 @@ void ExecutionGenerator::createTemporaryCatalogRelation(
   DCHECK(!partition_equivalent_expr_ids.empty());
   const E::ExprId partition_expr_id = 
*partition_equivalent_expr_ids.begin();
   DCHECK(attribute_substitution_map_.find(partition_expr_id) != 
attribute_substitution_map_.end());
+  // Use the attribute id from the input relation.
+  // NOTE(zuyu): The following line should be before changing
+  // 'attribute_substitution_map_' with the output attributes.
   
output_partition_attr_ids.push_back(attribute_substitution_map_[partition_expr_id]->getID());
 }
 
@@ -544,6 +529,20 @@ void ExecutionGenerator::createTemporaryCatalogRelation(
 
insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
   }
 
+  attribute_id aid = 0;
+  for (const E::NamedExpressionPtr _expression :
+   physical->getOutputAttributes()) {
+// The attribute name is simply set to the attribute id to make it 
distinct.
+auto catalog_attribute =
+make_unique(catalog_relation.get(), 
std::to_string(aid),
+  project_expression->getValueType(), aid,
+  project_expression->attribute_alias());
+attribute_substitution_map_[project_expression->id()] =
+catalog_attribute.get();
+catalog_relation->addAttribute(catalog_attribute.release());
+++aid;
+  }
+
   *catalog_relation_output = catalog_relation.get();
   const relation_id output_rel_id = catalog_database_->addRelation(
   catalog_relation.release());

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d6347724/query_optimizer/tests/execution_generator/Partition.test
--
diff --git a/query_optimizer/tests/execution_generator/Partition.test 
b/query_optimizer/tests/execution_generator/Partition.test
index da9b6b8..747b969 100644
--- a/query_optimizer/tests/execution_generator/Partition.test
+++ b/query_optimizer/tests/execution_generator/Partition.test
@@ -297,30 +297,7 @@ SELECT 

[01/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 8a84039e1 -> c43107d1f


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/windows/raw_logging.cc
--
diff --git a/third_party/src/glog/src/windows/raw_logging.cc 
b/third_party/src/glog/src/windows/raw_logging.cc
deleted file mode 100644
index 7a7409b..000
--- a/third_party/src/glog/src/windows/raw_logging.cc
+++ /dev/null
@@ -1,172 +0,0 @@
-// Copyright (c) 2006, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Maxim Lifantsev
-//
-// logging_unittest.cc covers the functionality herein
-
-#include "utilities.h"
-
-#include 
-#include 
-#include 
-#ifdef HAVE_UNISTD_H
-# include// for close() and write()
-#endif
-#include  // for open()
-#include 
-#include "config.h"
-#include "glog/logging.h"  // To pick up flag settings etc.
-#include "glog/raw_logging.h"
-#include "base/commandlineflags.h"
-
-#ifdef HAVE_STACKTRACE
-# include "stacktrace.h"
-#endif
-
-#if defined(HAVE_SYSCALL_H)
-#include  // for syscall()
-#elif defined(HAVE_SYS_SYSCALL_H)
-#include  // for syscall()
-#endif
-#ifdef HAVE_UNISTD_H
-# include 
-#endif
-
-#if defined(HAVE_SYSCALL_H) || defined(HAVE_SYS_SYSCALL_H)
-# define safe_write(fd, s, len)  syscall(SYS_write, fd, s, len)
-#else
-  // Not so safe, but what can you do?
-# define safe_write(fd, s, len)  write(fd, s, len)
-#endif
-
-_START_GOOGLE_NAMESPACE_
-
-// Data for RawLog__ below. We simply pick up the latest
-// time data created by a normal log message to avoid calling
-// localtime_r which can allocate memory.
-static struct ::tm last_tm_time_for_raw_log;
-static int last_usecs_for_raw_log;
-
-void RawLog__SetLastTime(const struct ::tm& t, int usecs) {
-  memcpy(_tm_time_for_raw_log, , sizeof(last_tm_time_for_raw_log));
-  last_usecs_for_raw_log = usecs;
-}
-
-// CAVEAT: vsnprintf called from *DoRawLog below has some (exotic) code paths
-// that invoke malloc() and getenv() that might acquire some locks.
-// If this becomes a problem we should reimplement a subset of vsnprintf
-// that does not need locks and malloc.
-
-// Helper for RawLog__ below.
-// *DoRawLog writes to *buf of *size and move them past the written portion.
-// It returns true iff there was no overflow or error.
-static bool DoRawLog(char** buf, int* size, const char* format, ...) {
-  va_list ap;
-  va_start(ap, format);
-  int n = vsnprintf(*buf, *size, format, ap);
-  va_end(ap);
-  if (n < 0 || n > *size) return false;
-  *size -= n;
-  *buf += n;
-  return true;
-}
-
-// Helper for RawLog__ below.
-inline static bool VADoRawLog(char** buf, int* size,
-  const char* format, va_list ap) {
-  int n = vsnprintf(*buf, *size, format, ap);
-  if (n < 0 || n > *size) return false;
-  *size -= n;
-  *buf += n;
-  return true;
-}
-
-static const int kLogBufSize = 3000;
-static bool crashed = false;
-static CrashReason crash_reason;
-static char crash_buf[kLogBufSize + 1] = { 0 };  // Will end in '\0'
-
-void RawLog__(LogSeverity severity, const char* file, int line,
-  const char* format, ...) {
-  if (!(FLAGS_logtostderr || severity >= FLAGS_stderrthreshold ||
-FLAGS_alsologtostderr || !IsGoogleLoggingInitialized())) {
-return;  // this stderr log message is suppressed
-  }
-  // can't call 

[06/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/logging_unittest.cc
--
diff --git a/third_party/src/glog/src/logging_unittest.cc 
b/third_party/src/glog/src/logging_unittest.cc
deleted file mode 100644
index d7e95cf..000
--- a/third_party/src/glog/src/logging_unittest.cc
+++ /dev/null
@@ -1,1215 +0,0 @@
-// Copyright (c) 2002, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Ray Sidney
-
-#include "config_for_unittests.h"
-#include "utilities.h"
-
-#include 
-#ifdef HAVE_GLOB_H
-# include 
-#endif
-#include 
-#ifdef HAVE_UNISTD_H
-# include 
-#endif
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
-
-#include "base/commandlineflags.h"
-#include "glog/logging.h"
-#include "glog/raw_logging.h"
-#include "googletest.h"
-
-DECLARE_string(log_backtrace_at);  // logging.cc
-
-#ifdef HAVE_LIB_GFLAGS
-#include 
-#endif
-
-#ifdef HAVE_LIB_GMOCK
-#include 
-#include "mock-log.h"
-// Introduce several symbols from gmock.
-using testing::_;
-using testing::AnyNumber;
-using testing::HasSubstr;
-using testing::AllOf;
-using testing::StrNe;
-using testing::StrictMock;
-using testing::InitGoogleMock;
-using GOOGLE_NAMESPACE::glog_testing::ScopedMockLog;
-#endif
-
-using namespace std;
-using namespace GOOGLE_NAMESPACE;
-
-// Some non-advertised functions that we want to test or use.
-_START_GOOGLE_NAMESPACE_
-namespace base {
-namespace internal {
-bool GetExitOnDFatal();
-void SetExitOnDFatal(bool value);
-}  // namespace internal
-}  // namespace base
-_END_GOOGLE_NAMESPACE_
-
-static void TestLogging(bool check_counts);
-static void TestRawLogging();
-static void LogWithLevels(int v, int severity, bool err, bool alsoerr);
-static void TestLoggingLevels();
-static void TestLogString();
-static void TestLogSink();
-static void TestLogToString();
-static void TestLogSinkWaitTillSent();
-static void TestCHECK();
-static void TestDCHECK();
-static void TestSTREQ();
-static void TestBasename();
-static void TestSymlink();
-static void TestExtension();
-static void TestWrapper();
-static void TestErrno();
-static void TestTruncate();
-
-static int x = -1;
-static void BM_Check1(int n) {
-  while (n-- > 0) {
-CHECK_GE(n, x);
-CHECK_GE(n, x);
-CHECK_GE(n, x);
-CHECK_GE(n, x);
-CHECK_GE(n, x);
-CHECK_GE(n, x);
-CHECK_GE(n, x);
-CHECK_GE(n, x);
-  }
-}
-BENCHMARK(BM_Check1);
-
-static void CheckFailure(int a, int b, const char* file, int line, const char* 
msg);
-static void BM_Check3(int n) {
-  while (n-- > 0) {
-if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x");
-if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x");
-if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x");
-if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x");
-if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x");
-if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x");
-if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x");
-if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x");
-  }
-}
-BENCHMARK(BM_Check3);
-
-static void BM_Check2(int n) {
-  if (n == 17) {
-x = 5;
-  }
-  while (n-- > 0) {
-CHECK(n >= x);
-CHECK(n >= x);
-CHECK(n >= x);
-CHECK(n >= x);
-CHECK(n >= x);
-CHECK(n >= x);
-CHECK(n >= x);
-CHECK(n >= x);
-  }
-}

[08/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/glog/vlog_is_on.h
--
diff --git a/third_party/src/glog/src/glog/vlog_is_on.h 
b/third_party/src/glog/src/glog/vlog_is_on.h
deleted file mode 100644
index 02b0b86..000
--- a/third_party/src/glog/src/glog/vlog_is_on.h
+++ /dev/null
@@ -1,129 +0,0 @@
-// Copyright (c) 1999, 2007, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Ray Sidney and many others
-//
-// Defines the VLOG_IS_ON macro that controls the variable-verbosity
-// conditional logging.
-//
-// It's used by VLOG and VLOG_IF in logging.h
-// and by RAW_VLOG in raw_logging.h to trigger the logging.
-//
-// It can also be used directly e.g. like this:
-//   if (VLOG_IS_ON(2)) {
-// // do some logging preparation and logging
-// // that can't be accomplished e.g. via just VLOG(2) << ...;
-//   }
-//
-// The truth value that VLOG_IS_ON(level) returns is determined by 
-// the three verbosity level flags:
-//   --v=  Gives the default maximal active V-logging level;
-//0 is the default.
-//Normally positive values are used for V-logging levels.
-//   --vmodule=  Gives the per-module maximal V-logging levels to override
-//the value given by --v.
-//E.g. "my_module=2,foo*=3" would change the logging level
-//for all code in source files "my_module.*" and "foo*.*"
-//("-inl" suffixes are also disregarded for this matching).
-//
-// SetVLOGLevel helper function is provided to do limited dynamic control over
-// V-logging by overriding the per-module settings given via --vmodule flag.
-//
-// CAVEAT: --vmodule functionality is not available in non gcc compilers.
-//
-
-#ifndef BASE_VLOG_IS_ON_H_
-#define BASE_VLOG_IS_ON_H_
-
-#include "glog/log_severity.h"
-
-// Annoying stuff for windows -- makes sure clients can import these functions
-#ifndef GOOGLE_GLOG_DLL_DECL
-# if defined(_WIN32) && !defined(__CYGWIN__)
-#   define GOOGLE_GLOG_DLL_DECL  __declspec(dllimport)
-# else
-#   define GOOGLE_GLOG_DLL_DECL
-# endif
-#endif
-
-#if defined(__GNUC__)
-// We emit an anonymous static int* variable at every VLOG_IS_ON(n) site.
-// (Normally) the first time every VLOG_IS_ON(n) site is hit,
-// we determine what variable will dynamically control logging at this site:
-// it's either FLAGS_v or an appropriate internal variable
-// matching the current source file that represents results of
-// parsing of --vmodule flag and/or SetVLOGLevel calls.
-#define VLOG_IS_ON(verboselevel)\
-  __extension__  \
-  ({ static google::int32* vlocal__ = ::kLogSiteUninitialized;  
 \
- google::int32 verbose_level__ = (verboselevel);\
- (*vlocal__ >= verbose_level__) &&  \
- ((vlocal__ != ::kLogSiteUninitialized) ||   \
-  (google::InitVLOG3__(__, _v, \
-   __FILE__, verbose_level__))); })
-#else
-// GNU extensions not available, so we do not support --vmodule.
-// Dynamic value of FLAGS_v always controls the logging level.
-#define VLOG_IS_ON(verboselevel) (FLAGS_v >= (verboselevel))
-#endif
-
-// Set VLOG(_IS_ON) level for module_pattern to log_level.
-// This lets us dynamically control what is normally set by the --vmodule flag.
-// Returns the 

[10/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/base/mutex.h
--
diff --git a/third_party/src/glog/src/base/mutex.h 
b/third_party/src/glog/src/base/mutex.h
deleted file mode 100644
index 37527d5..000
--- a/third_party/src/glog/src/base/mutex.h
+++ /dev/null
@@ -1,331 +0,0 @@
-// Copyright (c) 2007, Google Inc.
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-// 
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-// ---
-// Author: Craig Silverstein.
-//
-// A simple mutex wrapper, supporting locks and read-write locks.
-// You should assume the locks are *not* re-entrant.
-//
-// To use: you should define the following macros in your configure.ac:
-//   ACX_PTHREAD
-//   AC_RWLOCK
-// The latter is defined in ../autoconf.
-//
-// This class is meant to be internal-only and should be wrapped by an
-// internal namespace.  Before you use this module, please give the
-// name of your internal namespace for this module.  Or, if you want
-// to expose it, you'll want to move it to the Google namespace.  We
-// cannot put this class in global namespace because there can be some
-// problems when we have multiple versions of Mutex in each shared object.
-//
-// NOTE: by default, we have #ifdef'ed out the TryLock() method.
-//   This is for two reasons:
-// 1) TryLock() under Windows is a bit annoying (it requires a
-//#define to be defined very early).
-// 2) TryLock() is broken for NO_THREADS mode, at least in NDEBUG
-//mode.
-// If you need TryLock(), and either these two caveats are not a
-// problem for you, or you're willing to work around them, then
-// feel free to #define GMUTEX_TRYLOCK, or to remove the #ifdefs
-// in the code below.
-//
-// CYGWIN NOTE: Cygwin support for rwlock seems to be buggy:
-//http://www.cygwin.com/ml/cygwin/2008-12/msg00017.html
-// Because of that, we might as well use windows locks for
-// cygwin.  They seem to be more reliable than the cygwin pthreads layer.
-//
-// TRICKY IMPLEMENTATION NOTE:
-// This class is designed to be safe to use during
-// dynamic-initialization -- that is, by global constructors that are
-// run before main() starts.  The issue in this case is that
-// dynamic-initialization happens in an unpredictable order, and it
-// could be that someone else's dynamic initializer could call a
-// function that tries to acquire this mutex -- but that all happens
-// before this mutex's constructor has run.  (This can happen even if
-// the mutex and the function that uses the mutex are in the same .cc
-// file.)  Basically, because Mutex does non-trivial work in its
-// constructor, it's not, in the naive implementation, safe to use
-// before dynamic initialization has run on it.
-//
-// The solution used here is to pair the actual mutex primitive with a
-// bool that is set to true when the mutex is dynamically initialized.
-// (Before that it's false.)  Then we modify all mutex routines to
-// look at the bool, and not try to lock/unlock until the bool makes
-// it to true (which happens after the Mutex constructor has run.)
-//
-// This works because before main() starts -- particularly, during
-// dynamic initialization -- there are no threads, so a) it's ok that
-// the mutex operations are a no-op, since we don't need locking then
-// anyway; and b) we can be quite confident our bool won't change
-// state between a call to Lock() and a call to Unlock() (that would
-// require a global 

[03/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/windows/glog/logging.h
--
diff --git a/third_party/src/glog/src/windows/glog/logging.h 
b/third_party/src/glog/src/windows/glog/logging.h
deleted file mode 100644
index 1d91b12..000
--- a/third_party/src/glog/src/windows/glog/logging.h
+++ /dev/null
@@ -1,1603 +0,0 @@
-// This file is automatically generated from src/glog/logging.h.in
-// using src/windows/preprocess.sh.
-// DO NOT EDIT!
-
-// Copyright (c) 1999, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Ray Sidney
-//
-// This file contains #include information about logging-related stuff.
-// Pretty much everybody needs to #include this file so that they can
-// log various happenings.
-//
-#ifndef _LOGGING_H_
-#define _LOGGING_H_
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#if 0
-# include 
-#endif
-#include 
-
-// Annoying stuff for windows -- makes sure clients can import these functions
-#ifndef GOOGLE_GLOG_DLL_DECL
-# if defined(_WIN32) && !defined(__CYGWIN__)
-#   define GOOGLE_GLOG_DLL_DECL  __declspec(dllimport)
-# else
-#   define GOOGLE_GLOG_DLL_DECL
-# endif
-#endif
-#if defined(_MSC_VER)
-#define GLOG_MSVC_PUSH_DISABLE_WARNING(n) __pragma(warning(push)) \
- __pragma(warning(disable:n))
-#define GLOG_MSVC_POP_WARNING() __pragma(warning(pop))
-#else
-#define GLOG_MSVC_PUSH_DISABLE_WARNING(n)
-#define GLOG_MSVC_POP_WARNING()
-#endif
-
-// We care a lot about number of bits things take up.  Unfortunately,
-// systems define their bit-specific ints in a lot of different ways.
-// We use our own way, and have a typedef to get there.
-// Note: these commands below may look like "#if 1" or "#if 0", but
-// that's because they were constructed that way at ./configure time.
-// Look at logging.h.in to see how they're calculated (based on your config).
-#if 0
-#include  // the normal place uint16_t is defined
-#endif
-#if 0
-#include   // the normal place u_int16_t is defined
-#endif
-#if 0
-#include// a third place for uint16_t or u_int16_t
-#endif
-
-#if 0
-#include 
-#endif
-
-namespace google {
-
-#if 0  // the C99 format
-typedef int32_t int32;
-typedef uint32_t uint32;
-typedef int64_t int64;
-typedef uint64_t uint64;
-#elif 0   // the BSD format
-typedef int32_t int32;
-typedef u_int32_t uint32;
-typedef int64_t int64;
-typedef u_int64_t uint64;
-#elif 1// the windows (vc7) format
-typedef __int32 int32;
-typedef unsigned __int32 uint32;
-typedef __int64 int64;
-typedef unsigned __int64 uint64;
-#else
-#error Do not know how to define a 32-bit integer quantity on your system
-#endif
-
-}
-
-// The global value of GOOGLE_STRIP_LOG. All the messages logged to
-// LOG(XXX) with severity less than GOOGLE_STRIP_LOG will not be displayed.
-// If it can be determined at compile time that the message will not be
-// printed, the statement will be compiled out.
-//
-// Example: to strip out all INFO and WARNING messages, use the value
-// of 2 below. To make an exception for WARNING messages from a single
-// file, add "#define GOOGLE_STRIP_LOG 1" to that file _before_ including
-// base/logging.h
-#ifndef GOOGLE_STRIP_LOG
-#define GOOGLE_STRIP_LOG 0
-#endif
-
-// GCC can be told that a certain branch is not likely to be taken (for
-// instance, a CHECK failure), and 

[11/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

Remove glog source code from third party

- glog source code is now downloaded through the download script.
- Added patches for glog.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c43107d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c43107d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c43107d1

Branch: refs/heads/master
Commit: c43107d1fed5f96770024d52c10cd7bd061bb7c1
Parents: 8a84039
Author: Harshad Deshmukh 
Authored: Fri Dec 15 15:06:49 2017 -0600
Committer: Harshad Deshmukh 
Committed: Sat Dec 16 18:53:45 2017 -0600

--
 CMakeLists.txt  |3 +-
 third_party/download_and_patch_prerequisites.sh |8 +
 .../patches/glog/glogCMakeLists.txt.patch   |   23 +
 third_party/patches/glog/utilities.cc.patch |   11 +
 third_party/reset_third_party_dir.sh|1 +
 third_party/src/glog/AUTHORS|2 -
 third_party/src/glog/CMakeLists.txt |  208 --
 third_party/src/glog/COPYING|   65 -
 third_party/src/glog/ChangeLog  |   84 -
 third_party/src/glog/INSTALL|  297 ---
 third_party/src/glog/NEWS   |0
 third_party/src/glog/README |5 -
 third_party/src/glog/README.windows |   26 -
 third_party/src/glog/doc/designstyle.css|  115 -
 third_party/src/glog/doc/glog.html  |  613 --
 .../src/glog/src/base/commandlineflags.h|  133 --
 third_party/src/glog/src/base/googleinit.h  |   51 -
 third_party/src/glog/src/base/mutex.h   |  331 ---
 third_party/src/glog/src/config.h.in|  171 --
 third_party/src/glog/src/config_cmake.h.in  |  169 --
 third_party/src/glog/src/config_for_unittests.h |   66 -
 third_party/src/glog/src/demangle.cc| 1304 ---
 third_party/src/glog/src/demangle.h |   84 -
 third_party/src/glog/src/demangle_unittest.cc   |  142 --
 third_party/src/glog/src/demangle_unittest.sh   |   95 -
 third_party/src/glog/src/demangle_unittest.txt  |  137 --
 third_party/src/glog/src/glog/log_severity.h|   92 -
 third_party/src/glog/src/glog/logging.h | 1619 --
 third_party/src/glog/src/glog/raw_logging.h |  191 --
 third_party/src/glog/src/glog/stl_logging.h |  183 --
 third_party/src/glog/src/glog/vlog_is_on.h  |  129 --
 third_party/src/glog/src/googletest.h   |  604 --
 third_party/src/glog/src/logging.cc | 2049 -
 .../src/glog/src/logging_striplog_test.sh   |   79 -
 third_party/src/glog/src/logging_striptest10.cc |   35 -
 third_party/src/glog/src/logging_striptest2.cc  |   35 -
 .../src/glog/src/logging_striptest_main.cc  |   73 -
 third_party/src/glog/src/logging_unittest.cc| 1215 ---
 third_party/src/glog/src/logging_unittest.err   |  305 ---
 third_party/src/glog/src/mock-log.h |  155 --
 third_party/src/glog/src/mock-log_test.cc   |  106 -
 third_party/src/glog/src/raw_logging.cc |  172 --
 third_party/src/glog/src/signalhandler.cc   |  350 ---
 .../src/glog/src/signalhandler_unittest.cc  |   97 -
 .../src/glog/src/signalhandler_unittest.sh  |  131 --
 third_party/src/glog/src/stacktrace.h   |   60 -
 .../src/glog/src/stacktrace_generic-inl.h   |   59 -
 .../src/glog/src/stacktrace_libunwind-inl.h |   87 -
 .../src/glog/src/stacktrace_powerpc-inl.h   |  130 --
 third_party/src/glog/src/stacktrace_unittest.cc |  208 --
 third_party/src/glog/src/stacktrace_x86-inl.h   |  139 --
 .../src/glog/src/stacktrace_x86_64-inl.h|  109 -
 .../src/glog/src/stl_logging_unittest.cc|  182 --
 third_party/src/glog/src/symbolize.cc   |  681 --
 third_party/src/glog/src/symbolize.h|  116 -
 third_party/src/glog/src/symbolize_unittest.cc  |  365 
 third_party/src/glog/src/utilities.cc   |  347 ---
 third_party/src/glog/src/utilities.h|  226 --
 third_party/src/glog/src/utilities_unittest.cc  |   54 -
 third_party/src/glog/src/vlog_is_on.cc  |  249 ---
 .../glog/src/windows/base/commandlineflags.h|  133 --
 .../src/glog/src/windows/base/googleinit.h  |   51 -
 third_party/src/glog/src/windows/base/mutex.h   |  331 ---
 .../src/glog/src/windows/config_cmake.h.in  |  142 --
 .../src/glog/src/windows/glog/log_severity.h|   96 -
 third_party/src/glog/src/windows/glog/logging.h | 1603 --
 .../src/glog/src/windows/glog/raw_logging.h |  189 --
 .../src/glog/src/windows/glog/stl_logging.h |  187 --
 .../src/glog/src/windows/glog/vlog_is_on.h  |  133 --
 third_party/src/glog/src/windows/logging.cc | 2050 --
 

[05/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/signalhandler.cc
--
diff --git a/third_party/src/glog/src/signalhandler.cc 
b/third_party/src/glog/src/signalhandler.cc
deleted file mode 100644
index d6c203b..000
--- a/third_party/src/glog/src/signalhandler.cc
+++ /dev/null
@@ -1,350 +0,0 @@
-// Copyright (c) 2008, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Satoru Takabayashi
-//
-// Implementation of InstallFailureSignalHandler().
-
-#include "utilities.h"
-#include "stacktrace.h"
-#include "symbolize.h"
-#include "glog/logging.h"
-
-#include 
-#include 
-#ifdef HAVE_UCONTEXT_H
-# include 
-#endif
-#ifdef HAVE_SYS_UCONTEXT_H
-# include 
-#endif
-#include 
-
-_START_GOOGLE_NAMESPACE_
-
-namespace {
-
-// We'll install the failure signal handler for these signals.  We could
-// use strsignal() to get signal names, but we don't use it to avoid
-// introducing yet another #ifdef complication.
-//
-// The list should be synced with the comment in signalhandler.h.
-const struct {
-  int number;
-  const char *name;
-} kFailureSignals[] = {
-  { SIGSEGV, "SIGSEGV" },
-  { SIGILL, "SIGILL" },
-  { SIGFPE, "SIGFPE" },
-  { SIGABRT, "SIGABRT" },
-  { SIGBUS, "SIGBUS" },
-  { SIGTERM, "SIGTERM" },
-};
-
-// Returns the program counter from signal context, NULL if unknown.
-void* GetPC(void* ucontext_in_void) {
-#if (defined(HAVE_UCONTEXT_H) || defined(HAVE_SYS_UCONTEXT_H)) && 
defined(PC_FROM_UCONTEXT)
-  if (ucontext_in_void != NULL) {
-ucontext_t *context = reinterpret_cast(ucontext_in_void);
-return (void*)context->PC_FROM_UCONTEXT;
-  }
-#endif
-  return NULL;
-}
-
-// The class is used for formatting error messages.  We don't use printf()
-// as it's not async signal safe.
-class MinimalFormatter {
- public:
-  MinimalFormatter(char *buffer, int size)
-  : buffer_(buffer),
-cursor_(buffer),
-end_(buffer + size) {
-  }
-
-  // Returns the number of bytes written in the buffer.
-  int num_bytes_written() const { return cursor_ - buffer_; }
-
-  // Appends string from "str" and updates the internal cursor.
-  void AppendString(const char* str) {
-int i = 0;
-while (str[i] != '\0' && cursor_ + i < end_) {
-  cursor_[i] = str[i];
-  ++i;
-}
-cursor_ += i;
-  }
-
-  // Formats "number" in "radix" and updates the internal cursor.
-  // Lowercase letters are used for 'a' - 'z'.
-  void AppendUint64(uint64 number, int radix) {
-int i = 0;
-while (cursor_ + i < end_) {
-  const int tmp = number % radix;
-  number /= radix;
-  cursor_[i] = (tmp < 10 ? '0' + tmp : 'a' + tmp - 10);
-  ++i;
-  if (number == 0) {
-break;
-  }
-}
-// Reverse the bytes written.
-std::reverse(cursor_, cursor_ + i);
-cursor_ += i;
-  }
-
-  // Formats "number" as hexadecimal number, and updates the internal
-  // cursor.  Padding will be added in front if needed.
-  void AppendHexWithPadding(uint64 number, int width) {
-char* start = cursor_;
-AppendString("0x");
-AppendUint64(number, 16);
-// Move to right and add padding in front if needed.
-if (cursor_ < start + width) {
-  const int64 delta = start + width - cursor_;
-  std::copy(start, cursor_, start + delta);
-  std::fill(start, start + delta, ' ');
-  cursor_ = start + width;
-}
-  }
-
- private:
-  char *buffer_;
-  

[02/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/windows/logging.cc
--
diff --git a/third_party/src/glog/src/windows/logging.cc 
b/third_party/src/glog/src/windows/logging.cc
deleted file mode 100644
index 5c09a66..000
--- a/third_party/src/glog/src/windows/logging.cc
+++ /dev/null
@@ -1,2050 +0,0 @@
-// Copyright (c) 1999, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#define _GNU_SOURCE 1 // needed for O_NOFOLLOW and pread()/pwrite()
-
-#include "utilities.h"
-
-#include 
-#include 
-#include 
-#include 
-#ifdef HAVE_UNISTD_H
-# include   // For _exit.
-#endif
-#include 
-#include 
-#include 
-#ifdef HAVE_SYS_UTSNAME_H
-# include   // For uname.
-#endif
-#include 
-#include 
-#include 
-#include 
-#include 
-#ifdef HAVE_PWD_H
-# include 
-#endif
-#ifdef HAVE_SYSLOG_H
-# include 
-#endif
-#include 
-#include// for errno
-#include 
-#include "base/commandlineflags.h"// to get the program name
-#include "glog/logging.h"
-#include "glog/raw_logging.h"
-#include "base/googleinit.h"
-
-#ifdef HAVE_STACKTRACE
-# include "stacktrace.h"
-#endif
-
-using std::string;
-using std::vector;
-using std::setw;
-using std::setfill;
-using std::hex;
-using std::dec;
-using std::min;
-using std::ostream;
-using std::ostringstream;
-
-using std::FILE;
-using std::fwrite;
-using std::fclose;
-using std::fflush;
-using std::fprintf;
-using std::perror;
-
-#ifdef __QNX__
-using std::fdopen;
-#endif
-
-// There is no thread annotation support.
-#define EXCLUSIVE_LOCKS_REQUIRED(mu)
-
-static bool BoolFromEnv(const char *varname, bool defval) {
-  const char* const valstr = getenv(varname);
-  if (!valstr) {
-return defval;
-  }
-  return memchr("tTyY1\0", valstr[0], 6) != NULL;
-}
-
-GLOG_DEFINE_bool(logtostderr, BoolFromEnv("GOOGLE_LOGTOSTDERR", false),
- "log messages go to stderr instead of logfiles");
-GLOG_DEFINE_bool(alsologtostderr, BoolFromEnv("GOOGLE_ALSOLOGTOSTDERR", false),
- "log messages go to stderr in addition to logfiles");
-GLOG_DEFINE_bool(colorlogtostderr, false,
- "color messages logged to stderr (if supported by terminal)");
-#ifdef OS_LINUX
-GLOG_DEFINE_bool(drop_log_memory, true, "Drop in-memory buffers of log 
contents. "
- "Logs can grow very quickly and they are rarely read before 
they "
- "need to be evicted from memory. Instead, drop them from 
memory "
- "as soon as they are flushed to disk.");
-_START_GOOGLE_NAMESPACE_
-namespace logging {
-static const int64 kPageSize = getpagesize();
-}
-_END_GOOGLE_NAMESPACE_
-#endif
-
-// By default, errors (including fatal errors) get logged to stderr as
-// well as the file.
-//
-// The default is ERROR instead of FATAL so that users can see problems
-// when they run a program without having to look in another file.
-DEFINE_int32(stderrthreshold,
- GOOGLE_NAMESPACE::GLOG_ERROR,
- "log messages at or above this level are copied to stderr in "
- "addition to logfiles.  This flag obsoletes --alsologtostderr.");
-
-GLOG_DEFINE_string(alsologtoemail, "",
-   "log messages go to these email addresses "
-   "in addition to logfiles");
-GLOG_DEFINE_bool(log_prefix, true,
- "Prepend the log prefix to the start of each log line");

[09/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/glog/log_severity.h
--
diff --git a/third_party/src/glog/src/glog/log_severity.h 
b/third_party/src/glog/src/glog/log_severity.h
deleted file mode 100644
index 99945a4..000
--- a/third_party/src/glog/src/glog/log_severity.h
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright (c) 2007, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef BASE_LOG_SEVERITY_H__
-#define BASE_LOG_SEVERITY_H__
-
-// Annoying stuff for windows -- makes sure clients can import these functions
-#ifndef GOOGLE_GLOG_DLL_DECL
-# if defined(_WIN32) && !defined(__CYGWIN__)
-#   define GOOGLE_GLOG_DLL_DECL  __declspec(dllimport)
-# else
-#   define GOOGLE_GLOG_DLL_DECL
-# endif
-#endif
-
-// Variables of type LogSeverity are widely taken to lie in the range
-// [0, NUM_SEVERITIES-1].  Be careful to preserve this assumption if
-// you ever need to change their values or add a new severity.
-typedef int LogSeverity;
-
-const int GLOG_INFO = 0, GLOG_WARNING = 1, GLOG_ERROR = 2, GLOG_FATAL = 3,
-  NUM_SEVERITIES = 4;
-#ifndef GLOG_NO_ABBREVIATED_SEVERITIES
-# ifdef ERROR
-#  error ERROR macro is defined. Define GLOG_NO_ABBREVIATED_SEVERITIES before 
including logging.h. See the document for detail.
-# endif
-const int INFO = GLOG_INFO, WARNING = GLOG_WARNING,
-  ERROR = GLOG_ERROR, FATAL = GLOG_FATAL;
-#endif
-
-// DFATAL is FATAL in debug mode, ERROR in normal mode
-#ifdef NDEBUG
-#define DFATAL_LEVEL ERROR
-#else
-#define DFATAL_LEVEL FATAL
-#endif
-
-extern GOOGLE_GLOG_DLL_DECL const char* const LogSeverityNames[NUM_SEVERITIES];
-
-// NDEBUG usage helpers related to (RAW_)DCHECK:
-//
-// DEBUG_MODE is for small !NDEBUG uses like
-//   if (DEBUG_MODE) foo.CheckThatFoo();
-// instead of substantially more verbose
-//   #ifndef NDEBUG
-// foo.CheckThatFoo();
-//   #endif
-//
-// IF_DEBUG_MODE is for small !NDEBUG uses like
-//   IF_DEBUG_MODE( string error; )
-//   DCHECK(Foo()) << error;
-// instead of substantially more verbose
-//   #ifndef NDEBUG
-// string error;
-// DCHECK(Foo()) << error;
-//   #endif
-//
-#ifdef NDEBUG
-enum { DEBUG_MODE = 0 };
-#define IF_DEBUG_MODE(x)
-#else
-enum { DEBUG_MODE = 1 };
-#define IF_DEBUG_MODE(x) x
-#endif
-
-#endif  // BASE_LOG_SEVERITY_H__

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/glog/logging.h
--
diff --git a/third_party/src/glog/src/glog/logging.h 
b/third_party/src/glog/src/glog/logging.h
deleted file mode 100644
index 8a6dca0..000
--- a/third_party/src/glog/src/glog/logging.h
+++ /dev/null
@@ -1,1619 +0,0 @@
-// Copyright (c) 1999, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote 

[04/11] incubator-quickstep git commit: Remove glog source code from third party

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c43107d1/third_party/src/glog/src/symbolize.h
--
diff --git a/third_party/src/glog/src/symbolize.h 
b/third_party/src/glog/src/symbolize.h
deleted file mode 100644
index 1ebe4dd..000
--- a/third_party/src/glog/src/symbolize.h
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright (c) 2006, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Author: Satoru Takabayashi
-//
-// This library provides Symbolize() function that symbolizes program
-// counters to their corresponding symbol names on linux platforms.
-// This library has a minimal implementation of an ELF symbol table
-// reader (i.e. it doesn't depend on libelf, etc.).
-//
-// The algorithm used in Symbolize() is as follows.
-//
-//   1. Go through a list of maps in /proc/self/maps and find the map
-//   containing the program counter.
-//
-//   2. Open the mapped file and find a regular symbol table inside.
-//   Iterate over symbols in the symbol table and look for the symbol
-//   containing the program counter.  If such a symbol is found,
-//   obtain the symbol name, and demangle the symbol if possible.
-//   If the symbol isn't found in the regular symbol table (binary is
-//   stripped), try the same thing with a dynamic symbol table.
-//
-// Note that Symbolize() is originally implemented to be used in
-// FailureSignalHandler() in base/google.cc.  Hence it doesn't use
-// malloc() and other unsafe operations.  It should be both
-// thread-safe and async-signal-safe.
-
-#ifndef BASE_SYMBOLIZE_H_
-#define BASE_SYMBOLIZE_H_
-
-#include "utilities.h"
-#include "config.h"
-#include "glog/logging.h"
-
-#ifdef HAVE_SYMBOLIZE
-
-#if defined(__ELF__)  // defined by gcc on Linux
-#include 
-#include   // For ElfW() macro.
-
-// If there is no ElfW macro, let's define it by ourself.
-#ifndef ElfW
-# if SIZEOF_VOID_P == 4
-#  define ElfW(type) Elf32_##type
-# elif SIZEOF_VOID_P == 8
-#  define ElfW(type) Elf64_##type
-# else
-#  error "Unknown sizeof(void *)"
-# endif
-#endif
-
-_START_GOOGLE_NAMESPACE_
-
-// Gets the section header for the given name, if it exists. Returns true on
-// success. Otherwise, returns false.
-bool GetSectionHeaderByName(int fd, const char *name, size_t name_len,
-ElfW(Shdr) *out);
-
-_END_GOOGLE_NAMESPACE_
-
-#endif  /* __ELF__ */
-
-_START_GOOGLE_NAMESPACE_
-
-// Installs a callback function, which will be called right before a symbol 
name
-// is printed. The callback is intended to be used for showing a file name and 
a
-// line number preceding a symbol name.
-// "fd" is a file descriptor of the object file containing the program
-// counter "pc". The callback function should write output to "out"
-// and return the size of the output written. On error, the callback
-// function should return -1.
-typedef int (*SymbolizeCallback)(int fd, void *pc, char *out, size_t out_size,
- uint64 relocation);
-void InstallSymbolizeCallback(SymbolizeCallback callback);
-
-_END_GOOGLE_NAMESPACE_
-
-#endif
-
-_START_GOOGLE_NAMESPACE_
-
-// Symbolizes a program counter.  On success, returns true and write the
-// symbol name to "out".  The symbol name is demangled if possible
-// (supports symbols generated by GCC 3.x or newer).  Otherwise,
-// returns false.
-bool Symbolize(void *pc, char *out, int out_size);
-
-_END_GOOGLE_NAMESPACE_
-
-#endif  // 

[2/2] incubator-quickstep git commit: Improvements and bug fixes

[jianqiao]

Improvements and bug fixes


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7e8b33f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7e8b33f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7e8b33f8

Branch: refs/heads/trace
Commit: 7e8b33f8a26aba9e66dd21c149d41f4f2971d36f
Parents: aec7623
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Sat Dec 16 22:32:05 2017 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Sat Dec 16 22:32:05 2017 -0600

--
 cli/QuickstepCli.cpp|  10 +-
 query_execution/QueryContext.hpp|  16 ++
 query_optimizer/ExecutionGenerator.cpp  | 214 ++---
 .../cost_model/StarSchemaSimpleCostModel.cpp|  39 +++-
 .../cost_model/StarSchemaSimpleCostModel.hpp|   4 +-
 query_optimizer/resolver/Resolver.cpp   |   5 +-
 query_optimizer/rules/FuseAggregateJoin.cpp |   6 -
 query_optimizer/rules/ReorderColumns.cpp|   6 +-
 .../FinalizeAggregationOperator.cpp |  29 +--
 .../FinalizeAggregationOperator.hpp |   3 -
 .../InitializeAggregationOperator.cpp   |  24 +-
 .../InitializeAggregationOperator.hpp   |   7 +-
 relational_operators/InsertOperator.cpp |  18 +-
 relational_operators/InsertOperator.hpp |  13 +-
 relational_operators/WorkOrder.proto|   2 +-
 relational_operators/WorkOrderFactory.cpp   |  28 ++-
 .../tests/AggregationOperator_unittest.cpp  |   2 -
 storage/AggregationOperationState.cpp   | 133 ++-
 storage/AggregationOperationState.hpp   |  38 +--
 storage/AggregationOperationState.proto |   3 -
 storage/CMakeLists.txt  |  24 ++
 storage/CollisionFreeVectorTable.cpp|  58 -
 storage/CollisionFreeVectorTable.hpp|  42 ++--
 storage/CompactKeySeparateChainingHashTable.cpp | 195 
 storage/CompactKeySeparateChainingHashTable.hpp | 234 +++
 storage/Flags.hpp   |   1 -
 storage/HashTable.proto |  15 +-
 storage/HashTableBase.hpp   |   1 +
 storage/HashTableFactory.hpp|  24 +-
 utility/CMakeLists.txt  |  10 +
 utility/ExecutionDAGVisualizer.cpp  |   4 +
 utility/Range.hpp   | 188 +++
 utility/ScopedArray.hpp |  78 +++
 33 files changed, 1113 insertions(+), 361 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7e8b33f8/cli/QuickstepCli.cpp
--
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 5db5dfc..b84b13b 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -378,7 +378,15 @@ int main(int argc, char* argv[]) {
   query_handle.release(),
   );
 } catch (const quickstep::SqlError _error) {
-  fprintf(io_handle->err(), "%s", 
sql_error.formatMessage(*command_string).c_str());
+  switch (statement.getStatementType()) {
+case ParseStatement::kDropTable:
+  // Quick hack for QuickGrail for cleaner log information
+  // since we don't have DROP TABLE IF EXISTS yet.
+  break;
+default:
+  fprintf(io_handle->err(), "%s",
+  sql_error.formatMessage(*command_string).c_str());
+  }
   reset_parser = true;
   break;
 }

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7e8b33f8/query_execution/QueryContext.hpp
--
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 7876821..e65f096 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -489,6 +489,22 @@ class QueryContext {
   }
 
   /**
+   * @brief Whether the given vector of Tuple ids is valid.
+   *
+   * @param ids The vector of Tuple ids.
+   *
+   * @return True if valid, otherwise false.
+   **/
+  bool areValidTupleIds(const std::vector ) const {
+for (const tuple_id id : ids) {
+  if (id >= tuples_.size()) {
+return false;
+  }
+}
+return true;
+  }
+
+  /**
* @brief Release the ownership of the Tuple referenced by the id.
*
* @note Each id should use only once.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7e8b33f8/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/

[1/2] incubator-quickstep git commit: Improvements and bug fixes

[jianqiao]

_t;
+  using BucketIndex = std::uint32_t;
+
+  inline BucketIndex locateBucketInternal(const KeyCode key_code);
+
+  template 
+  inline void constructCompactKeyCodeComponent(const std::size_t num_tuples,
+   const std::size_t offset,
+   const std::size_t key_size,
+   ValueAccessorT *accessor,
+   const attribute_id attr,
+   KeyCode *key_codes);
+
+  static constexpr std::size_t kInitMinPartitionLength = 1024uL * 256uL;
+  static constexpr std::size_t kFinalMinPartitionLength = 1024uL * 4uL;
+
+  struct KeyBucket {
+KeyCode key_code;
+std::atomic next;
+  };
+
+  static constexpr std::size_t kSlotDataSize = 
sizeof(std::atomic);
+  static constexpr std::size_t kKeyBucketDataSize = sizeof(KeyBucket);
+  static constexpr BucketIndex kExclusiveState = 
std::numeric_limits::max();
+
+  const std::vector key_types_;
+  std::vector key_sizes_;
+
+  ScopedArray<std::atomic> slots_;
+  ScopedArray key_buckets_;
+
+  std::size_t num_slots_;
+  std::size_t num_key_buckets_;
+  std::atomic buckets_allocated_;
+
+  std::unique_ptr slots_init_splitter_;
+  std::unique_ptr key_buckets_init_splitter_;
+  mutable std::unique_ptr final_splitter_;
+
+  DISALLOW_COPY_AND_ASSIGN(CompactKeySeparateChainingHashTable);
+};
+
+// 
+// Implementations of class methods follow.
+
+inline CompactKeySeparateChainingHashTable::BucketIndex
+CompactKeySeparateChainingHashTable::locateBucketInternal(const KeyCode 
key_code) {
+  std::atomic *pending_chain = _[key_code % num_slots_];
+
+  for (;;) {
+BucketIndex existing_chain = 0;
+
+// Check if current node is the end of the chain.
+if (pending_chain->compare_exchange_strong(existing_chain,
+   kExclusiveState,
+   std::memory_order_acq_rel)) {
+  const BucketIndex bucket_index =
+  buckets_allocated_.fetch_add(1, std::memory_order_relaxed);
+
+  // TODO(jianqiao): Resize.
+  if (bucket_index > num_key_buckets_) {
+LOG(FATAL) << "Need resize, not handled";
+  }
+
+  // Store key code into key bucket.
+  key_buckets_[bucket_index].key_code = key_code;
+
+  // Update the chaing pointer to point to the new node.
+  pending_chain->store(bucket_index + 1, std::memory_order_release);
+
+  return bucket_index;
+}
+
+// Spin until the pointer is available.
+while (existing_chain == kExclusiveState) {
+  existing_chain = pending_chain->load(std::memory_order_acquire);
+}
+
+if (existing_chain == 0) {
+  // Other thread had to roll back, so try again.
+  continue;
+}
+
+const BucketIndex bucket_index = existing_chain - 1;
+KeyBucket _bucket = key_buckets_[bucket_index];
+if (key_bucket.key_code == key_code) {
+  return bucket_index;
+} else {
+  pending_chain = _bucket.next;
+}
+  }
+}
+
+template 
+inline void CompactKeySeparateChainingHashTable
+::constructCompactKeyCodeComponent(const std::size_t num_tuples,
+   const std::size_t offset,
+   const std::size_t key_size,
+   ValueAccessorT *accessor,
+   const attribute_id attr,
+   KeyCode *key_codes) {
+  accessor->beginIteration();
+  for (std::size_t i = 0; i < num_tuples; ++i) {
+accessor->next();
+std::memcpy(reinterpret_cast<char*>(key_codes + i) + offset,
+accessor->template getUntypedValue(attr),
+key_size);
+  }
+}
+
+}  // namespace quickstep
+
+#endif  // QUICKSTEP_STORAGE_COMPACT_KEY_SEPARATE_CHAINING_HASH_TABLE_HPP_

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7e8b33f8/storage/Flags.hpp
--
diff --git a/storage/Flags.hpp b/storage/Flags.hpp
index 1d5527c..87f7da4 100644
--- a/storage/Flags.hpp
+++ b/storage/Flags.hpp
@@ -41,7 +41,6 @@ DECLARE_bool(use_hdfs);
 DECLARE_string(hdfs_namenode_host);
 DECLARE_int32(hdfs_namenode_port);
 DECLARE_int32(hdfs_num_replications);
-
 #endif
 
 /** @} */

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7e8b33f8/storage/HashTable.proto
--
diff --git a/storage/HashTable.proto b/storage/HashTable.proto
index d489b9f..40c8e32 100644
--- a/storage/HashTable.proto
+++ b/storage/HashTable.proto
@@ -23,16 +23,11 @@ import "types/Type.proto";
 
 enum HashTableImplType {
   COLLISION_FREE_VECTOR = 0;
-  LINEAR_OPEN_ADDRESSING = 1;
-  SEPARATE_

[02/11] incubator-quickstep git commit: Updates to save blocks and analyze

[jianqiao]

Updates to save blocks and analyze


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b45cdbc6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b45cdbc6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b45cdbc6

Branch: refs/heads/transitive-closure
Commit: b45cdbc63be488aad3211b5146dfe6f8fbfdc024
Parents: df20e4c
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Mon Nov 27 16:21:03 2017 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Mon Nov 27 16:21:03 2017 -0600

--
 catalog/CatalogRelationStatistics.hpp |   9 ++
 cli/CommandExecutor.cpp   | 249 -
 cli/Constants.hpp |   2 +
 storage/StorageManager.cpp|  12 +-
 4 files changed, 222 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b45cdbc6/catalog/CatalogRelationStatistics.hpp
--
diff --git a/catalog/CatalogRelationStatistics.hpp 
b/catalog/CatalogRelationStatistics.hpp
index df95231..55fc747 100644
--- a/catalog/CatalogRelationStatistics.hpp
+++ b/catalog/CatalogRelationStatistics.hpp
@@ -68,6 +68,15 @@ class CatalogRelationStatistics {
   serialization::CatalogRelationStatistics getProto() const;
 
   /**
+   * @brief Clear all statistics.
+   */
+  void clear() {
+num_tuples_ = kNullValue;
+column_stats_.clear();
+is_exact_ = true;
+  }
+
+  /**
* @brief Check whether the statistics are exact for the relation.
*
* return True if the statistics are exact for the relation, false otherwise.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b45cdbc6/cli/CommandExecutor.cpp
--
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 6a84672..7976d7d 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -201,9 +201,28 @@ void ExecuteAnalyze(const PtrVector 
,
 CatalogRelationStatistics *mutable_stat =
 mutable_relation->getStatisticsMutable();
 
+mutable_stat->clear();
+
 const std::string rel_name = EscapeQuotes(relation.getName(), '"');
 
-// Get the number of distinct values for each column.
+// Get the number of tuples for the relation.
+std::string query_string = "SELECT COUNT(*) FROM \"";
+query_string.append(rel_name);
+query_string.append("\";");
+
+TypedValue num_tuples =
+ExecuteQueryForSingleResult(main_thread_client_id,
+foreman_client_id,
+query_string,
+bus,
+storage_manager,
+query_processor,
+parser_wrapper.get());
+
+DCHECK_EQ(TypeID::kLong, num_tuples.getTypeID());
+mutable_stat->setNumTuples(num_tuples.getLiteral());
+
+// Get the min/max values for each column.
 for (const CatalogAttribute  : relation) {
   const std::string attr_name = EscapeQuotes(attribute.getName(), '"');
   const Type _type = attribute.getType();
@@ -211,24 +230,15 @@ void ExecuteAnalyze(const PtrVector 
,
   AggregateFunctionMin::Instance().canApplyToTypes({_type});
   bool is_max_applicable =
   AggregateFunctionMax::Instance().canApplyToTypes({_type});
+  if (!is_min_applicable || !is_max_applicable) {
+continue;
+  }
 
-  // NOTE(jianqiao): Note that the relation name and the attribute names 
may
-  // contain non-letter characters, e.g. CREATE TABLE "with space"("1" 
int).
-  // So here we need to format the names with double quotes (").
-  std::string query_string = "SELECT COUNT(DISTINCT \"";
+  std::string query_string = "SELECT MIN(\"";
   query_string.append(attr_name);
-  query_string.append("\")");
-  if (is_min_applicable) {
-query_string.append(", MIN(\"");
-query_string.append(attr_name);
-query_string.append("\")");
-  }
-  if (is_max_applicable) {
-query_string.append(", MAX(\"");
-query_string.append(attr_name);
-query_string.append("\")");
-  }
-  query_string.append(" FROM \"");
+  query_string.append("\"), MAX(\"");
+  query_string.append(attr_name);
+  query_string.append("\") FROM \"");
   query_string.append(rel_name);
   query_string.append(&quo

[07/11] incubator-quickstep git commit: Initialize updates for transitive closure

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/734ddc1e/parser/preprocessed/SqlParser_gen.hpp
--
diff --git a/parser/preprocessed/SqlParser_gen.hpp 
b/parser/preprocessed/SqlParser_gen.hpp
index 142059d..130f353 100644
--- a/parser/preprocessed/SqlParser_gen.hpp
+++ b/parser/preprocessed/SqlParser_gen.hpp
@@ -78,107 +78,108 @@ extern int quickstep_yydebug;
 TOKEN_BLOCKPROPERTIES = 288,
 TOKEN_BLOCKSAMPLE = 289,
 TOKEN_BLOOM_FILTER = 290,
-TOKEN_CSB_TREE = 291,
-TOKEN_BY = 292,
-TOKEN_CASE = 293,
-TOKEN_CHARACTER = 294,
-TOKEN_CHECK = 295,
-TOKEN_COLUMN = 296,
+TOKEN_BY = 291,
+TOKEN_CASE = 292,
+TOKEN_CHARACTER = 293,
+TOKEN_CHECK = 294,
+TOKEN_COLUMN = 295,
+TOKEN_CONNECT = 296,
 TOKEN_CONSTRAINT = 297,
 TOKEN_COPY = 298,
 TOKEN_CREATE = 299,
-TOKEN_CURRENT = 300,
-TOKEN_DATE = 301,
-TOKEN_DATETIME = 302,
-TOKEN_DAY = 303,
-TOKEN_DECIMAL = 304,
-TOKEN_DEFAULT = 305,
-TOKEN_DELETE = 306,
-TOKEN_DESC = 307,
-TOKEN_DISTINCT = 308,
-TOKEN_DOUBLE = 309,
-TOKEN_DROP = 310,
-TOKEN_ELSE = 311,
-TOKEN_END = 312,
-TOKEN_EXISTS = 313,
-TOKEN_EXTRACT = 314,
-TOKEN_FALSE = 315,
-TOKEN_FIRST = 316,
-TOKEN_FLOAT = 317,
-TOKEN_FOLLOWING = 318,
-TOKEN_FOR = 319,
-TOKEN_FOREIGN = 320,
-TOKEN_FROM = 321,
-TOKEN_FULL = 322,
-TOKEN_GROUP = 323,
-TOKEN_HASH = 324,
-TOKEN_HAVING = 325,
-TOKEN_HOUR = 326,
-TOKEN_IN = 327,
-TOKEN_INDEX = 328,
-TOKEN_INNER = 329,
-TOKEN_INSERT = 330,
-TOKEN_INTEGER = 331,
-TOKEN_INTERVAL = 332,
-TOKEN_INTO = 333,
-TOKEN_JOIN = 334,
-TOKEN_KEY = 335,
-TOKEN_LAST = 336,
-TOKEN_LEFT = 337,
-TOKEN_LIMIT = 338,
-TOKEN_LONG = 339,
-TOKEN_MINUTE = 340,
-TOKEN_MONTH = 341,
-TOKEN_NULL = 342,
-TOKEN_NULLS = 343,
-TOKEN_OFF = 344,
-TOKEN_ON = 345,
-TOKEN_ORDER = 346,
-TOKEN_OUTER = 347,
-TOKEN_OVER = 348,
-TOKEN_PARTITION = 349,
-TOKEN_PARTITIONS = 350,
-TOKEN_PERCENT = 351,
-TOKEN_PRECEDING = 352,
-TOKEN_PRIMARY = 353,
-TOKEN_PRIORITY = 354,
-TOKEN_QUIT = 355,
-TOKEN_RANGE = 356,
-TOKEN_REAL = 357,
-TOKEN_REFERENCES = 358,
-TOKEN_RIGHT = 359,
-TOKEN_ROW = 360,
-TOKEN_ROW_DELIMITER = 361,
-TOKEN_ROWS = 362,
-TOKEN_SECOND = 363,
-TOKEN_SELECT = 364,
-TOKEN_SET = 365,
-TOKEN_SMA = 366,
-TOKEN_SMALLINT = 367,
-TOKEN_STDERR = 368,
-TOKEN_STDOUT = 369,
-TOKEN_SUBSTRING = 370,
-TOKEN_TABLE = 371,
-TOKEN_THEN = 372,
-TOKEN_TIME = 373,
-TOKEN_TIMESTAMP = 374,
-TOKEN_TO = 375,
-TOKEN_TRUE = 376,
-TOKEN_TUPLESAMPLE = 377,
-TOKEN_UNBOUNDED = 378,
-TOKEN_UNIQUE = 379,
-TOKEN_UPDATE = 380,
-TOKEN_USING = 381,
-TOKEN_VALUES = 382,
-TOKEN_VARCHAR = 383,
-TOKEN_WHEN = 384,
-TOKEN_WHERE = 385,
-TOKEN_WINDOW = 386,
-TOKEN_WITH = 387,
-TOKEN_YEAR = 388,
-TOKEN_YEARMONTH = 389,
-TOKEN_EOF = 390,
-TOKEN_LEX_ERROR = 391
+TOKEN_CSB_TREE = 300,
+TOKEN_CURRENT = 301,
+TOKEN_DATE = 302,
+TOKEN_DATETIME = 303,
+TOKEN_DAY = 304,
+TOKEN_DECIMAL = 305,
+TOKEN_DEFAULT = 306,
+TOKEN_DELETE = 307,
+TOKEN_DESC = 308,
+TOKEN_DISTINCT = 309,
+TOKEN_DOUBLE = 310,
+TOKEN_DROP = 311,
+TOKEN_ELSE = 312,
+TOKEN_END = 313,
+TOKEN_EXISTS = 314,
+TOKEN_EXTRACT = 315,
+TOKEN_FALSE = 316,
+TOKEN_FIRST = 317,
+TOKEN_FLOAT = 318,
+TOKEN_FOLLOWING = 319,
+TOKEN_FOR = 320,
+TOKEN_FOREIGN = 321,
+TOKEN_FROM = 322,
+TOKEN_FULL = 323,
+TOKEN_GROUP = 324,
+TOKEN_HASH = 325,
+TOKEN_HAVING = 326,
+TOKEN_HOUR = 327,
+TOKEN_IN = 328,
+TOKEN_INDEX = 329,
+TOKEN_INNER = 330,
+TOKEN_INSERT = 331,
+TOKEN_INTEGER = 332,
+TOKEN_INTERVAL = 333,
+TOKEN_INTO = 334,
+TOKEN_JOIN = 335,
+TOKEN_KEY = 336,
+TOKEN_LAST = 337,
+TOKEN_LEFT = 338,
+TOKEN_LIMIT = 339,
+TOKEN_LONG = 340,
+TOKEN_MINUTE = 341,
+TOKEN_MONTH = 342,
+TOKEN_NULL = 343,
+TOKEN_NULLS = 344,
+TOKEN_OFF = 345,
+TOKEN_ON = 346,
+TOKEN_ORDER = 347,
+TOKEN_OUTER = 348,
+TOKEN_OVER = 349,
+TOKEN_PARTITION = 350,
+TOKEN_PARTITIONS = 351,
+TOKEN_PERCENT = 352,
+TOKEN_PRECEDING = 353,
+TOKEN_PRIMARY = 354,
+TOKEN_PRIORITY = 355,
+TOKEN_QUIT = 356,
+TOKEN_RANGE = 357,
+TOKEN_REAL = 358,
+TOKEN_REFERENCES = 359,
+TOKEN_RIGHT = 360,
+TOKEN_ROW = 361,
+TOKEN_ROW_DELIMITER = 362,
+TOKEN_ROWS = 363,
+TOKEN_SECOND = 364,
+TOKEN_SELECT = 365,
+TOKEN_SET = 366,
+TOKEN_SMA = 367,
+TOKEN_SMALLINT = 368,
+TOKEN_STDERR = 369,
+TOKEN_STDOUT = 370,
+TOKEN_SUBSTRING = 371,
+TOKEN_TABLE = 372,
+TOKEN_THEN = 373,
+TOKEN_TIME 

[01/11] incubator-quickstep git commit: Updates to compact key hash table

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/transitive-closure [created] 2aefd7bce


Updates to compact key hash table


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/df20e4c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/df20e4c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/df20e4c0

Branch: refs/heads/transitive-closure
Commit: df20e4c09ec3a8490b1d1ddcd590d813a873181e
Parents: 3595bc1
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Sun Nov 26 21:16:46 2017 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Mon Nov 27 16:06:44 2017 -0600

--
 query_optimizer/CMakeLists.txt  |   1 +
 query_optimizer/ExecutionGenerator.cpp  | 109 ++---
 .../cost_model/StarSchemaSimpleCostModel.cpp|  39 +++-
 .../cost_model/StarSchemaSimpleCostModel.hpp|   4 +-
 query_optimizer/rules/FuseAggregateJoin.cpp |   6 -
 .../FinalizeAggregationOperator.cpp |  29 +--
 .../FinalizeAggregationOperator.hpp |   3 -
 .../InitializeAggregationOperator.cpp   |  24 +-
 .../InitializeAggregationOperator.hpp   |   7 +-
 storage/AggregationOperationState.cpp   | 134 ++-
 storage/AggregationOperationState.hpp   |  38 +--
 storage/AggregationOperationState.proto |   3 -
 storage/CMakeLists.txt  |  24 ++
 storage/CollisionFreeVectorTable.cpp|  58 -
 storage/CollisionFreeVectorTable.hpp|  42 ++--
 storage/CompactKeySeparateChainingHashTable.cpp | 195 
 storage/CompactKeySeparateChainingHashTable.hpp | 234 +++
 storage/Flags.hpp   |   1 -
 storage/HashTable.proto |  15 +-
 storage/HashTableBase.hpp   |   1 +
 storage/HashTableFactory.hpp|  24 +-
 utility/CMakeLists.txt  |  10 +
 utility/Range.hpp   | 188 +++
 utility/ScopedArray.hpp |  78 +++
 24 files changed, 977 insertions(+), 290 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df20e4c0/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 5e0db44..011cecb 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -148,6 +148,7 @@ 
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
   quickstep_relationaloperators_UpdateOperator
   quickstep_relationaloperators_WindowAggregationOperator
   quickstep_storage_AggregationOperationState_proto
+  quickstep_storage_Flags
   quickstep_storage_HashTableFactory
   quickstep_storage_HashTable_proto
   quickstep_storage_InsertDestination_proto

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df20e4c0/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index b0d3c48..3ef74ee 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -132,6 +132,7 @@
 #include "relational_operators/UpdateOperator.hpp"
 #include "relational_operators/WindowAggregationOperator.hpp"
 #include "storage/AggregationOperationState.pb.h"
+#include "storage/Flags.hpp"
 #include "storage/HashTable.pb.h"
 #include "storage/HashTableFactory.hpp"
 #include "storage/InsertDestination.pb.h"
@@ -199,70 +200,6 @@ namespace S = ::quickstep::serialization;
 
 namespace {
 
-size_t CacheLineAlignedBytes(const size_t actual_bytes) {
-  return (actual_bytes + kCacheLineBytes - 1) / kCacheLineBytes * 
kCacheLineBytes;
-}
-
-size_t CalculateNumInitializationPartitionsForCollisionFreeVectorTable(const 
size_t memory_size) {
-  // At least 1 partition, at most (#workers * 2) partitions.
-  return std::max(1uL, std::min(memory_size / kCollisonFreeVectorInitBlobSize,
-static_cast(2 * FLAGS_num_workers)));
-}
-
-void CalculateCollisionFreeAggregationInfo(
-const size_t num_entries, const vector<pair<AggregationID, vector>> _by_aggrs_info,
-S::CollisionFreeVectorInfo *collision_free_vector_info) {
-  size_t memory_size = CacheLineAlignedBytes(
-  BarrieredReadWriteConcurrentBitVector::BytesNeeded(num_entries));
-
-

[08/11] incubator-quickstep git commit: Initialize updates for transitive closure

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/734ddc1e/parser/preprocessed/SqlParser_gen.cpp
--
diff --git a/parser/preprocessed/SqlParser_gen.cpp 
b/parser/preprocessed/SqlParser_gen.cpp
index 9b77875..b1b4547 100644
--- a/parser/preprocessed/SqlParser_gen.cpp
+++ b/parser/preprocessed/SqlParser_gen.cpp
@@ -133,6 +133,7 @@ typedef struct YYLTYPE {
 #include "parser/ParseSubqueryExpression.hpp"
 #include "parser/ParseSubqueryTableReference.hpp"
 #include "parser/ParseTableReference.hpp"
+#include "parser/ParseTransitiveClosureTableReference.hpp"
 #include "parser/ParseWindow.hpp"
 #include "storage/StorageBlockInfo.hpp"
 #include "types/Type.hpp"
@@ -153,7 +154,7 @@ typedef struct YYLTYPE {
 // Needed for Bison 2.6 and higher, which do not automatically provide this 
typedef.
 typedef void* yyscan_t;
 
-#line 157 "SqlParser_gen.cpp" /* yacc.c:339  */
+#line 158 "SqlParser_gen.cpp" /* yacc.c:339  */
 
 # ifndef YY_NULLPTR
 #  if defined __cplusplus && 201103L <= __cplusplus
@@ -221,107 +222,108 @@ extern int quickstep_yydebug;
 TOKEN_BLOCKPROPERTIES = 288,
 TOKEN_BLOCKSAMPLE = 289,
 TOKEN_BLOOM_FILTER = 290,
-TOKEN_CSB_TREE = 291,
-TOKEN_BY = 292,
-TOKEN_CASE = 293,
-TOKEN_CHARACTER = 294,
-TOKEN_CHECK = 295,
-TOKEN_COLUMN = 296,
+TOKEN_BY = 291,
+TOKEN_CASE = 292,
+TOKEN_CHARACTER = 293,
+TOKEN_CHECK = 294,
+TOKEN_COLUMN = 295,
+TOKEN_CONNECT = 296,
 TOKEN_CONSTRAINT = 297,
 TOKEN_COPY = 298,
 TOKEN_CREATE = 299,
-TOKEN_CURRENT = 300,
-TOKEN_DATE = 301,
-TOKEN_DATETIME = 302,
-TOKEN_DAY = 303,
-TOKEN_DECIMAL = 304,
-TOKEN_DEFAULT = 305,
-TOKEN_DELETE = 306,
-TOKEN_DESC = 307,
-TOKEN_DISTINCT = 308,
-TOKEN_DOUBLE = 309,
-TOKEN_DROP = 310,
-TOKEN_ELSE = 311,
-TOKEN_END = 312,
-TOKEN_EXISTS = 313,
-TOKEN_EXTRACT = 314,
-TOKEN_FALSE = 315,
-TOKEN_FIRST = 316,
-TOKEN_FLOAT = 317,
-TOKEN_FOLLOWING = 318,
-TOKEN_FOR = 319,
-TOKEN_FOREIGN = 320,
-TOKEN_FROM = 321,
-TOKEN_FULL = 322,
-TOKEN_GROUP = 323,
-TOKEN_HASH = 324,
-TOKEN_HAVING = 325,
-TOKEN_HOUR = 326,
-TOKEN_IN = 327,
-TOKEN_INDEX = 328,
-TOKEN_INNER = 329,
-TOKEN_INSERT = 330,
-TOKEN_INTEGER = 331,
-TOKEN_INTERVAL = 332,
-TOKEN_INTO = 333,
-TOKEN_JOIN = 334,
-TOKEN_KEY = 335,
-TOKEN_LAST = 336,
-TOKEN_LEFT = 337,
-TOKEN_LIMIT = 338,
-TOKEN_LONG = 339,
-TOKEN_MINUTE = 340,
-TOKEN_MONTH = 341,
-TOKEN_NULL = 342,
-TOKEN_NULLS = 343,
-TOKEN_OFF = 344,
-TOKEN_ON = 345,
-TOKEN_ORDER = 346,
-TOKEN_OUTER = 347,
-TOKEN_OVER = 348,
-TOKEN_PARTITION = 349,
-TOKEN_PARTITIONS = 350,
-TOKEN_PERCENT = 351,
-TOKEN_PRECEDING = 352,
-TOKEN_PRIMARY = 353,
-TOKEN_PRIORITY = 354,
-TOKEN_QUIT = 355,
-TOKEN_RANGE = 356,
-TOKEN_REAL = 357,
-TOKEN_REFERENCES = 358,
-TOKEN_RIGHT = 359,
-TOKEN_ROW = 360,
-TOKEN_ROW_DELIMITER = 361,
-TOKEN_ROWS = 362,
-TOKEN_SECOND = 363,
-TOKEN_SELECT = 364,
-TOKEN_SET = 365,
-TOKEN_SMA = 366,
-TOKEN_SMALLINT = 367,
-TOKEN_STDERR = 368,
-TOKEN_STDOUT = 369,
-TOKEN_SUBSTRING = 370,
-TOKEN_TABLE = 371,
-TOKEN_THEN = 372,
-TOKEN_TIME = 373,
-TOKEN_TIMESTAMP = 374,
-TOKEN_TO = 375,
-TOKEN_TRUE = 376,
-TOKEN_TUPLESAMPLE = 377,
-TOKEN_UNBOUNDED = 378,
-TOKEN_UNIQUE = 379,
-TOKEN_UPDATE = 380,
-TOKEN_USING = 381,
-TOKEN_VALUES = 382,
-TOKEN_VARCHAR = 383,
-TOKEN_WHEN = 384,
-TOKEN_WHERE = 385,
-TOKEN_WINDOW = 386,
-TOKEN_WITH = 387,
-TOKEN_YEAR = 388,
-TOKEN_YEARMONTH = 389,
-TOKEN_EOF = 390,
-TOKEN_LEX_ERROR = 391
+TOKEN_CSB_TREE = 300,
+TOKEN_CURRENT = 301,
+TOKEN_DATE = 302,
+TOKEN_DATETIME = 303,
+TOKEN_DAY = 304,
+TOKEN_DECIMAL = 305,
+TOKEN_DEFAULT = 306,
+TOKEN_DELETE = 307,
+TOKEN_DESC = 308,
+TOKEN_DISTINCT = 309,
+TOKEN_DOUBLE = 310,
+TOKEN_DROP = 311,
+TOKEN_ELSE = 312,
+TOKEN_END = 313,
+TOKEN_EXISTS = 314,
+TOKEN_EXTRACT = 315,
+TOKEN_FALSE = 316,
+TOKEN_FIRST = 317,
+TOKEN_FLOAT = 318,
+TOKEN_FOLLOWING = 319,
+TOKEN_FOR = 320,
+TOKEN_FOREIGN = 321,
+TOKEN_FROM = 322,
+TOKEN_FULL = 323,
+TOKEN_GROUP = 324,
+TOKEN_HASH = 325,
+TOKEN_HAVING = 326,
+TOKEN_HOUR = 327,
+TOKEN_IN = 328,
+TOKEN_INDEX = 329,
+TOKEN_INNER = 330,
+TOKEN_INSERT = 331,
+TOKEN_INTEGER = 332,
+TOKEN_INTERVAL = 333,
+TOKEN_INTO = 334,
+TOKEN_JOIN = 335,
+TOKEN_KEY = 336,
+TOKEN_LAST = 337,
+TOKEN_LEFT = 338,
+TOKEN_LIMIT = 339,
+TOKEN_LONG = 340,
+TOKEN_MINUTE = 341,
+TOKEN_MONTH = 342,
+TOKEN_NULL = 343,
+TOKEN_NULLS = 344,
+TOKEN_OFF = 345,
+TOKEN_ON = 346,
+

[03/11] incubator-quickstep git commit: Fix dag visualizer

[jianqiao]

Fix dag visualizer


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a57638e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a57638e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a57638e4

Branch: refs/heads/transitive-closure
Commit: a57638e400d6174be9f644b89f8b9353582c5889
Parents: b45cdbc
Author: jianqiao <jianq...@node-2.jianqiao.quickstep-pg0.wisc.cloudlab.us>
Authored: Mon Nov 27 18:27:27 2017 -0600
Committer: jianqiao <jianq...@node-2.jianqiao.quickstep-pg0.wisc.cloudlab.us>
Committed: Mon Nov 27 18:27:27 2017 -0600

--
 utility/ExecutionDAGVisualizer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a57638e4/utility/ExecutionDAGVisualizer.cpp
--
diff --git a/utility/ExecutionDAGVisualizer.cpp 
b/utility/ExecutionDAGVisualizer.cpp
index 8059ef3..bb8fe8f 100644
--- a/utility/ExecutionDAGVisualizer.cpp
+++ b/utility/ExecutionDAGVisualizer.cpp
@@ -293,7 +293,7 @@ void ExecutionDAGVisualizer::bindProfilingStats(
   "effective concurrency: " + FormatDigits(concurrency, 2));
 
   DCHECK(workorders_count.find(node_index) != workorders_count.end());
-  const std::size_t workorders_count_for_node = 
workorders_count.at(node_index);
+  const std::size_t workorders_count_for_node = 
workorders_count[node_index];
   if (workorders_count_for_node > 0) {
 mean_time_per_workorder[node_index] =
 mean_time_per_workorder[node_index] /

[05/11] incubator-quickstep git commit: Updates to cost model

[jianqiao]

Updates to cost model


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d725bcaf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d725bcaf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d725bcaf

Branch: refs/heads/transitive-closure
Commit: d725bcafec86b7637ce08a24c309b2ea1981a618
Parents: a08044e
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Fri Dec 1 17:20:25 2017 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Fri Dec 1 17:20:25 2017 -0600

--
 query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d725bcaf/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
--
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp 
b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index f7417b6..729a563 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -486,7 +486,7 @@ std::size_t StarSchemaSimpleCostModel::getNumDistinctValues(
   return stat.getNumDistinctValues(rel_attr_id);
 }
   }
-  return estimateCardinalityForTableReference(table_reference);
+  return estimateCardinalityForTableReference(table_reference) * 0.5;
 }
 
 bool StarSchemaSimpleCostModel::impliesUniqueAttributes(
@@ -520,7 +520,7 @@ bool StarSchemaSimpleCostModel::impliesUniqueAttributes(
   std::static_pointer_cast(physical_plan);
   const CatalogRelationStatistics  =
   table_reference->relation()->getStatistics();
-  if (stat.hasNumTuples()) {
+  if (stat.isExact() && stat.hasNumTuples()) {
 const std::size_t num_tuples = stat.getNumTuples();
 for (const auto  : attributes) {
   const attribute_id rel_attr_id =

[11/11] incubator-quickstep git commit: Updates to transitive closure

[jianqiao]

Updates to transitive closure


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2aefd7bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2aefd7bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2aefd7bc

Branch: refs/heads/transitive-closure
Commit: 2aefd7bce5bad9bb6063b4fd71ec37876d58662d
Parents: 734ddc1
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Mon Dec 11 14:45:08 2017 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Mon Dec 11 16:07:23 2017 -0600

--
 query_optimizer/ExecutionGenerator.cpp  |  19 ++-
 query_optimizer/PhysicalGenerator.cpp   |   2 +-
 .../BuildTransitiveClosureOperator.cpp  |   2 -
 relational_operators/CMakeLists.txt |   5 +
 .../InitializeTransitiveClosureOperator.cpp |   6 +-
 .../TransitiveClosureOperator.cpp   | 158 +++
 .../TransitiveClosureOperator.hpp   |  86 +-
 storage/TransitiveClosureState.hpp  |   8 +
 types/containers/ColumnVector.hpp   |   7 +
 .../BarrieredReadWriteConcurrentBitVector.hpp   |   7 +
 utility/BitVector.hpp   |  28 +++-
 11 files changed, 314 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2aefd7bc/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index 8f29271..648b937 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -2488,18 +2488,23 @@ void ExecutionGenerator::convertTransitiveClosure(
  _relation,
  insert_destination_proto);
 
+  const QueryPlan::DAGNodeIndex tc_operator_index =
+  execution_plan_->addRelationalOperator(
+  new TransitiveClosureOperator(query_handle_->query_id(),
+transitive_closure_state_index,
+*output_relation,
+insert_destination_index));
+  insert_destination_proto->set_relational_op_index(tc_operator_index);
 
-  (void)insert_destination_index;
+  execution_plan_->addDirectDependency(tc_operator_index,
+   build_tc_operator_index,
+   true /* is_pipeline_breaker */);
 
-  // TODO: fix
-  insert_destination_proto->set_relational_op_index(build_tc_operator_index /* 
FIX */);
   physical_to_output_relation_map_.emplace(
   std::piecewise_construct,
   std::forward_as_tuple(physical_plan),
-  std::forward_as_tuple(build_tc_operator_index /* FIX */, 
output_relation));
-
-  temporary_relation_info_vec_.emplace_back(build_tc_operator_index /* FIX */,
-output_relation);
+  std::forward_as_tuple(tc_operator_index, output_relation));
+  temporary_relation_info_vec_.emplace_back(tc_operator_index, 
output_relation);
 }
 
 }  // namespace optimizer

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2aefd7bc/query_optimizer/PhysicalGenerator.cpp
--
diff --git a/query_optimizer/PhysicalGenerator.cpp 
b/query_optimizer/PhysicalGenerator.cpp
index b7b0db0..865cd11 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -194,7 +194,7 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
  << physical_plan_->toString();
   }
 
-  std::cerr << "Optimized physical plan:\n" << physical_plan_->toString();
+  DVLOG(4) << "Optimized physical plan:\n" << physical_plan_->toString();
 
   if (FLAGS_visualize_plan) {
 quickstep::PlanVisualizer plan_visualizer;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2aefd7bc/relational_operators/BuildTransitiveClosureOperator.cpp
--
diff --git a/relational_operators/BuildTransitiveClosureOperator.cpp 
b/relational_operators/BuildTransitiveClosureOperator.cpp
index e151756..919a974 100644
--- a/relational_operators/BuildTransitiveClosureOperator.cpp
+++ b/relational_operators/BuildTransitiveClosureOperator.cpp
@@ -107,7 +107,6 @@ void BuildTransitiveClosureWorkOrder::execute() {
 }
 
 void BuildTransitiveClosureWorkOrder::buildStartRelation(ValueAccessor 
*accessor) {
-  std::cout << "BuildStartRelation: " << block_ << "\n";
   InvokeOnAnyValueAccessor(
   

[09/11] incubator-quickstep git commit: Initialize updates for transitive closure

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/734ddc1e/parser/preprocessed/SqlLexer_gen.cpp
--
diff --git a/parser/preprocessed/SqlLexer_gen.cpp 
b/parser/preprocessed/SqlLexer_gen.cpp
index 4800cde..6847a14 100644
--- a/parser/preprocessed/SqlLexer_gen.cpp
+++ b/parser/preprocessed/SqlLexer_gen.cpp
@@ -592,8 +592,8 @@ static void yynoreturn yy_fatal_error ( const char* msg , 
yyscan_t yyscanner );
yyg->yy_hold_char = *yy_cp; \
*yy_cp = '\0'; \
yyg->yy_c_buf_p = yy_cp;
-#define YY_NUM_RULES 164
-#define YY_END_OF_BUFFER 165
+#define YY_NUM_RULES 165
+#define YY_END_OF_BUFFER 166
 /* This struct is not used in this scanner,
but its presence is necessary. */
 struct yy_trans_info
@@ -601,72 +601,74 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
-static const flex_int16_t yy_accept[589] =
+static const flex_int16_t yy_accept[593] =
 {   0,
 0,0,0,0,0,0,0,0,0,0,
-0,0,  165,2,2,  163,  163,  162,  161,  163,
-  140,  136,  139,  136,  136,  159,  132,  129,  133,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  137,4,5,5,3,  155,
-  155,  152,  156,  156,  150,  157,  157,  154,1,  162,
-  130,  160,  159,  159,  159,0,  134,  131,  135,  158,
-  158,  158,  158,   10,  158,  158,  158,   22,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  138,
-
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,   58,   67,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,   81,   82,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  113,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,4,5,3,  155,  151,  156,
-  149,  149,  141,  143,  144,  145,  146,  147,  148,  149,
-  157,  153,  160,  159,0,  159,6,7,  158,9,
-   11,  158,  158,   15,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,   33,  158,  158,  158,  158,
-
-  158,  158,  158,  158,   43,  158,  158,  158,  158,  158,
-  158,   50,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,   62,  158,   69,  158,  158,  158,  158,  158,  158,
-  158,   77,  158,   80,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,   98,  158,  158,
-  103,  104,  158,  158,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,  158,  141,
-  143,  142,  158,  158,  158,  158,  158,  158,  158,   20,
-   23,  158,  158,  158,   28,  158,  158,  158,   31,  158,
-  158,  158,   37,  158,  158,   41,   42,  158,  158,  158,
-
-  158,  158,  158,  158,   52,   53,  158,   55,  158,   57,
-  158,  158,  158,  158,   66,   68,   70,   71,   72,  158,
-   74,  158,  158,   78,  158,  158,   85,  158,  158,  158,
-  158,  158,   92,  158,   94,  158,  158,  158,  100,  158,
-  158,  158,  158,  158,  158,  158,  158,  110,  111,  114,
-  158,  158,  158,  158,  158,  158,  158,  158,  123,  158,
-  158,  126,  127,  141,  142,8,  158,  158,  158,  158,
-  158,  158,  158,   25,  158,  158,  158,  158,  158,  158,
-  158,  158,  158,  158,  158,  158,  158,  158,   46,   47,
-   48,  158,  158,   54,  158,   59,   60,  158,  158,  158,
-
-   73,  158,   76,   79,   83,   84,  158,  158,  158,  158,
-  158,   93,  158,  158,   97,  158,  158,  158,  158,  158,
-  158,  158,  109,  158,  158,  158,  117,  158,  158,  120,
-  158,  158,  124,  158,  158,  158,  158,   14,  158,  158,
-  158,  158,  158,   26,  158,   29,  158,  158,  158,  158,
-  158,   36,  158,  158,   40,   44,  158,  158,  158,   56,
-   61,  158,  158,  158,   75,  158,  158,  158,  158,  158,
-  158,   96,  158,  101,  102,  158,  106,  107,  158,  158,
-  158,  158,  118,  119,  121,  158,  125,  158,  158,   13,
-  158,  158,  158,  158,  158,  158,   21,   30,  158,   34,
-
-   35,  158,  158,   45,  158,   51,   63,  158,  158,  158,
-   88,  158,   90,  158,  158,  158,  158,  158,  158,  158,
-  158,  122,  158,  158,  158,  158,  158,  158,  158,  158,
-   32,  158,   39,  158,  158,   65,  158,  158,   91,  158,
-  158,  105,  158,  158,  158,  158,  158,   12,  158,  158,
-  158,  158,   24,  158,  158,   49,   64,   86,   89,  158,
-  158,  108,  112,  158,  116,  128,   16,  158,  158,  158,
-   27,   38,   87,   95,  158,  158,  158,   18,   19,  158,
-  115,  158,  158,  158,   99,  158,   17,0
+0,0,  166,2,2,  164,  164,  163,  162,  164,
+  141,  137,  140,  

[06/11] incubator-quickstep git commit: Initialize updates for transitive closure

[jianqiao]

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/734ddc1e/relational_operators/CMakeLists.txt
--
diff --git a/relational_operators/CMakeLists.txt 
b/relational_operators/CMakeLists.txt
index 7b9ed96..6cc7f08 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -38,6 +38,9 @@ 
add_library(quickstep_relationaloperators_BuildAggregationExistenceMapOperator
 BuildAggregationExistenceMapOperator.hpp)
 add_library(quickstep_relationaloperators_BuildHashOperator 
BuildHashOperator.cpp BuildHashOperator.hpp)
 add_library(quickstep_relationaloperators_BuildLIPFilterOperator 
BuildLIPFilterOperator.cpp BuildLIPFilterOperator.hpp)
+add_library(quickstep_relationaloperators_BuildTransitiveClosureOperator
+BuildTransitiveClosureOperator.cpp
+BuildTransitiveClosureOperator.hpp)
 add_library(quickstep_relationaloperators_CreateIndexOperator ../empty_src.cpp 
CreateIndexOperator.hpp)
 add_library(quickstep_relationaloperators_CreateTableOperator ../empty_src.cpp 
CreateTableOperator.hpp)
 add_library(quickstep_relationaloperators_DestroyAggregationStateOperator
@@ -53,6 +56,9 @@ add_library(quickstep_relationaloperators_HashJoinOperator 
HashJoinOperator.cpp
 add_library(quickstep_relationaloperators_InitializeAggregationOperator
 InitializeAggregationOperator.cpp
 InitializeAggregationOperator.hpp)
+add_library(quickstep_relationaloperators_InitializeTransitiveClosureOperator
+InitializeTransitiveClosureOperator.cpp
+InitializeTransitiveClosureOperator.hpp)
 add_library(quickstep_relationaloperators_InsertOperator InsertOperator.cpp 
InsertOperator.hpp)
 add_library(quickstep_relationaloperators_NestedLoopsJoinOperator
 NestedLoopsJoinOperator.cpp
@@ -73,6 +79,9 @@ 
add_library(quickstep_relationaloperators_SortRunGenerationOperator SortRunGener
 SortRunGenerationOperator.hpp)
 add_library(quickstep_relationaloperators_TableExportOperator 
TableExportOperator.cpp TableExportOperator.hpp)
 add_library(quickstep_relationaloperators_TableGeneratorOperator 
TableGeneratorOperator.cpp TableGeneratorOperator.hpp)
+add_library(quickstep_relationaloperators_TransitiveClosureOperator
+TransitiveClosureOperator.cpp
+TransitiveClosureOperator.hpp)
 add_library(quickstep_relationaloperators_TextScanOperator 
TextScanOperator.cpp TextScanOperator.hpp)
 add_library(quickstep_relationaloperators_UnionAllOperator 
UnionAllOperator.cpp UnionAllOperator.hpp)
 add_library(quickstep_relationaloperators_UpdateOperator UpdateOperator.cpp 
UpdateOperator.hpp)
@@ -171,6 +180,28 @@ 
target_link_libraries(quickstep_relationaloperators_BuildLIPFilterOperator
   quickstep_utility_lipfilter_LIPFilterBuilder
   quickstep_utility_lipfilter_LIPFilterUtil
   tmb)
+target_link_libraries(quickstep_relationaloperators_BuildTransitiveClosureOperator
+  glog
+  quickstep_catalog_CatalogRelation
+  quickstep_catalog_CatalogTypedefs
+  quickstep_cli_Flags
+  quickstep_queryexecution_QueryContext
+  quickstep_queryexecution_WorkOrderProtosContainer
+  quickstep_queryexecution_WorkOrdersContainer
+  quickstep_relationaloperators_RelationalOperator
+  quickstep_relationaloperators_WorkOrder
+  quickstep_relationaloperators_WorkOrder_proto
+  quickstep_storage_StorageBlock
+  quickstep_storage_StorageBlockInfo
+  quickstep_storage_StorageManager
+  quickstep_storage_TransitiveClosureState
+  quickstep_storage_TupleStorageSubBlock
+  quickstep_storage_ValueAccessor
+  quickstep_storage_ValueAccessorUtil
+  quickstep_utility_BlockIDStream
+  quickstep_utility_Macros
+  quickstep_utility_Range
+  tmb)
 target_link_libraries(quickstep_relationaloperators_CreateIndexOperator
   glog
   quickstep_catalog_CatalogRelation
@@ -304,6 +335,20 @@ 
target_link_libraries(quickstep_relationaloperators_InitializeAggregationOperato
   quickstep_storage_AggregationOperationState
   quickstep_utility_Macros
   tmb)
+target_link_libraries(quickstep_relationaloperators_InitializeTransitiveClosureOperator
+  glog
+  quickstep_cli_Flags
+  quickstep_queryexecution_QueryContext
+  quickstep_queryexecution_WorkOrderProtosContainer
+  

incubator-quickstep git commit: Get the list of referenced base relations

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 39c6214a3 -> 8a84039e1


Get the list of referenced base relations

- Find the base relations that are referenced in the query.
- The referenced relations are stored in the QueryHandle.
- Separate method in QueryProcessor class to just get this list of relations,
  without needing to fully optimize the query.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8a84039e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8a84039e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8a84039e

Branch: refs/heads/master
Commit: 8a84039e140ed80376505b6da950c08a6112470e
Parents: 39c6214
Author: Harshad Deshmukh 
Authored: Tue Nov 28 16:16:59 2017 -0600
Committer: Harshad Deshmukh 
Committed: Fri Dec 1 14:26:19 2017 -0600

--
 query_optimizer/CMakeLists.txt  |  2 +
 query_optimizer/LogicalGenerator.cpp|  1 -
 query_optimizer/Optimizer.cpp   | 11 +++
 query_optimizer/Optimizer.hpp   | 15 
 query_optimizer/QueryHandle.hpp | 21 +
 query_optimizer/QueryProcessor.cpp  |  7 ++
 query_optimizer/QueryProcessor.hpp  | 11 +++
 query_optimizer/resolver/CMakeLists.txt |  1 +
 query_optimizer/resolver/Resolver.cpp   |  7 ++
 query_optimizer/resolver/Resolver.hpp   |  5 ++
 query_optimizer/rules/CMakeLists.txt| 13 
 .../rules/ReferencedBaseRelations.cpp   | 66 
 .../rules/ReferencedBaseRelations.hpp   | 80 
 13 files changed, 239 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 1e4e346..9128c63 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -206,7 +206,9 @@ 
target_link_libraries(quickstep_queryoptimizer_LogicalToPhysicalMapper
 target_link_libraries(quickstep_queryoptimizer_Optimizer
   quickstep_queryoptimizer_ExecutionGenerator
   quickstep_queryoptimizer_LogicalGenerator
+  quickstep_queryoptimizer_OptimizerContext
   quickstep_queryoptimizer_PhysicalGenerator
+  quickstep_queryoptimizer_resolver_Resolver
   quickstep_utility_Macros)
 target_link_libraries(quickstep_queryoptimizer_OptimizerContext
   quickstep_queryoptimizer_expressions_ExprId

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/LogicalGenerator.cpp
--
diff --git a/query_optimizer/LogicalGenerator.cpp 
b/query_optimizer/LogicalGenerator.cpp
index abeca53..111fa1f 100644
--- a/query_optimizer/LogicalGenerator.cpp
+++ b/query_optimizer/LogicalGenerator.cpp
@@ -23,7 +23,6 @@
 #include 
 
 #include "parser/ParseStatement.hpp"
-
 #include "query_optimizer/OptimizerContext.hpp"
 #include "query_optimizer/Validator.hpp"
 #include "query_optimizer/logical/Logical.hpp"

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/Optimizer.cpp
--
diff --git a/query_optimizer/Optimizer.cpp b/query_optimizer/Optimizer.cpp
index 1b91574..d002ca1 100644
--- a/query_optimizer/Optimizer.cpp
+++ b/query_optimizer/Optimizer.cpp
@@ -21,6 +21,8 @@
 
 #include "query_optimizer/ExecutionGenerator.hpp"
 #include "query_optimizer/LogicalGenerator.hpp"
+#include "query_optimizer/OptimizerContext.hpp"
+#include "query_optimizer/resolver/Resolver.hpp"
 
 namespace quickstep {
 namespace optimizer {
@@ -38,5 +40,14 @@ void Optimizer::generateQueryHandle(const ParseStatement 
_statement,
   logical_generator.generatePlan(*catalog_database, parse_statement)));
 }
 
+void Optimizer::findReferencedBaseRelations(const ParseStatement 
_statement,
+CatalogDatabase *catalog_database,
+QueryHandle *query_handle) {
+  OptimizerContext optimizer_context;
+  resolver::Resolver resolver(*catalog_database, _context);
+  resolver.resolve(parse_statement);
+  
query_handle->setReferencedBaseRelations(resolver.getReferencedBaseRelations());
+}
+
 }  // namespace optimizer
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/Optimizer.hpp

incubator-quickstep git commit: Disable compiler flags that incur error for higher versions of gcc/clang.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/disable-flags [created] f94c7e31c


Disable compiler flags that incur error for higher versions of gcc/clang.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f94c7e31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f94c7e31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f94c7e31

Branch: refs/heads/disable-flags
Commit: f94c7e31c1f5aa28f6706d075b753cf864be5431
Parents: 39c6214
Author: Jianqiao Zhu <jianq...@cs.wisc.edu>
Authored: Wed Nov 29 11:35:01 2017 -0600
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Wed Nov 29 11:35:29 2017 -0600

--
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f94c7e31/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 60d8a2d..e7c7354 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -291,7 +291,7 @@ else()
   # Treat warnings as errors, 'cause we hardcore.
   CHECK_CXX_COMPILER_FLAG("-Werror" COMPILER_HAS_WERROR)
   if (COMPILER_HAS_WERROR)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
+# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
   endif()
 
   # Clang reports such warning when using Protoc 3.0 beta.
@@ -585,7 +585,7 @@ if(USE_TCMALLOC)
   CHECK_CXX_COMPILER_FLAG("-Wno-return-type-c-linkage"
   COMPILER_HAS_WNO_RETURN_TYPE_C_LINKAGE)
   if (COMPILER_HAS_WNO_RETURN_TYPE_C_LINKAGE)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type-c-linkage")
+# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-type-c-linkage")
   endif()
 endif()
 

[incubator-quickstep] Git Push Summary

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/Hash-Join-Fuse [deleted] 114d04852

incubator-quickstep git commit: Hash-Join-Fuse: Feature added and tests modified.

[jianqiao]

t fuse will result in the first check being true.
+  if (build_relation == probe_relation && physical_plan->build_predicate() == 
nullptr) {
 THROW_SQL_ERROR() << "Self-join is not supported";
   }
 
@@ -1006,7 +1017,8 @@ void ExecutionGenerator::convertHashJoin(const 
P::HashJoinPtr _plan) {
   build_attribute_ids,
   any_build_attributes_nullable,
   probe_num_partitions,
-  join_hash_table_index));
+  join_hash_table_index,
+  build_predicate_index));
 
   // Create InsertDestination proto.
   const CatalogRelation *output_relation = nullptr;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/39c6214a/query_optimizer/PhysicalGenerator.cpp
--
diff --git a/query_optimizer/PhysicalGenerator.cpp 
b/query_optimizer/PhysicalGenerator.cpp
index 865cd11..0d15a66 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -30,6 +30,7 @@
 #include "query_optimizer/rules/CollapseSelection.hpp"
 #include "query_optimizer/rules/ExtractCommonSubexpression.hpp"
 #include "query_optimizer/rules/FuseAggregateJoin.hpp"
+#include "query_optimizer/rules/FuseHashSelect.hpp"
 #include "query_optimizer/rules/InjectJoinFilters.hpp"
 #include "query_optimizer/rules/Partition.hpp"
 #include "query_optimizer/rules/PruneColumns.hpp"
@@ -67,6 +68,10 @@ DEFINE_bool(use_partition_rule, true,
 "If true, apply an optimization to support partitioned inputs. The 
"
 "optimization may add additional Selection for repartitioning.");
 
+DEFINE_bool(use_fuse_hash_select, true,
+"If true, apply an optimization that moves build-side Selection 
nodes"
+"into the hash join operator instead.");
+
 DEFINE_bool(use_filter_joins, true,
 "If true, apply an optimization that strength-reduces HashJoins to 
"
 "FilterJoins (implemented as LIPFilters attached to some anchoring 
"
@@ -176,6 +181,10 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
 rules.push_back(std::make_unique());
   }
 
+  if (FLAGS_use_fuse_hash_select) {
+rules.emplace_back(new FuseHashSelect());
+  }
+
   // NOTE(jianqiao): Adding rules after InjectJoinFilters (or AttachLIPFilters)
   // requires extra handling of LIPFilterConfiguration for transformed nodes.
   // So currently it is suggested that all the new rules be placed before this

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/39c6214a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
--
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp 
b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index e0e3dff..5aec4b4 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -179,8 +179,9 @@ std::size_t 
StarSchemaSimpleCostModel::estimateCardinalityForHashJoin(
   std::size_t right_cardinality = estimateCardinality(physical_plan->right());
   double left_selectivity = estimateSelectivity(physical_plan->left());
   double right_selectivity = estimateSelectivity(physical_plan->right());
-  return std::max(static_cast(left_cardinality * 
right_selectivity + 0.5),
-  static_cast(right_cardinality * 
left_selectivity + 0.5));
+  double build_selectivity = 
estimateSelectivityForPredicate(physical_plan->build_predicate(), 
physical_plan);
+  return std::max(static_cast(left_cardinality * 
right_selectivity * build_selectivity + 0.5),
+  static_cast(right_cardinality * 
left_selectivity * build_selectivity + 0.5));
 }
 
 std::size_t StarSchemaSimpleCostModel::estimateCardinalityForNestedLoopsJoin(
@@ -295,16 +296,20 @@ std::size_t 
StarSchemaSimpleCostModel::estimateNumDistinctValues(
 estimateNumDistinctValues(attribute_id, hash_join->left());
 double right_child_selectivity =
 estimateSelectivity(hash_join->right());
+double build_selectivity =
+estimateSelectivityForPredicate(hash_join->build_predicate(), 
hash_join);
 return static_cast(
-left_child_num_distinct_values * right_child_selectivity * 
filter_selectivity + 0.5);
+left_child_num_distinct_values * right_child_selectivity * 
filter_selectivity * build_selectivity + 0.5);
   }
   if (E::ContainsExprId(hash_join->right()->getOutputAttributes(), 
attribute_id)) {
 std::size_t right_child_num_distinct_values =
 estimateNumDistinctValues(attribute_id, hash_join->right());
 double left_child_selectivity =
 estimateSelectivity(hash_join->left());
+dou

incubator-quickstep git commit: Hash-Join-Fuse: Feature added and tests modified.

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/Hash-Join-Fuse [created] 114d04852


Hash-Join-Fuse: Feature added and tests modified.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/114d0485
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/114d0485
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/114d0485

Branch: refs/heads/Hash-Join-Fuse
Commit: 114d04852243cfca2e392d61083d55457d30d0c3
Parents: 3595bc1
Author: Dylan Bacon <dylanpba...@gmail.com>
Authored: Wed Sep 20 14:09:32 2017 -0500
Committer: Jianqiao Zhu <jianq...@cs.wisc.edu>
Committed: Mon Nov 20 14:39:57 2017 -0600

--
 query_optimizer/CMakeLists.txt  |  1 +
 query_optimizer/ExecutionGenerator.cpp  | 16 +++-
 query_optimizer/PhysicalGenerator.cpp   |  9 ++
 .../cost_model/StarSchemaSimpleCostModel.cpp| 17 ++--
 query_optimizer/physical/HashJoin.cpp   | 12 +++
 query_optimizer/physical/HashJoin.hpp   | 16 +++-
 query_optimizer/rules/CMakeLists.txt| 10 +++
 .../rules/ExtractCommonSubexpression.cpp|  1 +
 query_optimizer/rules/FuseHashSelect.cpp| 88 
 query_optimizer/rules/FuseHashSelect.hpp| 64 ++
 query_optimizer/rules/InjectJoinFilters.cpp | 16 ++--
 query_optimizer/rules/Partition.cpp |  1 +
 .../rules/ReduceGroupByAttributes.cpp   |  1 +
 query_optimizer/rules/ReorderColumns.cpp|  1 +
 .../StarSchemaHashJoinOrderOptimization.cpp |  2 +
 query_optimizer/rules/SwapProbeBuild.cpp|  1 +
 .../rules/tests/PruneColumns_unittest.cpp   |  2 +
 query_optimizer/strategy/Join.cpp   |  1 +
 .../strategy/tests/Join_unittest.cpp|  6 ++
 .../tests/physical_generator/Join.test  | 32 +++
 .../tests/physical_generator/Select.test| 31 ---
 relational_operators/BuildHashOperator.cpp  | 28 ++-
 relational_operators/BuildHashOperator.hpp  | 13 ++-
 relational_operators/WorkOrder.proto|  3 +-
 relational_operators/WorkOrderFactory.cpp   |  2 +
 .../tests/HashJoinOperator_unittest.cpp | 27 --
 26 files changed, 336 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/114d0485/query_optimizer/CMakeLists.txt
--
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 5e0db44..1e4e346 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -226,6 +226,7 @@ 
target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
   quickstep_queryoptimizer_rules_CollapseSelection
   quickstep_queryoptimizer_rules_ExtractCommonSubexpression
   quickstep_queryoptimizer_rules_FuseAggregateJoin
+  quickstep_queryoptimizer_rules_FuseHashSelect
   quickstep_queryoptimizer_rules_InjectJoinFilters
   quickstep_queryoptimizer_rules_Partition
   quickstep_queryoptimizer_rules_PruneColumns

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/114d0485/query_optimizer/ExecutionGenerator.cpp
--
diff --git a/query_optimizer/ExecutionGenerator.cpp 
b/query_optimizer/ExecutionGenerator.cpp
index b0d3c48..5ef58a9 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -939,6 +939,15 @@ void ExecutionGenerator::convertHashJoin(const 
P::HashJoinPtr _plan) {
 
query_context_proto_->add_predicates()->CopyFrom(residual_predicate->getProto());
   }
 
+  // Convert the build predicate proto.
+  QueryContext::predicate_id build_predicate_index = 
QueryContext::kInvalidPredicateId;
+  if (physical_plan->build_predicate()) {
+build_predicate_index = query_context_proto_->predicates_size();
+
+unique_ptr 
build_predicate(convertPredicate(physical_plan->build_predicate()));
+
query_context_proto_->add_predicates()->MergeFrom(build_predicate->getProto());
+  }
+
   // Convert the project expressions proto.
   const QueryContext::scalar_group_id project_expressions_group_index =
   query_context_proto_->scalar_groups_size();
@@ -966,7 +975,9 @@ void ExecutionGenerator::convertHashJoin(const 
P::HashJoinPtr _plan) {
   const CatalogRelation *probe_relation = probe_relation_info->relation;
 
   // FIXME(quickstep-team): Add support for self-join.
-  if (build_relation == probe_relation) {
+  // We check to see if the build_predicate is null as certain queries that
+  // support ha

incubator-quickstep git commit: Temporary Build Support for OS X 10.13

[jianqiao]

Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 0fe838dfe -> b237969cb


Temporary Build Support for OS X 10.13

Change of Regular Expression


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b237969c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b237969c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b237969c

Branch: refs/heads/master
Commit: b237969cb490519ffc9a1403dc327432c3625e7b
Parents: 0fe838d
Author: Yuanchen Li 
Authored: Thu Nov 9 02:18:42 2017 -0600
Committer: Yuanchen Li 
Committed: Sat Nov 18 16:01:15 2017 -0600

--
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b237969c/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 071f8fc..60d8a2d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -302,11 +302,11 @@ else()
 endif()
   endif()
 
-  # OSX 10.12 has deprecated certain system-level APIs which causes protobuf & 
glog
+  # OSX 10.12+ has deprecated certain system-level APIs which causes protobuf 
& glog
   # builds to fail. As a short-term workaround for now, we turn off deprecated
   # warnings so that they do not cause build failures anymore.
   # TODO: Remove this workaround by fixing the protobuf_cmake and glog_cmake.
-  if (${CMAKE_SYSTEM} MATCHES "Darwin-16.[0-9]*.[0-9]*")
+  if (${CMAKE_SYSTEM} MATCHES "Darwin-1[67].[0-9]*.[0-9]*")
 if (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
   CHECK_CXX_COMPILER_FLAG("-Wno-error=deprecated-declarations" 
COMPILER_HAS_WNO_DEPRECATED)
   if (COMPILER_HAS_WNO_DEPRECATED)