[6/9] incubator-quickstep git commit: Refectored bulk insertion to the SplitRow store

2016-10-20 Thread zuyuz 
Refectored bulk insertion to the SplitRow store

The inner loop of the insert algorithm has been changed to reduce
function calls to only those that are absolutely necessary. Also, we
merge copies which come from other rowstore source, speeding up
insertion time.

Also adds support for the idea of 'partial inserts'. Partial
inserts are when you are only inserting a subset of the columns at a
time. Partial inserts will be used in a later commit.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d3a09205
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d3a09205
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d3a09205

Branch: refs/heads/multiple_shiftboss
Commit: d3a0920595e4c40ba5e86907359b6e254b5b4958
Parents: 55480d8
Author: cramja 
Authored: Wed Oct 5 16:40:30 2016 -0500
Committer: jianqiao 
Committed: Wed Oct 19 00:32:08 2016 -0500

--
 storage/SplitRowStoreTupleStorageSubBlock.cpp   | 691 +--
 storage/SplitRowStoreTupleStorageSubBlock.hpp   | 186 +
 ...litRowStoreTupleStorageSubBlock_unittest.cpp | 449 ++--
 utility/BitVector.hpp   |  14 +
 4 files changed, 906 insertions(+), 434 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d3a09205/storage/SplitRowStoreTupleStorageSubBlock.cpp
--
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.cpp 
b/storage/SplitRowStoreTupleStorageSubBlock.cpp
index f955c99..068e975 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.cpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.cpp
@@ -41,54 +41,61 @@ namespace quickstep {
 
 QUICKSTEP_REGISTER_TUPLE_STORE(SplitRowStoreTupleStorageSubBlock, 
SPLIT_ROW_STORE);
 
+using splitrow_internal::CopyGroupList;
+using splitrow_internal::ContiguousAttrs;
+using splitrow_internal::NullableAttr;
+using splitrow_internal::VarLenAttr;
+
+const std::size_t SplitRowStoreTupleStorageSubBlock::kVarLenSlotSize = 
sizeof(std::uint32_t) * 2;
+
 namespace {
 
-template 
-inline std::size_t CalculateVariableSize(
+  template
+  inline std::size_t CalculateVariableSize(
 const CatalogRelationSchema ,
 const ValueAccessorT ) {
-  std::size_t total_size = 0;
-  attribute_id accessor_attr_id = 0;
-  for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
-   attr_it != relation.end();
-   ++attr_it, ++accessor_attr_id) {
-if (!attr_it->getType().isVariableLength()) {
-  continue;
-}
+std::size_t total_size = 0;
+attribute_id accessor_attr_id = 0;
+for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
+ attr_it != relation.end();
+ ++attr_it, ++accessor_attr_id) {
+  if (!attr_it->getType().isVariableLength()) {
+continue;
+  }
 
-TypedValue value(accessor.getTypedValue(accessor_attr_id));
-if (nullable_attrs && value.isNull()) {
-  continue;
+  TypedValue value(accessor.getTypedValue(accessor_attr_id));
+  if (nullable_attrs && value.isNull()) {
+continue;
+  }
+  total_size += value.getDataSize();
 }
-total_size += value.getDataSize();
+return total_size;
   }
-  return total_size;
-}
 
-template 
-inline std::size_t CalculateVariableSizeWithRemappedAttributes(
+  template
+  inline std::size_t CalculateVariableSizeWithRemappedAttributes(
 const CatalogRelationSchema ,
 const ValueAccessorT ,
 const std::vector _map) {
-  std::size_t total_size = 0;
-  std::vector::const_iterator attr_map_it = 
attribute_map.begin();
-  for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
-   attr_it != relation.end();
-   ++attr_it, ++attr_map_it) {
-if (!attr_it->getType().isVariableLength()) {
-  continue;
-}
+std::size_t total_size = 0;
+std::vector::const_iterator attr_map_it = 
attribute_map.begin();
+for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
+ attr_it != relation.end();
+ ++attr_it, ++attr_map_it) {
+  if (!attr_it->getType().isVariableLength()) {
+continue;
+  }
 
-TypedValue value(accessor.getTypedValue(*attr_map_it));
-if (nullable_attrs && value.isNull()) {
-  continue;
+  TypedValue value(accessor.getTypedValue(*attr_map_it));
+  if (nullable_attrs && value.isNull()) {
+continue;
+  }
+  total_size += value.getDataSize();
 }
-total_size += value.getDataSize();
+return total_size;
   }
-  return total_size;
-}
 
-}  // namespace
+}  // anonymous namespace

[3/5] incubator-quickstep git commit: Refectored bulk insertion to the SplitRow store

2016-10-20 Thread jianqiao 
Refectored bulk insertion to the SplitRow store

The inner loop of the insert algorithm has been changed to reduce
function calls to only those that are absolutely necessary. Also, we
merge copies which come from other rowstore source, speeding up
insertion time.

Also adds support for the idea of 'partial inserts'. Partial
inserts are when you are only inserting a subset of the columns at a
time. Partial inserts will be used in a later commit.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d3a09205
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d3a09205
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d3a09205

Branch: refs/heads/lip-refactor
Commit: d3a0920595e4c40ba5e86907359b6e254b5b4958
Parents: 55480d8
Author: cramja 
Authored: Wed Oct 5 16:40:30 2016 -0500
Committer: jianqiao 
Committed: Wed Oct 19 00:32:08 2016 -0500

--
 storage/SplitRowStoreTupleStorageSubBlock.cpp   | 691 +--
 storage/SplitRowStoreTupleStorageSubBlock.hpp   | 186 +
 ...litRowStoreTupleStorageSubBlock_unittest.cpp | 449 ++--
 utility/BitVector.hpp   |  14 +
 4 files changed, 906 insertions(+), 434 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d3a09205/storage/SplitRowStoreTupleStorageSubBlock.cpp
--
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.cpp 
b/storage/SplitRowStoreTupleStorageSubBlock.cpp
index f955c99..068e975 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.cpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.cpp
@@ -41,54 +41,61 @@ namespace quickstep {
 
 QUICKSTEP_REGISTER_TUPLE_STORE(SplitRowStoreTupleStorageSubBlock, 
SPLIT_ROW_STORE);
 
+using splitrow_internal::CopyGroupList;
+using splitrow_internal::ContiguousAttrs;
+using splitrow_internal::NullableAttr;
+using splitrow_internal::VarLenAttr;
+
+const std::size_t SplitRowStoreTupleStorageSubBlock::kVarLenSlotSize = 
sizeof(std::uint32_t) * 2;
+
 namespace {
 
-template 
-inline std::size_t CalculateVariableSize(
+  template
+  inline std::size_t CalculateVariableSize(
 const CatalogRelationSchema ,
 const ValueAccessorT ) {
-  std::size_t total_size = 0;
-  attribute_id accessor_attr_id = 0;
-  for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
-   attr_it != relation.end();
-   ++attr_it, ++accessor_attr_id) {
-if (!attr_it->getType().isVariableLength()) {
-  continue;
-}
+std::size_t total_size = 0;
+attribute_id accessor_attr_id = 0;
+for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
+ attr_it != relation.end();
+ ++attr_it, ++accessor_attr_id) {
+  if (!attr_it->getType().isVariableLength()) {
+continue;
+  }
 
-TypedValue value(accessor.getTypedValue(accessor_attr_id));
-if (nullable_attrs && value.isNull()) {
-  continue;
+  TypedValue value(accessor.getTypedValue(accessor_attr_id));
+  if (nullable_attrs && value.isNull()) {
+continue;
+  }
+  total_size += value.getDataSize();
 }
-total_size += value.getDataSize();
+return total_size;
   }
-  return total_size;
-}
 
-template 
-inline std::size_t CalculateVariableSizeWithRemappedAttributes(
+  template
+  inline std::size_t CalculateVariableSizeWithRemappedAttributes(
 const CatalogRelationSchema ,
 const ValueAccessorT ,
 const std::vector _map) {
-  std::size_t total_size = 0;
-  std::vector::const_iterator attr_map_it = 
attribute_map.begin();
-  for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
-   attr_it != relation.end();
-   ++attr_it, ++attr_map_it) {
-if (!attr_it->getType().isVariableLength()) {
-  continue;
-}
+std::size_t total_size = 0;
+std::vector::const_iterator attr_map_it = 
attribute_map.begin();
+for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
+ attr_it != relation.end();
+ ++attr_it, ++attr_map_it) {
+  if (!attr_it->getType().isVariableLength()) {
+continue;
+  }
 
-TypedValue value(accessor.getTypedValue(*attr_map_it));
-if (nullable_attrs && value.isNull()) {
-  continue;
+  TypedValue value(accessor.getTypedValue(*attr_map_it));
+  if (nullable_attrs && value.isNull()) {
+continue;
+  }
+  total_size += value.getDataSize();
 }
-total_size += value.getDataSize();
+return total_size;
   }
-  return total_size;
-}
 
-}  // namespace
+}  // anonymous namespace
 
 SplitRowStoreTupleStorageSubBlock::SplitRowStoreTupleStorageSubBlock(

[13/14] incubator-quickstep git commit: Refectored bulk insertion to the SplitRow store

2016-10-19 Thread hbdeshmukh 
Refectored bulk insertion to the SplitRow store

The inner loop of the insert algorithm has been changed to reduce
function calls to only those that are absolutely necessary. Also, we
merge copies which come from other rowstore source, speeding up
insertion time.

Also adds support for the idea of 'partial inserts'. Partial
inserts are when you are only inserting a subset of the columns at a
time. Partial inserts will be used in a later commit.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d3a09205
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d3a09205
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d3a09205

Branch: refs/heads/fix-fasthash-destructor
Commit: d3a0920595e4c40ba5e86907359b6e254b5b4958
Parents: 55480d8
Author: cramja 
Authored: Wed Oct 5 16:40:30 2016 -0500
Committer: jianqiao 
Committed: Wed Oct 19 00:32:08 2016 -0500

--
 storage/SplitRowStoreTupleStorageSubBlock.cpp   | 691 +--
 storage/SplitRowStoreTupleStorageSubBlock.hpp   | 186 +
 ...litRowStoreTupleStorageSubBlock_unittest.cpp | 449 ++--
 utility/BitVector.hpp   |  14 +
 4 files changed, 906 insertions(+), 434 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d3a09205/storage/SplitRowStoreTupleStorageSubBlock.cpp
--
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.cpp 
b/storage/SplitRowStoreTupleStorageSubBlock.cpp
index f955c99..068e975 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.cpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.cpp
@@ -41,54 +41,61 @@ namespace quickstep {
 
 QUICKSTEP_REGISTER_TUPLE_STORE(SplitRowStoreTupleStorageSubBlock, 
SPLIT_ROW_STORE);
 
+using splitrow_internal::CopyGroupList;
+using splitrow_internal::ContiguousAttrs;
+using splitrow_internal::NullableAttr;
+using splitrow_internal::VarLenAttr;
+
+const std::size_t SplitRowStoreTupleStorageSubBlock::kVarLenSlotSize = 
sizeof(std::uint32_t) * 2;
+
 namespace {
 
-template 
-inline std::size_t CalculateVariableSize(
+  template
+  inline std::size_t CalculateVariableSize(
 const CatalogRelationSchema ,
 const ValueAccessorT ) {
-  std::size_t total_size = 0;
-  attribute_id accessor_attr_id = 0;
-  for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
-   attr_it != relation.end();
-   ++attr_it, ++accessor_attr_id) {
-if (!attr_it->getType().isVariableLength()) {
-  continue;
-}
+std::size_t total_size = 0;
+attribute_id accessor_attr_id = 0;
+for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
+ attr_it != relation.end();
+ ++attr_it, ++accessor_attr_id) {
+  if (!attr_it->getType().isVariableLength()) {
+continue;
+  }
 
-TypedValue value(accessor.getTypedValue(accessor_attr_id));
-if (nullable_attrs && value.isNull()) {
-  continue;
+  TypedValue value(accessor.getTypedValue(accessor_attr_id));
+  if (nullable_attrs && value.isNull()) {
+continue;
+  }
+  total_size += value.getDataSize();
 }
-total_size += value.getDataSize();
+return total_size;
   }
-  return total_size;
-}
 
-template 
-inline std::size_t CalculateVariableSizeWithRemappedAttributes(
+  template
+  inline std::size_t CalculateVariableSizeWithRemappedAttributes(
 const CatalogRelationSchema ,
 const ValueAccessorT ,
 const std::vector _map) {
-  std::size_t total_size = 0;
-  std::vector::const_iterator attr_map_it = 
attribute_map.begin();
-  for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
-   attr_it != relation.end();
-   ++attr_it, ++attr_map_it) {
-if (!attr_it->getType().isVariableLength()) {
-  continue;
-}
+std::size_t total_size = 0;
+std::vector::const_iterator attr_map_it = 
attribute_map.begin();
+for (CatalogRelationSchema::const_iterator attr_it = relation.begin();
+ attr_it != relation.end();
+ ++attr_it, ++attr_map_it) {
+  if (!attr_it->getType().isVariableLength()) {
+continue;
+  }
 
-TypedValue value(accessor.getTypedValue(*attr_map_it));
-if (nullable_attrs && value.isNull()) {
-  continue;
+  TypedValue value(accessor.getTypedValue(*attr_map_it));
+  if (nullable_attrs && value.isNull()) {
+continue;
+  }
+  total_size += value.getDataSize();
 }
-total_size += value.getDataSize();
+return total_size;
   }
-  return total_size;
-}
 
-}  // namespace
+}  // anonymous namespace