http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/file/reader.h ---------------------------------------------------------------------- diff --git a/src/parquet/file/reader.h b/src/parquet/file/reader.h index 58df9a6..565bb5c 100644 --- a/src/parquet/file/reader.h +++ b/src/parquet/file/reader.h @@ -50,8 +50,8 @@ class RowGroupReader { virtual std::unique_ptr<PageReader> GetColumnPageReader(int i) = 0; }; - RowGroupReader(const SchemaDescriptor* schema, - std::unique_ptr<Contents> contents, MemoryAllocator* allocator); + RowGroupReader(const SchemaDescriptor* schema, std::unique_ptr<Contents> contents, + MemoryAllocator* allocator); // Construct a ColumnReader for the indicated row group-relative // column. Ownership is shared with the RowGroupReader. @@ -73,7 +73,6 @@ class RowGroupReader { MemoryAllocator* allocator_; }; - class ParquetFileReader { public: // Forward declare the PIMPL @@ -89,9 +88,7 @@ class ParquetFileReader { virtual int num_row_groups() const = 0; // Return const-poitner to make it clear that this object is not to be copied - const SchemaDescriptor* schema() const { - return &schema_; - } + const SchemaDescriptor* schema() const { return &schema_; } SchemaDescriptor schema_; }; @@ -117,16 +114,12 @@ class ParquetFileReader { int num_row_groups() const; // Returns the file schema descriptor - const SchemaDescriptor* descr() { - return schema_; - } + const SchemaDescriptor* descr() { return schema_; } - const ColumnDescriptor* column_schema(int i) const { - return schema_->Column(i); - } + const ColumnDescriptor* column_schema(int i) const { return schema_->Column(i); } - void DebugPrint(std::ostream& stream, std::list<int> selected_columns, - bool print_values = true); + void DebugPrint( + std::ostream& stream, std::list<int> selected_columns, bool print_values = true); private: // PIMPL idiom @@ -138,7 +131,6 @@ class ParquetFileReader { const SchemaDescriptor* schema_; }; +} // namespace parquet -} // namespace parquet - -#endif // PARQUET_FILE_READER_H +#endif // PARQUET_FILE_READER_H
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/file/writer-internal.cc ---------------------------------------------------------------------- diff --git a/src/parquet/file/writer-internal.cc b/src/parquet/file/writer-internal.cc index f087752..d87d521 100644 --- a/src/parquet/file/writer-internal.cc +++ b/src/parquet/file/writer-internal.cc @@ -33,10 +33,9 @@ static constexpr uint8_t PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'}; // ---------------------------------------------------------------------- // SerializedPageWriter -SerializedPageWriter::SerializedPageWriter(OutputStream* sink, - Compression::type codec, format::ColumnChunk* metadata, - MemoryAllocator* allocator) : sink_(sink), metadata_(metadata), - allocator_(allocator) { +SerializedPageWriter::SerializedPageWriter(OutputStream* sink, Compression::type codec, + format::ColumnChunk* metadata, MemoryAllocator* allocator) + : sink_(sink), metadata_(metadata), allocator_(allocator) { compressor_ = Codec::Create(codec); // Currently we directly start with the data page metadata_->meta_data.__set_data_page_offset(sink_->Tell()); @@ -57,16 +56,16 @@ int64_t SerializedPageWriter::WriteDataPage(int32_t num_rows, int32_t num_values const std::shared_ptr<Buffer>& definition_levels, Encoding::type definition_level_encoding, const std::shared_ptr<Buffer>& repetition_levels, - Encoding::type repetition_level_encoding, - const std::shared_ptr<Buffer>& values, Encoding::type encoding) { - int64_t uncompressed_size = definition_levels->size() + repetition_levels->size() - + values->size(); + Encoding::type repetition_level_encoding, const std::shared_ptr<Buffer>& values, + Encoding::type encoding) { + int64_t uncompressed_size = + definition_levels->size() + repetition_levels->size() + values->size(); // Concatenate data into a single buffer // TODO: In the uncompressed case, directly write this to the sink // TODO: Reuse the (un)compressed_data buffer instead of recreating it each time. std::shared_ptr<OwnedMutableBuffer> uncompressed_data = - std::make_shared<OwnedMutableBuffer>(uncompressed_size, allocator_); + std::make_shared<OwnedMutableBuffer>(uncompressed_size, allocator_); uint8_t* uncompressed_ptr = uncompressed_data->mutable_data(); memcpy(uncompressed_ptr, repetition_levels->data(), repetition_levels->size()); uncompressed_ptr += repetition_levels->size(); @@ -124,7 +123,7 @@ int64_t RowGroupSerializer::num_rows() const { return num_rows_; } -const SchemaDescriptor* RowGroupSerializer::schema() const { +const SchemaDescriptor* RowGroupSerializer::schema() const { return schema_; } @@ -134,20 +133,17 @@ ColumnWriter* RowGroupSerializer::NextColumn() { } current_column_index_++; - if (current_column_writer_) { - total_bytes_written_ += current_column_writer_->Close(); - } + if (current_column_writer_) { total_bytes_written_ += current_column_writer_->Close(); } const ColumnDescriptor* column_descr = schema_->Column(current_column_index_); format::ColumnChunk* col_meta = &metadata_->columns[current_column_index_]; col_meta->__isset.meta_data = true; col_meta->meta_data.__set_type(ToThrift(column_descr->physical_type())); col_meta->meta_data.__set_path_in_schema(column_descr->path()->ToDotVector()); - std::unique_ptr<PageWriter> pager(new SerializedPageWriter(sink_, - Compression::UNCOMPRESSED, col_meta, - allocator_)); - current_column_writer_ = ColumnWriter::Make(column_descr, - std::move(pager), num_rows_, allocator_); + std::unique_ptr<PageWriter> pager( + new SerializedPageWriter(sink_, Compression::UNCOMPRESSED, col_meta, allocator_)); + current_column_writer_ = + ColumnWriter::Make(column_descr, std::move(pager), num_rows_, allocator_); return current_column_writer_.get(); } @@ -177,9 +173,7 @@ std::unique_ptr<ParquetFileWriter::Contents> FileSerializer::Open( } void FileSerializer::Close() { - if (row_group_writer_) { - row_group_writer_->Close(); - } + if (row_group_writer_) { row_group_writer_->Close(); } row_group_writer_.reset(); // Write magic bytes and metadata @@ -201,9 +195,7 @@ int64_t FileSerializer::num_rows() const { } RowGroupWriter* FileSerializer::AppendRowGroup(int64_t num_rows) { - if (row_group_writer_) { - row_group_writer_->Close(); - } + if (row_group_writer_) { row_group_writer_->Close(); } num_rows_ += num_rows; num_row_groups_++; @@ -224,8 +216,8 @@ void FileSerializer::WriteMetaData() { // Write MetaData uint32_t metadata_len = sink_->Tell(); - SchemaFlattener flattener(static_cast<GroupNode*>(schema_.schema().get()), - &metadata_.schema); + SchemaFlattener flattener( + static_cast<GroupNode*>(schema_.schema().get()), &metadata_.schema); flattener.Flatten(); // TODO: Currently we only write version 1 files @@ -244,12 +236,9 @@ void FileSerializer::WriteMetaData() { sink_->Write(PARQUET_MAGIC, 4); } -FileSerializer::FileSerializer( - std::shared_ptr<OutputStream> sink, - std::shared_ptr<GroupNode>& schema, - MemoryAllocator* allocator = default_allocator()) : - sink_(sink), allocator_(allocator), - num_row_groups_(0), num_rows_(0) { +FileSerializer::FileSerializer(std::shared_ptr<OutputStream> sink, + std::shared_ptr<GroupNode>& schema, MemoryAllocator* allocator = default_allocator()) + : sink_(sink), allocator_(allocator), num_row_groups_(0), num_rows_(0) { schema_.Init(schema); StartFile(); } @@ -259,4 +248,4 @@ void FileSerializer::StartFile() { sink_->Write(PARQUET_MAGIC, 4); } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/file/writer-internal.h ---------------------------------------------------------------------- diff --git a/src/parquet/file/writer-internal.h b/src/parquet/file/writer-internal.h index e88348b..8c5db68 100644 --- a/src/parquet/file/writer-internal.h +++ b/src/parquet/file/writer-internal.h @@ -35,9 +35,8 @@ namespace parquet { // TODO: Currently only writes DataPage pages. class SerializedPageWriter : public PageWriter { public: - SerializedPageWriter(OutputStream* sink, - Compression::type codec, format::ColumnChunk* metadata, - MemoryAllocator* allocator = default_allocator()); + SerializedPageWriter(OutputStream* sink, Compression::type codec, + format::ColumnChunk* metadata, MemoryAllocator* allocator = default_allocator()); virtual ~SerializedPageWriter() {} @@ -47,8 +46,8 @@ class SerializedPageWriter : public PageWriter { const std::shared_ptr<Buffer>& definition_levels, Encoding::type definition_level_encoding, const std::shared_ptr<Buffer>& repetition_levels, - Encoding::type repetition_level_encoding, - const std::shared_ptr<Buffer>& values, Encoding::type encoding) override; + Encoding::type repetition_level_encoding, const std::shared_ptr<Buffer>& values, + Encoding::type encoding) override; void Close() override; @@ -67,16 +66,15 @@ class SerializedPageWriter : public PageWriter { // RowGroupWriter::Contents implementation for the Parquet file specification class RowGroupSerializer : public RowGroupWriter::Contents { public: - RowGroupSerializer(int64_t num_rows, - const SchemaDescriptor* schema, - OutputStream* sink, - format::RowGroup* metadata, - MemoryAllocator* allocator) : - num_rows_(num_rows), schema_(schema), sink_(sink), - metadata_(metadata), - allocator_(allocator), - total_bytes_written_(0), - current_column_index_(-1) { + RowGroupSerializer(int64_t num_rows, const SchemaDescriptor* schema, OutputStream* sink, + format::RowGroup* metadata, MemoryAllocator* allocator) + : num_rows_(num_rows), + schema_(schema), + sink_(sink), + metadata_(metadata), + allocator_(allocator), + total_bytes_written_(0), + current_column_index_(-1) { metadata_->__set_num_rows(num_rows_); metadata_->columns.resize(schema->num_columns()); } @@ -109,8 +107,7 @@ class RowGroupSerializer : public RowGroupWriter::Contents { class FileSerializer : public ParquetFileWriter::Contents { public: static std::unique_ptr<ParquetFileWriter::Contents> Open( - std::shared_ptr<OutputStream> sink, - std::shared_ptr<schema::GroupNode>& schema, + std::shared_ptr<OutputStream> sink, std::shared_ptr<schema::GroupNode>& schema, MemoryAllocator* allocator = default_allocator()); void Close() override; @@ -125,8 +122,7 @@ class FileSerializer : public ParquetFileWriter::Contents { private: explicit FileSerializer(std::shared_ptr<OutputStream> sink, - std::shared_ptr<schema::GroupNode>& schema, - MemoryAllocator* allocator); + std::shared_ptr<schema::GroupNode>& schema, MemoryAllocator* allocator); std::shared_ptr<OutputStream> sink_; format::FileMetaData metadata_; @@ -140,6 +136,6 @@ class FileSerializer : public ParquetFileWriter::Contents { void WriteMetaData(); }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_FILE_WRITER_INTERNAL_H +#endif // PARQUET_FILE_WRITER_INTERNAL_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/file/writer.cc ---------------------------------------------------------------------- diff --git a/src/parquet/file/writer.cc b/src/parquet/file/writer.cc index 6e9f4f9..c516a5e 100644 --- a/src/parquet/file/writer.cc +++ b/src/parquet/file/writer.cc @@ -27,9 +27,9 @@ namespace parquet { // ---------------------------------------------------------------------- // RowGroupWriter public API -RowGroupWriter::RowGroupWriter(std::unique_ptr<Contents> contents, - MemoryAllocator* allocator): - contents_(std::move(contents)), allocator_(allocator) { +RowGroupWriter::RowGroupWriter( + std::unique_ptr<Contents> contents, MemoryAllocator* allocator) + : contents_(std::move(contents)), allocator_(allocator) { schema_ = contents_->schema(); } @@ -80,4 +80,4 @@ RowGroupWriter* ParquetFileWriter::AppendRowGroup(int64_t num_rows) { return contents_->AppendRowGroup(num_rows); } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/file/writer.h ---------------------------------------------------------------------- diff --git a/src/parquet/file/writer.h b/src/parquet/file/writer.h index 9ad2539..22ee856 100644 --- a/src/parquet/file/writer.h +++ b/src/parquet/file/writer.h @@ -94,17 +94,14 @@ class ParquetFileWriter { virtual int num_row_groups() const = 0; // Return const-poitner to make it clear that this object is not to be copied - const SchemaDescriptor* schema() const { - return &schema_; - } + const SchemaDescriptor* schema() const { return &schema_; } SchemaDescriptor schema_; }; ParquetFileWriter(); ~ParquetFileWriter(); - static std::unique_ptr<ParquetFileWriter> Open( - std::shared_ptr<OutputStream> sink, + static std::unique_ptr<ParquetFileWriter> Open(std::shared_ptr<OutputStream> sink, std::shared_ptr<schema::GroupNode>& schema, MemoryAllocator* allocator = default_allocator()); @@ -144,13 +141,9 @@ class ParquetFileWriter { /** * Returns the file schema descriptor */ - const SchemaDescriptor* descr() { - return schema_; - } + const SchemaDescriptor* descr() { return schema_; } - const ColumnDescriptor* column_schema(int i) const { - return schema_->Column(i); - } + const ColumnDescriptor* column_schema(int i) const { return schema_->Column(i); } private: // This is declared in the .cc file so that we can hide compiled Thrift @@ -161,7 +154,6 @@ class ParquetFileWriter { const SchemaDescriptor* schema_; }; -} // namespace parquet - -#endif // PARQUET_FILE_WRITER_H +} // namespace parquet +#endif // PARQUET_FILE_WRITER_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/public-api-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/public-api-test.cc b/src/parquet/public-api-test.cc index 4edf79b..1dc7621 100644 --- a/src/parquet/public-api-test.cc +++ b/src/parquet/public-api-test.cc @@ -29,4 +29,4 @@ TEST(TestPublicAPI, DoesNotIncludeThrift) { #endif } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/reader-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/reader-test.cc b/src/parquet/reader-test.cc index a09891c..9a8fcf6 100644 --- a/src/parquet/reader-test.cc +++ b/src/parquet/reader-test.cc @@ -36,14 +36,14 @@ namespace parquet { const char* data_dir = std::getenv("PARQUET_TEST_DATA"); - class TestAllTypesPlain : public ::testing::Test { public: void SetUp() { std::string dir_string(data_dir); std::stringstream ss; - ss << dir_string << "/" << "alltypes_plain.parquet"; + ss << dir_string << "/" + << "alltypes_plain.parquet"; reader_ = ParquetFileReader::OpenFile(ss.str()); } @@ -54,15 +54,14 @@ class TestAllTypesPlain : public ::testing::Test { std::unique_ptr<ParquetFileReader> reader_; }; -TEST_F(TestAllTypesPlain, NoopConstructDestruct) { -} +TEST_F(TestAllTypesPlain, NoopConstructDestruct) {} TEST_F(TestAllTypesPlain, TestBatchRead) { std::shared_ptr<RowGroupReader> group = reader_->RowGroup(0); // column 0, id std::shared_ptr<Int32Reader> col = - std::dynamic_pointer_cast<Int32Reader>(group->Column(0)); + std::dynamic_pointer_cast<Int32Reader>(group->Column(0)); int16_t def_levels[4]; int16_t rep_levels[4]; @@ -94,8 +93,7 @@ TEST_F(TestAllTypesPlain, TestFlatScannerInt32) { std::shared_ptr<RowGroupReader> group = reader_->RowGroup(0); // column 0, id - std::shared_ptr<Int32Scanner> scanner( - new Int32Scanner(group->Column(0))); + std::shared_ptr<Int32Scanner> scanner(new Int32Scanner(group->Column(0))); int32_t val; bool is_null; for (int i = 0; i < 8; ++i) { @@ -107,20 +105,17 @@ TEST_F(TestAllTypesPlain, TestFlatScannerInt32) { ASSERT_FALSE(scanner->NextValue(&val, &is_null)); } - TEST_F(TestAllTypesPlain, TestSetScannerBatchSize) { std::shared_ptr<RowGroupReader> group = reader_->RowGroup(0); // column 0, id - std::shared_ptr<Int32Scanner> scanner( - new Int32Scanner(group->Column(0))); + std::shared_ptr<Int32Scanner> scanner(new Int32Scanner(group->Column(0))); ASSERT_EQ(128, scanner->batch_size()); scanner->SetBatchSize(1024); ASSERT_EQ(1024, scanner->batch_size()); } - TEST_F(TestAllTypesPlain, DebugPrintWorks) { std::stringstream ss; @@ -156,14 +151,14 @@ TEST_F(TestAllTypesPlain, ColumnSelectionOutOfRange) { ASSERT_THROW(reader_->DebugPrint(ss, columns), ParquetException); } - class TestLocalFileSource : public ::testing::Test { public: void SetUp() { std::string dir_string(data_dir); std::stringstream ss; - ss << dir_string << "/" << "alltypes_plain.parquet"; + ss << dir_string << "/" + << "alltypes_plain.parquet"; file.reset(new LocalFileSource()); file->Open(ss.str()); @@ -186,5 +181,4 @@ TEST_F(TestLocalFileSource, FileClosedOnDestruction) { ASSERT_EQ(EBADF, errno); } - -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/converter.cc ---------------------------------------------------------------------- diff --git a/src/parquet/schema/converter.cc b/src/parquet/schema/converter.cc index bece340..08eeb66 100644 --- a/src/parquet/schema/converter.cc +++ b/src/parquet/schema/converter.cc @@ -77,8 +77,7 @@ std::shared_ptr<SchemaDescriptor> FromParquet(const std::vector<SchemaElement>& std::unique_ptr<Node> root = converter.Convert(); std::shared_ptr<SchemaDescriptor> descr = std::make_shared<SchemaDescriptor>(); - descr->Init(std::shared_ptr<GroupNode>( - static_cast<GroupNode*>(root.release()))); + descr->Init(std::shared_ptr<GroupNode>(static_cast<GroupNode*>(root.release()))); return descr; } @@ -91,7 +90,7 @@ void ToParquet(const GroupNode* schema, std::vector<format::SchemaElement>* out) class SchemaVisitor : public Node::ConstVisitor { public: explicit SchemaVisitor(std::vector<format::SchemaElement>* elements) - : elements_(elements) {} + : elements_(elements) {} virtual ~SchemaVisitor() {} void Visit(const Node* node) override { @@ -113,8 +112,8 @@ class SchemaVisitor : public Node::ConstVisitor { std::vector<format::SchemaElement>* elements_; }; -SchemaFlattener::SchemaFlattener(const GroupNode* schema, - std::vector<format::SchemaElement>* out) +SchemaFlattener::SchemaFlattener( + const GroupNode* schema, std::vector<format::SchemaElement>* out) : root_(schema), elements_(out) {} void SchemaFlattener::Flatten() { @@ -122,6 +121,6 @@ void SchemaFlattener::Flatten() { root_->VisitConst(&visitor); } -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/converter.h ---------------------------------------------------------------------- diff --git a/src/parquet/schema/converter.h b/src/parquet/schema/converter.h index 2742b98..617d985 100644 --- a/src/parquet/schema/converter.h +++ b/src/parquet/schema/converter.h @@ -30,7 +30,9 @@ namespace parquet { -namespace format { class SchemaElement;} +namespace format { +class SchemaElement; +} class SchemaDescriptor; @@ -47,11 +49,8 @@ std::shared_ptr<SchemaDescriptor> FromParquet( class FlatSchemaConverter { public: - FlatSchemaConverter(const format::SchemaElement* elements, int length) : - elements_(elements), - length_(length), - pos_(0), - current_id_(0) {} + FlatSchemaConverter(const format::SchemaElement* elements, int length) + : elements_(elements), length_(length), pos_(0), current_id_(0) {} std::unique_ptr<Node> Convert(); @@ -61,9 +60,7 @@ class FlatSchemaConverter { int pos_; int current_id_; - int next_id() { - return current_id_++; - } + int next_id() { return current_id_++; } const format::SchemaElement& Next(); @@ -87,8 +84,8 @@ class SchemaFlattener { std::vector<format::SchemaElement>* elements_; }; -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet -#endif // PARQUET_SCHEMA_CONVERTER_H +#endif // PARQUET_SCHEMA_CONVERTER_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/descriptor.cc ---------------------------------------------------------------------- diff --git a/src/parquet/schema/descriptor.cc b/src/parquet/schema/descriptor.cc index 4b3bb50..01f0421 100644 --- a/src/parquet/schema/descriptor.cc +++ b/src/parquet/schema/descriptor.cc @@ -46,8 +46,8 @@ void SchemaDescriptor::Init(const NodePtr& schema) { } } -void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level, - int16_t max_rep_level) { +void SchemaDescriptor::BuildTree( + const NodePtr& node, int16_t max_def_level, int16_t max_rep_level) { if (node->is_optional()) { ++max_def_level; } else if (node->is_repeated()) { @@ -71,14 +71,12 @@ void SchemaDescriptor::BuildTree(const NodePtr& node, int16_t max_def_level, ColumnDescriptor::ColumnDescriptor(const schema::NodePtr& node, int16_t max_definition_level, int16_t max_repetition_level, - const SchemaDescriptor* schema_descr) : - node_(node), + const SchemaDescriptor* schema_descr) + : node_(node), max_definition_level_(max_definition_level), max_repetition_level_(max_repetition_level), schema_descr_(schema_descr) { - if (!node_->is_primitive()) { - throw ParquetException("Must be a primitive type"); - } + if (!node_->is_primitive()) { throw ParquetException("Must be a primitive type"); } primitive_node_ = static_cast<const PrimitiveNode*>(node_.get()); } @@ -113,4 +111,4 @@ const std::shared_ptr<ColumnPath> ColumnDescriptor::path() const { return std::make_shared<ColumnPath>(std::move(path_)); } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/descriptor.h ---------------------------------------------------------------------- diff --git a/src/parquet/schema/descriptor.h b/src/parquet/schema/descriptor.h index bb9cd4d..1686b90 100644 --- a/src/parquet/schema/descriptor.h +++ b/src/parquet/schema/descriptor.h @@ -39,28 +39,17 @@ class SchemaDescriptor; class ColumnDescriptor { public: ColumnDescriptor(const schema::NodePtr& node, int16_t max_definition_level, - int16_t max_repetition_level, - const SchemaDescriptor* schema_descr = nullptr); + int16_t max_repetition_level, const SchemaDescriptor* schema_descr = nullptr); - int16_t max_definition_level() const { - return max_definition_level_; - } + int16_t max_definition_level() const { return max_definition_level_; } - int16_t max_repetition_level() const { - return max_repetition_level_; - } + int16_t max_repetition_level() const { return max_repetition_level_; } - Type::type physical_type() const { - return primitive_node_->physical_type(); - } + Type::type physical_type() const { return primitive_node_->physical_type(); } - LogicalType::type logical_type() const { - return primitive_node_->logical_type(); - } + LogicalType::type logical_type() const { return primitive_node_->logical_type(); } - const std::string& name() const { - return primitive_node_->name(); - } + const std::string& name() const { return primitive_node_->name(); } const std::shared_ptr<schema::ColumnPath> path() const; @@ -106,13 +95,9 @@ class SchemaDescriptor { const ColumnDescriptor* Column(int i) const; // The number of physical columns appearing in the file - int num_columns() const { - return leaves_.size(); - } + int num_columns() const { return leaves_.size(); } - const schema::NodePtr& schema() const { - return schema_; - } + const schema::NodePtr& schema() const { return schema_; } private: friend class ColumnDescriptor; @@ -120,8 +105,8 @@ class SchemaDescriptor { schema::NodePtr schema_; const schema::GroupNode* group_; - void BuildTree(const schema::NodePtr& node, int16_t max_def_level, - int16_t max_rep_level); + void BuildTree( + const schema::NodePtr& node, int16_t max_def_level, int16_t max_rep_level); // Result of leaf node / tree analysis std::vector<ColumnDescriptor> leaves_; @@ -138,6 +123,6 @@ class SchemaDescriptor { std::unordered_map<int, schema::NodePtr> leaf_to_base_; }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_SCHEMA_DESCRIPTOR_H +#endif // PARQUET_SCHEMA_DESCRIPTOR_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/printer.cc ---------------------------------------------------------------------- diff --git a/src/parquet/schema/printer.cc b/src/parquet/schema/printer.cc index 6de696b..b190398 100644 --- a/src/parquet/schema/printer.cc +++ b/src/parquet/schema/printer.cc @@ -29,10 +29,8 @@ namespace schema { class SchemaPrinter : public Node::ConstVisitor { public: - explicit SchemaPrinter(std::ostream& stream, int indent_width) : - stream_(stream), - indent_(0), - indent_width_(2) {} + explicit SchemaPrinter(std::ostream& stream, int indent_width) + : stream_(stream), indent_(0), indent_width_(2) {} void Visit(const Node* node) override; @@ -132,12 +130,11 @@ void SchemaPrinter::Visit(const Node* node) { } } -void PrintSchema(const Node* schema, std::ostream& stream, - int indent_width) { +void PrintSchema(const Node* schema, std::ostream& stream, int indent_width) { SchemaPrinter printer(stream, indent_width); printer.Visit(schema); } -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/printer.h ---------------------------------------------------------------------- diff --git a/src/parquet/schema/printer.h b/src/parquet/schema/printer.h index 18c48e1..e3e2f32 100644 --- a/src/parquet/schema/printer.h +++ b/src/parquet/schema/printer.h @@ -28,11 +28,10 @@ namespace schema { class Node; -void PrintSchema(const Node* schema, std::ostream& stream, - int indent_width = 2); +void PrintSchema(const Node* schema, std::ostream& stream, int indent_width = 2); -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet -#endif // PARQUET_SCHEMA_PRINTER_H +#endif // PARQUET_SCHEMA_PRINTER_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/schema-converter-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/schema/schema-converter-test.cc b/src/parquet/schema/schema-converter-test.cc index 55b8439..0a03dc6 100644 --- a/src/parquet/schema/schema-converter-test.cc +++ b/src/parquet/schema/schema-converter-test.cc @@ -45,9 +45,7 @@ namespace schema { class TestSchemaConverter : public ::testing::Test { public: - void setUp() { - name_ = "parquet_schema"; - } + void setUp() { name_ = "parquet_schema"; } void Convert(const parquet::format::SchemaElement* elements, int length) { FlatSchemaConverter converter(elements, length); @@ -66,14 +64,10 @@ bool check_for_parent_consistency(const GroupNode* node) { // Each node should have the group as parent for (int i = 0; i < node->field_count(); i++) { const NodePtr& field = node->field(i); - if (field->parent() != node) { - return false; - } + if (field->parent() != node) { return false; } if (field->is_group()) { const GroupNode* group = static_cast<GroupNode*>(field.get()); - if (!check_for_parent_consistency(group)) { - return false; - } + if (!check_for_parent_consistency(group)) { return false; } } } return true; @@ -85,8 +79,8 @@ TEST_F(TestSchemaConverter, NestedExample) { elements.push_back(NewGroup(name_, FieldRepetitionType::REPEATED, 2, 0)); // A primitive one - elements.push_back(NewPrimitive("a", FieldRepetitionType::REQUIRED, - format::Type::INT32, 1)); + elements.push_back( + NewPrimitive("a", FieldRepetitionType::REQUIRED, format::Type::INT32, 1)); // A group elements.push_back(NewGroup("bag", FieldRepetitionType::OPTIONAL, 1, 2)); @@ -95,8 +89,8 @@ TEST_F(TestSchemaConverter, NestedExample) { elt = NewGroup("b", FieldRepetitionType::REPEATED, 1, 3); elt.__set_converted_type(ConvertedType::LIST); elements.push_back(elt); - elements.push_back(NewPrimitive("item", FieldRepetitionType::OPTIONAL, - format::Type::INT64, 4)); + elements.push_back( + NewPrimitive("item", FieldRepetitionType::OPTIONAL, format::Type::INT64, 4)); Convert(&elements[0], elements.size()); @@ -126,8 +120,8 @@ TEST_F(TestSchemaConverter, InvalidRoot) { // element is not a group, it is a malformed Parquet file. SchemaElement elements[2]; - elements[0] = NewPrimitive("not-a-group", FieldRepetitionType::REQUIRED, - format::Type::INT32, 0); + elements[0] = + NewPrimitive("not-a-group", FieldRepetitionType::REQUIRED, format::Type::INT32, 0); ASSERT_THROW(Convert(elements, 2), ParquetException); // While the Parquet spec indicates that the root group should have REPEATED @@ -135,8 +129,7 @@ TEST_F(TestSchemaConverter, InvalidRoot) { // groups as the first element. These tests check that this is okay as a // practicality matter. elements[0] = NewGroup("not-repeated", FieldRepetitionType::REQUIRED, 1, 0); - elements[1] = NewPrimitive("a", FieldRepetitionType::REQUIRED, - format::Type::INT32, 1); + elements[1] = NewPrimitive("a", FieldRepetitionType::REQUIRED, format::Type::INT32, 1); Convert(elements, 2); elements[0] = NewGroup("not-repeated", FieldRepetitionType::OPTIONAL, 1, 0); @@ -156,13 +149,9 @@ TEST_F(TestSchemaConverter, NotEnoughChildren) { class TestSchemaFlatten : public ::testing::Test { public: - void setUp() { - name_ = "parquet_schema"; - } + void setUp() { name_ = "parquet_schema"; } - void Flatten(const GroupNode* schema) { - ToParquet(schema, &elements_); - } + void Flatten(const GroupNode* schema) { ToParquet(schema, &elements_); } protected: std::string name_; @@ -175,8 +164,8 @@ TEST_F(TestSchemaFlatten, NestedExample) { elements.push_back(NewGroup(name_, FieldRepetitionType::REPEATED, 2, 0)); // A primitive one - elements.push_back(NewPrimitive("a", FieldRepetitionType::REQUIRED, - format::Type::INT32, 1)); + elements.push_back( + NewPrimitive("a", FieldRepetitionType::REQUIRED, format::Type::INT32, 1)); // A group elements.push_back(NewGroup("bag", FieldRepetitionType::OPTIONAL, 1, 2)); @@ -185,8 +174,8 @@ TEST_F(TestSchemaFlatten, NestedExample) { elt = NewGroup("b", FieldRepetitionType::REPEATED, 1, 3); elt.__set_converted_type(ConvertedType::LIST); elements.push_back(elt); - elements.push_back(NewPrimitive("item", FieldRepetitionType::OPTIONAL, - format::Type::INT64, 4)); + elements.push_back( + NewPrimitive("item", FieldRepetitionType::OPTIONAL, format::Type::INT64, 4)); // Construct the schema NodeVector fields; @@ -207,6 +196,6 @@ TEST_F(TestSchemaFlatten, NestedExample) { } } -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/schema-descriptor-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/schema/schema-descriptor-test.cc b/src/parquet/schema/schema-descriptor-test.cc index 9f0c70f..dd552be 100644 --- a/src/parquet/schema/schema-descriptor-test.cc +++ b/src/parquet/schema/schema-descriptor-test.cc @@ -36,8 +36,8 @@ namespace parquet { namespace schema { TEST(TestColumnDescriptor, TestAttrs) { - NodePtr node = PrimitiveNode::Make("name", Repetition::OPTIONAL, - Type::BYTE_ARRAY, LogicalType::UTF8); + NodePtr node = PrimitiveNode::Make( + "name", Repetition::OPTIONAL, Type::BYTE_ARRAY, LogicalType::UTF8); ColumnDescriptor descr(node, 4, 1); ASSERT_EQ("name", descr.name()); @@ -49,8 +49,8 @@ TEST(TestColumnDescriptor, TestAttrs) { ASSERT_EQ(-1, descr.type_length()); // Test FIXED_LEN_BYTE_ARRAY - node = PrimitiveNode::Make("name", Repetition::OPTIONAL, - Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 12, 10, 4); + node = PrimitiveNode::Make("name", Repetition::OPTIONAL, Type::FIXED_LEN_BYTE_ARRAY, + LogicalType::DECIMAL, 12, 10, 4); descr = ColumnDescriptor(node, 4, 1); ASSERT_EQ(Type::FIXED_LEN_BYTE_ARRAY, descr.physical_type()); @@ -59,16 +59,14 @@ TEST(TestColumnDescriptor, TestAttrs) { class TestSchemaDescriptor : public ::testing::Test { public: - void setUp() { - } + void setUp() {} protected: SchemaDescriptor descr_; }; TEST_F(TestSchemaDescriptor, InitNonGroup) { - NodePtr node = PrimitiveNode::Make("field", Repetition::OPTIONAL, - Type::INT32); + NodePtr node = PrimitiveNode::Make("field", Repetition::OPTIONAL, Type::INT32); ASSERT_THROW(descr_.Init(node), ParquetException); } @@ -85,8 +83,8 @@ TEST_F(TestSchemaDescriptor, BuildTree) { NodePtr item1 = Int64("item1", Repetition::REQUIRED); NodePtr item2 = Boolean("item2", Repetition::OPTIONAL); NodePtr item3 = Int32("item3", Repetition::REPEATED); - NodePtr list(GroupNode::Make("records", Repetition::REPEATED, - {item1, item2, item3}, LogicalType::LIST)); + NodePtr list(GroupNode::Make( + "records", Repetition::REPEATED, {item1, item2, item3}, LogicalType::LIST)); NodePtr bag(GroupNode::Make("bag", Repetition::OPTIONAL, {list})); fields.push_back(bag); @@ -129,6 +127,6 @@ TEST_F(TestSchemaDescriptor, BuildTree) { ASSERT_EQ(nleaves, descr_.num_columns()); } -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/schema-printer-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/schema/schema-printer-test.cc b/src/parquet/schema/schema-printer-test.cc index e41413b..286aea9 100644 --- a/src/parquet/schema/schema-printer-test.cc +++ b/src/parquet/schema/schema-printer-test.cc @@ -46,8 +46,8 @@ TEST(TestSchemaPrinter, Examples) { // 3-level list encoding NodePtr item1 = Int64("item1"); NodePtr item2 = Boolean("item2", Repetition::REQUIRED); - NodePtr list(GroupNode::Make("b", Repetition::REPEATED, {item1, item2}, - LogicalType::LIST)); + NodePtr list( + GroupNode::Make("b", Repetition::REPEATED, {item1, item2}, LogicalType::LIST)); NodePtr bag(GroupNode::Make("bag", Repetition::OPTIONAL, {list})); fields.push_back(bag); @@ -67,6 +67,6 @@ TEST(TestSchemaPrinter, Examples) { ASSERT_EQ(expected, result); } -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/schema-types-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/schema/schema-types-test.cc b/src/parquet/schema/schema-types-test.cc index 8215613..37c8b14 100644 --- a/src/parquet/schema/schema-types-test.cc +++ b/src/parquet/schema/schema-types-test.cc @@ -76,8 +76,7 @@ class TestPrimitiveNode : public ::testing::Test { TEST_F(TestPrimitiveNode, Attrs) { PrimitiveNode node1("foo", Repetition::REPEATED, Type::INT32); - PrimitiveNode node2("bar", Repetition::OPTIONAL, Type::BYTE_ARRAY, - LogicalType::UTF8); + PrimitiveNode node2("bar", Repetition::OPTIONAL, Type::BYTE_ARRAY, LogicalType::UTF8); ASSERT_EQ("foo", node1.name()); @@ -111,8 +110,8 @@ TEST_F(TestPrimitiveNode, Attrs) { } TEST_F(TestPrimitiveNode, FromParquet) { - SchemaElement elt = NewPrimitive(name_, FieldRepetitionType::OPTIONAL, - format::Type::INT32, 0); + SchemaElement elt = + NewPrimitive(name_, FieldRepetitionType::OPTIONAL, format::Type::INT32, 0); Convert(&elt); ASSERT_EQ(name_, prim_node_->name()); ASSERT_EQ(id_, prim_node_->id()); @@ -130,8 +129,8 @@ TEST_F(TestPrimitiveNode, FromParquet) { ASSERT_EQ(LogicalType::UTF8, prim_node_->logical_type()); // FIXED_LEN_BYTE_ARRAY - elt = NewPrimitive(name_, FieldRepetitionType::OPTIONAL, - format::Type::FIXED_LEN_BYTE_ARRAY, 0); + elt = NewPrimitive( + name_, FieldRepetitionType::OPTIONAL, format::Type::FIXED_LEN_BYTE_ARRAY, 0); elt.__set_type_length(16); Convert(&elt); @@ -142,8 +141,8 @@ TEST_F(TestPrimitiveNode, FromParquet) { ASSERT_EQ(16, prim_node_->type_length()); // ConvertedType::Decimal - elt = NewPrimitive(name_, FieldRepetitionType::OPTIONAL, - format::Type::FIXED_LEN_BYTE_ARRAY, 0); + elt = NewPrimitive( + name_, FieldRepetitionType::OPTIONAL, format::Type::FIXED_LEN_BYTE_ARRAY, 0); elt.__set_converted_type(ConvertedType::DECIMAL); elt.__set_type_length(6); elt.__set_scale(2); @@ -194,42 +193,54 @@ TEST_F(TestPrimitiveNode, Equals) { } TEST_F(TestPrimitiveNode, PhysicalLogicalMapping) { - ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::INT32, LogicalType::INT_32)); - ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::BYTE_ARRAY, LogicalType::JSON)); - ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::INT32, LogicalType::JSON), ParquetException); - ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::INT64, LogicalType::TIMESTAMP_MILLIS)); - ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::INT32, LogicalType::INT_64), ParquetException); - ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::BYTE_ARRAY, LogicalType::INT_8), ParquetException); - ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::BYTE_ARRAY, LogicalType::INTERVAL), ParquetException); - ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::FIXED_LEN_BYTE_ARRAY, LogicalType::ENUM), ParquetException); - ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::BYTE_ARRAY, LogicalType::ENUM)); + ASSERT_NO_THROW( + PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::INT32, LogicalType::INT_32)); + ASSERT_NO_THROW(PrimitiveNode::Make( + "foo", Repetition::REQUIRED, Type::BYTE_ARRAY, LogicalType::JSON)); + ASSERT_THROW( + PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::INT32, LogicalType::JSON), + ParquetException); + ASSERT_NO_THROW(PrimitiveNode::Make( + "foo", Repetition::REQUIRED, Type::INT64, LogicalType::TIMESTAMP_MILLIS)); + ASSERT_THROW( + PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::INT32, LogicalType::INT_64), + ParquetException); + ASSERT_THROW(PrimitiveNode::Make( + "foo", Repetition::REQUIRED, Type::BYTE_ARRAY, LogicalType::INT_8), + ParquetException); + ASSERT_THROW(PrimitiveNode::Make( + "foo", Repetition::REQUIRED, Type::BYTE_ARRAY, LogicalType::INTERVAL), + ParquetException); ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 0, 2, 4), ParquetException); + Type::FIXED_LEN_BYTE_ARRAY, LogicalType::ENUM), + ParquetException); + ASSERT_NO_THROW(PrimitiveNode::Make( + "foo", Repetition::REQUIRED, Type::BYTE_ARRAY, LogicalType::ENUM)); ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::FLOAT, LogicalType::DECIMAL, 0, 2, 4), ParquetException); + Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 0, 2, 4), + ParquetException); + ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::FLOAT, + LogicalType::DECIMAL, 0, 2, 4), + ParquetException); ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 0, 4, 0), ParquetException); + Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 0, 4, 0), + ParquetException); ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 10, 0, 4), ParquetException); + Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 10, 0, 4), + ParquetException); ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 10, 4, -1), ParquetException); + Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 10, 4, -1), + ParquetException); ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 10, 2, 4), ParquetException); + Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 10, 2, 4), + ParquetException); ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY, LogicalType::DECIMAL, 10, 6, 4)); ASSERT_NO_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, Type::FIXED_LEN_BYTE_ARRAY, LogicalType::INTERVAL, 12)); ASSERT_THROW(PrimitiveNode::Make("foo", Repetition::REQUIRED, - Type::FIXED_LEN_BYTE_ARRAY, LogicalType::INTERVAL, 10), ParquetException); + Type::FIXED_LEN_BYTE_ARRAY, LogicalType::INTERVAL, 10), + ParquetException); } // ---------------------------------------------------------------------- @@ -295,6 +306,6 @@ TEST_F(TestGroupNode, Equals) { ASSERT_FALSE(group5.Equals(&group4)); } -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/test-util.h ---------------------------------------------------------------------- diff --git a/src/parquet/schema/test-util.h b/src/parquet/schema/test-util.h index 25dacb0..ee9e225 100644 --- a/src/parquet/schema/test-util.h +++ b/src/parquet/schema/test-util.h @@ -62,8 +62,8 @@ static inline SchemaElement NewGroup(const std::string& name, return result; } -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet -#endif // PARQUET_COLUMN_TEST_UTIL_H +#endif // PARQUET_COLUMN_TEST_UTIL_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/types.cc ---------------------------------------------------------------------- diff --git a/src/parquet/schema/types.cc b/src/parquet/schema/types.cc index e5b7ed2..abb031d 100644 --- a/src/parquet/schema/types.cc +++ b/src/parquet/schema/types.cc @@ -54,9 +54,7 @@ std::shared_ptr<ColumnPath> ColumnPath::extend(const std::string& node_name) con std::string ColumnPath::ToDotString() const { std::stringstream ss; for (auto it = path_.cbegin(); it != path_.cend(); ++it) { - if (it != path_.cbegin()) { - ss << "."; - } + if (it != path_.cbegin()) { ss << "."; } ss << *it; } return ss.str(); @@ -70,24 +68,23 @@ const std::vector<std::string>& ColumnPath::ToDotVector() const { // Base node bool Node::EqualsInternal(const Node* other) const { - return type_ == other->type_ && - name_ == other->name_ && - repetition_ == other->repetition_ && - logical_type_ == other->logical_type_; + return type_ == other->type_ && name_ == other->name_ && + repetition_ == other->repetition_ && logical_type_ == other->logical_type_; } void Node::SetParent(const Node* parent) { - parent_ = parent; + parent_ = parent; } // ---------------------------------------------------------------------- // Primitive node PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetition, - Type::type type, LogicalType::type logical_type, - int length, int precision, int scale, int id) : - Node(Node::PRIMITIVE, name, repetition, logical_type, id), - physical_type_(type), type_length_(length) { + Type::type type, LogicalType::type logical_type, int length, int precision, int scale, + int id) + : Node(Node::PRIMITIVE, name, repetition, logical_type, id), + physical_type_(type), + type_length_(length) { std::stringstream ss; // Check if the physical and logical types match // Mapping referred from Apache parquet-mr as on 2016-02-22 @@ -108,10 +105,8 @@ PrimitiveNode::PrimitiveNode(const std::string& name, Repetition::type repetitio } break; case LogicalType::DECIMAL: - if ((type != Type::INT32) && - (type != Type::INT64) && - (type != Type::BYTE_ARRAY) && - (type != Type::FIXED_LEN_BYTE_ARRAY)) { + if ((type != Type::INT32) && (type != Type::INT64) && (type != Type::BYTE_ARRAY) && + (type != Type::FIXED_LEN_BYTE_ARRAY)) { ss << "DECIMAL can only annotate INT32, INT64, BYTE_ARRAY, and FIXED"; throw ParquetException(ss.str()); } @@ -188,7 +183,7 @@ bool PrimitiveNode::EqualsInternal(const PrimitiveNode* other) const { } if (logical_type_ == LogicalType::DECIMAL) { is_equal &= (decimal_metadata_.precision == other->decimal_metadata_.precision) && - (decimal_metadata_.scale == other->decimal_metadata_.scale); + (decimal_metadata_.scale == other->decimal_metadata_.scale); } if (physical_type_ == Type::FIXED_LEN_BYTE_ARRAY) { is_equal &= (type_length_ == other->type_length_); @@ -197,9 +192,7 @@ bool PrimitiveNode::EqualsInternal(const PrimitiveNode* other) const { } bool PrimitiveNode::Equals(const Node* other) const { - if (!Node::EqualsInternal(other)) { - return false; - } + if (!Node::EqualsInternal(other)) { return false; } return EqualsInternal(static_cast<const PrimitiveNode*>(other)); } @@ -215,24 +208,16 @@ void PrimitiveNode::VisitConst(Node::ConstVisitor* visitor) const { // Group node bool GroupNode::EqualsInternal(const GroupNode* other) const { - if (this == other) { - return true; - } - if (this->field_count() != other->field_count()) { - return false; - } + if (this == other) { return true; } + if (this->field_count() != other->field_count()) { return false; } for (int i = 0; i < this->field_count(); ++i) { - if (!this->field(i)->Equals(other->field(i).get())) { - return false; - } + if (!this->field(i)->Equals(other->field(i).get())) { return false; } } return true; } bool GroupNode::Equals(const Node* other) const { - if (!Node::EqualsInternal(other)) { - return false; - } + if (!Node::EqualsInternal(other)) { return false; } return EqualsInternal(static_cast<const GroupNode*>(other)); } @@ -248,8 +233,7 @@ void GroupNode::VisitConst(Node::ConstVisitor* visitor) const { // Node construction from Parquet metadata struct NodeParams { - explicit NodeParams(const std::string& name) : - name(name) {} + explicit NodeParams(const std::string& name) : name(name) {} const std::string& name; Repetition::type repetition; @@ -268,33 +252,32 @@ static inline NodeParams GetNodeParams(const format::SchemaElement* element) { return params; } -std::unique_ptr<Node> GroupNode::FromParquet(const void* opaque_element, int node_id, - const NodeVector& fields) { +std::unique_ptr<Node> GroupNode::FromParquet( + const void* opaque_element, int node_id, const NodeVector& fields) { const format::SchemaElement* element = - static_cast<const format::SchemaElement*>(opaque_element); + static_cast<const format::SchemaElement*>(opaque_element); NodeParams params = GetNodeParams(element); - return std::unique_ptr<Node>(new GroupNode(params.name, params.repetition, fields, - params.logical_type, node_id)); + return std::unique_ptr<Node>(new GroupNode( + params.name, params.repetition, fields, params.logical_type, node_id)); } -std::unique_ptr<Node> PrimitiveNode::FromParquet(const void* opaque_element, - int node_id) { +std::unique_ptr<Node> PrimitiveNode::FromParquet( + const void* opaque_element, int node_id) { const format::SchemaElement* element = - static_cast<const format::SchemaElement*>(opaque_element); + static_cast<const format::SchemaElement*>(opaque_element); NodeParams params = GetNodeParams(element); - std::unique_ptr<PrimitiveNode> result = std::unique_ptr<PrimitiveNode>( - new PrimitiveNode(params.name, params.repetition, - FromThrift(element->type), params.logical_type, - element->type_length, element->precision, element->scale, node_id)); + std::unique_ptr<PrimitiveNode> result = + std::unique_ptr<PrimitiveNode>(new PrimitiveNode(params.name, params.repetition, + FromThrift(element->type), params.logical_type, element->type_length, + element->precision, element->scale, node_id)); // Return as unique_ptr to the base type return std::unique_ptr<Node>(result.release()); } void GroupNode::ToParquet(void* opaque_element) const { - format::SchemaElement* element = - static_cast<format::SchemaElement*>(opaque_element); + format::SchemaElement* element = static_cast<format::SchemaElement*>(opaque_element); element->__set_name(name_); element->__set_num_children(field_count()); element->__set_repetition_type(ToThrift(repetition_)); @@ -305,8 +288,7 @@ void GroupNode::ToParquet(void* opaque_element) const { } void PrimitiveNode::ToParquet(void* opaque_element) const { - format::SchemaElement* element = - static_cast<format::SchemaElement*>(opaque_element); + format::SchemaElement* element = static_cast<format::SchemaElement*>(opaque_element); element->__set_name(name_); element->__set_num_children(0); @@ -321,6 +303,6 @@ void PrimitiveNode::ToParquet(void* opaque_element) const { element->__set_scale(decimal_metadata_.scale); } -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/schema/types.h ---------------------------------------------------------------------- diff --git a/src/parquet/schema/types.h b/src/parquet/schema/types.h index 452c23a..c9df1b7 100644 --- a/src/parquet/schema/types.h +++ b/src/parquet/schema/types.h @@ -66,11 +66,7 @@ namespace schema { // of these encodings (versus a struct containing an array). We should refuse // the temptation to guess, as they say. struct ListEncoding { - enum type { - ONE_LEVEL, - TWO_LEVEL, - THREE_LEVEL - }; + enum type { ONE_LEVEL, TWO_LEVEL, THREE_LEVEL }; }; struct DecimalMetadata { @@ -100,69 +96,42 @@ class GroupNode; // and optionally a logical type (ConvertedType in Parquet metadata parlance) class Node { public: - enum type { - PRIMITIVE, - GROUP - }; + enum type { PRIMITIVE, GROUP }; - Node(Node::type type, const std::string& name, - Repetition::type repetition, - LogicalType::type logical_type = LogicalType::NONE, - int id = -1) : - type_(type), - name_(name), - repetition_(repetition), - logical_type_(logical_type), - id_(id), - parent_(nullptr) {} + Node(Node::type type, const std::string& name, Repetition::type repetition, + LogicalType::type logical_type = LogicalType::NONE, int id = -1) + : type_(type), + name_(name), + repetition_(repetition), + logical_type_(logical_type), + id_(id), + parent_(nullptr) {} virtual ~Node() {} - bool is_primitive() const { - return type_ == Node::PRIMITIVE; - } + bool is_primitive() const { return type_ == Node::PRIMITIVE; } - bool is_group() const { - return type_ == Node::GROUP; - } + bool is_group() const { return type_ == Node::GROUP; } - bool is_optional() const { - return repetition_ == Repetition::OPTIONAL; - } + bool is_optional() const { return repetition_ == Repetition::OPTIONAL; } - bool is_repeated() const { - return repetition_ == Repetition::REPEATED; - } + bool is_repeated() const { return repetition_ == Repetition::REPEATED; } - bool is_required() const { - return repetition_ == Repetition::REQUIRED; - } + bool is_required() const { return repetition_ == Repetition::REQUIRED; } virtual bool Equals(const Node* other) const = 0; - const std::string& name() const { - return name_; - } + const std::string& name() const { return name_; } - Node::type node_type() const { - return type_; - } + Node::type node_type() const { return type_; } - Repetition::type repetition() const { - return repetition_; - } + Repetition::type repetition() const { return repetition_; } - LogicalType::type logical_type() const { - return logical_type_; - } + LogicalType::type logical_type() const { return logical_type_; } - int id() const { - return id_; - } + int id() const { return id_; } - const Node* parent() const { - return parent_; - } + const Node* parent() const { return parent_; } // ToParquet returns an opaque void* to avoid exporting // parquet::SchemaElement into the public API @@ -214,46 +183,36 @@ class PrimitiveNode : public Node { // parquet::SchemaElement into the public API static std::unique_ptr<Node> FromParquet(const void* opaque_element, int id); - static inline NodePtr Make(const std::string& name, - Repetition::type repetition, Type::type type, - LogicalType::type logical_type = LogicalType::NONE, + static inline NodePtr Make(const std::string& name, Repetition::type repetition, + Type::type type, LogicalType::type logical_type = LogicalType::NONE, int length = -1, int precision = -1, int scale = -1) { - return NodePtr(new PrimitiveNode(name, repetition, type, logical_type, - length, precision, scale)); + return NodePtr(new PrimitiveNode( + name, repetition, type, logical_type, length, precision, scale)); } virtual bool Equals(const Node* other) const; - Type::type physical_type() const { - return physical_type_; - } + Type::type physical_type() const { return physical_type_; } - int32_t type_length() const { - return type_length_; - } + int32_t type_length() const { return type_length_; } - const DecimalMetadata& decimal_metadata() const { - return decimal_metadata_; - } + const DecimalMetadata& decimal_metadata() const { return decimal_metadata_; } void ToParquet(void* opaque_element) const override; virtual void Visit(Visitor* visitor); void VisitConst(ConstVisitor* visitor) const override; private: - PrimitiveNode(const std::string& name, Repetition::type repetition, - Type::type type, LogicalType::type logical_type = LogicalType::NONE, - int length = -1, int precision = -1, int scale = -1, int id = -1); + PrimitiveNode(const std::string& name, Repetition::type repetition, Type::type type, + LogicalType::type logical_type = LogicalType::NONE, int length = -1, + int precision = -1, int scale = -1, int id = -1); Type::type physical_type_; int32_t type_length_; DecimalMetadata decimal_metadata_; // For FIXED_LEN_BYTE_ARRAY - void SetTypeLength(int32_t length) { - type_length_ = length; - } - + void SetTypeLength(int32_t length) { type_length_ = length; } // For Decimal logical type: Precision and scale void SetDecimalMetadata(int32_t scale, int32_t precision) { @@ -273,24 +232,19 @@ class GroupNode : public Node { public: // Like PrimitiveNode, GroupNode::FromParquet accepts an opaque void* to avoid exporting // parquet::SchemaElement into the public API - static std::unique_ptr<Node> FromParquet(const void* opaque_element, int id, - const NodeVector& fields); + static std::unique_ptr<Node> FromParquet( + const void* opaque_element, int id, const NodeVector& fields); - static inline NodePtr Make(const std::string& name, - Repetition::type repetition, const NodeVector& fields, - LogicalType::type logical_type = LogicalType::NONE) { + static inline NodePtr Make(const std::string& name, Repetition::type repetition, + const NodeVector& fields, LogicalType::type logical_type = LogicalType::NONE) { return NodePtr(new GroupNode(name, repetition, fields, logical_type)); } virtual bool Equals(const Node* other) const; - const NodePtr& field(int i) const { - return fields_[i]; - } + const NodePtr& field(int i) const { return fields_[i]; } - int field_count() const { - return fields_.size(); - } + int field_count() const { return fields_.size(); } void ToParquet(void* opaque_element) const override; virtual void Visit(Visitor* visitor); @@ -298,15 +252,13 @@ class GroupNode : public Node { private: GroupNode(const std::string& name, Repetition::type repetition, - const NodeVector& fields, - LogicalType::type logical_type = LogicalType::NONE, - int id = -1) : - Node(Node::GROUP, name, repetition, logical_type, id), - fields_(fields) { - for (NodePtr& field : fields_) { - field->SetParent(this); - } + const NodeVector& fields, LogicalType::type logical_type = LogicalType::NONE, + int id = -1) + : Node(Node::GROUP, name, repetition, logical_type, id), fields_(fields) { + for (NodePtr& field : fields_) { + field->SetParent(this); } + } NodeVector fields_; bool EqualsInternal(const GroupNode* other) const; @@ -318,10 +270,10 @@ class GroupNode : public Node { // ---------------------------------------------------------------------- // Convenience primitive type factory functions -#define PRIMITIVE_FACTORY(FuncName, TYPE) \ - static inline NodePtr FuncName(const std::string& name, \ - Repetition::type repetition = Repetition::OPTIONAL) { \ - return PrimitiveNode::Make(name, repetition, Type::TYPE); \ +#define PRIMITIVE_FACTORY(FuncName, TYPE) \ + static inline NodePtr FuncName( \ + const std::string& name, Repetition::type repetition = Repetition::OPTIONAL) { \ + return PrimitiveNode::Make(name, repetition, Type::TYPE); \ } PRIMITIVE_FACTORY(Boolean, BOOLEAN); @@ -332,8 +284,8 @@ PRIMITIVE_FACTORY(Float, FLOAT); PRIMITIVE_FACTORY(Double, DOUBLE); PRIMITIVE_FACTORY(ByteArray, BYTE_ARRAY); -} // namespace schema +} // namespace schema -} // namespace parquet +} // namespace parquet -#endif // PARQUET_SCHEMA_TYPES_H +#endif // PARQUET_SCHEMA_TYPES_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/thrift/util.h ---------------------------------------------------------------------- diff --git a/src/parquet/thrift/util.h b/src/parquet/thrift/util.h index 287884e..38b1a15 100644 --- a/src/parquet/thrift/util.h +++ b/src/parquet/thrift/util.h @@ -122,6 +122,6 @@ inline void SerializeThriftMsg(T* obj, uint32_t len, OutputStream* out) { out->Write(out_buffer, out_length); } -} // namespace parquet +} // namespace parquet -#endif // PARQUET_THRIFT_UTIL_H +#endif // PARQUET_THRIFT_UTIL_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/types-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/types-test.cc b/src/parquet/types-test.cc index 487e9d5..f1f829d 100644 --- a/src/parquet/types-test.cc +++ b/src/parquet/types-test.cc @@ -31,52 +31,33 @@ TEST(TestTypeToString, PhysicalTypes) { ASSERT_STREQ("FLOAT", type_to_string(Type::FLOAT).c_str()); ASSERT_STREQ("DOUBLE", type_to_string(Type::DOUBLE).c_str()); ASSERT_STREQ("BYTE_ARRAY", type_to_string(Type::BYTE_ARRAY).c_str()); - ASSERT_STREQ("FIXED_LEN_BYTE_ARRAY", - type_to_string(Type::FIXED_LEN_BYTE_ARRAY).c_str()); + ASSERT_STREQ( + "FIXED_LEN_BYTE_ARRAY", type_to_string(Type::FIXED_LEN_BYTE_ARRAY).c_str()); } TEST(TestLogicalTypeToString, LogicalTypes) { - ASSERT_STREQ("NONE", - logical_type_to_string(LogicalType::NONE).c_str()); - ASSERT_STREQ("UTF8", - logical_type_to_string(LogicalType::UTF8).c_str()); - ASSERT_STREQ("MAP_KEY_VALUE", - logical_type_to_string(LogicalType::MAP_KEY_VALUE).c_str()); - ASSERT_STREQ("LIST", - logical_type_to_string(LogicalType::LIST).c_str()); - ASSERT_STREQ("ENUM", - logical_type_to_string(LogicalType::ENUM).c_str()); - ASSERT_STREQ("DECIMAL", - logical_type_to_string(LogicalType::DECIMAL).c_str()); - ASSERT_STREQ("DATE", - logical_type_to_string(LogicalType::DATE).c_str()); - ASSERT_STREQ("TIME_MILLIS", - logical_type_to_string(LogicalType::TIME_MILLIS).c_str()); - ASSERT_STREQ("TIMESTAMP_MILLIS", - logical_type_to_string(LogicalType::TIMESTAMP_MILLIS).c_str()); - ASSERT_STREQ("UINT_8", - logical_type_to_string(LogicalType::UINT_8).c_str()); - ASSERT_STREQ("UINT_16", - logical_type_to_string(LogicalType::UINT_16).c_str()); - ASSERT_STREQ("UINT_32", - logical_type_to_string(LogicalType::UINT_32).c_str()); - ASSERT_STREQ("UINT_64", - logical_type_to_string(LogicalType::UINT_64).c_str()); - ASSERT_STREQ("INT_8", - logical_type_to_string(LogicalType::INT_8).c_str()); - ASSERT_STREQ("INT_16", - logical_type_to_string(LogicalType::INT_16).c_str()); - ASSERT_STREQ("INT_32", - logical_type_to_string(LogicalType::INT_32).c_str()); - ASSERT_STREQ("INT_64", - logical_type_to_string(LogicalType::INT_64).c_str()); - ASSERT_STREQ("JSON", - logical_type_to_string(LogicalType::JSON).c_str()); - ASSERT_STREQ("BSON", - logical_type_to_string(LogicalType::BSON).c_str()); - ASSERT_STREQ("INTERVAL", - logical_type_to_string(LogicalType::INTERVAL).c_str()); + ASSERT_STREQ("NONE", logical_type_to_string(LogicalType::NONE).c_str()); + ASSERT_STREQ("UTF8", logical_type_to_string(LogicalType::UTF8).c_str()); + ASSERT_STREQ( + "MAP_KEY_VALUE", logical_type_to_string(LogicalType::MAP_KEY_VALUE).c_str()); + ASSERT_STREQ("LIST", logical_type_to_string(LogicalType::LIST).c_str()); + ASSERT_STREQ("ENUM", logical_type_to_string(LogicalType::ENUM).c_str()); + ASSERT_STREQ("DECIMAL", logical_type_to_string(LogicalType::DECIMAL).c_str()); + ASSERT_STREQ("DATE", logical_type_to_string(LogicalType::DATE).c_str()); + ASSERT_STREQ("TIME_MILLIS", logical_type_to_string(LogicalType::TIME_MILLIS).c_str()); + ASSERT_STREQ( + "TIMESTAMP_MILLIS", logical_type_to_string(LogicalType::TIMESTAMP_MILLIS).c_str()); + ASSERT_STREQ("UINT_8", logical_type_to_string(LogicalType::UINT_8).c_str()); + ASSERT_STREQ("UINT_16", logical_type_to_string(LogicalType::UINT_16).c_str()); + ASSERT_STREQ("UINT_32", logical_type_to_string(LogicalType::UINT_32).c_str()); + ASSERT_STREQ("UINT_64", logical_type_to_string(LogicalType::UINT_64).c_str()); + ASSERT_STREQ("INT_8", logical_type_to_string(LogicalType::INT_8).c_str()); + ASSERT_STREQ("INT_16", logical_type_to_string(LogicalType::INT_16).c_str()); + ASSERT_STREQ("INT_32", logical_type_to_string(LogicalType::INT_32).c_str()); + ASSERT_STREQ("INT_64", logical_type_to_string(LogicalType::INT_64).c_str()); + ASSERT_STREQ("JSON", logical_type_to_string(LogicalType::JSON).c_str()); + ASSERT_STREQ("BSON", logical_type_to_string(LogicalType::BSON).c_str()); + ASSERT_STREQ("INTERVAL", logical_type_to_string(LogicalType::INTERVAL).c_str()); } - -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/types.h ---------------------------------------------------------------------- diff --git a/src/parquet/types.h b/src/parquet/types.h index 72017f9..a2d3622 100644 --- a/src/parquet/types.h +++ b/src/parquet/types.h @@ -84,11 +84,7 @@ struct LogicalType { // Mirrors parquet::FieldRepetitionType struct Repetition { - enum type { - REQUIRED = 0, - OPTIONAL = 1, - REPEATED = 2 - }; + enum type { REQUIRED = 0, OPTIONAL = 1, REPEATED = 2 }; }; // Data encodings. Mirrors parquet::Encoding @@ -107,22 +103,12 @@ struct Encoding { // Compression, mirrors parquet::CompressionCodec struct Compression { - enum type { - UNCOMPRESSED, - SNAPPY, - GZIP, - LZO - }; + enum type { UNCOMPRESSED, SNAPPY, GZIP, LZO }; }; // parquet::PageType struct PageType { - enum type { - DATA_PAGE, - INDEX_PAGE, - DICTIONARY_PAGE, - DATA_PAGE_V2 - }; + enum type { DATA_PAGE, INDEX_PAGE, DICTIONARY_PAGE, DATA_PAGE_V2 }; }; // ---------------------------------------------------------------------- @@ -134,8 +120,7 @@ struct ByteArray { const uint8_t* ptr; bool operator==(const ByteArray& other) const { - return this->len == other.len && - 0 == memcmp(this->ptr, other.ptr, this->len); + return this->len == other.len && 0 == memcmp(this->ptr, other.ptr, this->len); } bool operator!=(const ByteArray& other) const { @@ -158,9 +143,7 @@ MANUALLY_ALIGNED_STRUCT(1) Int96 { return 0 == memcmp(this->value, other.value, 3 * sizeof(uint32_t)); } - bool operator!=(const Int96& other) const { - return !(*this == other); - } + bool operator!=(const Int96& other) const { return !(*this == other); } }; STRUCT_END(Int96, 12); @@ -171,16 +154,16 @@ static inline std::string ByteArrayToString(const ByteArray& a) { static inline std::string Int96ToString(const Int96& a) { std::stringstream result; for (int i = 0; i < 3; i++) { - result << a.value[i] << " "; + result << a.value[i] << " "; } return result.str(); } static inline std::string FixedLenByteArrayToString(const FixedLenByteArray& a, int len) { - const uint8_t *bytes = reinterpret_cast<const uint8_t*>(a.ptr); + const uint8_t* bytes = reinterpret_cast<const uint8_t*>(a.ptr); std::stringstream result; for (int i = 0; i < len; i++) { - result << (uint32_t)bytes[i] << " "; + result << (uint32_t)bytes[i] << " "; } return result.str(); } @@ -195,8 +178,7 @@ static inline int ByteCompare(const ByteArray& x1, const ByteArray& x2) { } template <int TYPE> -struct type_traits { -}; +struct type_traits {}; template <> struct type_traits<Type::BOOLEAN> { @@ -383,6 +365,6 @@ static inline std::string logical_type_to_string(LogicalType::type t) { break; } } -} // namespace parquet +} // namespace parquet -#endif // PARQUET_TYPES_H +#endif // PARQUET_TYPES_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/bit-stream-utils.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/bit-stream-utils.h b/src/parquet/util/bit-stream-utils.h index aee9c93..dd0c9e2 100644 --- a/src/parquet/util/bit-stream-utils.h +++ b/src/parquet/util/bit-stream-utils.h @@ -37,9 +37,7 @@ class BitWriter { public: /// buffer: buffer to write bits to. Buffer should be preallocated with /// 'buffer_len' bytes. - BitWriter(uint8_t* buffer, int buffer_len) : - buffer_(buffer), - max_bytes_(buffer_len) { + BitWriter(uint8_t* buffer, int buffer_len) : buffer_(buffer), max_bytes_(buffer_len) { Clear(); } @@ -62,7 +60,7 @@ class BitWriter { /// Writes v to the next aligned byte using num_bytes. If T is larger than /// num_bytes, the extra high-order bytes will be ignored. Returns false if /// there was not enough space. - template<typename T> + template <typename T> bool PutAligned(T v, int num_bytes); /// Write a Vlq encoded int to the buffer. Returns false if there was not enough @@ -92,8 +90,8 @@ class BitWriter { /// buffer_. This is faster than writing values byte by byte directly to buffer_. uint64_t buffered_values_; - int byte_offset_; // Offset in buffer_ - int bit_offset_; // Offset in buffered_values_ + int byte_offset_; // Offset in buffer_ + int bit_offset_; // Offset in buffered_values_ }; /// Utility class to read bit/byte stream. This class can read bits or bytes @@ -102,11 +100,8 @@ class BitWriter { class BitReader { public: /// 'buffer' is the buffer to read from. The buffer's length is 'buffer_len'. - BitReader(const uint8_t* buffer, int buffer_len) : - buffer_(buffer), - max_bytes_(buffer_len), - byte_offset_(0), - bit_offset_(0) { + BitReader(const uint8_t* buffer, int buffer_len) + : buffer_(buffer), max_bytes_(buffer_len), byte_offset_(0), bit_offset_(0) { int num_bytes = std::min(8, max_bytes_ - byte_offset_); memcpy(&buffered_values_, buffer_ + byte_offset_, num_bytes); } @@ -124,7 +119,7 @@ class BitReader { /// Gets the next value from the buffer. Returns true if 'v' could be read or false if /// there are not enough bytes left. num_bits must be <= 32. - template<typename T> + template <typename T> bool GetValue(int num_bits, T* v); /// Reads a 'num_bytes'-sized value from the buffer and stores it in 'v'. T @@ -132,7 +127,7 @@ class BitReader { /// 'num_bytes'. The value is assumed to be byte-aligned so the stream will /// be advanced to the start of the next byte before 'v' is read. Returns /// false if there are not enough bytes left. - template<typename T> + template <typename T> bool GetAligned(int num_bytes, T* v); /// Reads a vlq encoded int from the stream. The encoded int must start at @@ -158,10 +153,10 @@ class BitReader { /// faster than reading values byte by byte directly from buffer_. uint64_t buffered_values_; - int byte_offset_; // Offset in buffer_ - int bit_offset_; // Offset in buffered_values_ + int byte_offset_; // Offset in buffer_ + int bit_offset_; // Offset in buffered_values_ }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_BIT_STREAM_UTILS_H +#endif // PARQUET_UTIL_BIT_STREAM_UTILS_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/bit-stream-utils.inline.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/bit-stream-utils.inline.h b/src/parquet/util/bit-stream-utils.inline.h index cd7d984..02c0e25 100644 --- a/src/parquet/util/bit-stream-utils.inline.h +++ b/src/parquet/util/bit-stream-utils.inline.h @@ -67,7 +67,7 @@ inline uint8_t* BitWriter::GetNextBytePtr(int num_bytes) { return ptr; } -template<typename T> +template <typename T> inline bool BitWriter::PutAligned(T val, int num_bytes) { uint8_t* ptr = GetNextBytePtr(num_bytes); if (ptr == NULL) return false; @@ -85,7 +85,7 @@ inline bool BitWriter::PutVlqInt(uint32_t v) { return result; } -template<typename T> +template <typename T> inline bool BitReader::GetValue(int num_bits, T* v) { DCHECK(buffer_ != NULL); // TODO: revisit this limit if necessary @@ -116,7 +116,7 @@ inline bool BitReader::GetValue(int num_bits, T* v) { return true; } -template<typename T> +template <typename T> inline bool BitReader::GetAligned(int num_bytes, T* v) { DCHECK_LE(num_bytes, static_cast<int>(sizeof(T))); int bytes_read = BitUtil::Ceil(bit_offset_, 8); @@ -165,6 +165,6 @@ inline bool BitReader::GetZigZagVlqInt(int32_t* v) { return true; } -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_BIT_STREAM_UTILS_INLINE_H +#endif // PARQUET_UTIL_BIT_STREAM_UTILS_INLINE_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/bit-util-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/bit-util-test.cc b/src/parquet/util/bit-util-test.cc index bf1ee9d..f305b9c 100644 --- a/src/parquet/util/bit-util-test.cc +++ b/src/parquet/util/bit-util-test.cc @@ -33,9 +33,7 @@ namespace parquet { static void ensure_cpu_info_initialized() { - if (!CpuInfo::initialized()) { - CpuInfo::Init(); - } + if (!CpuInfo::initialized()) { CpuInfo::Init(); } } TEST(BitUtil, Ceil) { @@ -93,9 +91,9 @@ TEST(BitUtil, TrailingBits) { EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 0), 0); EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 1), 1); EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 64), - BOOST_BINARY(1 1 1 1 1 1 1 1)); + BOOST_BINARY(1 1 1 1 1 1 1 1)); EXPECT_EQ(BitUtil::TrailingBits(BOOST_BINARY(1 1 1 1 1 1 1 1), 100), - BOOST_BINARY(1 1 1 1 1 1 1 1)); + BOOST_BINARY(1 1 1 1 1 1 1 1)); EXPECT_EQ(BitUtil::TrailingBits(0, 1), 0); EXPECT_EQ(BitUtil::TrailingBits(0, 64), 0); EXPECT_EQ(BitUtil::TrailingBits(1LL << 63, 0), 0); @@ -111,12 +109,12 @@ TEST(BitUtil, ByteSwap) { EXPECT_EQ(BitUtil::ByteSwap(static_cast<int32_t>(0x11223344)), 0x44332211); EXPECT_EQ(BitUtil::ByteSwap(static_cast<uint64_t>(0)), 0); - EXPECT_EQ(BitUtil::ByteSwap( - static_cast<uint64_t>(0x1122334455667788)), 0x8877665544332211); + EXPECT_EQ( + BitUtil::ByteSwap(static_cast<uint64_t>(0x1122334455667788)), 0x8877665544332211); EXPECT_EQ(BitUtil::ByteSwap(static_cast<int64_t>(0)), 0); - EXPECT_EQ(BitUtil::ByteSwap( - static_cast<int64_t>(0x1122334455667788)), 0x8877665544332211); + EXPECT_EQ( + BitUtil::ByteSwap(static_cast<int64_t>(0x1122334455667788)), 0x8877665544332211); EXPECT_EQ(BitUtil::ByteSwap(static_cast<int16_t>(0)), 0); EXPECT_EQ(BitUtil::ByteSwap(static_cast<int16_t>(0x1122)), 0x2211); @@ -189,4 +187,4 @@ TEST(BitStreamUtil, ZigZag) { TestZigZag(-std::numeric_limits<int32_t>::max()); } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/bit-util.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/bit-util.h b/src/parquet/util/bit-util.h index d591aef..b75f5a1 100644 --- a/src/parquet/util/bit-util.h +++ b/src/parquet/util/bit-util.h @@ -39,8 +39,7 @@ namespace parquet { // We add a partial stub implementation here template <typename T> -struct make_unsigned { -}; +struct make_unsigned {}; template <> struct make_unsigned<int8_t> { @@ -111,41 +110,30 @@ class BitUtil { /// Specialized round up and down functions for frequently used factors, /// like 8 (bits->bytes), 32 (bits->i32), and 64 (bits->i64). /// Returns the rounded up number of bytes that fit the number of bits. - static inline uint32_t RoundUpNumBytes(uint32_t bits) { - return (bits + 7) >> 3; - } + static inline uint32_t RoundUpNumBytes(uint32_t bits) { return (bits + 7) >> 3; } /// Returns the rounded down number of bytes that fit the number of bits. - static inline uint32_t RoundDownNumBytes(uint32_t bits) { - return bits >> 3; - } + static inline uint32_t RoundDownNumBytes(uint32_t bits) { return bits >> 3; } /// Returns the rounded up to 32 multiple. Used for conversions of bits to i32. - static inline uint32_t RoundUpNumi32(uint32_t bits) { - return (bits + 31) >> 5; - } + static inline uint32_t RoundUpNumi32(uint32_t bits) { return (bits + 31) >> 5; } /// Returns the rounded up 32 multiple. - static inline uint32_t RoundDownNumi32(uint32_t bits) { - return bits >> 5; - } + static inline uint32_t RoundDownNumi32(uint32_t bits) { return bits >> 5; } /// Returns the rounded up to 64 multiple. Used for conversions of bits to i64. - static inline uint32_t RoundUpNumi64(uint32_t bits) { - return (bits + 63) >> 6; - } + static inline uint32_t RoundUpNumi64(uint32_t bits) { return (bits + 63) >> 6; } /// Returns the rounded down to 64 multiple. - static inline uint32_t RoundDownNumi64(uint32_t bits) { - return bits >> 6; - } + static inline uint32_t RoundDownNumi64(uint32_t bits) { return bits >> 6; } /// Non hw accelerated pop count. /// TODO: we don't use this in any perf sensitive code paths currently. There /// might be a much faster way to implement this. static inline int PopcountNoHw(uint64_t x) { int count = 0; - for (; x != 0; ++count) x &= x-1; + for (; x != 0; ++count) + x &= x - 1; return count; } @@ -163,7 +151,7 @@ class BitUtil { } // Compute correct population count for various-width signed integers - template<typename T> + template <typename T> static inline int PopcountSigned(T v) { // Converting to same-width unsigned then extending preserves the bit pattern. return BitUtil::Popcount(static_cast<typename make_unsigned<T>::type>(v)); @@ -189,20 +177,17 @@ class BitUtil { // (floor(log2(n)) = MSB(n) (0-indexed)) --x; int result = 1; - while (x >>= 1) ++result; + while (x >>= 1) + ++result; return result; } /// Swaps the byte order (i.e. endianess) - static inline int64_t ByteSwap(int64_t value) { - return __builtin_bswap64(value); - } + static inline int64_t ByteSwap(int64_t value) { return __builtin_bswap64(value); } static inline uint64_t ByteSwap(uint64_t value) { return static_cast<uint64_t>(__builtin_bswap64(value)); } - static inline int32_t ByteSwap(int32_t value) { - return __builtin_bswap32(value); - } + static inline int32_t ByteSwap(int32_t value) { return __builtin_bswap32(value); } static inline uint32_t ByteSwap(uint32_t value) { return static_cast<uint32_t>(__builtin_bswap32(value)); } @@ -231,7 +216,8 @@ class BitUtil { *reinterpret_cast<int64_t*>(dst) = ByteSwap(*reinterpret_cast<const int64_t*>(src)); return; - default: break; + default: + break; } uint8_t* d = reinterpret_cast<uint8_t*>(dst); @@ -241,52 +227,52 @@ class BitUtil { } } - /// Converts to big endian format (if not already in big endian) from the - /// machine's native endian format. +/// Converts to big endian format (if not already in big endian) from the +/// machine's native endian format. #if __BYTE_ORDER == __LITTLE_ENDIAN - static inline int64_t ToBigEndian(int64_t value) { return ByteSwap(value); } + static inline int64_t ToBigEndian(int64_t value) { return ByteSwap(value); } static inline uint64_t ToBigEndian(uint64_t value) { return ByteSwap(value); } - static inline int32_t ToBigEndian(int32_t value) { return ByteSwap(value); } + static inline int32_t ToBigEndian(int32_t value) { return ByteSwap(value); } static inline uint32_t ToBigEndian(uint32_t value) { return ByteSwap(value); } - static inline int16_t ToBigEndian(int16_t value) { return ByteSwap(value); } + static inline int16_t ToBigEndian(int16_t value) { return ByteSwap(value); } static inline uint16_t ToBigEndian(uint16_t value) { return ByteSwap(value); } #else - static inline int64_t ToBigEndian(int64_t val) { return val; } + static inline int64_t ToBigEndian(int64_t val) { return val; } static inline uint64_t ToBigEndian(uint64_t val) { return val; } - static inline int32_t ToBigEndian(int32_t val) { return val; } + static inline int32_t ToBigEndian(int32_t val) { return val; } static inline uint32_t ToBigEndian(uint32_t val) { return val; } - static inline int16_t ToBigEndian(int16_t val) { return val; } + static inline int16_t ToBigEndian(int16_t val) { return val; } static inline uint16_t ToBigEndian(uint16_t val) { return val; } #endif - /// Converts from big endian format to the machine's native endian format. +/// Converts from big endian format to the machine's native endian format. #if __BYTE_ORDER == __LITTLE_ENDIAN - static inline int64_t FromBigEndian(int64_t value) { return ByteSwap(value); } + static inline int64_t FromBigEndian(int64_t value) { return ByteSwap(value); } static inline uint64_t FromBigEndian(uint64_t value) { return ByteSwap(value); } - static inline int32_t FromBigEndian(int32_t value) { return ByteSwap(value); } + static inline int32_t FromBigEndian(int32_t value) { return ByteSwap(value); } static inline uint32_t FromBigEndian(uint32_t value) { return ByteSwap(value); } - static inline int16_t FromBigEndian(int16_t value) { return ByteSwap(value); } + static inline int16_t FromBigEndian(int16_t value) { return ByteSwap(value); } static inline uint16_t FromBigEndian(uint16_t value) { return ByteSwap(value); } #else - static inline int64_t FromBigEndian(int64_t val) { return val; } + static inline int64_t FromBigEndian(int64_t val) { return val; } static inline uint64_t FromBigEndian(uint64_t val) { return val; } - static inline int32_t FromBigEndian(int32_t val) { return val; } + static inline int32_t FromBigEndian(int32_t val) { return val; } static inline uint32_t FromBigEndian(uint32_t val) { return val; } - static inline int16_t FromBigEndian(int16_t val) { return val; } + static inline int16_t FromBigEndian(int16_t val) { return val; } static inline uint16_t FromBigEndian(uint16_t val) { return val; } #endif // Logical right shift for signed integer types // This is needed because the C >> operator does arithmetic right shift // Negative shift amounts lead to undefined behavior - template<typename T> + template <typename T> static T ShiftRightLogical(T v, int shift) { // Conversion to unsigned ensures most significant bits always filled with 0's return static_cast<typename make_unsigned<T>::type>(v) >> shift; } // Get an specific bit of a numeric type - template<typename T> + template <typename T> static inline int8_t GetBit(T v, int bitpos) { T masked = v & (static_cast<T>(0x1) << bitpos); return static_cast<int8_t>(ShiftRightLogical(masked, bitpos)); @@ -294,7 +280,7 @@ class BitUtil { // Set a specific bit to 1 // Behavior when bitpos is negative is undefined - template<typename T> + template <typename T> static T SetBit(T v, int bitpos) { return v | (static_cast<T>(0x1) << bitpos); } @@ -309,7 +295,7 @@ class BitUtil { // Set a specific bit to 0 // Behavior when bitpos is negative is undefined - template<typename T> + template <typename T> static T UnsetBit(T v, int bitpos) { return v & ~(static_cast<T>(0x1) << bitpos); } @@ -323,6 +309,6 @@ class BitUtil { } }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_BIT_UTIL_H +#endif // PARQUET_UTIL_BIT_UTIL_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/buffer-builder.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/buffer-builder.h b/src/parquet/util/buffer-builder.h index 5c388a7..26f134e 100644 --- a/src/parquet/util/buffer-builder.h +++ b/src/parquet/util/buffer-builder.h @@ -29,20 +29,17 @@ namespace parquet { class BufferBuilder { public: BufferBuilder(uint8_t* dst_buffer, int dst_len) - : buffer_(dst_buffer), capacity_(dst_len), size_(0) { - } + : buffer_(dst_buffer), capacity_(dst_len), size_(0) {} BufferBuilder(char* dst_buffer, int dst_len) - : buffer_(reinterpret_cast<uint8_t*>(dst_buffer)), - capacity_(dst_len), size_(0) { - } + : buffer_(reinterpret_cast<uint8_t*>(dst_buffer)), capacity_(dst_len), size_(0) {} inline void Append(const void* buffer, int len) { memcpy(buffer_ + size_, buffer, len); size_ += len; } - template<typename T> + template <typename T> inline void Append(const T& v) { Append(&v, sizeof(T)); } @@ -56,6 +53,6 @@ class BufferBuilder { int size_; }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_BUFFER_BUILDER_H +#endif // PARQUET_UTIL_BUFFER_BUILDER_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/buffer-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/buffer-test.cc b/src/parquet/util/buffer-test.cc index c871f91..a71a1f6 100644 --- a/src/parquet/util/buffer-test.cc +++ b/src/parquet/util/buffer-test.cc @@ -30,8 +30,7 @@ using std::string; namespace parquet { -class TestBuffer : public ::testing::Test { -}; +class TestBuffer : public ::testing::Test {}; TEST_F(TestBuffer, Resize) { OwnedMutableBuffer buf; @@ -48,8 +47,8 @@ TEST_F(TestBuffer, Resize) { } TEST_F(TestBuffer, ResizeOOM) { - // Tests that deliberately throw Exceptions foul up valgrind and report - // red herring memory leaks +// Tests that deliberately throw Exceptions foul up valgrind and report +// red herring memory leaks #ifndef PARQUET_VALGRIND OwnedMutableBuffer buf; ASSERT_NO_THROW(buf.Resize(100)); @@ -59,10 +58,8 @@ TEST_F(TestBuffer, ResizeOOM) { FAIL() << "Exception not thrown"; } catch (const ParquetException& e) { // pass - } catch(const std::exception& e) { - FAIL() << "Different exception thrown"; - } + } catch (const std::exception& e) { FAIL() << "Different exception thrown"; } #endif } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/buffer.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/buffer.cc b/src/parquet/util/buffer.cc index 84c20fb..0b7100c 100644 --- a/src/parquet/util/buffer.cc +++ b/src/parquet/util/buffer.cc @@ -25,8 +25,7 @@ namespace parquet { -Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, - int64_t size) { +Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size) { data_ = parent->data() + offset; size_ = size; parent_ = parent; @@ -36,15 +35,13 @@ std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() { return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size()); } -OwnedMutableBuffer::OwnedMutableBuffer(int64_t size, MemoryAllocator* allocator) : - ResizableBuffer(nullptr, 0), allocator_(allocator) { +OwnedMutableBuffer::OwnedMutableBuffer(int64_t size, MemoryAllocator* allocator) + : ResizableBuffer(nullptr, 0), allocator_(allocator) { Resize(size); } OwnedMutableBuffer::~OwnedMutableBuffer() { - if (mutable_data_) { - allocator_->Free(mutable_data_, capacity_); - } + if (mutable_data_) { allocator_->Free(mutable_data_, capacity_); } } void OwnedMutableBuffer::Reserve(int64_t new_capacity) { @@ -72,9 +69,10 @@ uint8_t& OwnedMutableBuffer::operator[](int64_t i) { } template <class T> -Vector<T>::Vector(int64_t size, MemoryAllocator* allocator) : - buffer_(new OwnedMutableBuffer(size * sizeof(T), allocator)), - size_(size), capacity_(size) { +Vector<T>::Vector(int64_t size, MemoryAllocator* allocator) + : buffer_(new OwnedMutableBuffer(size * sizeof(T), allocator)), + size_(size), + capacity_(size) { if (size > 0) { data_ = reinterpret_cast<T*>(buffer_->mutable_data()); } else { @@ -122,4 +120,4 @@ template class Vector<Int96>; template class Vector<ByteArray>; template class Vector<FixedLenByteArray>; -} // namespace parquet +} // namespace parquet
