kangpinghuang commented on a change in pull request #1633: add zone map page(#1390) URL: https://github.com/apache/incubator-doris/pull/1633#discussion_r316962352
########## File path: gensrc/proto/segment_v2.proto ########## @@ -1,155 +1,162 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// Define file format struct, like data header, index header. - -syntax="proto2"; - -package doris.segment_v2; - -message ColumnSchemaPB { - optional uint32 column_id = 1; - optional string type = 2; - optional string aggregation = 3; - optional uint32 length = 4; - optional bool is_key = 5; - optional string default_value = 6; - optional uint32 precision = 9 [default = 27]; - optional uint32 frac = 10 [default = 9]; - optional bool is_nullable = 11 [default=false]; - optional bool is_bf_column = 15 [default=false]; // is bloom filter indexed column - optional bool is_bitmap_column = 16 [default=false]; -} - -// page position info -message PagePointerPB { - required uint64 offset = 1; // offset in segment file - required uint32 size = 2; // size of page in byte -} - -message MetadataPairPB { - optional string key = 1; - optional bytes value = 2; -} - -enum EncodingTypePB { - UNKNOWN_ENCODING = 0; - DEFAULT_ENCODING = 1; - PLAIN_ENCODING = 2; - PREFIX_ENCODING = 3; - RLE = 4; - DICT_ENCODING = 5; - BIT_SHUFFLE = 6; -} - -enum CompressionTypePB { - UNKNOWN_COMPRESSION = 0; - DEFAULT_COMPRESSION = 1; - NO_COMPRESSION = 2; - SNAPPY = 3; - LZ4 = 4; - LZ4F = 5; - ZLIB = 6; - ZSTD = 7; -} - -message ZoneMapPB { - optional bytes min = 1; - optional bytes max = 2; - optional bool null_flag = 3; -} - -message ColumnMetaPB { - // column id in table schema - optional uint32 column_id = 1; - // unique column id - optional uint32 unique_id = 2; - // this field is FieldType's value - optional int32 type = 3; - optional EncodingTypePB encoding = 4; - // compress type for column - optional CompressionTypePB compression = 5; - // if this column can be nullable - optional bool is_nullable = 6; - // ordinal index page - optional PagePointerPB ordinal_index_page = 7; - - // // dictionary page for DICT_ENCODING - // optional PagePointerPB dict_page = 2; - - // // bloom filter pages for bloom filter column - // repeated PagePointerPB bloom_filter_pages = 3; - - // optional PagePointerPB page_zonemap_page = 5; // page zonemap info of column - - // optional PagePointerPB bitmap_index_page = 6; // bitmap index page - - // // data footprint of column after encoding and compress - // optional uint64 data_footprint = 7; - // // index footprint of column after encoding and compress - // optional uint64 index_footprint = 8; - // // raw column data footprint - // optional uint64 raw_data_footprint = 9; - - // optional ZoneMapPB column_zonemap = 11; // column zonemap info - // repeated MetadataPairPB column_meta_datas = 12; -} - -message FileFooterPB { - optional uint32 version = 1 [default = 1]; // file version - repeated ColumnSchemaPB schema = 2; // tablet schema - optional uint64 num_values = 3; // number of values - optional uint64 index_footprint = 4; // total idnex footprint of all columns - optional uint64 data_footprint = 5; // total data footprint of all columns - optional uint64 raw_data_footprint = 6; // raw data footprint - - optional CompressionTypePB compress_type = 7 [default = LZ4F]; // default compression type for file columns - repeated MetadataPairPB file_meta_datas = 8; // meta data of file - optional PagePointerPB key_index_page = 9; // short key index page -} - -message ShortKeyFooterPB { - // How many index item in this index. - optional uint32 num_items = 1; - // The total bytes occupied by the index key - optional uint32 key_bytes = 2; - // The total bytes occupied by the key offsets - optional uint32 offset_bytes = 3; - // Segment id which this index is belong to - optional uint32 segment_id = 4; - // number rows in each block - optional uint32 num_rows_per_block = 5; - // How many rows in this segment - optional uint32 num_segment_rows = 6; - // Total bytes for this segment - optional uint32 segment_bytes = 7; -} - -message SegmentFooterPB { - optional uint32 version = 1 [default = 1]; // file version - repeated ColumnMetaPB columns = 2; // tablet schema - optional uint64 num_rows = 3; // number of values - optional uint64 index_footprint = 4; // total idnex footprint of all columns - optional uint64 data_footprint = 5; // total data footprint of all columns - optional uint64 raw_data_footprint = 6; // raw data footprint - - optional CompressionTypePB compress_type = 7 [default = LZ4F]; // default compression type for file columns - repeated MetadataPairPB file_meta_datas = 8; // meta data of file - - // Short key index's page - optional PagePointerPB short_key_index_page = 9; -} - +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// Define file format struct, like data header, index header. + +syntax="proto2"; + +package doris.segment_v2; + +message ColumnSchemaPB { + optional uint32 column_id = 1; + optional string type = 2; + optional string aggregation = 3; + optional uint32 length = 4; + optional bool is_key = 5; + optional string default_value = 6; + optional uint32 precision = 9 [default = 27]; + optional uint32 frac = 10 [default = 9]; + optional bool is_nullable = 11 [default=false]; + optional bool is_bf_column = 15 [default=false]; // is bloom filter indexed column + optional bool is_bitmap_column = 16 [default=false]; +} + +// page position info +message PagePointerPB { + required uint64 offset = 1; // offset in segment file + required uint32 size = 2; // size of page in byte +} + +message MetadataPairPB { + optional string key = 1; + optional bytes value = 2; +} + +enum EncodingTypePB { + UNKNOWN_ENCODING = 0; + DEFAULT_ENCODING = 1; + PLAIN_ENCODING = 2; + PREFIX_ENCODING = 3; + RLE = 4; + DICT_ENCODING = 5; + BIT_SHUFFLE = 6; +} + +enum CompressionTypePB { + UNKNOWN_COMPRESSION = 0; + DEFAULT_COMPRESSION = 1; + NO_COMPRESSION = 2; + SNAPPY = 3; + LZ4 = 4; + LZ4F = 5; + ZLIB = 6; + ZSTD = 7; +} + +message ZoneMapPB { + // min value without null flag prefix + optional bytes min = 1; + // max value without null flag prefix + optional bytes max = 2; + // whether the zone has null value + optional bool exist_null = 3; + // whether the zone has none-null value + optional bool exist_none_null = 4; Review comment: ok ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@doris.apache.org For additional commands, e-mail: dev-h...@doris.apache.org
