http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/07c6c21d/protocol/src/main/protobuf/Coordination.proto ---------------------------------------------------------------------- diff --git a/protocol/src/main/protobuf/Coordination.proto b/protocol/src/main/protobuf/Coordination.proto new file mode 100644 index 0000000..5cc5cab --- /dev/null +++ b/protocol/src/main/protobuf/Coordination.proto @@ -0,0 +1,26 @@ +package exec; + +option java_package = "org.apache.drill.exec.proto"; +option java_outer_classname = "CoordinationProtos"; +option optimize_for = SPEED; + +message DrillbitEndpoint{ + optional string address = 1; + optional int32 user_port = 2; + optional int32 bit_port = 3; + optional Roles roles = 4; +} + +message DrillServiceInstance{ + optional string id = 1; + optional int64 registrationTimeUTC = 2; + optional DrillbitEndpoint endpoint = 3; +} + +message Roles{ + optional bool sql_query = 1 [default = true]; + optional bool logical_plan = 2 [default = true]; + optional bool physical_plan = 3 [default = true]; + optional bool java_executor = 4 [default = true]; + optional bool distributed_cache = 5 [default = true]; +}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/07c6c21d/protocol/src/main/protobuf/ExecutionProtos.proto ---------------------------------------------------------------------- diff --git a/protocol/src/main/protobuf/ExecutionProtos.proto b/protocol/src/main/protobuf/ExecutionProtos.proto new file mode 100644 index 0000000..7501d7c --- /dev/null +++ b/protocol/src/main/protobuf/ExecutionProtos.proto @@ -0,0 +1,98 @@ +package exec.bit; + +option java_package = "org.apache.drill.exec.proto"; +option java_outer_classname = "ExecProtos"; +option optimize_for = SPEED; + +import "Coordination.proto"; +import "UserBitShared.proto"; + + + +////// UserToBit RPC /////// +enum RpcType { + HANDSHAKE = 0; + ACK = 1; + GOODBYE = 2; + + // bit requests + REQ_INIATILIZE_FRAGMENT = 3; // Returns Handle + REQ_RECORD_BATCH = 4; // send record batch overview, returns Ack + + REQ_CANCEL_FRAGMENT = 6; // send a cancellation message for a fragment, returns Ack + REQ_FRAGMENT_STATUS = 7; // get a fragment status, returns FragmentStatus + REQ_BIT_STATUS = 8; // get bit status. + + // bit responses + RESP_FRAGMENT_HANDLE = 9; + RESP_FRAGMENT_STATUS = 10; + RESP_BIT_STATUS = 11; +} + + +message BitHandshake{ + optional int32 rpc_version = 1; + optional DrillbitEndpoint endpoint = 2; +} + +message BitBatchChunk {} + +message BitStatus { + repeated FragmentStatus fragment_status = 1; +} + +message FragmentStatus { + + enum FragmentState { + SENDING = 0; + AWAITING_ALLOCATION = 1; + RUNNING = 2; + FINISHED = 3; + CANCELLED = 4; + FAILED = 5; + } + + optional int64 memory_use = 1; + optional int64 batches_completed = 2; + optional int64 records_completed = 3; + optional int32 estimated_completion_percentage = 4; + optional FragmentState state = 5; + optional int64 data_processed = 6; + + optional FragmentHandle handle = 7; + optional exec.shared.DrillPBError error = 8; + optional int64 running_time = 9; +} + +message FragmentRecordBatch{ + optional FragmentHandle handle = 1; + optional int32 sending_major_fragment_id = 2; + optional int32 sending_minor_fragment_id = 3; + optional exec.shared.RecordBatchDef def = 4; + optional bool isLastBatch = 5; +} + +message PlanFragment { + optional FragmentHandle handle = 1; + optional float network_cost = 4; + optional float cpu_cost = 5; + optional float disk_cost = 6; + optional float memory_cost = 7; + optional string fragment_json = 8; + optional DrillbitEndpoint assignment = 10; + optional bool leaf_fragment = 9; + optional DrillbitEndpoint foreman = 11; + +} + +message FragmentHandle { + optional exec.shared.QueryId query_id = 1; + optional int32 major_fragment_id = 2; + optional int32 minor_fragment_id = 3; +} + +message WorkQueueStatus{ + optional DrillbitEndpoint endpoint = 1; + optional int32 queue_length = 2; + optional int64 report_time = 3; +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/07c6c21d/protocol/src/main/protobuf/GeneralRPC.proto ---------------------------------------------------------------------- diff --git a/protocol/src/main/protobuf/GeneralRPC.proto b/protocol/src/main/protobuf/GeneralRPC.proto new file mode 100644 index 0000000..48011bf --- /dev/null +++ b/protocol/src/main/protobuf/GeneralRPC.proto @@ -0,0 +1,39 @@ +package exec.rpc; + +option java_package = "org.apache.drill.exec.proto"; +option java_outer_classname = "GeneralRPCProtos"; +option optimize_for = SPEED; + +import "Coordination.proto"; + +message Ack{ + optional bool ok = 1; +} + +enum RpcMode { + REQUEST = 0; + RESPONSE = 1; + RESPONSE_FAILURE = 2; +} + +message RpcHeader{ + optional RpcMode mode = 1; + optional int32 coordination_id = 2; // reusable coordination identifier. Sender defines. Server returns on return. Irrelevant for purely single direction rpc. + optional int32 rpc_type = 3; // a rpc mode specific rpc type. +} + +message CompleteRpcMessage { + optional RpcHeader header = 1; // required + optional bytes protobuf_body = 2; // required + optional bytes raw_body = 3; // optional +} + +// Class to be used when an unexpected exception occurs while a rpc call is being evaluated. +message RpcFailure { + optional int64 error_id = 1; // for server trackback. + optional int32 error_code = 2; // system defined error code. + optional string short_error = 3; + optional string long_error = 4; +} + + http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/07c6c21d/protocol/src/main/protobuf/SchemaDef.proto ---------------------------------------------------------------------- diff --git a/protocol/src/main/protobuf/SchemaDef.proto b/protocol/src/main/protobuf/SchemaDef.proto new file mode 100644 index 0000000..2d6575b --- /dev/null +++ b/protocol/src/main/protobuf/SchemaDef.proto @@ -0,0 +1,31 @@ +package exec; + +option java_package = "org.apache.drill.exec.proto"; +option java_outer_classname = "SchemaDefProtos"; +option optimize_for = SPEED; + +import "Types.proto"; + + +enum ValueMode { + VALUE_VECTOR = 0; + RLE = 1; + DICT = 2; +} + +message NamePart { + enum Type{ + NAME = 0; + ARRAY = 1; + } + + optional Type type = 1; + optional string name = 2; // only required if this is a named type. +} + +message FieldDef { + repeated NamePart name = 1; // multipart description of entire field name + optional common.MajorType major_type = 2; // the type associated with this field. + //repeated FieldDef field = 3; // only in the cases of type == MAP or REPEATMAP + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/07c6c21d/protocol/src/main/protobuf/Types.proto ---------------------------------------------------------------------- diff --git a/protocol/src/main/protobuf/Types.proto b/protocol/src/main/protobuf/Types.proto new file mode 100644 index 0000000..3434110 --- /dev/null +++ b/protocol/src/main/protobuf/Types.proto @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http:// www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common; + +option java_package = "org.apache.drill.common.types"; +option java_outer_classname = "TypeProtos"; +option optimize_for = SPEED; + + +enum MinorType { + LATE = 0; // late binding type + MAP = 1; // an empty map column. Useful for conceptual setup. Children listed within here + REPEATMAP = 2; // a repeated map column (means that multiple children sit below this) + TINYINT = 3; // single byte signed integer + SMALLINT = 4; // two byte signed integer + INT = 5; // four byte signed integer + BIGINT = 6; // eight byte signed integer + DECIMAL4 = 7; // a decimal supporting precision between 1 and 8 (4 bits for decimal location, 1 sign) + DECIMAL8 = 8; // a decimal supporting precision between 9 and 18 (5 bits for decimal location, 1 sign) + DECIMAL12 = 9; // a decimal supporting precision between 19 and 28 (5 bits for decimal location, 1 sign) + DECIMAL16 = 10; // a decimal supporting precision between 29 and 37 (6 bits for decimal location, 1 sign) + MONEY = 11; // signed decimal with two digit precision + DATE = 12; // days since 4713bc + TIME = 13; // time in micros before or after 2000/1/1 + TIMETZ = 14; // time in micros before or after 2000/1/1 with timezone + TIMESTAMP = 15; // unix epoch time in millis + DATETIME = 16; // TBD + INTERVAL = 17; // TBD + FLOAT4 = 18; // 4 byte ieee 754 + FLOAT8 = 19; // 8 byte ieee 754 + BIT = 20; // single bit value (boolean) + FIXEDCHAR = 21; // utf8 fixed length string, padded with spaces + FIXED16CHAR = 22; + FIXEDBINARY = 23; // fixed length binary, padded with 0 bytes + VARCHAR = 24; // utf8 variable length string + VAR16CHAR = 25; // utf16 variable length string + VARBINARY = 26; // variable length binary + UINT1 = 29; // unsigned 1 byte integer + UINT2 = 30; // unsigned 2 byte integer + UINT4 = 31; // unsigned 4 byte integer + UINT8 = 32; // unsigned 8 byte integer +// PROTO2 = 33; // protobuf encoded complex type. (up to 2^16 in length) +// PROTO4 = 34; // protobuf encoded complex type. (up to 2^32 in length) +// MSGPACK2 = 35; // msgpack encoded complex type. (up to 2^16 in length) +// MSGPACK4 = 36; // msgpack encoded complex type. (up to 2^32 in length) + NULL = 37; // a value of unknown type (e.g. a missing reference). + +} + +message MajorType { + optional MinorType minor_type = 1; + optional DataMode mode = 2; + optional int32 width = 3; // optional width for fixed size values. + optional int32 precision = 4; // used for decimal types + optional int32 scale = 5; // used for decimal types +} + + + +enum DataMode { + OPTIONAL = 0; // nullable + REQUIRED = 1; // non-nullable + REPEATED = 2; // single, repeated-field +} + http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/07c6c21d/protocol/src/main/protobuf/User.proto ---------------------------------------------------------------------- diff --git a/protocol/src/main/protobuf/User.proto b/protocol/src/main/protobuf/User.proto new file mode 100644 index 0000000..cbf5b4c --- /dev/null +++ b/protocol/src/main/protobuf/User.proto @@ -0,0 +1,90 @@ +package exec.user; + +option java_package = "org.apache.drill.exec.proto"; +option java_outer_classname = "UserProtos"; +option optimize_for = SPEED; + +import "SchemaDef.proto"; +import "UserBitShared.proto"; + + + +////// UserToBit RPC /////// +enum RpcType { + HANDSHAKE = 0; + ACK = 1; + GOODBYE = 2; + + // user to bit + RUN_QUERY = 3; + CANCEL_QUERY = 4; + REQUEST_RESULTS = 5; + + // bit to user + QUERY_RESULT = 6; + QUERY_HANDLE = 7; +} + +message UserToBitHandshake { + optional bool support_listening = 2; + optional int32 rpc_version = 3; +} + +message RequestResults { + optional exec.shared.QueryId query_id = 1; + optional int32 maximum_responses = 2; +} + +message RunQuery { + optional QueryResultsMode results_mode = 1; + optional QueryType type = 2; + optional string plan = 3; +} + +enum QueryType { + SQL = 1; + LOGICAL = 2; + PHYSICAL = 3; +} + +enum QueryResultsMode { + STREAM_FULL = 1; // Server will inform the client regularly on the status of the query. Once the query is completed, service will inform the client as each query chunk is made available. + // STREAM_FIRST = 2; // Server will inform the client regularly on the status of the query. Once the query is completed, server will inform the client of the first query chunk. + // QUERY_FOR_STATUS = 3; // Client will need to query for status of query. +} + + +message BitToUserHandshake { + optional int32 rpc_version = 2; +} + +message NodeStatus { + optional int32 node_id = 1; + optional int64 memory_footprint = 2; +} + +message QueryResult { + enum QueryState { + PENDING = 0; + RUNNING = 1; + COMPLETED = 2; + CANCELED = 3; + FAILED = 4; + UNKNOWN_QUERY = 5; + } + + optional QueryState query_state = 1; + optional exec.shared.QueryId query_id = 2; + optional bool is_last_chunk = 3; + optional int32 row_count = 4; + optional int64 records_scan = 5; + optional int64 records_error = 6; + optional int64 submission_time = 7; + repeated NodeStatus node_status = 8; + repeated exec.shared.DrillPBError error = 9; + optional exec.shared.RecordBatchDef def = 10; + optional bool schema_changed = 11; +} + + + http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/07c6c21d/protocol/src/main/protobuf/UserBitShared.proto ---------------------------------------------------------------------- diff --git a/protocol/src/main/protobuf/UserBitShared.proto b/protocol/src/main/protobuf/UserBitShared.proto new file mode 100644 index 0000000..7e2506c --- /dev/null +++ b/protocol/src/main/protobuf/UserBitShared.proto @@ -0,0 +1,47 @@ +package exec.shared; + +option java_package = "org.apache.drill.exec.proto"; +option java_outer_classname = "UserBitShared"; +option optimize_for = SPEED; + +import "Coordination.proto"; +import "SchemaDef.proto"; + +message QueryId { + optional sfixed64 part1 = 1; + optional sfixed64 part2 = 2; +} + +message DrillPBError{ + optional string error_id = 1; // for debug tracing purposes + optional DrillbitEndpoint endpoint = 2; + optional int32 error_type = 3; + optional string message = 4; + repeated ParsingError parsing_error = 5; //optional, used when providing location of error within a piece of text. +} + +message ParsingError{ + optional int32 start_column = 2; + optional int32 start_row = 3; + optional int32 end_column = 4; + optional int32 end_row = 5; +} + +message RecordBatch{ + +} + +message RecordBatchDef { + repeated FieldMetadata field = 1; + optional int32 record_count = 2; + +} + +message FieldMetadata { + optional FieldDef def = 1; + optional int32 value_count = 2; + optional int32 var_byte_length = 3; + optional int32 group_count = 4; // number of groups. (number of repeated records) + optional int32 buffer_length = 5; + repeated FieldMetadata child = 6; +} \ No newline at end of file
