http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/SecureBulkLoad.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/if/SecureBulkLoad.proto b/hbase-native-client/src/hbase/if/SecureBulkLoad.proto new file mode 100644 index 0000000..814735b --- /dev/null +++ b/hbase-native-client/src/hbase/if/SecureBulkLoad.proto @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "SecureBulkLoadProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import 'HBase.proto'; +import 'Client.proto'; + +message SecureBulkLoadHFilesRequest { + repeated BulkLoadHFileRequest.FamilyPath family_path = 1; + optional bool assign_seq_num = 2; + required DelegationToken fs_token = 3; + required string bulk_token = 4; +} + +message SecureBulkLoadHFilesResponse { + required bool loaded = 1; +} + +message DelegationToken { + optional bytes identifier = 1; + optional bytes password = 2; + optional string kind = 3; + optional string service = 4; +} + +message PrepareBulkLoadRequest { + required TableName table_name = 1; +} + +message PrepareBulkLoadResponse { + required string bulk_token = 1; +} + +message CleanupBulkLoadRequest { + required string bulk_token = 1; + +} + +message CleanupBulkLoadResponse { +} + +service SecureBulkLoadService { + rpc PrepareBulkLoad(PrepareBulkLoadRequest) + returns (PrepareBulkLoadResponse); + + rpc SecureBulkLoadHFiles(SecureBulkLoadHFilesRequest) + returns (SecureBulkLoadHFilesResponse); + + rpc CleanupBulkLoad(CleanupBulkLoadRequest) + returns (CleanupBulkLoadResponse); +}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/Snapshot.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/if/Snapshot.proto b/hbase-native-client/src/hbase/if/Snapshot.proto new file mode 100644 index 0000000..ae1a1e6 --- /dev/null +++ b/hbase-native-client/src/hbase/if/Snapshot.proto @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "SnapshotProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "FS.proto"; +import "HBase.proto"; + +message SnapshotFileInfo { + enum Type { + HFILE = 1; + WAL = 2; + } + + required Type type = 1; + + optional string hfile = 3; + + optional string wal_server = 4; + optional string wal_name = 5; +} + +message SnapshotRegionManifest { + optional int32 version = 1; + + required RegionInfo region_info = 2; + repeated FamilyFiles family_files = 3; + + message StoreFile { + required string name = 1; + optional Reference reference = 2; + + // TODO: Add checksums or other fields to verify the file + optional uint64 file_size = 3; + } + + message FamilyFiles { + required bytes family_name = 1; + repeated StoreFile store_files = 2; + } +} + +message SnapshotDataManifest { + required TableSchema table_schema = 1; + repeated SnapshotRegionManifest region_manifests = 2; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/Tracing.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/if/Tracing.proto b/hbase-native-client/src/hbase/if/Tracing.proto new file mode 100644 index 0000000..5a64cfc --- /dev/null +++ b/hbase-native-client/src/hbase/if/Tracing.proto @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "TracingProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +//Used to pass through the information necessary to continue +//a trace after an RPC is made. All we need is the traceid +//(so we know the overarching trace this message is a part of), and +//the id of the current span when this message was sent, so we know +//what span caused the new span we will create when this message is received. +message RPCTInfo { + optional int64 trace_id = 1; + optional int64 parent_id = 2; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/VisibilityLabels.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/if/VisibilityLabels.proto b/hbase-native-client/src/hbase/if/VisibilityLabels.proto new file mode 100644 index 0000000..d2dc44d --- /dev/null +++ b/hbase-native-client/src/hbase/if/VisibilityLabels.proto @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "VisibilityLabelsProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "Client.proto"; + +message VisibilityLabelsRequest { + repeated VisibilityLabel visLabel = 1; +} + +message VisibilityLabel { + required bytes label = 1; + optional uint32 ordinal = 2; +} + +message VisibilityLabelsResponse { + repeated RegionActionResult result = 1; +} + +message SetAuthsRequest { + required bytes user = 1; + repeated bytes auth = 2; +} + +message UserAuthorizations { + required bytes user = 1; + repeated uint32 auth = 2; +} + +message MultiUserAuthorizations { + repeated UserAuthorizations userAuths = 1; +} + +message GetAuthsRequest { + required bytes user = 1; +} + +message GetAuthsResponse { + required bytes user = 1; + repeated bytes auth = 2; +} + +message ListLabelsRequest { + optional string regex = 1; +} + +message ListLabelsResponse { + repeated bytes label = 1; +} + +service VisibilityLabelsService { + rpc addLabels(VisibilityLabelsRequest) + returns (VisibilityLabelsResponse); + rpc setAuths(SetAuthsRequest) + returns (VisibilityLabelsResponse); + rpc clearAuths(SetAuthsRequest) + returns (VisibilityLabelsResponse); + rpc getAuths(GetAuthsRequest) + returns (GetAuthsResponse); + rpc listLabels(ListLabelsRequest) + returns (ListLabelsResponse); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/WAL.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/if/WAL.proto b/hbase-native-client/src/hbase/if/WAL.proto new file mode 100644 index 0000000..2061b22 --- /dev/null +++ b/hbase-native-client/src/hbase/if/WAL.proto @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "WALProtos"; +option java_generic_services = false; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "Client.proto"; + +message WALHeader { + optional bool has_compression = 1; + optional bytes encryption_key = 2; + optional bool has_tag_compression = 3; + optional string writer_cls_name = 4; + optional string cell_codec_cls_name = 5; +} + +/* + * Protocol buffer version of WALKey; see WALKey comment, not really a key but WALEdit header + * for some KVs + */ +message WALKey { + required bytes encoded_region_name = 1; + required bytes table_name = 2; + required uint64 log_sequence_number = 3; + required uint64 write_time = 4; + /* + This parameter is deprecated in favor of clusters which + contains the list of clusters that have consumed the change. + It is retained so that the log created by earlier releases (0.94) + can be read by the newer releases. + */ + optional UUID cluster_id = 5 [deprecated=true]; + + repeated FamilyScope scopes = 6; + optional uint32 following_kv_count = 7; + + /* + This field contains the list of clusters that have + consumed the change + */ + repeated UUID cluster_ids = 8; + + optional uint64 nonceGroup = 9; + optional uint64 nonce = 10; + optional uint64 orig_sequence_number = 11; + +/* + optional CustomEntryType custom_entry_type = 9; + + enum CustomEntryType { + COMPACTION = 0; + } +*/ +} + +enum ScopeType { + REPLICATION_SCOPE_LOCAL = 0; + REPLICATION_SCOPE_GLOBAL = 1; +} + +message FamilyScope { + required bytes family = 1; + required ScopeType scope_type = 2; +} + +/** + * Custom WAL entries + */ + +/** + * Special WAL entry to hold all related to a compaction. + * Written to WAL before completing compaction. There is + * sufficient info in the below message to complete later + * the * compaction should we fail the WAL write. + */ +message CompactionDescriptor { + required bytes table_name = 1; // TODO: WALKey already stores these, might remove + required bytes encoded_region_name = 2; + required bytes family_name = 3; + repeated string compaction_input = 4; // relative to store dir + repeated string compaction_output = 5; + required string store_home_dir = 6; // relative to region dir + optional bytes region_name = 7; // full region name +} + +/** + * Special WAL entry to hold all related to a flush. + */ +message FlushDescriptor { + enum FlushAction { + START_FLUSH = 0; + COMMIT_FLUSH = 1; + ABORT_FLUSH = 2; + CANNOT_FLUSH = 3; // marker for indicating that a flush has been requested but cannot complete + } + + message StoreFlushDescriptor { + required bytes family_name = 1; + required string store_home_dir = 2; //relative to region dir + repeated string flush_output = 3; // relative to store dir (if this is a COMMIT_FLUSH) + } + + required FlushAction action = 1; + required bytes table_name = 2; + required bytes encoded_region_name = 3; + optional uint64 flush_sequence_number = 4; + repeated StoreFlushDescriptor store_flushes = 5; + optional bytes region_name = 6; // full region name +} + +message StoreDescriptor { + required bytes family_name = 1; + required string store_home_dir = 2; //relative to region dir + repeated string store_file = 3; // relative to store dir + optional uint64 store_file_size_bytes = 4; // size of store file +} + +/** + * Special WAL entry used for writing bulk load events to WAL + */ +message BulkLoadDescriptor { + required TableName table_name = 1; + required bytes encoded_region_name = 2; + repeated StoreDescriptor stores = 3; + required int64 bulkload_seq_num = 4; +} + +/** + * Special WAL entry to hold all related to a region event (open/close). + */ +message RegionEventDescriptor { + enum EventType { + REGION_OPEN = 0; + REGION_CLOSE = 1; + } + + required EventType event_type = 1; + required bytes table_name = 2; + required bytes encoded_region_name = 3; + optional uint64 log_sequence_number = 4; + repeated StoreDescriptor stores = 5; + optional ServerName server = 6; // Server who opened the region + optional bytes region_name = 7; // full region name +} + +/** + * A trailer that is appended to the end of a properly closed WAL file. + * If missing, this is either a legacy or a corrupted WAL file. + * N.B. This trailer currently doesn't contain any information and we + * purposefully don't expose it in the WAL APIs. It's for future growth. + */ +message WALTrailer { +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/ZooKeeper.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/if/ZooKeeper.proto b/hbase-native-client/src/hbase/if/ZooKeeper.proto new file mode 100644 index 0000000..41c0e0e --- /dev/null +++ b/hbase-native-client/src/hbase/if/ZooKeeper.proto @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ZNode data in hbase are serialized protobufs with a four byte +// 'magic' 'PBUF' prefix. +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "ZooKeeperProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "ClusterStatus.proto"; + +/** + * Content of the meta-region-server znode. + */ +message MetaRegionServer { + // The ServerName hosting the meta region currently, or destination server, + // if meta region is in transition. + required ServerName server = 1; + // The major version of the rpc the server speaks. This is used so that + // clients connecting to the cluster can have prior knowledge of what version + // to send to a RegionServer. AsyncHBase will use this to detect versions. + optional uint32 rpc_version = 2; + + // State of the region transition. OPEN means fully operational 'hbase:meta' + optional RegionState.State state = 3; +} + +/** + * Content of the master znode. + */ +message Master { + // The ServerName of the current Master + required ServerName master = 1; + // Major RPC version so that clients can know what version the master can accept. + optional uint32 rpc_version = 2; + optional uint32 info_port = 3; +} + +/** + * Content of the '/hbase/running', cluster state, znode. + */ +message ClusterUp { + // If this znode is present, cluster is up. Currently + // the data is cluster start_date. + required string start_date = 1; +} + +/** + * WAL SplitLog directory znodes have this for content. Used doing distributed + * WAL splitting. Holds current state and name of server that originated split. + */ +message SplitLogTask { + enum State { + UNASSIGNED = 0; + OWNED = 1; + RESIGNED = 2; + DONE = 3; + ERR = 4; + } + enum RecoveryMode { + UNKNOWN = 0; + LOG_SPLITTING = 1; + LOG_REPLAY = 2; + } + required State state = 1; + required ServerName server_name = 2; + optional RecoveryMode mode = 3 [default = UNKNOWN]; +} + +/** + * The znode that holds state of table. + * Deprected, table state is stored in table descriptor on HDFS. + */ +message DeprecatedTableState { + // Table's current state + enum State { + ENABLED = 0; + DISABLED = 1; + DISABLING = 2; + ENABLING = 3; + } + // This is the table's state. If no znode for a table, + // its state is presumed enabled. See o.a.h.h.zookeeper.ZKTable class + // for more. + required State state = 1 [default = ENABLED]; +} + +message TableCF { + optional TableName table_name = 1; + repeated bytes families = 2; +} + +/** + * Used by replication. Holds a replication peer key. + */ +message ReplicationPeer { + // clusterkey is the concatenation of the slave cluster's + // hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent + required string clusterkey = 1; + optional string replicationEndpointImpl = 2; + repeated BytesBytesPair data = 3; + repeated NameStringPair configuration = 4; + repeated TableCF table_cfs = 5; +} + +/** + * Used by replication. Holds whether enabled or disabled + */ +message ReplicationState { + enum State { + ENABLED = 0; + DISABLED = 1; + } + required State state = 1; +} + +/** + * Used by replication. Holds the current position in an WAL file. + */ +message ReplicationHLogPosition { + required int64 position = 1; +} + +/** + * Used by replication. Used to lock a region server during failover. + */ +message ReplicationLock { + required string lock_owner = 1; +} + +/** + * Metadata associated with a table lock in zookeeper + */ +message TableLock { + optional TableName table_name = 1; + optional ServerName lock_owner = 2; + optional int64 thread_id = 3; + optional bool is_shared = 4; + optional string purpose = 5; + optional int64 create_time = 6; +} + +/** + * State of the switch. + */ +message SwitchState { + optional bool enabled = 1; +} + +/** + * State for split and merge, used in hbck + */ +message SplitAndMergeState { + optional bool split_enabled = 1; + optional bool merge_enabled = 2; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/test.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/if/test.proto b/hbase-native-client/src/hbase/if/test.proto new file mode 100644 index 0000000..72b68e9 --- /dev/null +++ b/hbase-native-client/src/hbase/if/test.proto @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.hbase.ipc.protobuf.generated"; +option java_outer_classname = "TestProtos"; +option java_generate_equals_and_hash = true; + +message EmptyRequestProto { +} + +message EmptyResponseProto { +} + +message EchoRequestProto { + required string message = 1; +} + +message EchoResponseProto { + required string message = 1; +} + +message PauseRequestProto { + required uint32 ms = 1; +} + +message AddrResponseProto { + required string addr = 1; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/test_rpc_service.proto ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/if/test_rpc_service.proto b/hbase-native-client/src/hbase/if/test_rpc_service.proto new file mode 100644 index 0000000..2730403 --- /dev/null +++ b/hbase-native-client/src/hbase/if/test_rpc_service.proto @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +option java_package = "org.apache.hadoop.hbase.ipc.protobuf.generated"; +option java_outer_classname = "TestRpcServiceProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; + +import "test.proto"; + + +/** + * A protobuf service for use in tests + */ +service TestProtobufRpcProto { + rpc ping(EmptyRequestProto) returns (EmptyResponseProto); + rpc echo(EchoRequestProto) returns (EchoResponseProto); + rpc error(EmptyRequestProto) returns (EmptyResponseProto); + rpc pause(PauseRequestProto) returns (EmptyResponseProto); + rpc addr(EmptyRequestProto) returns (AddrResponseProto); + rpc socketNotOpen(EmptyRequestProto) returns (EmptyResponseProto); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/security/BUCK ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/security/BUCK b/hbase-native-client/src/hbase/security/BUCK new file mode 100644 index 0000000..c329f30 --- /dev/null +++ b/hbase-native-client/src/hbase/security/BUCK @@ -0,0 +1,27 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the library dealing with a single connection +# to a single server. +cxx_library( + name="security", + srcs=[], + deps=["//include/hbase/security:security", "//src/hbase/client:conf"], + compiler_flags=['-Weffc++'], + visibility=[ + 'PUBLIC', + ],) http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/BUCK ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/serde/BUCK b/hbase-native-client/src/hbase/serde/BUCK new file mode 100644 index 0000000..6b39e0b --- /dev/null +++ b/hbase-native-client/src/hbase/serde/BUCK @@ -0,0 +1,86 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cxx_library( + name="serde", + srcs=[ + "rpc-serde.cc", + "zk.cc", + ], + deps=[ + "//include/hbase/serde:serde", "//src/hbase/if:if", "//third-party:folly", "//src/hbase/utils:utils", "//src/hbase/security:security" + ], + tests=[ + ":client-deserializer-test", + ":client-serializer-test", + ":server-name-test", + ":table-name-test", + ":zk-deserializer-test", + ":region-info-deserializer-test", + ], + compiler_flags=['-Weffc++'], + visibility=[ + 'PUBLIC', + ],) +cxx_test( + name="table-name-test", + srcs=[ + "table-name-test.cc", + ], + deps=[ + ":serde", + ],) +cxx_test( + name="server-name-test", + srcs=[ + "server-name-test.cc", + ], + deps=[ + ":serde", + ],) +cxx_test( + name="client-serializer-test", + srcs=[ + "client-serializer-test.cc", + ], + deps=[ + ":serde", + ],) +cxx_test( + name="client-deserializer-test", + srcs=[ + "client-deserializer-test.cc", + ], + deps=[ + ":serde", + ],) +cxx_test( + name="zk-deserializer-test", + srcs=[ + "zk-deserializer-test.cc", + ], + deps=[ + ":serde", + ],) +cxx_test( + name="region-info-deserializer-test", + srcs=[ + "region-info-deserializer-test.cc", + ], + deps=[ + ":serde", + ],) http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/client-deserializer-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/serde/client-deserializer-test.cc b/hbase-native-client/src/hbase/serde/client-deserializer-test.cc new file mode 100644 index 0000000..3e4c42b --- /dev/null +++ b/hbase-native-client/src/hbase/serde/client-deserializer-test.cc @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include <folly/io/IOBuf.h> +#include <gtest/gtest.h> + +#include "hbase/if/Client.pb.h" +#include "hbase/serde/rpc-serde.h" + +using namespace hbase; +using folly::IOBuf; +using hbase::pb::GetRequest; +using hbase::pb::RegionSpecifier; +using hbase::pb::RegionSpecifier_RegionSpecifierType; + +TEST(TestRpcSerde, TestReturnFalseOnNullPtr) { + RpcSerde deser{nullptr}; + ASSERT_LT(deser.ParseDelimited(nullptr, nullptr), 0); +} + +TEST(TestRpcSerde, TestReturnFalseOnBadInput) { + RpcSerde deser{nullptr}; + auto buf = IOBuf::copyBuffer("test"); + GetRequest gr; + + ASSERT_LT(deser.ParseDelimited(buf.get(), &gr), 0); +} + +TEST(TestRpcSerde, TestGoodGetRequestFullRoundTrip) { + GetRequest in; + RpcSerde ser{nullptr}; + RpcSerde deser{nullptr}; + + // fill up the GetRequest. + in.mutable_region()->set_value("test_region_id"); + in.mutable_region()->set_type( + RegionSpecifier_RegionSpecifierType::RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME); + in.mutable_get()->set_row("test_row"); + + // Create the buffer + auto buf = ser.SerializeDelimited(in); + + GetRequest out; + + int used_bytes = deser.ParseDelimited(buf.get(), &out); + + ASSERT_GT(used_bytes, 0); + ASSERT_EQ(used_bytes, buf->length()); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/client-serializer-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/serde/client-serializer-test.cc b/hbase-native-client/src/hbase/serde/client-serializer-test.cc new file mode 100644 index 0000000..8279caa --- /dev/null +++ b/hbase-native-client/src/hbase/serde/client-serializer-test.cc @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include <gtest/gtest.h> + +#include <folly/io/Cursor.h> + +#include <string> + +#include "hbase/if/HBase.pb.h" +#include "hbase/if/RPC.pb.h" +#include "hbase/serde/rpc-serde.h" + +using namespace hbase; +using namespace hbase::pb; +using namespace folly; +using namespace folly::io; + +TEST(RpcSerdeTest, PreambleIncludesHBas) { + RpcSerde ser{nullptr}; + auto buf = ser.Preamble(false); + const char *p = reinterpret_cast<const char *>(buf->data()); + // Take the first for chars and make sure they are the + // magic string + EXPECT_EQ("HBas", std::string(p, 4)); + + EXPECT_EQ(6, buf->computeChainDataLength()); +} + +TEST(RpcSerdeTest, PreambleIncludesVersion) { + RpcSerde ser{nullptr}; + auto buf = ser.Preamble(false); + EXPECT_EQ(0, static_cast<const uint8_t *>(buf->data())[4]); + EXPECT_EQ(80, static_cast<const uint8_t *>(buf->data())[5]); +} + +TEST(RpcSerdeTest, TestHeaderLengthPrefixed) { + RpcSerde ser{nullptr}; + auto header = ser.Header("elliott"); + + // The header should be prefixed by 4 bytes of length. + EXPECT_EQ(4, header->length()); + EXPECT_TRUE(header->length() < header->computeChainDataLength()); + EXPECT_TRUE(header->isChained()); + + // Now make sure the length is correct. + Cursor cursor(header.get()); + auto prefixed_len = cursor.readBE<uint32_t>(); + EXPECT_EQ(prefixed_len, header->next()->length()); +} + +TEST(RpcSerdeTest, TestHeaderDecode) { + RpcSerde ser{nullptr}; + auto buf = ser.Header("elliott"); + auto header_buf = buf->next(); + ConnectionHeader h; + + EXPECT_TRUE(h.ParseFromArray(header_buf->data(), header_buf->length())); + EXPECT_EQ("elliott", h.user_info().effective_user()); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/region-info-deserializer-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/serde/region-info-deserializer-test.cc b/hbase-native-client/src/hbase/serde/region-info-deserializer-test.cc new file mode 100644 index 0000000..84219d8 --- /dev/null +++ b/hbase-native-client/src/hbase/serde/region-info-deserializer-test.cc @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "hbase/serde/region-info.h" + +#include <gtest/gtest.h> + +#include <string> + +#include "hbase/if/HBase.pb.h" +#include "hbase/serde/table-name.h" + +using std::string; +using hbase::pb::RegionInfo; +using hbase::pb::TableName; + +TEST(TestRegionInfoDesializer, TestDeserialize) { + string ns{"test_ns"}; + string tn{"table_name"}; + string start_row{"AAAAAA"}; + string stop_row{"BBBBBBBBBBBB"}; + uint64_t region_id = 2345678; + + RegionInfo ri_out; + ri_out.set_region_id(region_id); + ri_out.mutable_table_name()->set_namespace_(ns); + ri_out.mutable_table_name()->set_qualifier(tn); + ri_out.set_start_key(start_row); + ri_out.set_end_key(stop_row); + + string header{"PBUF"}; + string ser = header + ri_out.SerializeAsString(); + + auto out = folly::to<RegionInfo>(ser); + + EXPECT_EQ(region_id, out.region_id()); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/rpc-serde.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/serde/rpc-serde.cc b/hbase-native-client/src/hbase/serde/rpc-serde.cc new file mode 100644 index 0000000..5f2920d --- /dev/null +++ b/hbase-native-client/src/hbase/serde/rpc-serde.cc @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <folly/Conv.h> +#include <folly/Logging.h> +#include <folly/io/Cursor.h> +#include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/io/zero_copy_stream_impl_lite.h> +#include <google/protobuf/message.h> +#include <boost/algorithm/string.hpp> + +#include <utility> + +#include "hbase/if/RPC.pb.h" +#include "hbase/serde/rpc-serde.h" +#include "hbase/utils/version.h" + +using folly::IOBuf; +using folly::io::RWPrivateCursor; +using google::protobuf::Message; +using google::protobuf::io::ArrayInputStream; +using google::protobuf::io::ArrayOutputStream; +using google::protobuf::io::CodedInputStream; +using google::protobuf::io::CodedOutputStream; +using google::protobuf::io::ZeroCopyOutputStream; + +using namespace hbase::pb; + +namespace hbase { + +static const std::string PREAMBLE = "HBas"; +static const std::string INTERFACE = "ClientService"; +static const uint8_t RPC_VERSION = 0; +static const uint8_t DEFAULT_AUTH_TYPE = 80; +static const uint8_t KERBEROS_AUTH_TYPE = 81; + +int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) { + if (buf == nullptr || msg == nullptr) { + return -2; + } + + DCHECK(!buf->isChained()); + + ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())}; + CodedInputStream coded_stream{&ais}; + + uint32_t msg_size; + + // Try and read the varint. + if (coded_stream.ReadVarint32(&msg_size) == false) { + FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t"; + return -3; + } + + coded_stream.PushLimit(msg_size); + // Parse the message. + if (msg->MergeFromCodedStream(&coded_stream) == false) { + FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a protobuf message from data."; + return -4; + } + + // Make sure all the data was consumed. + if (coded_stream.ConsumedEntireMessage() == false) { + FB_LOG_EVERY_MS(ERROR, 1000) << "Orphaned data left after reading protobuf message"; + return -5; + } + + return coded_stream.CurrentPosition(); +} + +RpcSerde::RpcSerde() {} + +RpcSerde::RpcSerde(std::shared_ptr<Codec> codec) : codec_(codec) {} + +std::unique_ptr<IOBuf> RpcSerde::Preamble(bool secure) { + auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2); + magic->append(2); + RWPrivateCursor c(magic.get()); + c.skip(4); + // Version + c.write(RPC_VERSION); + if (secure) { + // for now support only KERBEROS (DIGEST is not supported) + c.write(KERBEROS_AUTH_TYPE); + } else { + c.write(DEFAULT_AUTH_TYPE); + } + return magic; +} + +std::unique_ptr<IOBuf> RpcSerde::Header(const std::string &user) { + pb::ConnectionHeader h; + + // TODO(eclark): Make this not a total lie. + h.mutable_user_info()->set_effective_user(user); + // The service name that we want to talk to. + // + // Right now we're completely ignoring the service interface. + // That may or may not be the correct thing to do. + // It worked for a while with the java client; until it + // didn't. + // TODO: send the service name and user from the RpcClient + h.set_service_name(INTERFACE); + + std::unique_ptr<pb::VersionInfo> version_info = CreateVersionInfo(); + + h.set_allocated_version_info(version_info.release()); + + if (codec_ != nullptr) { + h.set_cell_block_codec_class(codec_->java_class_name()); + } + return PrependLength(SerializeMessage(h)); +} + +std::unique_ptr<pb::VersionInfo> RpcSerde::CreateVersionInfo() { + std::unique_ptr<pb::VersionInfo> version_info = std::make_unique<pb::VersionInfo>(); + version_info->set_user(Version::user); + version_info->set_revision(Version::revision); + version_info->set_url(Version::url); + version_info->set_date(Version::date); + version_info->set_src_checksum(Version::src_checksum); + version_info->set_version(Version::version); + + std::string version{Version::version}; + std::vector<std::string> version_parts; + boost::split(version_parts, version, boost::is_any_of("."), boost::token_compress_on); + uint32_t major_version = 0, minor_version = 0; + if (version_parts.size() >= 2) { + version_info->set_version_major(folly::to<uint32_t>(version_parts[0])); + version_info->set_version_minor(folly::to<uint32_t>(version_parts[1])); + } + + VLOG(1) << "Client VersionInfo:" << version_info->ShortDebugString(); + return version_info; +} + +std::unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id, const std::string &method, + const Message *msg) { + pb::RequestHeader rq; + rq.set_method_name(method); + rq.set_call_id(call_id); + rq.set_request_param(msg != nullptr); + auto ser_header = SerializeDelimited(rq); + if (msg != nullptr) { + auto ser_req = SerializeDelimited(*msg); + ser_header->appendChain(std::move(ser_req)); + } + + return PrependLength(std::move(ser_header)); +} + +std::unique_ptr<folly::IOBuf> RpcSerde::Response(const uint32_t call_id, + const google::protobuf::Message *msg) { + pb::ResponseHeader rh; + rh.set_call_id(call_id); + auto ser_header = SerializeDelimited(rh); + auto ser_resp = SerializeDelimited(*msg); + ser_header->appendChain(std::move(ser_resp)); + + return PrependLength(std::move(ser_header)); +} + +std::unique_ptr<folly::IOBuf> RpcSerde::Response(const uint32_t call_id, + const google::protobuf::Message *msg, + const folly::exception_wrapper &exception) { + /* create ResponseHeader */ + pb::ResponseHeader rh; + rh.set_call_id(call_id); + + /* create ExceptionResponse */ + if (bool(exception)) { + VLOG(1) << "packing ExceptionResponse"; + auto exception_response = new pb::ExceptionResponse(); + exception_response->set_exception_class_name(exception.class_name().c_str()); + exception_response->set_stack_trace(exception.what().c_str()); + rh.set_allocated_exception(exception_response); + } + + /* serialize Response header and body */ + auto ser_header = SerializeDelimited(rh); + auto ser_resp = SerializeDelimited(*msg); + ser_header->appendChain(std::move(ser_resp)); + + VLOG(3) << "Converted hbase::Response to folly::IOBuf"; + return PrependLength(std::move(ser_header)); +} + +std::unique_ptr<CellScanner> RpcSerde::CreateCellScanner(std::unique_ptr<folly::IOBuf> buf, + uint32_t offset, uint32_t length) { + if (codec_ == nullptr) { + return nullptr; + } + return codec_->CreateDecoder(std::move(buf), offset, length); +} + +std::unique_ptr<IOBuf> RpcSerde::PrependLength(std::unique_ptr<IOBuf> msg) { + // Java ints are 4 long. So create a buffer that large + auto len_buf = IOBuf::create(4); + // Then make those bytes visible. + len_buf->append(4); + + RWPrivateCursor c(len_buf.get()); + // Get the size of the data to be pushed out the network. + auto size = msg->computeChainDataLength(); + + // Write the length to this IOBuf. + c.writeBE(static_cast<uint32_t>(size)); + + // Then attach the origional to the back of len_buf + len_buf->appendChain(std::move(msg)); + return len_buf; +} + +std::unique_ptr<IOBuf> RpcSerde::SerializeDelimited(const Message &msg) { + // Get the buffer size needed for just the message. + int msg_size = msg.ByteSize(); + int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size; + + // Create a buffer big enough to hold the varint and the object. + auto buf = IOBuf::create(buf_size); + buf->append(buf_size); + + // Create the array output stream. + ArrayOutputStream aos{buf->writableData(), static_cast<int>(buf->length())}; + // Wrap the ArrayOuputStream in the coded output stream to allow writing + // Varint32 + CodedOutputStream cos{&aos}; + + // Write out the size. + cos.WriteVarint32(msg_size); + + // Now write the rest out. + // We're using the protobuf output streams here to keep track + // of where in the output array we are rather than IOBuf. + msg.SerializeWithCachedSizesToArray(cos.GetDirectBufferForNBytesAndAdvance(msg_size)); + + // Return the buffer. + return buf; +} +// TODO(eclark): Make this 1 copy. +std::unique_ptr<IOBuf> RpcSerde::SerializeMessage(const Message &msg) { + auto buf = IOBuf::copyBuffer(msg.SerializeAsString()); + return buf; +} +} // namespace hbase http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/server-name-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/serde/server-name-test.cc b/hbase-native-client/src/hbase/serde/server-name-test.cc new file mode 100644 index 0000000..12d3287 --- /dev/null +++ b/hbase-native-client/src/hbase/serde/server-name-test.cc @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "hbase/serde/server-name.h" + +#include <gtest/gtest.h> +#include <string> + +using hbase::pb::ServerName; + +TEST(TestServerName, TestMakeServerName) { + auto sn = folly::to<ServerName>("test:123"); + + ASSERT_EQ("test", sn.host_name()); + ASSERT_EQ(123, sn.port()); +} + +TEST(TestServerName, TestIps) { + auto sn = folly::to<ServerName>("127.0.0.1:999"); + ASSERT_EQ("127.0.0.1", sn.host_name()); + ASSERT_EQ(999, sn.port()); +} + +TEST(TestServerName, TestThrow) { ASSERT_ANY_THROW(folly::to<ServerName>("Ther's no colon here")); } + +TEST(TestServerName, TestIPV6) { + auto sn = folly::to<ServerName>("[::::1]:123"); + + ASSERT_EQ("[::::1]", sn.host_name()); + ASSERT_EQ(123, sn.port()); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/table-name-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/serde/table-name-test.cc b/hbase-native-client/src/hbase/serde/table-name-test.cc new file mode 100644 index 0000000..77bd6c2 --- /dev/null +++ b/hbase-native-client/src/hbase/serde/table-name-test.cc @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <folly/Conv.h> +#include <gtest/gtest.h> + +#include <string> + +#include "hbase/serde/table-name.h" + +using namespace hbase; +using hbase::pb::TableName; + +TEST(TestTableName, TestToStringNoDefault) { + TableName tn; + tn.set_qualifier("TestTableName"); + std::string result = folly::to<std::string>(tn); + ASSERT_EQ(result.find("default"), std::string::npos); + ASSERT_EQ("TestTableName", result); +} + +TEST(TestTableName, TestToStringNoDefaltWhenSet) { + TableName tn; + tn.set_namespace_("default"); + tn.set_qualifier("TestTableName"); + std::string result = folly::to<std::string>(tn); + ASSERT_EQ(result.find("default"), std::string::npos); + ASSERT_EQ("TestTableName", result); +} + +TEST(TestTableName, TestToStringIncludeNS) { + TableName tn; + tn.set_namespace_("hbase"); + tn.set_qualifier("acl"); + std::string result = folly::to<std::string>(tn); + ASSERT_EQ(result.find("hbase"), 0); + ASSERT_EQ("hbase:acl", result); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/zk-deserializer-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/serde/zk-deserializer-test.cc b/hbase-native-client/src/hbase/serde/zk-deserializer-test.cc new file mode 100644 index 0000000..141efce --- /dev/null +++ b/hbase-native-client/src/hbase/serde/zk-deserializer-test.cc @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "hbase/serde/zk.h" + +#include <folly/Logging.h> +#include <folly/io/Cursor.h> +#include <folly/io/IOBuf.h> +#include <gtest/gtest.h> + +#include "hbase/if/ZooKeeper.pb.h" + +using namespace hbase; +using namespace hbase::pb; +using namespace folly; +using namespace std; +using namespace folly::io; + +// Test that would test if there's nothing there. +TEST(TestZkDesializer, TestThrowNoMagicNum) { + ZkDeserializer deser; + MetaRegionServer mrs; + + auto buf = IOBuf::create(100); + buf->append(100); + RWPrivateCursor c{buf.get()}; + c.write<uint8_t>(99); + ASSERT_THROW(deser.Parse(buf.get(), &mrs), runtime_error); +} + +// Test if the protobuf is in a format that we can't decode +TEST(TestZkDesializer, TestBadProtoThrow) { + ZkDeserializer deser; + MetaRegionServer mrs; + string magic{"PBUF"}; + + // Set ServerName + mrs.mutable_server()->set_host_name("test"); + mrs.mutable_server()->set_port(567); + mrs.mutable_server()->set_start_code(9567); + + // One byte magic number + // four bytes for id length + // four bytes for id + // four bytes for PBUF + uint32_t start_len = 1 + 4 + 4 + 4; + // How large the protobuf will be + uint32_t pbuf_size = mrs.ByteSize(); + + auto buf = IOBuf::create(start_len + pbuf_size); + buf->append(start_len + pbuf_size); + RWPrivateCursor c{buf.get()}; + + // Write the magic number + c.write<uint8_t>(255); + // Write the id len + c.writeBE<uint32_t>(4); + // Write the id + c.write<uint32_t>(13); + // Write the PBUF string + c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4); + + // Create the protobuf + MetaRegionServer out; + ASSERT_THROW(deser.Parse(buf.get(), &out), runtime_error); +} + +// Test to make sure the whole thing works. +TEST(TestZkDesializer, TestNoThrow) { + ZkDeserializer deser; + MetaRegionServer mrs; + string magic{"PBUF"}; + + // Set ServerName + mrs.mutable_server()->set_host_name("test"); + mrs.mutable_server()->set_port(567); + mrs.mutable_server()->set_start_code(9567); + + // One byte magic number + // four bytes for id length + // four bytes for id + // four bytes for PBUF + uint32_t start_len = 1 + 4 + 4 + 4; + // How large the protobuf will be + uint32_t pbuf_size = mrs.ByteSize(); + + auto buf = IOBuf::create(start_len + pbuf_size); + buf->append(start_len + pbuf_size); + RWPrivateCursor c{buf.get()}; + + // Write the magic number + c.write<uint8_t>(255); + // Write the id len + c.writeBE<uint32_t>(4); + // Write the id + c.write<uint32_t>(13); + // Write the PBUF string + c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4); + + // Now write the serialized protobuf + mrs.SerializeWithCachedSizesToArray(buf->writableData() + start_len); + + // Create the protobuf + MetaRegionServer out; + ASSERT_TRUE(deser.Parse(buf.get(), &out)); + ASSERT_EQ(mrs.server().host_name(), out.server().host_name()); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/zk.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/serde/zk.cc b/hbase-native-client/src/hbase/serde/zk.cc new file mode 100644 index 0000000..b962cc5 --- /dev/null +++ b/hbase-native-client/src/hbase/serde/zk.cc @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "hbase/serde/zk.h" + +#include <folly/io/Cursor.h> +#include <folly/io/IOBuf.h> +#include <google/protobuf/message.h> + +#include <string> + +using std::runtime_error; + +namespace hbase { + +static const std::string MAGIC_STRING = "PBUF"; + +bool ZkDeserializer::Parse(folly::IOBuf *buf, google::protobuf::Message *out) { + // The format is like this + // 1 byte of magic number. 255 + // 4 bytes of id length. + // id_length number of bytes for the id of who put up the znode + // 4 bytes of a magic string PBUF + // Then the protobuf serialized without a varint header. + + folly::io::Cursor c{buf}; + + // There should be a magic number for recoverable zk + uint8_t magic_num = c.read<uint8_t>(); + if (magic_num != 255) { + LOG(ERROR) << "Magic number not in ZK znode data expected 255 got =" << unsigned(magic_num); + throw runtime_error("Magic number not in znode data"); + } + // How long is the id? + uint32_t id_len = c.readBE<uint32_t>(); + + if (id_len >= c.length()) { + LOG(ERROR) << "After skiping the if from zookeeper data there's not enough " + "left to read anything else"; + throw runtime_error("Not enough bytes to decode from zookeeper"); + } + + // Skip the id + c.skip(id_len); + + // Make sure that the magic string is there. + if (MAGIC_STRING != c.readFixedString(4)) { + LOG(ERROR) << "There was no PBUF magic string."; + throw runtime_error("No PBUF magic string in the zookpeeper data."); + } + + // Try to decode the protobuf. + // If there's an error bail out. + if (out->ParseFromArray(c.data(), c.length()) == false) { + LOG(ERROR) << "Error parsing Protobuf Message"; + throw runtime_error("Error parsing protobuf"); + } + + return true; +} +} // namespace hbase http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/BUCK ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/test-util/BUCK b/hbase-native-client/src/hbase/test-util/BUCK new file mode 100644 index 0000000..f1aedab --- /dev/null +++ b/hbase-native-client/src/hbase/test-util/BUCK @@ -0,0 +1,53 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +cxx_library( + name="test-util", + header_namespace="hbase/test-util", + exported_headers=["test-util.h", "mini-cluster.h"], + srcs=["test-util.cc", "mini-cluster.cc"], + deps=[ + "//third-party:folly", + "//src/hbase/client:client", + ], + preprocessor_flags=[ + '-I' + os.environ['JAVA_HOME'] + '/include', + '-I' + os.environ['JAVA_HOME'] + '/include/darwin', + '-I' + os.environ['JAVA_HOME'] + '/include/linux' + ], + exported_preprocessor_flags=[ + '-I' + os.environ['JAVA_HOME'] + '/include', + '-I' + os.environ['JAVA_HOME'] + '/include/darwin', + '-I' + os.environ['JAVA_HOME'] + '/include/linux' + ], + compiler_flags=[ + '-I' + os.environ['JAVA_HOME'] + '/include', + '-I' + os.environ['JAVA_HOME'] + '/include/darwin', + '-I' + os.environ['JAVA_HOME'] + '/include/linux', '-ggdb' + ], + linker_flags=[ + '-ljvm', '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server', + '-ggdb' + ], + exported_linker_flags=[ + '-ljvm', '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server', + '-Wl,-rpath=' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server' + ], + visibility=[ + 'PUBLIC', + ],) http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/mini-cluster.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/test-util/mini-cluster.cc b/hbase-native-client/src/hbase/test-util/mini-cluster.cc new file mode 100644 index 0000000..1e491a2 --- /dev/null +++ b/hbase-native-client/src/hbase/test-util/mini-cluster.cc @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "hbase/test-util/mini-cluster.h" +#include <fcntl.h> +#include <glog/logging.h> +#include <boost/filesystem/fstream.hpp> +#include <boost/filesystem/operations.hpp> +#include <fstream> + +using hbase::MiniCluster; + +JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) { + JavaVMInitArgs args; + JavaVMOption jvm_options; + args.version = JNI_VERSION_1_6; + args.nOptions = 1; + char *classpath = getenv("CLASSPATH"); + std::string clspath; + if (classpath == NULL || strstr(classpath, "-tests.jar") == NULL) { + std::string clsPathFilePath("../target/cached_classpath.txt"); + std::ifstream fd(clsPathFilePath); + std::string prefix(""); + if (fd.is_open()) { + if (classpath == NULL) { + LOG(INFO) << "got empty classpath"; + } else { + // prefix bootstrapper.jar + prefix.assign(classpath); + } + std::string line; + if (getline(fd, line)) { + clspath = prefix + ":" + line; + int ret = setenv("CLASSPATH", clspath.c_str(), 1); + LOG(INFO) << "set clspath " << ret; + } else { + LOG(INFO) << "nothing read from " << clsPathFilePath; + exit(-1); + } + } else { + LOG(INFO) << "nothing read from " << clsPathFilePath; + exit(-1); + } + fd.close(); + } + + auto options = std::string{"-Djava.class.path="} + clspath; + jvm_options.optionString = const_cast<char *>(options.c_str()); + args.options = &jvm_options; + args.ignoreUnrecognized = 0; + int rv; + rv = JNI_CreateJavaVM(jvm, reinterpret_cast<void **>(&env_), &args); + CHECK(rv >= 0 && env_); + return env_; +} + +MiniCluster::~MiniCluster() { + if (jvm_ != NULL) { + jvm_->DestroyJavaVM(); + jvm_ = NULL; + } + env_ = nullptr; +} + +void MiniCluster::Setup() { + jmethodID constructor; + pthread_mutex_lock(&count_mutex_); + if (env_ == NULL) { + env_ = CreateVM(&jvm_); + if (env_ == NULL) { + exit(-1); + } + testing_util_class_ = env_->FindClass("org/apache/hadoop/hbase/HBaseTestingUtility"); + // this should be converted to a globalref I think to avoid the underlying java obj getting + // GC'ed + if (testing_util_class_ == NULL) { + LOG(INFO) << "Couldn't find class HBaseTestingUtility"; + exit(-1); + } + jmethodID mid = env_->GetStaticMethodID(testing_util_class_, "createLocalHTU", + "()Lorg/apache/hadoop/hbase/HBaseTestingUtility;"); + htu_ = env_->CallStaticObjectMethod(testing_util_class_, mid); + // this should be converted to a globalref I think to avoid the underlying java obj getting + // GC'ed + if (htu_ == NULL) { + LOG(INFO) << "Couldn't invoke method createLocalHTU in HBaseTestingUtility"; + exit(-1); + } + get_conn_mid_ = env_->GetMethodID(testing_util_class_, "getConnection", + "()Lorg/apache/hadoop/hbase/client/Connection;"); + jclass conn_class = env_->FindClass("org/apache/hadoop/hbase/client/Connection"); + get_admin_mid_ = + env_->GetMethodID(conn_class, "getAdmin", "()Lorg/apache/hadoop/hbase/client/Admin;"); + get_table_mid_ = env_->GetMethodID( + conn_class, "getTable", + "(Lorg/apache/hadoop/hbase/TableName;)Lorg/apache/hadoop/hbase/client/Table;"); + if (get_table_mid_ == NULL) { + LOG(INFO) << "Couldn't find getConnection"; + exit(-1); + } + jclass adminClass = env_->FindClass("org/apache/hadoop/hbase/client/Admin"); + move_mid_ = env_->GetMethodID(adminClass, "move", "([B[B)V"); + if (move_mid_ == NULL) { + LOG(INFO) << "Couldn't find move"; + exit(-1); + } + create_table_mid_ = + env_->GetMethodID(testing_util_class_, "createTable", + "(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/" + "apache/hadoop/hbase/client/Table;"); + create_table_families_mid_ = env_->GetMethodID(testing_util_class_, "createTable", + "(Lorg/apache/hadoop/hbase/TableName;[[B)Lorg/" + "apache/hadoop/hbase/client/Table;"); + create_table_with_split_mid_ = env_->GetMethodID( + testing_util_class_, "createTable", + "(Lorg/apache/hadoop/hbase/TableName;[[B[[B)Lorg/apache/hadoop/hbase/client/Table;"); + if (create_table_with_split_mid_ == NULL) { + LOG(INFO) << "Couldn't find method createTable with split"; + exit(-1); + } + + table_name_class_ = env_->FindClass("org/apache/hadoop/hbase/TableName"); + tbl_name_value_of_mid_ = env_->GetStaticMethodID( + table_name_class_, "valueOf", "(Ljava/lang/String;)Lorg/apache/hadoop/hbase/TableName;"); + if (tbl_name_value_of_mid_ == NULL) { + LOG(INFO) << "Couldn't find method valueOf in TableName"; + exit(-1); + } + jclass hbaseMiniClusterClass = env_->FindClass("org/apache/hadoop/hbase/MiniHBaseCluster"); + stop_rs_mid_ = + env_->GetMethodID(hbaseMiniClusterClass, "stopRegionServer", + "(I)Lorg/apache/hadoop/hbase/util/JVMClusterUtil$RegionServerThread;"); + get_conf_mid_ = env_->GetMethodID(hbaseMiniClusterClass, "getConfiguration", + "()Lorg/apache/hadoop/conf/Configuration;"); + + conf_class_ = env_->FindClass("org/apache/hadoop/conf/Configuration"); + set_conf_mid_ = + env_->GetMethodID(conf_class_, "set", "(Ljava/lang/String;Ljava/lang/String;)V"); + if (set_conf_mid_ == NULL) { + LOG(INFO) << "Couldn't find method getConf in MiniHBaseCluster"; + exit(-1); + } + conf_get_mid_ = env_->GetMethodID(conf_class_, "get", "(Ljava/lang/String;)Ljava/lang/String;"); + + jclass tableClass = env_->FindClass("org/apache/hadoop/hbase/client/Table"); + put_mid_ = env_->GetMethodID(tableClass, "put", "(Lorg/apache/hadoop/hbase/client/Put;)V"); + jclass connFactoryClass = env_->FindClass("org/apache/hadoop/hbase/client/ConnectionFactory"); + create_conn_mid_ = env_->GetStaticMethodID(connFactoryClass, "createConnection", + "()Lorg/apache/hadoop/hbase/client/Connection;"); + if (create_conn_mid_ == NULL) { + LOG(INFO) << "Couldn't find createConnection"; + exit(-1); + } + put_class_ = env_->FindClass("org/apache/hadoop/hbase/client/Put"); + put_ctor_ = env_->GetMethodID(put_class_, "<init>", "([B)V"); + add_col_mid_ = + env_->GetMethodID(put_class_, "addColumn", "([B[B[B)Lorg/apache/hadoop/hbase/client/Put;"); + if (add_col_mid_ == NULL) { + LOG(INFO) << "Couldn't find method addColumn"; + exit(-1); + } + } + pthread_mutex_unlock(&count_mutex_); +} + +jobject MiniCluster::htu() { + Setup(); + return htu_; +} + +JNIEnv *MiniCluster::env() { + Setup(); + return env_; +} +// converts C char* to Java byte[] +jbyteArray MiniCluster::StrToByteChar(const std::string &str) { + if (str.length() == 0) { + return nullptr; + } + int n = str.length(); + jbyteArray arr = env_->NewByteArray(n); + env_->SetByteArrayRegion(arr, 0, n, reinterpret_cast<const jbyte *>(str.c_str())); + return arr; +} + +jobject MiniCluster::CreateTable(const std::string &table, const std::string &family) { + jstring table_name_str = env_->NewStringUTF(table.c_str()); + jobject table_name = + env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str); + jstring family_str = env_->NewStringUTF(family.c_str()); + jobject table_obj = env_->CallObjectMethod(htu_, create_table_mid_, table_name, family_str); + return table_obj; +} + +jobject MiniCluster::CreateTable(const std::string &table, + const std::vector<std::string> &families) { + jstring table_name_str = env_->NewStringUTF(table.c_str()); + jobject table_name = + env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str); + jclass array_element_type = env_->FindClass("[B"); + jobjectArray family_array = env_->NewObjectArray(families.size(), array_element_type, nullptr); + int i = 0; + for (auto family : families) { + env_->SetObjectArrayElement(family_array, i++, StrToByteChar(family)); + } + jobject table_obj = + env_->CallObjectMethod(htu_, create_table_families_mid_, table_name, family_array); + return table_obj; +} + +jobject MiniCluster::CreateTable(const std::string &table, const std::string &family, + const std::vector<std::string> &keys) { + std::vector<std::string> families{}; + families.push_back(std::string{family}); + return CreateTable(table, families, keys); +} + +jobject MiniCluster::CreateTable(const std::string &table, const std::vector<std::string> &families, + const std::vector<std::string> &keys) { + jstring table_name_str = env_->NewStringUTF(table.c_str()); + jobject table_name = + env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str); + jclass array_element_type = env_->FindClass("[B"); + + int i = 0; + jobjectArray family_array = env_->NewObjectArray(families.size(), array_element_type, nullptr); + for (auto family : families) { + env_->SetObjectArrayElement(family_array, i++, StrToByteChar(family)); + } + + jobjectArray key_array = env_->NewObjectArray(keys.size(), array_element_type, nullptr); + + i = 0; + for (auto key : keys) { + env_->SetObjectArrayElement(key_array, i++, StrToByteChar(key)); + } + + jobject tbl = env_->CallObjectMethod(htu_, create_table_with_split_mid_, table_name, family_array, + key_array); + return tbl; +} + +jobject MiniCluster::StopRegionServer(int idx) { + env(); + return env_->CallObjectMethod(cluster_, stop_rs_mid_, (jint)idx); +} + +// returns the Configuration for the cluster +jobject MiniCluster::GetConf() { + env(); + return env_->CallObjectMethod(cluster_, get_conf_mid_); +} +// return the Admin instance for the local cluster +jobject MiniCluster::admin() { + env(); + jobject conn = env_->CallObjectMethod(htu(), get_conn_mid_); + jobject admin = env_->CallObjectMethod(conn, get_admin_mid_); + return admin; +} + +// moves region to server +void MiniCluster::MoveRegion(const std::string ®ion, const std::string &server) { + jobject admin_ = admin(); + env_->CallObjectMethod(admin_, move_mid_, StrToByteChar(region), StrToByteChar(server)); +} + +jobject MiniCluster::StartCluster(int num_region_servers) { + env(); + jmethodID mid = env_->GetMethodID(testing_util_class_, "startMiniCluster", + "(I)Lorg/apache/hadoop/hbase/MiniHBaseCluster;"); + if (mid == NULL) { + LOG(INFO) << "Couldn't find method startMiniCluster in the class HBaseTestingUtility"; + exit(-1); + } + cluster_ = env_->CallObjectMethod(htu(), mid, static_cast<jint>(num_region_servers)); + return cluster_; +} + +void MiniCluster::StopCluster() { + env(); + jmethodID mid = env_->GetMethodID(testing_util_class_, "shutdownMiniCluster", "()V"); + env_->CallVoidMethod(htu(), mid); + if (jvm_ != NULL) { + jvm_->DestroyJavaVM(); + jvm_ = NULL; + } +} + +const std::string MiniCluster::GetConfValue(const std::string &key) { + jobject conf = GetConf(); + jstring jval = + (jstring)env_->CallObjectMethod(conf, conf_get_mid_, env_->NewStringUTF(key.c_str())); + const char *val = env_->GetStringUTFChars(jval, 0); + return val; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/mini-cluster.h ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/test-util/mini-cluster.h b/hbase-native-client/src/hbase/test-util/mini-cluster.h new file mode 100644 index 0000000..6b4547c --- /dev/null +++ b/hbase-native-client/src/hbase/test-util/mini-cluster.h @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include <jni.h> +#include <string> +#include <vector> + +namespace hbase { + +class MiniCluster { + public: + virtual ~MiniCluster(); + jobject StartCluster(int32_t num_region_servers); + void StopCluster(); + jobject CreateTable(const std::string &table, const std::string &family); + jobject CreateTable(const std::string &table, const std::vector<std::string> &families); + jobject CreateTable(const std::string &table, const std::string &family, + const std::vector<std::string> &keys); + jobject CreateTable(const std::string &table, const std::vector<std::string> &families, + const std::vector<std::string> &keys); + jobject StopRegionServer(int idx); + + // moves region to server + void MoveRegion(const std::string ®ion, const std::string &server); + // returns the Configuration instance for the cluster + jobject GetConf(); + // returns the value for config key retrieved from cluster + const std::string GetConfValue(const std::string &key); + + private: + JNIEnv *env_; + jclass testing_util_class_; + jclass table_name_class_; + jclass put_class_; + jclass conf_class_; + jmethodID stop_rs_mid_; + jmethodID get_conf_mid_; + jmethodID set_conf_mid_; + jmethodID tbl_name_value_of_mid_; + jmethodID create_table_mid_; + jmethodID create_table_families_mid_; + jmethodID create_table_with_split_mid_; + jmethodID put_mid_; + jmethodID put_ctor_; + jmethodID add_col_mid_; + jmethodID create_conn_mid_; + jmethodID get_conn_mid_; + jmethodID get_table_mid_; + jmethodID conf_get_mid_; + jmethodID get_admin_mid_; + jmethodID move_mid_; + jmethodID str_ctor_mid_; + jobject htu_; + jobject cluster_; + pthread_mutex_t count_mutex_; + JavaVM *jvm_; + JNIEnv *CreateVM(JavaVM **jvm); + void Setup(); + jobject htu(); + JNIEnv *env(); + jbyteArray StrToByteChar(const std::string &str); + jobject admin(); +}; +} /*namespace hbase*/ http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/test-util.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/test-util/test-util.cc b/hbase-native-client/src/hbase/test-util/test-util.cc new file mode 100644 index 0000000..ebaf701 --- /dev/null +++ b/hbase-native-client/src/hbase/test-util/test-util.cc @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "hbase/test-util/test-util.h" +#include <string.h> + +#include <folly/Format.h> + +#include "hbase/client/zk-util.h" + +using hbase::TestUtil; +using folly::Random; + +std::string TestUtil::RandString(int len) { + // Create the whole string. + // Filling everything with z's + auto s = std::string(len, 'z'); + + // Now pick a bunch of random numbers + for (int i = 0; i < len; i++) { + // use Folly's random to get the numbers + // as I don't want to have to learn + // all the cpp rand invocation magic. + auto r = Random::rand32('a', 'z'); + // Cast that to ascii. + s[i] = static_cast<char>(r); + } + return s; +} + +TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {} + +TestUtil::~TestUtil() { + if (mini_) { + StopMiniCluster(); + mini_ = nullptr; + } +} + +void TestUtil::StartMiniCluster(int32_t num_region_servers) { + mini_ = std::make_unique<MiniCluster>(); + mini_->StartCluster(num_region_servers); + + conf()->Set(ZKUtil::kHBaseZookeeperQuorum_, mini_->GetConfValue(ZKUtil::kHBaseZookeeperQuorum_)); + conf()->Set(ZKUtil::kHBaseZookeeperClientPort_, + mini_->GetConfValue(ZKUtil::kHBaseZookeeperClientPort_)); +} + +void TestUtil::StopMiniCluster() { mini_->StopCluster(); } + +void TestUtil::CreateTable(const std::string &table, const std::string &family) { + mini_->CreateTable(table, family); +} + +void TestUtil::CreateTable(const std::string &table, const std::vector<std::string> &families) { + mini_->CreateTable(table, families); +} + +void TestUtil::CreateTable(const std::string &table, const std::string &family, + const std::vector<std::string> &keys) { + mini_->CreateTable(table, family, keys); +} + +void TestUtil::CreateTable(const std::string &table, const std::vector<std::string> &families, + const std::vector<std::string> &keys) { + mini_->CreateTable(table, families, keys); +} + +void TestUtil::MoveRegion(const std::string ®ion, const std::string &server) { + mini_->MoveRegion(region, server); +} + +void TestUtil::StartStandAloneInstance() { + auto p = temp_dir_.path().string(); + auto cmd = std::string{"bin/start-local-hbase.sh " + p}; + auto res_code = std::system(cmd.c_str()); + CHECK_EQ(res_code, 0); +} + +void TestUtil::StopStandAloneInstance() { + auto res_code = std::system("bin/stop-local-hbase.sh"); + CHECK_EQ(res_code, 0); +} + +void TestUtil::RunShellCmd(const std::string &command) { + auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command); + auto res_code = std::system(cmd_string.c_str()); + CHECK_EQ(res_code, 0); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/test-util.h ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/test-util/test-util.h b/hbase-native-client/src/hbase/test-util/test-util.h new file mode 100644 index 0000000..7c57c28 --- /dev/null +++ b/hbase-native-client/src/hbase/test-util/test-util.h @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#pragma once + +#include <folly/Random.h> +#include <folly/experimental/TestUtil.h> + +#include <cstdlib> +#include <memory> +#include <string> +#include <vector> +#include "hbase/client/configuration.h" +#include "hbase/test-util/mini-cluster.h" + +namespace hbase { +/** + * @brief Class to deal with a local instance cluster for testing. + */ +class TestUtil { + public: + TestUtil(); + + /** + * Destroying a TestUtil will spin down a cluster and remove the test dir. + */ + ~TestUtil(); + + /** + * Create a random string. This random string is all letters, as such it is + * very good for use as a directory name. + */ + static std::string RandString(int len = 32); + + /** + * Returns the configuration to talk to the local cluster + */ + std::shared_ptr<Configuration> conf() const { return conf_; } + + /** + * Starts mini hbase cluster with specified number of region servers + */ + void StartMiniCluster(int32_t num_region_servers); + + void StopMiniCluster(); + void CreateTable(const std::string &table, const std::string &family); + void CreateTable(const std::string &table, const std::vector<std::string> &families); + void CreateTable(const std::string &table, const std::string &family, + const std::vector<std::string> &keys); + void CreateTable(const std::string &table, const std::vector<std::string> &families, + const std::vector<std::string> &keys); + + void StartStandAloneInstance(); + void StopStandAloneInstance(); + void RunShellCmd(const std::string &); + void MoveRegion(const std::string ®ion, const std::string &server); + + private: + std::unique_ptr<MiniCluster> mini_; + folly::test::TemporaryDirectory temp_dir_; + std::shared_ptr<Configuration> conf_ = std::make_shared<Configuration>(); +}; +} // namespace hbase http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/utils/BUCK ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/utils/BUCK b/hbase-native-client/src/hbase/utils/BUCK new file mode 100644 index 0000000..ab55d8f --- /dev/null +++ b/hbase-native-client/src/hbase/utils/BUCK @@ -0,0 +1,57 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cxx_library( + name="utils", + exported_headers=[ + ], + srcs=["bytes-util.cc", "connection-util.cc", "user-util.cc"], + deps=[ + '//include/hbase/utils:utils', + '//third-party:folly', + ], + tests=[":user-util-test"], + linker_flags=['-L/usr/local/lib', '-lkrb5'], + exported_linker_flags=['-L/usr/local/lib', '-lkrb5'], + visibility=[ + 'PUBLIC', + ], + compiler_flags=['-Weffc++'],) +cxx_test( + name="bytes-util-test", + srcs=[ + "bytes-util-test.cc", + ], + deps=[ + ":utils", + ],) +cxx_test( + name="concurrent-map-test", + srcs=[ + "concurrent-map-test.cc", + ], + deps=[ + ":utils", + ],) +cxx_test( + name="user-util-test", + srcs=[ + "user-util-test.cc", + ], + deps=[ + ":utils", + ],) http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/utils/bytes-util-test.cc ---------------------------------------------------------------------- diff --git a/hbase-native-client/src/hbase/utils/bytes-util-test.cc b/hbase-native-client/src/hbase/utils/bytes-util-test.cc new file mode 100644 index 0000000..e601d2d --- /dev/null +++ b/hbase-native-client/src/hbase/utils/bytes-util-test.cc @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <folly/Logging.h> +#include <gtest/gtest.h> +#include <string> + +#include "hbase/utils/bytes-util.h" + +using hbase::BytesUtil; + +TEST(TestBytesUtil, TestToStringBinary) { + std::string empty{""}; + EXPECT_EQ(empty, BytesUtil::ToStringBinary(empty)); + + std::string foo_bar{"foo bar"}; + EXPECT_EQ(foo_bar, BytesUtil::ToStringBinary(foo_bar)); + + std::string foo_bar2{"foo bar_/!@#$%^&*(){}[]|1234567890"}; + EXPECT_EQ(foo_bar2, BytesUtil::ToStringBinary(foo_bar2)); + + char zero = 0; + EXPECT_EQ("\\x00", BytesUtil::ToStringBinary(std::string{zero})); + + char max = 255; + EXPECT_EQ("\\xFF", BytesUtil::ToStringBinary(std::string{max})); + + EXPECT_EQ("\\x00\\xFF", BytesUtil::ToStringBinary(std::string{zero} + std::string{max})); + + EXPECT_EQ("foo_\\x00\\xFF_bar", + BytesUtil::ToStringBinary("foo_" + std::string{zero} + std::string{max} + "_bar")); +} + +TEST(TestBytesUtil, TestToStringToInt64) { + int64_t num = 761235; + EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num))); + + num = -56125; + EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num))); + + num = 0; + EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num))); +} + +TEST(TestBytesUtil, TestCreateClosestRowAfter) { + std::string empty{""}; + EXPECT_EQ(BytesUtil::CreateClosestRowAfter(empty), std::string{'\0'}); + + std::string foo{"foo"}; + EXPECT_EQ(BytesUtil::CreateClosestRowAfter(foo), std::string{"foo"} + '\0'); + + EXPECT_EQ("f\\x00", BytesUtil::ToStringBinary(BytesUtil::CreateClosestRowAfter("f"))); +}