This is an automated email from the ASF dual-hosted git repository. zghao pushed a commit to branch HBASE-14850 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 302dbeb234bdde954d118313b4c6f3242a67b61b Author: tedyu <[email protected]> AuthorDate: Wed May 31 11:13:14 2017 -0700 HBASE-15903 Delete Object --- hbase-native-client/core/BUCK | 7 ++ hbase-native-client/core/client-test.cc | 71 +++++++++++++- hbase-native-client/core/delete-test.cc | 124 ++++++++++++++++++++++++ hbase-native-client/core/delete.cc | 133 ++++++++++++++++++++++++++ hbase-native-client/core/delete.h | 111 +++++++++++++++++++++ hbase-native-client/core/raw-async-table.cc | 15 +++ hbase-native-client/core/raw-async-table.h | 2 + hbase-native-client/core/request-converter.cc | 13 +++ hbase-native-client/core/request-converter.h | 3 + hbase-native-client/core/table.cc | 5 + hbase-native-client/core/table.h | 6 ++ 11 files changed, 489 insertions(+), 1 deletion(-) diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK index 9cea1f6..870c63f 100644 --- a/hbase-native-client/core/BUCK +++ b/hbase-native-client/core/BUCK @@ -38,6 +38,7 @@ cxx_library( "get.h", "mutation.h", "put.h", + "delete.h", "scan.h", "result.h", "request-converter.h", @@ -68,6 +69,7 @@ cxx_library( "get.cc", "mutation.cc", "put.cc", + "delete.cc", "scan.cc", "raw-async-table.cc", "result.cc", @@ -151,6 +153,11 @@ cxx_test( ], run_test_separately=True,) cxx_test( + name="delete-test", + srcs=["delete-test.cc",], + deps=[":core",], + run_test_separately=True,) +cxx_test( name="put-test", srcs=[ "put-test.cc", diff --git a/hbase-native-client/core/client-test.cc b/hbase-native-client/core/client-test.cc index 1c6ec4a..b58ca0a 100644 --- a/hbase-native-client/core/client-test.cc +++ b/hbase-native-client/core/client-test.cc @@ -22,6 +22,7 @@ #include "core/cell.h" #include "core/client.h" #include "core/configuration.h" +#include "core/delete.h" #include "core/get.h" #include "core/hbase-configuration-loader.h" #include "core/put.h" @@ -115,6 +116,75 @@ TEST_F(ClientTest, DefaultConfiguration) { client.Close(); } +TEST_F(ClientTest, PutGetDelete) { + // Using TestUtil to populate test data + std::string tableName = "t1"; + ClientTest::test_util->CreateTable(tableName, "d"); + + // Create TableName and Row to be fetched from HBase + auto tn = folly::to<hbase::pb::TableName>(tableName); + auto row = "test1"; + + // Create a client + hbase::Client client(*ClientTest::test_util->conf()); + + // Get connection to HBase Table + auto table = client.Table(tn); + ASSERT_TRUE(table) << "Unable to get connection to Table."; + + // Perform Puts + std::string valExtra = "value for extra"; + std::string valExt = "value for ext"; + table->Put(Put{row}.AddColumn("d", "1", "value1")); + // Put two values for column "extra" + table->Put(Put{row}.AddColumn("d", "extra", "1st val extra")); + table->Put(Put{row}.AddColumn("d", "extra", valExtra)); + table->Put(Put{row}.AddColumn("d", "ext", valExt)); + + // Perform the Get + hbase::Get get(row); + auto result = table->Get(get); + + // Test the values, should be same as in put executed on hbase shell + ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty."; + EXPECT_EQ("test1", result->Row()); + EXPECT_EQ("value1", *(result->Value("d", "1"))); + EXPECT_EQ(valExtra, *(result->Value("d", "extra"))); + auto cell = *(result->ColumnCells("d", "extra"))[0]; + auto tsExtra = cell.Timestamp(); + + // delete column "1" + table->Delete(hbase::Delete{row}.AddColumn("d", "1")); + result = table->Get(get); + ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty."; + ASSERT_TRUE(result->Value("d", "1") == nullptr) << "Column 1 should be gone"; + EXPECT_EQ(valExtra, *(result->Value("d", "extra"))); + + // delete cell from column "extra" with timestamp tsExtra + table->Delete(hbase::Delete{row}.AddColumn("d", "extra", tsExtra)); + result = table->Get(get); + ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty."; + ASSERT_TRUE(result->Value("d", "1") == nullptr) << "Column 1 should be gone"; + ASSERT_TRUE(result->Value("d", "extra") != nullptr) << "Column extra should have value"; + EXPECT_EQ(valExt, *(result->Value("d", "ext"))) << "Column ext should have value"; + + // delete all cells from "extra" column + table->Delete(hbase::Delete{row}.AddColumns("d", "extra")); + result = table->Get(get); + ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty."; + ASSERT_TRUE(result->Value("d", "1") == nullptr) << "Column 1 should be gone"; + ASSERT_TRUE(result->Value("d", "extra") == nullptr) << "Column extra should be gone"; + EXPECT_EQ(valExt, *(result->Value("d", "ext"))) << "Column ext should have value"; + + // Delete the row and verify that subsequent Get returns nothing + table->Delete(hbase::Delete{row}.AddFamily("d")); + result = table->Get(get); + ASSERT_TRUE(result->IsEmpty()) << "Result should be empty."; + + table->Close(); + client.Close(); +} + TEST_F(ClientTest, PutGet) { // Using TestUtil to populate test data ClientTest::test_util->CreateTable("t", "d"); @@ -147,7 +217,6 @@ TEST_F(ClientTest, PutGet) { table->Close(); client.Close(); } - TEST_F(ClientTest, GetForNonExistentTable) { // Create TableName and Row to be fetched from HBase auto tn = folly::to<hbase::pb::TableName>("t_not_exists"); diff --git a/hbase-native-client/core/delete-test.cc b/hbase-native-client/core/delete-test.cc new file mode 100644 index 0000000..19e844b --- /dev/null +++ b/hbase-native-client/core/delete-test.cc @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include <glog/logging.h> +#include <gtest/gtest.h> + +#include "core/mutation.h" +#include "core/delete.h" +#include "utils/time-util.h" + +using hbase::Delete; +using hbase::Cell; +using hbase::CellType; +using hbase::Mutation; +using hbase::TimeUtil; + +const constexpr int64_t Mutation::kLatestTimestamp; + +TEST(Delete, Row) { + Delete del{"foo"}; + EXPECT_EQ("foo", del.row()); +} + +TEST(Delete, Timestamp) { + Delete del{"row"}; + + // test default timestamp + EXPECT_EQ(Mutation::kLatestTimestamp, del.TimeStamp()); + + // set custom timestamp + auto ts = TimeUtil::ToMillis(TimeUtil::GetNowNanos()); + del.SetTimeStamp(ts); + EXPECT_EQ(ts, del.TimeStamp()); + + // Add a column with custom timestamp + del.AddColumn("f", "q"); + auto &cell = del.FamilyMap().at("f")[0]; + EXPECT_EQ(ts, cell->Timestamp()); +} + +TEST(Delete, HasFamilies) { + Delete del{"row"}; + + EXPECT_EQ(false, del.HasFamilies()); + + del.AddColumn("f", "q"); + EXPECT_EQ(true, del.HasFamilies()); +} + +TEST(Delete, Add) { + CellType cell_type = CellType::DELETE; + std::string row = "row"; + std::string family = "family"; + std::string column = "column"; + int64_t timestamp = std::numeric_limits<int64_t>::max(); + auto cell = std::make_unique<Cell>(row, family, column, timestamp, "", cell_type); + + // add first cell + Delete del{"row"}; + del.Add(std::move(cell)); + EXPECT_EQ(1, del.FamilyMap().size()); + EXPECT_EQ(1, del.FamilyMap().at(family).size()); + + // add a non-matching row + auto cell2 = std::make_unique<Cell>(row, family, column, timestamp, "", cell_type); + Delete del2{"foo"}; + ASSERT_THROW(del2.Add(std::move(cell2)), std::runtime_error); // rows don't match + + // add a second cell with same family + auto cell3 = std::make_unique<Cell>(row, family, "column-2", timestamp, "", cell_type); + del.Add(std::move(cell3)); + EXPECT_EQ(1, del.FamilyMap().size()); + EXPECT_EQ(2, del.FamilyMap().at(family).size()); + + // add a cell to a different family + auto cell4 = std::make_unique<Cell>(row, "family-2", "column-2", timestamp, "", cell_type); + del.Add(std::move(cell4)); + EXPECT_EQ(2, del.FamilyMap().size()); + EXPECT_EQ(1, del.FamilyMap().at("family-2").size()); +} + +TEST(Delete, AddColumn) { + std::string row = "row"; + std::string family = "family"; + std::string column = "column"; + + Delete del{"row"}; + del.AddColumn(family, column); + EXPECT_EQ(1, del.FamilyMap().size()); + EXPECT_EQ(1, del.FamilyMap().at(family).size()); + + // add a second cell with same family + del.AddColumn(family, "column-2"); + EXPECT_EQ(1, del.FamilyMap().size()); + EXPECT_EQ(2, del.FamilyMap().at(family).size()); + + // add a cell to a different family + del.AddColumn("family-2", column); + EXPECT_EQ(2, del.FamilyMap().size()); + EXPECT_EQ(1, del.FamilyMap().at("family-2").size()); + + // use the AddColumn overload + auto ts = TimeUtil::ToMillis(TimeUtil::GetNowNanos()); + del.AddColumn(family, column, ts); + EXPECT_EQ(2, del.FamilyMap().size()); + EXPECT_EQ(3, del.FamilyMap().at(family).size()); + auto &cell = del.FamilyMap().at(family)[2]; + EXPECT_EQ(ts, cell->Timestamp()); +} diff --git a/hbase-native-client/core/delete.cc b/hbase-native-client/core/delete.cc new file mode 100644 index 0000000..b44971f --- /dev/null +++ b/hbase-native-client/core/delete.cc @@ -0,0 +1,133 @@ + + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/delete.h" +#include <folly/Conv.h> +#include <algorithm> +#include <limits> +#include <stdexcept> +#include <utility> + +namespace hbase { + +/** + * @brief Add the specified column to this Delete operation. + * This is an expensive call in that on the server-side, it first does a + * get to find the latest versions timestamp. Then it adds a delete using + * the fetched cells timestamp. + * @param family family name + * @param qualifier column qualifier + */ +Delete& Delete::AddColumn(const std::string& family, const std::string& qualifier) { + return AddColumn(family, qualifier, timestamp_); +} + +/** + * @brief Add the specified column to this Delete operation. + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp + */ +Delete& Delete::AddColumn(const std::string& family, const std::string& qualifier, + int64_t timestamp) { + if (timestamp < 0) { + throw std::runtime_error("Timestamp cannot be negative. ts=" + + folly::to<std::string>(timestamp)); + } + + return Add(std::make_unique<Cell>(row_, family, qualifier, timestamp, "", + hbase::CellType::DELETE)); +} +/** + * Delete all versions of the specified column. + * @param family family name + * @param qualifier column qualifier + */ +Delete& Delete::AddColumns(const std::string& family, const std::string& qualifier) { + return AddColumns(family, qualifier, timestamp_); +} +/** + * Delete all versions of the specified column with a timestamp less than + * or equal to the specified timestamp. + * @param family family name + * @param qualifier column qualifier + * @param timestamp maximum version timestamp + */ +Delete& Delete::AddColumns(const std::string& family, const std::string& qualifier, + int64_t timestamp) { + if (timestamp < 0) { + throw std::runtime_error("Timestamp cannot be negative. ts=" + + folly::to<std::string>(timestamp)); + } + + return Add(std::make_unique<Cell>(row_, family, qualifier, timestamp, "", + hbase::CellType::DELETE_COLUMN)); +} +/** + * Delete all versions of all columns of the specified family. + * <p> + * Overrides previous calls to deleteColumn and deleteColumns for the + * specified family. + * @param family family name + */ +Delete& Delete::AddFamily(const std::string& family) { + return AddFamily(family, timestamp_); +} + +/** + * Delete all columns of the specified family with a timestamp less than + * or equal to the specified timestamp. + * <p> + * Overrides previous calls to deleteColumn and deleteColumns for the + * specified family. + * @param family family name + * @param timestamp maximum version timestamp + */ +Delete& Delete::AddFamily(const std::string& family, int64_t timestamp) { + const auto &it = family_map_.find(family); + if (family_map_.end() != it) { + it->second.clear(); + } else { + family_map_[family]; + } + return Add(std::make_unique<Cell>(row_, family, "", timestamp, "", + hbase::CellType::DELETE_FAMILY)); +} +/** + * Delete all columns of the specified family with a timestamp equal to + * the specified timestamp. + * @param family family name + * @param timestamp version timestamp + */ +Delete& Delete::AddFamilyVersion(const std::string& family, int64_t timestamp) { + return Add(std::make_unique<Cell>(row_, family, "", timestamp, "", + hbase::CellType::DELETE_FAMILY_VERSION)); +} +Delete& Delete::Add(std::unique_ptr<Cell> cell) { + if (cell->Row() != row_) { + throw std::runtime_error("The row in " + cell->DebugString() + + " doesn't match the original one " + row_); + } + + family_map_[cell->Family()].push_back(std::move(cell)); + return *this; +} +} // namespace hbase diff --git a/hbase-native-client/core/delete.h b/hbase-native-client/core/delete.h new file mode 100644 index 0000000..9ebb5a6 --- /dev/null +++ b/hbase-native-client/core/delete.h @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include <cstdint> +#include <map> +#include <memory> +#include <string> +#include <vector> +#include "core/cell.h" +#include "core/mutation.h" + +namespace hbase { + +class Delete : public Mutation { + public: + /** + * Constructors + */ + /* + * If no further operations are done, this will delete everything + * associated with the specified row (all versions of all columns in all + * families), with timestamp from current point in time to the past. + * Cells defining timestamp for a future point in time + * (timestamp > current time) will not be deleted. + */ + explicit Delete(const std::string& row) : Mutation(row) {} + + Delete(const std::string& row, int64_t timestamp) : Mutation(row, timestamp) {} + Delete(const Delete& cdelete) : Mutation(cdelete) {} + Delete& operator=(const Delete& cdelete) { + Mutation::operator=(cdelete); + return *this; + } + + ~Delete() = default; + + /** + * @brief Add the specified column to this Delete operation. + * @param family family name + * @param qualifier column qualifier + */ + Delete& AddColumn(const std::string& family, const std::string& qualifier); + + /** + * @brief Add the specified column to this Delete operation. + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp + */ + Delete& AddColumn(const std::string& family, const std::string& qualifier, int64_t timestamp); + + /** + * @brief Deletes all versions of the specified column + * @param family family name + * @param qualifier column qualifier + */ + Delete& AddColumns(const std::string& family, const std::string& qualifier); + /** + * @brief Deletes all versions of the specified column with a timestamp less than + * or equal to the specified timestamp + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp + */ + Delete& AddColumns(const std::string& family, const std::string& qualifier, int64_t timestamp); + /** + * @brief Add the specified family to this Delete operation. + * @param family family name + */ + Delete& AddFamily(const std::string& family); + + /** + * @brief Deletes all columns of the specified family with a timestamp less than + * or equal to the specified timestamp + * @param family family name + * @param timestamp version timestamp + */ + Delete& AddFamily(const std::string& family, int64_t timestamp); + /** + * @brief Deletes all columns of the specified family with a timestamp + * equal to the specified timestamp + * @param family family name + * @param timestamp version timestamp + */ + Delete& AddFamilyVersion(const std::string& family, int64_t timestamp); + /** + * Advanced use only. + * Add an existing delete marker to this Delete object. + */ + Delete& Add(std::unique_ptr<Cell> cell); +}; + +} // namespace hbase diff --git a/hbase-native-client/core/raw-async-table.cc b/hbase-native-client/core/raw-async-table.cc index 1a66a04..26967eb 100644 --- a/hbase-native-client/core/raw-async-table.cc +++ b/hbase-native-client/core/raw-async-table.cc @@ -94,6 +94,21 @@ folly::Future<folly::Unit> RawAsyncTable::Put(const hbase::Put& put) { return caller->Call().then([caller](const auto r) { return r; }); } +folly::Future<folly::Unit> RawAsyncTable::Delete(const hbase::Delete& del) { + auto caller = + CreateCallerBuilder<folly::Unit>(del.row(), connection_conf_->write_rpc_timeout()) + ->action([=, &del](std::shared_ptr<hbase::HBaseRpcController> controller, + std::shared_ptr<hbase::RegionLocation> loc, + std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<folly::Unit> { + return Call<hbase::Delete, hbase::Request, hbase::Response, folly::Unit>( + rpc_client, controller, loc, del, &hbase::RequestConverter::DeleteToMutateRequest, + [](const Response& r) -> folly::Unit { return folly::unit; }); + }) + ->Build(); + + return caller->Call().then([caller](const auto r) { return r; }); +} + folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> RawAsyncTable::Get( const std::vector<hbase::Get>& gets) { return this->Batch(gets); diff --git a/hbase-native-client/core/raw-async-table.h b/hbase-native-client/core/raw-async-table.h index 9d9ed91..d042d27 100644 --- a/hbase-native-client/core/raw-async-table.h +++ b/hbase-native-client/core/raw-async-table.h @@ -29,6 +29,7 @@ #include "core/async-rpc-retrying-caller-factory.h" #include "core/async-rpc-retrying-caller.h" #include "core/connection-configuration.h" +#include "core/delete.h" #include "core/get.h" #include "core/put.h" #include "core/result.h" @@ -51,6 +52,7 @@ class RawAsyncTable { folly::Future<std::shared_ptr<Result>> Get(const hbase::Get& get); + folly::Future<folly::Unit> Delete(const hbase::Delete& del); folly::Future<folly::Unit> Put(const hbase::Put& put); void Close() {} diff --git a/hbase-native-client/core/request-converter.cc b/hbase-native-client/core/request-converter.cc index eb293f5..85b4d6d 100644 --- a/hbase-native-client/core/request-converter.cc +++ b/hbase-native-client/core/request-converter.cc @@ -219,4 +219,17 @@ std::unique_ptr<Request> RequestConverter::ToMutateRequest(const Put &put, VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString(); return pb_req; } + +std::unique_ptr<Request> RequestConverter::DeleteToMutateRequest(const Delete &del, + const std::string ®ion_name) { + auto pb_req = Request::mutate(); + auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg()); + RequestConverter::SetRegion(region_name, pb_msg->mutable_region()); + + pb_msg->set_allocated_mutation( + ToMutation(MutationType::MutationProto_MutationType_DELETE, del, -1).release()); + + VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString(); + return pb_req; +} } /* namespace hbase */ diff --git a/hbase-native-client/core/request-converter.h b/hbase-native-client/core/request-converter.h index b604d18..6db1962 100644 --- a/hbase-native-client/core/request-converter.h +++ b/hbase-native-client/core/request-converter.h @@ -25,6 +25,7 @@ #include "connection/request.h" #include "core/action.h" #include "core/cell.h" +#include "core/delete.h" #include "core/get.h" #include "core/mutation.h" #include "core/put.h" @@ -66,6 +67,8 @@ class RequestConverter { static std::unique_ptr<Request> ToMultiRequest(const ActionsByRegion ®ion_requests); + static std::unique_ptr<Request> DeleteToMutateRequest(const Delete &del,const std::string ®ion_name); + static std::unique_ptr<Request> ToMutateRequest(const Put &put, const std::string ®ion_name); static std::unique_ptr<pb::MutationProto> ToMutation(const MutationType type, diff --git a/hbase-native-client/core/table.cc b/hbase-native-client/core/table.cc index a2f31d9..dd26adf 100644 --- a/hbase-native-client/core/table.cc +++ b/hbase-native-client/core/table.cc @@ -57,6 +57,11 @@ void Table::Put(const hbase::Put &put) { future.get(operation_timeout()); } +void Table::Delete(const hbase::Delete &del) { + auto future = async_table_->Delete(del); + future.get(operation_timeout()); +} + milliseconds Table::operation_timeout() const { return TimeUtil::ToMillis(async_connection_->connection_conf()->operation_timeout()); } diff --git a/hbase-native-client/core/table.h b/hbase-native-client/core/table.h index 0ee8312..c4217d3 100644 --- a/hbase-native-client/core/table.h +++ b/hbase-native-client/core/table.h @@ -61,6 +61,12 @@ class Table { */ void Put(const hbase::Put &put); + /** + * @brief - Deletes some data in the table. + * @param - del Delete object to perform HBase Delete operation. + */ + void Delete(const hbase::Delete &del); + // TODO: Batch Puts /**
