This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 443b24a [TABLE SERVICE] [COMMON] add BytesHashRouter
443b24a is described below
commit 443b24a630500cd248041de4583f09dbce90c398
Author: Sijie Guo <guosi...@gmail.com>
AuthorDate: Sun Sep 30 11:39:08 2018 -0700
[TABLE SERVICE] [COMMON] add BytesHashRouter
Descriptions of the changes in this PR:
*Motivation*
Currently the hash router takes `ByteBuf`. At some cases,
we need to compute the hash of bytes array directly.
*Changes*
Add `BytesHashRouter` to compute hash value of bytes array directly,
avoiding wrapping bytes into `ByteBuf`
Author: Sijie Guo <guosi...@gmail.com>
Author: Enrico Olivelli <eolive...@gmail.com>
Author: Sijie Guo <si...@apache.org>
Reviewers: Enrico Olivelli <eolive...@gmail.com>
This closes #1719 from sijie/bytes_hash_router
---
.../clients/impl/routing/RangeRouter.java | 8 +++--
.../org/apache/bookkeeper/common/hash/Murmur3.java | 2 +-
.../bookkeeper/common/router/BytesHashRouter.java | 41 ++++++++++++++++++++++
.../bookkeeper/common/router/HashRouterTest.java | 13 +++++--
4 files changed, 59 insertions(+), 5 deletions(-)
diff --git
a/stream/clients/java/base/src/main/java/org/apache/bookkeeper/clients/impl/routing/RangeRouter.java
b/stream/clients/java/base/src/main/java/org/apache/bookkeeper/clients/impl/routing/RangeRouter.java
index 18e1e56..b1b5bd5 100644
---
a/stream/clients/java/base/src/main/java/org/apache/bookkeeper/clients/impl/routing/RangeRouter.java
+++
b/stream/clients/java/base/src/main/java/org/apache/bookkeeper/clients/impl/routing/RangeRouter.java
@@ -58,7 +58,11 @@ public class RangeRouter<K> {
* @return the range to write.
* @throws IllegalStateException if ranges is empty.
*/
- public Long getRange(@Nullable K key) {
+ public long getRange(@Nullable K key) {
+ return getRangeProperties(key).getRangeId();
+ }
+
+ public RangeProperties getRangeProperties(@Nullable K key) {
long routingKey;
if (null != key) {
routingKey = keyRouter.getRoutingKey(key);
@@ -79,7 +83,7 @@ public class RangeRouter<K> {
checkState(null != rs, "No range is available");
Map.Entry<Long, RangeProperties> ceilingEntry =
rs.getRanges().floorEntry(routingKey);
- return ceilingEntry.getValue().getRangeId();
+ return ceilingEntry.getValue();
}
public HashStreamRanges getRanges() {
diff --git
a/stream/common/src/main/java/org/apache/bookkeeper/common/hash/Murmur3.java
b/stream/common/src/main/java/org/apache/bookkeeper/common/hash/Murmur3.java
index 47852ba..b61a7c5 100644
--- a/stream/common/src/main/java/org/apache/bookkeeper/common/hash/Murmur3.java
+++ b/stream/common/src/main/java/org/apache/bookkeeper/common/hash/Murmur3.java
@@ -208,7 +208,7 @@ public class Murmur3 {
* @param seed - seed. (default is 0)
* @return - hashcode (2 longs)
*/
- public static long[] hash128(byte[] data, int offset, int length, int
seed) {
+ public static long[] hash128(byte[] data, int offset, int length, long
seed) {
long h1 = seed;
long h2 = seed;
final int nblocks = length >> 4;
diff --git
a/stream/common/src/main/java/org/apache/bookkeeper/common/router/BytesHashRouter.java
b/stream/common/src/main/java/org/apache/bookkeeper/common/router/BytesHashRouter.java
new file mode 100644
index 0000000..e8f61d9
--- /dev/null
+++
b/stream/common/src/main/java/org/apache/bookkeeper/common/router/BytesHashRouter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.bookkeeper.common.router;
+
+import org.apache.bookkeeper.common.hash.Murmur3;
+
+/**
+ * Hash router that computes the hash value of a byte array.
+ */
+public class BytesHashRouter implements HashRouter<byte[]> {
+
+ public static BytesHashRouter of() {
+ return ROUTER;
+ }
+
+ private static final BytesHashRouter ROUTER = new BytesHashRouter();
+
+ private BytesHashRouter() {}
+
+ @Override
+ public Long getRoutingKey(byte[] key) {
+ return Murmur3.hash128(key, 0, key.length,
AbstractHashRouter.HASH_SEED)[0];
+ }
+}
diff --git
a/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java
b/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java
index 5d4ed79..8fad535 100644
---
a/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java
+++
b/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java
@@ -15,6 +15,7 @@
package org.apache.bookkeeper.common.router;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import com.google.common.hash.Hashing;
@@ -32,33 +33,41 @@ public class HashRouterTest {
@Test
public void testByteBufHashRouter() {
- ByteBuf key = Unpooled.wrappedBuffer("foo".getBytes(UTF_8));
+ byte[] keyBytes = "foo".getBytes(UTF_8);
+ ByteBuf key = Unpooled.wrappedBuffer(keyBytes);
// murmur3 - 32 bits
int hash32 = Murmur3.hash32(
key, key.readerIndex(), key.readableBytes(), (int)
AbstractHashRouter.HASH_SEED);
+ int bytesHash32 = Murmur3.hash32(keyBytes, 0, keyBytes.length, (int)
AbstractHashRouter.HASH_SEED);
int guavaHash32 = Hashing.murmur3_32((int)
AbstractHashRouter.HASH_SEED)
.newHasher()
.putString("foo", UTF_8)
.hash()
.asInt();
+ assertEquals(hash32, bytesHash32);
assertEquals(hash32, guavaHash32);
// murmur3 - 128 bits
long[] hash128 = Murmur3.hash128(
key, key.readerIndex(), key.readableBytes(),
AbstractHashRouter.HASH_SEED);
- log.info("hash128: {}", hash128);
+ long[] bytesHash128 = Murmur3.hash128(keyBytes, 0, keyBytes.length,
AbstractHashRouter.HASH_SEED);
+ log.info("hash128: {}, bytes hash128: {}", hash128, bytesHash128);
long guavaHash128 = Hashing.murmur3_128((int)
AbstractHashRouter.HASH_SEED)
.newHasher()
.putString("foo", UTF_8)
.hash()
.asLong();
+ assertArrayEquals(hash128, bytesHash128);
assertEquals(hash128[0], guavaHash128);
ByteBufHashRouter router = ByteBufHashRouter.of();
long routingHash = router.getRoutingKey(key).longValue();
log.info("Routing hash = {}", routingHash);
assertEquals(hash128[0], routingHash);
+ BytesHashRouter bytesRouter = BytesHashRouter.of();
+ long bytesRoutingHash =
bytesRouter.getRoutingKey(keyBytes).longValue();
+ assertEquals(hash128[0], bytesRoutingHash);
}
}
