sijie closed pull request #856: ISSUE #590 (@bug W-4556980@) Reduce excessive
CPU usage on client side: add crc3?
URL: https://github.com/apache/bookkeeper/pull/856
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bookkeeper-proto/src/main/proto/DataFormats.proto
b/bookkeeper-proto/src/main/proto/DataFormats.proto
index cdade9563..5a5d65f79 100644
--- a/bookkeeper-proto/src/main/proto/DataFormats.proto
+++ b/bookkeeper-proto/src/main/proto/DataFormats.proto
@@ -45,6 +45,7 @@ message LedgerMetadataFormat {
enum DigestType {
CRC32 = 1;
HMAC = 2;
+ CRC32C = 3;
}
optional DigestType digestType = 7;
optional bytes password = 8;
diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml
index 8c057e6de..45e88eaf2 100644
--- a/bookkeeper-server/pom.xml
+++ b/bookkeeper-server/pom.xml
@@ -182,6 +182,11 @@
<artifactId>http-server</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.pulsar</groupId>
+ <artifactId>pulsar-checksum</artifactId>
+ <version>1.20.0-incubating</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
index 69e113993..e0959f45b 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeper.java
@@ -634,10 +634,19 @@ boolean isReorderReadSequence() {
* report fake bytes with a mathching MAC unless it knows the password
*/
public enum DigestType {
- MAC, CRC32;
+ MAC, CRC32, CRC32C;
public static DigestType
fromApiDigestType(org.apache.bookkeeper.client.api.DigestType digestType) {
- return digestType ==
org.apache.bookkeeper.client.api.DigestType.MAC ? MAC : CRC32;
+ switch (digestType) {
+ case MAC:
+ return DigestType.MAC;
+ case CRC32:
+ return DigestType.CRC32;
+ case CRC32C:
+ return DigestType.CRC32C;
+ default:
+ throw new IllegalArgumentException("Unable to convert
digest type " + digestType);
+ }
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32CDigestManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32CDigestManager.java
new file mode 100644
index 000000000..1923220d7
--- /dev/null
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/CRC32CDigestManager.java
@@ -0,0 +1,65 @@
+package org.apache.bookkeeper.client;
+
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+import com.scurrilous.circe.crc.Sse42Crc32C;
+import io.netty.buffer.ByteBuf;
+import org.apache.commons.lang3.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.pulsar.checksum.utils.Crc32cChecksum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class CRC32CDigestManager extends DigestManager {
+ static final Logger LOG =
LoggerFactory.getLogger(CRC32CDigestManager.class);
+
+ private final ThreadLocal<MutableInt> currentCrc = ThreadLocal
+ .withInitial(() -> new MutableInt(0));
+ private final ThreadLocal<MutableBoolean> isNewCrc = ThreadLocal
+ .withInitial(() -> new MutableBoolean(true));
+
+ public CRC32CDigestManager(long ledgerId) {
+ super(ledgerId);
+ if (!Sse42Crc32C.isSupported()) {
+ LOG.error("Sse42Crc32C is not supported, will use less slower
CRC32C implementation.");
+ }
+ }
+
+ @Override
+ int getMacCodeLength() {
+ return 4;
+ }
+
+ @Override
+ void populateValueAndReset(ByteBuf buf) {
+ buf.writeInt(currentCrc.get().intValue());
+ isNewCrc.get().setTrue();
+ }
+
+ @Override
+ void update(ByteBuf data) {
+ if (isNewCrc.get().isTrue()) {
+ isNewCrc.get().setFalse();
+ currentCrc.get().setValue(Crc32cChecksum.computeChecksum(data));
+ } else {
+ final int lastCrc = currentCrc.get().intValue();
+ currentCrc.get().setValue(Crc32cChecksum.resumeChecksum(lastCrc,
data));
+ }
+ }
+}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
index a5c734c51..21c9ae7a4 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DigestManager.java
@@ -69,6 +69,8 @@ static DigestManager instantiate(long ledgerId, byte[]
passwd, DigestType digest
return new MacDigestManager(ledgerId, passwd);
case CRC32:
return new CRC32DigestManager(ledgerId);
+ case CRC32C:
+ return new CRC32CDigestManager(ledgerId);
default:
throw new GeneralSecurityException("Unknown checksum type: " +
digestType);
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
index 62ca2833c..d37e7fc80 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
@@ -119,7 +119,7 @@ public LedgerMetadata(int ensembleSize,
this.metadataFormatVersion = CURRENT_METADATA_FORMAT_VERSION;
this.digestType = digestType.equals(BookKeeper.DigestType.MAC)
- ? LedgerMetadataFormat.DigestType.HMAC :
LedgerMetadataFormat.DigestType.CRC32;
+ ? LedgerMetadataFormat.DigestType.HMAC :
LedgerMetadataFormat.DigestType.valueOf(digestType.toString());
this.password = Arrays.copyOf(password, password.length);
this.hasPassword = true;
if (customMetadata != null) {
@@ -230,10 +230,15 @@ boolean hasPassword() {
@Override
public DigestType getDigestType() {
- if (digestType.equals(LedgerMetadataFormat.DigestType.HMAC)) {
- return DigestType.MAC;
- } else {
- return DigestType.CRC32;
+ switch (digestType) {
+ case HMAC:
+ return DigestType.MAC;
+ case CRC32:
+ return DigestType.CRC32;
+ case CRC32C:
+ return DigestType.CRC32C;
+ default:
+ throw new IllegalArgumentException("Unable to convert digest
type " + digestType);
}
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
index d7db214b0..728a24691 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/api/DigestType.java
@@ -39,5 +39,9 @@
/**
* Entries are verified by applied MAC algorithm.
*/
- MAC
+ MAC,
+ /**
+ * Entries are verified by applied CRC32C algorithm.
+ */
+ CRC32C
}
diff --git
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
index 3e069ca00..2100451ab 100644
---
a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
+++
b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieWriteLedgersWithDifferentDigestsTest.java
@@ -54,7 +54,7 @@
.getLogger(BookieWriteLedgersWithDifferentDigestsTest.class);
byte[] ledgerPassword = "aaa".getBytes();
- LedgerHandle lh, lh2;
+ LedgerHandle lh;
Enumeration<LedgerEntry> ls;
// test related variables
@@ -65,6 +65,7 @@
ArrayList<byte[]> entries2; // generated entries
private final DigestType digestType;
+ private final DigestType otherDigestType;
private static class SyncObj {
volatile int counter;
@@ -77,7 +78,7 @@ public SyncObj() {
@Parameterized.Parameters
public static Collection<Object[]> configs() {
- return Arrays.asList(new Object[][] { {DigestType.MAC },
{DigestType.CRC32}});
+ return Arrays.asList(new Object[][] { {DigestType.MAC },
{DigestType.CRC32}, {DigestType.CRC32C} });
}
@Override
@@ -93,21 +94,22 @@ public void setUp() throws Exception {
public BookieWriteLedgersWithDifferentDigestsTest(DigestType digestType) {
super(3);
this.digestType = digestType;
+ this.otherDigestType = digestType == DigestType.CRC32 ? DigestType.MAC
: DigestType.CRC32;
String ledgerManagerFactory =
"org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory";
// set ledger manager
baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
baseClientConf.setLedgerManagerFactoryClassName(ledgerManagerFactory);
}
- @Test
+ @Test()
public void testLedgersWithDifferentDigestTypesNoAutodetection() throws
Exception {
bkc.conf.setEnableDigestTypeAutodetection(false);
// Create ledgers
- lh = bkc.createLedgerAdv(3, 2, 2, DigestType.MAC, ledgerPassword);
+ lh = bkc.createLedgerAdv(3, 2, 2, digestType, ledgerPassword);
final long id = lh.ledgerId;
- LOG.info("Ledger ID-1: " + lh.getId());
+ LOG.info("Ledger ID: {}, digestType: {}", lh.getId(), digestType);
SyncObj syncObj1 = new SyncObj();
for (int i = numEntriesToWrite - 1; i >= 0; i--) {
ByteBuffer entry = ByteBuffer.allocate(4);
@@ -125,7 +127,7 @@ public void
testLedgersWithDifferentDigestTypesNoAutodetection() throws Exceptio
lh.close();
try {
- bkc.openLedgerNoRecovery(id, DigestType.CRC32,
ledgerPassword).close();
+ bkc.openLedgerNoRecovery(id, otherDigestType,
ledgerPassword).close();
fail("digest mismatch error is expected");
} catch (BKException bke) {
// expected
@@ -136,44 +138,32 @@ public void
testLedgersWithDifferentDigestTypesNoAutodetection() throws Exceptio
public void testLedgersWithDifferentDigestTypesWithAutodetection() throws
Exception {
bkc.conf.setEnableDigestTypeAutodetection(true);
// Create ledgers
- lh = bkc.createLedgerAdv(3, 2, 2, DigestType.MAC, ledgerPassword);
- lh2 = bkc.createLedgerAdv(3, 2, 2, DigestType.CRC32, ledgerPassword);
+ lh = bkc.createLedgerAdv(3, 2, 2, digestType, ledgerPassword);
final long id = lh.ledgerId;
- final long id2 = lh2.ledgerId;
LOG.info("Ledger ID-1: " + lh.getId());
- LOG.info("Ledger ID-2: " + lh2.getId());
SyncObj syncObj1 = new SyncObj();
- SyncObj syncObj2 = new SyncObj();
for (int i = numEntriesToWrite - 1; i >= 0; i--) {
ByteBuffer entry = ByteBuffer.allocate(4);
entry.putInt(rng.nextInt(maxInt));
entry.position(0);
entries1.add(0, entry.array());
- entries2.add(0, entry.array());
lh.asyncAddEntry(i, entry.array(), 0, entry.capacity(), this,
syncObj1);
- lh2.asyncAddEntry(i, entry.array(), 0, entry.capacity(), this,
syncObj2);
}
// Wait for all entries to be acknowledged
waitForEntriesAddition(syncObj1, numEntriesToWrite);
- waitForEntriesAddition(syncObj2, numEntriesToWrite);
// Reads here work ok because ledger uses digest type set during create
readEntries(lh, entries1);
- readEntries(lh2, entries2);
lh.close();
- lh2.close();
// open here would fail if provided digest type is used
// it passes because ledger just uses digest type from its
metadata/autodetects it
- lh = bkc.openLedgerNoRecovery(id, DigestType.CRC32, ledgerPassword);
- lh2 = bkc.openLedgerNoRecovery(id2, DigestType.MAC, ledgerPassword);
+ lh = bkc.openLedgerNoRecovery(id, otherDigestType, ledgerPassword);
readEntries(lh, entries1);
- readEntries(lh2, entries2);
lh.close();
- lh2.close();
}
private void waitForEntriesAddition(SyncObj syncObj, int
numEntriesToWrite) throws InterruptedException {
diff --git a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
index 2cc5ce766..866c946d6 100644
--- a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
+++ b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml
@@ -28,6 +28,10 @@
<!-- generated code, we can't be held responsible for findbugs in it //-->
<Class name="~org\.apache\.bookkeeper\.tests\.generated.*" />
</Match>
+ <Match>
+ <!-- generated code -->
+ <Class name="~org\.apache\.bookkeeper\.client\.generated.*"/>
+ </Match>
<Match>
<!-- it is safe to store external bytes reference here. since we are using
bytes from a slab. //-->
diff --git a/microbenchmarks/pom.xml b/microbenchmarks/pom.xml
new file mode 100644
index 000000000..4bdc7897e
--- /dev/null
+++ b/microbenchmarks/pom.xml
@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper</artifactId>
+ <version>4.7.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>microbenchmarks</artifactId>
+ <name>Apache BookKeeper :: microbenchmarks</name>
+ <url>http://maven.apache.org</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <jmh.version>1.19</jmh.version>
+ <javac.target>1.8</javac.target>
+ <uberjar.name>benchmarks</uberjar.name>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>${jmh.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>${jmh.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>${netty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bookkeeper</groupId>
+ <artifactId>bookkeeper-server</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>compile</scope>
+ <type>jar</type>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <compilerVersion>${javac.target}</compilerVersion>
+ <source>${javac.target}</source>
+ <target>${javac.target}</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.2</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>${uberjar.name}</finalName>
+ <transformers>
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.openjdk.jmh.Main</mainClass>
+ </transformer>
+ </transformers>
+ <filters>
+ <filter>
+ <!-- Shading signed JARs will fail without
+ this.
http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar
-->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>2.5</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.8.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-install-plugin</artifactId>
+ <version>2.5.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.9.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.3</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>2.2.1</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.17</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+</project>
diff --git a/microbenchmarks/results.txt b/microbenchmarks/results.txt
new file mode 100644
index 000000000..e9438aea1
--- /dev/null
+++ b/microbenchmarks/results.txt
@@ -0,0 +1,89 @@
+# on macbook pro, for 64KB entry
+# Run complete. Total time: 00:21:26
+
+Benchmark
(bufferType) (digest) Mode Cnt Score Error Units
+DigestTypeBenchmark.digestManager
ARRAY_BACKED MAC thrpt 5 8.586 ? 0.556 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate
ARRAY_BACKED MAC thrpt 5 0.692 ? 0.045 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm
ARRAY_BACKED MAC thrpt 5 88.010 ? 0.002 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space
ARRAY_BACKED MAC thrpt 5 1.615 ? 8.515 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm
ARRAY_BACKED MAC thrpt 5 203.670 ? 1073.904 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Survivor_Space
ARRAY_BACKED MAC thrpt 5 0.096 ? 0.826 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Survivor_Space.norm
ARRAY_BACKED MAC thrpt 5 12.176 ? 104.841 B/op
+DigestTypeBenchmark.digestManager:?gc.count
ARRAY_BACKED MAC thrpt 5 2.000 counts
+DigestTypeBenchmark.digestManager:?gc.time
ARRAY_BACKED MAC thrpt 5 16.000 ms
+DigestTypeBenchmark.digestManager:?stack
ARRAY_BACKED MAC thrpt NaN ---
+DigestTypeBenchmark.digestManager
ARRAY_BACKED CRC32 thrpt 5 445.869 ? 8.161 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate
ARRAY_BACKED CRC32 thrpt 5 ? 10?? MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm
ARRAY_BACKED CRC32 thrpt 5 ? 10?? B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space
ARRAY_BACKED CRC32 thrpt 5 0.815 ? 7.022 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm
ARRAY_BACKED CRC32 thrpt 5 1.997 ? 17.191 B/op
+DigestTypeBenchmark.digestManager:?gc.count
ARRAY_BACKED CRC32 thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:?gc.time
ARRAY_BACKED CRC32 thrpt 5 8.000 ms
+DigestTypeBenchmark.digestManager:?stack
ARRAY_BACKED CRC32 thrpt NaN ---
+DigestTypeBenchmark.digestManager
ARRAY_BACKED CRC32_C thrpt 5 832.499 ? 56.604 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate
ARRAY_BACKED CRC32_C thrpt 5 ? 10?? MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm
ARRAY_BACKED CRC32_C thrpt 5 ? 10?? B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space
ARRAY_BACKED CRC32_C thrpt 5 0.816 ? 7.023 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm
ARRAY_BACKED CRC32_C thrpt 5 1.064 ? 9.158 B/op
+DigestTypeBenchmark.digestManager:?gc.count
ARRAY_BACKED CRC32_C thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:?gc.time
ARRAY_BACKED CRC32_C thrpt 5 8.000 ms
+DigestTypeBenchmark.digestManager:?stack
ARRAY_BACKED CRC32_C thrpt NaN ---
+DigestTypeBenchmark.digestManager
NOT_ARRAY_BACKED MAC thrpt 5 8.367 ? 0.045 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate
NOT_ARRAY_BACKED MAC thrpt 5 502.288 ? 2.868 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm
NOT_ARRAY_BACKED MAC thrpt 5 65593.694 ? 12.966 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space
NOT_ARRAY_BACKED MAC thrpt 5 539.102 ? 0.215 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm
NOT_ARRAY_BACKED MAC thrpt 5 70401.331 ? 403.899 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen
NOT_ARRAY_BACKED MAC thrpt 5 0.006 ? 0.015 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen.norm
NOT_ARRAY_BACKED MAC thrpt 5 0.784 ? 1.962 B/op
+DigestTypeBenchmark.digestManager:?gc.count
NOT_ARRAY_BACKED MAC thrpt 5 55.000 counts
+DigestTypeBenchmark.digestManager:?gc.time
NOT_ARRAY_BACKED MAC thrpt 5 200.000 ms
+DigestTypeBenchmark.digestManager:?stack
NOT_ARRAY_BACKED MAC thrpt NaN ---
+DigestTypeBenchmark.digestManager
NOT_ARRAY_BACKED CRC32 thrpt 5 119.933 ? 7.129 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate
NOT_ARRAY_BACKED CRC32 thrpt 5 7206.443 ? 430.557 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm
NOT_ARRAY_BACKED CRC32 thrpt 5 65648.001 ? 0.001 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space
NOT_ARRAY_BACKED CRC32 thrpt 5 7684.923 ? 509.880 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm
NOT_ARRAY_BACKED CRC32 thrpt 5 70005.330 ? 532.187 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen
NOT_ARRAY_BACKED CRC32 thrpt 5 0.158 ? 0.078 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen.norm
NOT_ARRAY_BACKED CRC32 thrpt 5 1.439 ? 0.769 B/op
+DigestTypeBenchmark.digestManager:?gc.count
NOT_ARRAY_BACKED CRC32 thrpt 5 784.000 counts
+DigestTypeBenchmark.digestManager:?gc.time
NOT_ARRAY_BACKED CRC32 thrpt 5 2949.000 ms
+DigestTypeBenchmark.digestManager:?stack
NOT_ARRAY_BACKED CRC32 thrpt NaN ---
+DigestTypeBenchmark.digestManager
NOT_ARRAY_BACKED CRC32_C thrpt 5 130.444 ? 3.441 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate
NOT_ARRAY_BACKED CRC32_C thrpt 5 7824.727 ? 212.028 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm
NOT_ARRAY_BACKED CRC32_C thrpt 5 65552.001 ? 0.001 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space
NOT_ARRAY_BACKED CRC32_C thrpt 5 8349.266 ? 177.504 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm
NOT_ARRAY_BACKED CRC32_C thrpt 5 69947.075 ? 707.700 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen
NOT_ARRAY_BACKED CRC32_C thrpt 5 0.129 ? 0.083 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen.norm
NOT_ARRAY_BACKED CRC32_C thrpt 5 1.079 ? 0.691 B/op
+DigestTypeBenchmark.digestManager:?gc.count
NOT_ARRAY_BACKED CRC32_C thrpt 5 852.000 counts
+DigestTypeBenchmark.digestManager:?gc.time
NOT_ARRAY_BACKED CRC32_C thrpt 5 3229.000 ms
+DigestTypeBenchmark.digestManager:?stack
NOT_ARRAY_BACKED CRC32_C thrpt NaN ---
+DigestTypeBenchmark.digestManager
BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 8.642 ? 0.132 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate
BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 1.329 ? 0.021 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm
BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 168.010 ? 0.002 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space
BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 0.799 ? 6.880 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm
BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 101.377 ? 872.889 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Survivor_Space
BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 0.096 ? 0.826 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Survivor_Space.norm
BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 12.165 ? 104.747 B/op
+DigestTypeBenchmark.digestManager:?gc.count
BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:?gc.time
BYTE_BUF_DEFAULT_ALLOC MAC thrpt 5 8.000 ms
+DigestTypeBenchmark.digestManager:?stack
BYTE_BUF_DEFAULT_ALLOC MAC thrpt NaN ---
+DigestTypeBenchmark.digestManager
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 445.372 ? 14.813 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 26.083 ? 0.883 MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 64.000 ? 0.001 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 24.437 ? 93.861 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 60.215 ? 231.193 B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 0.002 ? 0.017 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Old_Gen.norm
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 0.005 ? 0.042 B/op
+DigestTypeBenchmark.digestManager:?gc.count
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 3.000 counts
+DigestTypeBenchmark.digestManager:?gc.time
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt 5 18.000 ms
+DigestTypeBenchmark.digestManager:?stack
BYTE_BUF_DEFAULT_ALLOC CRC32 thrpt NaN ---
+DigestTypeBenchmark.digestManager
BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 868.398 ? 9.782 ops/ms
+DigestTypeBenchmark.digestManager:?gc.alloc.rate
BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 ? 10?? MB/sec
+DigestTypeBenchmark.digestManager:?gc.alloc.rate.norm
BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 ? 10?? B/op
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space
BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 0.815 ? 7.018 MB/sec
+DigestTypeBenchmark.digestManager:?gc.churn.G1_Eden_Space.norm
BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 1.023 ? 8.808 B/op
+DigestTypeBenchmark.digestManager:?gc.count
BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 1.000 counts
+DigestTypeBenchmark.digestManager:?gc.time
BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt 5 7.000 ms
+DigestTypeBenchmark.digestManager:?stack
BYTE_BUF_DEFAULT_ALLOC CRC32_C thrpt NaN ---
+
diff --git a/microbenchmarks/run.sh b/microbenchmarks/run.sh
new file mode 100755
index 000000000..0d59321ee
--- /dev/null
+++ b/microbenchmarks/run.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+java -Xms1G -Xmx1G -Djdk.nio.maxCachedBufferSize=0
-Djava.net.preferIPv4Stack=true -Duser.timezone=UTC -XX:-MaxFDLimit
-XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ResizeTLAB -XX:-ResizePLAB
-XX:MetaspaceSize=128m -XX:MinMetaspaceFreeRatio=50
-XX:MaxMetaspaceFreeRatio=80 -XX:+ParallelRefProcEnabled
-XX:StackShadowPages=20 -XX:+UseCompressedOops -XX:+DisableExplicitGC
-XX:StringTableSize=1000003 -XX:InitiatingHeapOccupancyPercent=40 -jar
target/benchmarks.jar -prof gc -prof stack:lines=5;time=1;top=3
+
diff --git
a/microbenchmarks/src/main/java/org/apache/bookkeeper/client/DigestTypeBenchmark.java
b/microbenchmarks/src/main/java/org/apache/bookkeeper/client/DigestTypeBenchmark.java
new file mode 100644
index 000000000..bda4da57d
--- /dev/null
+++
b/microbenchmarks/src/main/java/org/apache/bookkeeper/client/DigestTypeBenchmark.java
@@ -0,0 +1,173 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.bookkeeper.client;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import io.netty.buffer.ByteBufAllocator;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.util.DoubleByteBuf;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
+/**
+ * Microbenchmarks for different digest type
+ * getting started:
+ * 1. http://tutorials.jenkov.com/java-performance/jmh.html
+ * 2.
http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/
+ * 3. google
+ *
+ * To run:
+ * build project from command line.
+ * execute ./run.sh
+ */
+public class DigestTypeBenchmark {
+
+ public enum BufferType {
+ ARRAY_BACKED,
+ NOT_ARRAY_BACKED,
+ BYTE_BUF_DEFAULT_ALLOC
+ }
+
+ public enum Digest {
+ MAC,
+ CRC32,
+ CRC32_C,
+ }
+
+ static byte[] randomBytes(int sz) {
+ byte[] b = new byte[sz];
+ ThreadLocalRandom.current().nextBytes(b);
+ return b;
+ }
+
+ @State(Scope.Thread)
+ public static class MyState {
+
+ @Param
+ public BufferType bufferType;
+ @Param
+ public Digest digest;
+ @Param({"1024", "4086", "8192", "16384", "65536"})
+ public int entrySize;
+
+ private DigestManager crc32;
+ private DigestManager crc32c;
+ private DigestManager mac;
+
+ private ByteBuf arrayBackedBuffer;
+ private ByteBuf notArrayBackedBuffer;
+ private ByteBuf byteBufDefaultAlloc;
+
+ public ByteBuf digestBuf;
+
+ @Setup(Level.Trial)
+ public void doSetup() throws Exception {
+ final byte[] password = "password".getBytes("UTF-8");
+ crc32 =
DigestManager.instantiate(ThreadLocalRandom.current().nextLong(0,
Long.MAX_VALUE),
+ password, DigestType.CRC32);
+
+ crc32c =
DigestManager.instantiate(ThreadLocalRandom.current().nextLong(0,
Long.MAX_VALUE),
+ password, DigestType.CRC32C);
+
+ mac =
DigestManager.instantiate(ThreadLocalRandom.current().nextLong(0,
Long.MAX_VALUE),
+ password, DigestType.MAC);
+
+ digestBuf =
Unpooled.buffer(getDigestManager(digest).getMacCodeLength());
+
+ arrayBackedBuffer = Unpooled.wrappedBuffer(randomBytes(entrySize));
+
+ final int headerSize = 32 +
getDigestManager(digest).getMacCodeLength();
+ notArrayBackedBuffer =
DoubleByteBuf.get(Unpooled.wrappedBuffer(randomBytes(headerSize)),
+ Unpooled.wrappedBuffer((randomBytes(entrySize -
headerSize))));
+
+ byteBufDefaultAlloc = ByteBufAllocator.DEFAULT.buffer(entrySize,
entrySize);
+ byteBufDefaultAlloc.writeBytes(randomBytes(entrySize));
+
+ if (!arrayBackedBuffer.hasArray() ||
notArrayBackedBuffer.hasArray()) {
+ throw new IllegalStateException("buffers in invalid state");
+ }
+ }
+
+ @TearDown(Level.Trial)
+ public void doTearDown() {
+ }
+
+ public ByteBuf getByteBuff(BufferType bType) {
+ switch (bType) {
+ case ARRAY_BACKED:
+ return arrayBackedBuffer;
+ case NOT_ARRAY_BACKED:
+ return notArrayBackedBuffer;
+ case BYTE_BUF_DEFAULT_ALLOC:
+ return byteBufDefaultAlloc;
+ default:
+ throw new IllegalArgumentException("unknown buffer type " +
bType);
+ }
+ }
+
+ public DigestManager getDigestManager(Digest digest) {
+ switch (digest) {
+ case CRC32:
+ return crc32;
+ case CRC32_C:
+ return crc32c;
+ case MAC:
+ return mac;
+ default:
+ throw new IllegalArgumentException("unknown digest " + digest);
+ }
+ }
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ @Warmup(iterations = 2, time = 3, timeUnit = TimeUnit.SECONDS)
+ @Measurement(iterations = 5, time = 12, timeUnit = TimeUnit.SECONDS)
+ @Threads(2)
+ @Fork(value = 1, warmups = 1)
+ public void digestManager(MyState state) {
+ final ByteBuf buff = state.getByteBuff(state.bufferType);
+ final DigestManager dm = state.getDigestManager(state.digest);
+ dm.update(buff);
+ state.digestBuf.clear();
+ dm.populateValueAndReset(state.digestBuf);
+ }
+
+}
diff --git a/pom.xml b/pom.xml
index 9c7a00270..0b15a8c16 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,6 +61,7 @@
<module>bookkeeper-http</module>
<module>tests</module>
<module>bookkeeper-dist</module>
+ <module>microbenchmarks</module>
</modules>
<mailingLists>
<mailingList>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
