szetszwo commented on code in PR #4424:
URL: https://github.com/apache/ozone/pull/4424#discussion_r1144195803
##########
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBBatchOperation.java:
##########
@@ -21,15 +21,193 @@
import org.apache.hadoop.hdds.utils.db.RocksDatabase.ColumnFamily;
import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions;
+import org.apache.ratis.util.StringUtils;
+import org.apache.ratis.util.TraditionalBinaryPrefix;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
/**
* Batch operation implementation for rocks db.
*/
public class RDBBatchOperation implements BatchOperation {
+ static final Logger LOG = LoggerFactory.getLogger(RDBBatchOperation.class);
+ static void debug(Supplier<String> message) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("\n{}", message.get());
+ }
+ }
+
+ static String byteSize2String(int length) {
+ return TraditionalBinaryPrefix.long2String(length, "B", 3);
+ }
+
+ static String countSize2String(int count, int size) {
+ return count + " (" + byteSize2String(size) + ")";
+ }
+
+ /**
+ * To implement {@link #equals(Object)} and {@link #hashCode()}
+ * based on the contents of {@link #bytes}.
+ * <p>
+ * Note that it is incorrect to use {@link #bytes#equals(Object)}
+ * and {@link #bytes#hashCode()} here since
+ * they do not use the contents of {@link #bytes} in the computations.
+ * These methods simply inherit from {@link Object).
+ */
+ static final class ByteArray {
+ private final byte[] bytes;
+
+ ByteArray(byte[] bytes) {
+ this.bytes = bytes;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ } else if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final ByteArray that = (ByteArray) obj;
+ return Arrays.equals(this.bytes, that.bytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(bytes);
+ }
+ }
+
+ class PutOpCache {
+ class FamilyCache {
+ private final ColumnFamily family;
+ /** A (key -> value) map. */
+ private final Map<ByteArray, byte[]> putOps = new HashMap<>();
+
+ private int batchSize;
+ private int discardedSize;
+ private int discardedCount;
+ private int putCount;
+ private int delCount;
+
+ FamilyCache(ColumnFamily family) {
+ this.family = family;
+ }
+
+ /** Batch put the entire family cache. */
+ void batchPut() throws IOException {
+ for (Map.Entry<ByteArray, byte[]> op : putOps.entrySet()) {
+ family.batchPut(writeBatch, op.getKey().bytes, op.getValue());
+ }
+ putOps.clear();
+ }
+
+ void put(byte[] key, byte[] value) {
+ batchSize += key.length + value.length;
Review Comment:
Good point. We should use `long` for sizes.
##########
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBBatchOperation.java:
##########
@@ -21,15 +21,193 @@
import org.apache.hadoop.hdds.utils.db.RocksDatabase.ColumnFamily;
import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions;
+import org.apache.ratis.util.StringUtils;
+import org.apache.ratis.util.TraditionalBinaryPrefix;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
/**
* Batch operation implementation for rocks db.
*/
public class RDBBatchOperation implements BatchOperation {
+ static final Logger LOG = LoggerFactory.getLogger(RDBBatchOperation.class);
+ static void debug(Supplier<String> message) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("\n{}", message.get());
+ }
+ }
+
+ static String byteSize2String(int length) {
+ return TraditionalBinaryPrefix.long2String(length, "B", 3);
+ }
+
+ static String countSize2String(int count, int size) {
+ return count + " (" + byteSize2String(size) + ")";
+ }
+
+ /**
+ * To implement {@link #equals(Object)} and {@link #hashCode()}
+ * based on the contents of {@link #bytes}.
+ * <p>
+ * Note that it is incorrect to use {@link #bytes#equals(Object)}
+ * and {@link #bytes#hashCode()} here since
+ * they do not use the contents of {@link #bytes} in the computations.
+ * These methods simply inherit from {@link Object).
+ */
+ static final class ByteArray {
+ private final byte[] bytes;
+
+ ByteArray(byte[] bytes) {
+ this.bytes = bytes;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ } else if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final ByteArray that = (ByteArray) obj;
+ return Arrays.equals(this.bytes, that.bytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(bytes);
+ }
+ }
+
+ class PutOpCache {
+ class FamilyCache {
+ private final ColumnFamily family;
+ /** A (key -> value) map. */
+ private final Map<ByteArray, byte[]> putOps = new HashMap<>();
+
+ private int batchSize;
+ private int discardedSize;
+ private int discardedCount;
+ private int putCount;
+ private int delCount;
+
+ FamilyCache(ColumnFamily family) {
+ this.family = family;
+ }
+
+ /** Batch put the entire family cache. */
+ void batchPut() throws IOException {
+ for (Map.Entry<ByteArray, byte[]> op : putOps.entrySet()) {
+ family.batchPut(writeBatch, op.getKey().bytes, op.getValue());
+ }
+ putOps.clear();
+ }
+
+ void put(byte[] key, byte[] value) {
+ batchSize += key.length + value.length;
+ putCount++;
+ final byte[] previousValue = putOps.put(new ByteArray(key), value);
+
+ if (previousValue != null) {
+ discardedSize += key.length + previousValue.length;
+ discardedCount++;
+ debug(() -> String.format("Overwriting a previous put(value: %s)",
+ byteSize2String(previousValue.length)));
+ }
+
+ debug(() -> String.format(
+ "%s: %s put(key: %s, value: %s), newPut=%s, %s; key=%s",
+ name, family.getName(), byteSize2String(key.length),
+ byteSize2String(value.length),
+ putCount, getBatchSizeDiscardedString(),
+ StringUtils.bytes2HexString(key).toUpperCase()));
+ }
+
+ void remove(byte[] key) {
+ batchSize += key.length;
+ delCount++;
+ final byte[] removed = putOps.remove(new ByteArray(key));
+ if (removed != null) {
+ discardedSize += key.length + removed.length;
+ discardedCount++;
+ debug(() -> String.format("Removed a previous put(value: %s)",
+ byteSize2String(removed.length)));
+ }
+
+ debug(() -> String.format(
+ "%s: %s delete(key: %s), newDel=%s, %s; key=%s",
+ name, family.getName(), byteSize2String(key.length),
+ delCount, getBatchSizeDiscardedString(),
+ StringUtils.bytes2HexString(key).toUpperCase()));
+ }
+
+ String getBatchSizeDiscardedString() {
+ return String.format("batchSize=%s, discarded: %s",
+ byteSize2String(batchSize),
+ countSize2String(discardedCount, discardedSize));
+ }
+ }
+
+ /** A (family name -> {@link FamilyCache}) map. */
+ private final Map<String, FamilyCache> map = new HashMap<>();
Review Comment:
Sure.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
