315157973 commented on a change in pull request #11490:
URL: https://github.com/apache/pulsar/pull/11490#discussion_r679175917
##########
File path:
managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
##########
@@ -264,4 +285,70 @@ private static MetaStoreException getException(Throwable
t) {
return new MetaStoreException(t);
}
}
+
+ /**
+ * Compress ManagedLedgerInfo data.
+ *
+ * compression data structure
+ * [MAGIC_NUMBER](2) + [METADATA_SIZE](4) + [METADATA_PAYLOAD] +
[MANAGED_LEDGER_INFO_PAYLOAD]
+ */
+ public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) {
+ if (compressionType == null ||
compressionType.equals(CompressionType.NONE)) {
+ return managedLedgerInfo.toByteArray();
+ }
+ byte[] originalBytes = managedLedgerInfo.toByteArray();
+ MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata =
MLDataFormats.ManagedLedgerInfoMetadata.newBuilder()
+ .setCompressionType(compressionType.name())
+ .setUnpressedSize(originalBytes.length)
+ .build();
+ ByteBuf metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer(
+ mlInfoMetadata.getSerializedSize() + 6,
mlInfoMetadata.getSerializedSize() + 6);
+ metadataByteBuf.writeShort(MAGIC_MANAGED_LEDGER_INFO_METADATA);
+ metadataByteBuf.writeInt(mlInfoMetadata.getSerializedSize());
+ metadataByteBuf.writeBytes(mlInfoMetadata.toByteArray());
+
+ ByteBuf originalByteBuf =
PulsarByteBufAllocator.DEFAULT.buffer(originalBytes.length,
originalBytes.length);
+ originalByteBuf.writeBytes(originalBytes);
+ ByteBuf encodeByteBuf =
CompressionCodecProvider.getCompressionCodec(compressionType).encode(originalByteBuf);
+
+ CompositeByteBuf compositeByteBuf =
PulsarByteBufAllocator.DEFAULT.compositeBuffer();
+ compositeByteBuf.addComponent(true, metadataByteBuf);
+ compositeByteBuf.addComponent(true, encodeByteBuf);
+
+ byte[] dataBytes = new byte[compositeByteBuf.readableBytes()];
+ compositeByteBuf.readBytes(dataBytes);
+ return dataBytes;
+ }
+
+ public ManagedLedgerInfo parseManagedLedgerInfo(byte[] data) throws
InvalidProtocolBufferException {
+ ByteBuf byteBuf = PulsarByteBufAllocator.DEFAULT.buffer(data.length,
data.length);
+ byteBuf.writeBytes(data);
Review comment:
It seems we can use wrappedBuffer to reduce one copy
##########
File path:
managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
##########
@@ -264,4 +285,70 @@ private static MetaStoreException getException(Throwable
t) {
return new MetaStoreException(t);
}
}
+
+ /**
+ * Compress ManagedLedgerInfo data.
+ *
+ * compression data structure
+ * [MAGIC_NUMBER](2) + [METADATA_SIZE](4) + [METADATA_PAYLOAD] +
[MANAGED_LEDGER_INFO_PAYLOAD]
+ */
+ public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) {
+ if (compressionType == null ||
compressionType.equals(CompressionType.NONE)) {
+ return managedLedgerInfo.toByteArray();
+ }
+ byte[] originalBytes = managedLedgerInfo.toByteArray();
+ MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata =
MLDataFormats.ManagedLedgerInfoMetadata.newBuilder()
+ .setCompressionType(compressionType.name())
+ .setUnpressedSize(originalBytes.length)
+ .build();
+ ByteBuf metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer(
+ mlInfoMetadata.getSerializedSize() + 6,
mlInfoMetadata.getSerializedSize() + 6);
+ metadataByteBuf.writeShort(MAGIC_MANAGED_LEDGER_INFO_METADATA);
+ metadataByteBuf.writeInt(mlInfoMetadata.getSerializedSize());
+ metadataByteBuf.writeBytes(mlInfoMetadata.toByteArray());
+
+ ByteBuf originalByteBuf =
PulsarByteBufAllocator.DEFAULT.buffer(originalBytes.length,
originalBytes.length);
+ originalByteBuf.writeBytes(originalBytes);
Review comment:
It seems we can use wrappedBuffer to reduce one copy
##########
File path:
managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
##########
@@ -264,4 +285,70 @@ private static MetaStoreException getException(Throwable
t) {
return new MetaStoreException(t);
}
}
+
+ /**
+ * Compress ManagedLedgerInfo data.
+ *
+ * compression data structure
+ * [MAGIC_NUMBER](2) + [METADATA_SIZE](4) + [METADATA_PAYLOAD] +
[MANAGED_LEDGER_INFO_PAYLOAD]
+ */
+ public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) {
+ if (compressionType == null ||
compressionType.equals(CompressionType.NONE)) {
+ return managedLedgerInfo.toByteArray();
+ }
+ byte[] originalBytes = managedLedgerInfo.toByteArray();
+ MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata =
MLDataFormats.ManagedLedgerInfoMetadata.newBuilder()
+ .setCompressionType(compressionType.name())
+ .setUnpressedSize(originalBytes.length)
+ .build();
+ ByteBuf metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer(
+ mlInfoMetadata.getSerializedSize() + 6,
mlInfoMetadata.getSerializedSize() + 6);
+ metadataByteBuf.writeShort(MAGIC_MANAGED_LEDGER_INFO_METADATA);
+ metadataByteBuf.writeInt(mlInfoMetadata.getSerializedSize());
+ metadataByteBuf.writeBytes(mlInfoMetadata.toByteArray());
+
+ ByteBuf originalByteBuf =
PulsarByteBufAllocator.DEFAULT.buffer(originalBytes.length,
originalBytes.length);
+ originalByteBuf.writeBytes(originalBytes);
+ ByteBuf encodeByteBuf =
CompressionCodecProvider.getCompressionCodec(compressionType).encode(originalByteBuf);
+
+ CompositeByteBuf compositeByteBuf =
PulsarByteBufAllocator.DEFAULT.compositeBuffer();
+ compositeByteBuf.addComponent(true, metadataByteBuf);
+ compositeByteBuf.addComponent(true, encodeByteBuf);
+
+ byte[] dataBytes = new byte[compositeByteBuf.readableBytes()];
+ compositeByteBuf.readBytes(dataBytes);
+ return dataBytes;
Review comment:
Do these ByteBuf need to be Released？
##########
File path:
managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java
##########
@@ -264,4 +285,70 @@ private static MetaStoreException getException(Throwable
t) {
return new MetaStoreException(t);
}
}
+
+ /**
+ * Compress ManagedLedgerInfo data.
+ *
+ * compression data structure
+ * [MAGIC_NUMBER](2) + [METADATA_SIZE](4) + [METADATA_PAYLOAD] +
[MANAGED_LEDGER_INFO_PAYLOAD]
+ */
+ public byte[] compressLedgerInfo(ManagedLedgerInfo managedLedgerInfo) {
+ if (compressionType == null ||
compressionType.equals(CompressionType.NONE)) {
+ return managedLedgerInfo.toByteArray();
+ }
+ byte[] originalBytes = managedLedgerInfo.toByteArray();
+ MLDataFormats.ManagedLedgerInfoMetadata mlInfoMetadata =
MLDataFormats.ManagedLedgerInfoMetadata.newBuilder()
+ .setCompressionType(compressionType.name())
+ .setUnpressedSize(originalBytes.length)
+ .build();
+ ByteBuf metadataByteBuf = PulsarByteBufAllocator.DEFAULT.buffer(
+ mlInfoMetadata.getSerializedSize() + 6,
mlInfoMetadata.getSerializedSize() + 6);
+ metadataByteBuf.writeShort(MAGIC_MANAGED_LEDGER_INFO_METADATA);
+ metadataByteBuf.writeInt(mlInfoMetadata.getSerializedSize());
+ metadataByteBuf.writeBytes(mlInfoMetadata.toByteArray());
+
+ ByteBuf originalByteBuf =
PulsarByteBufAllocator.DEFAULT.buffer(originalBytes.length,
originalBytes.length);
+ originalByteBuf.writeBytes(originalBytes);
+ ByteBuf encodeByteBuf =
CompressionCodecProvider.getCompressionCodec(compressionType).encode(originalByteBuf);
+
+ CompositeByteBuf compositeByteBuf =
PulsarByteBufAllocator.DEFAULT.compositeBuffer();
+ compositeByteBuf.addComponent(true, metadataByteBuf);
+ compositeByteBuf.addComponent(true, encodeByteBuf);
+
+ byte[] dataBytes = new byte[compositeByteBuf.readableBytes()];
+ compositeByteBuf.readBytes(dataBytes);
+ return dataBytes;
+ }
+
+ public ManagedLedgerInfo parseManagedLedgerInfo(byte[] data) throws
InvalidProtocolBufferException {
+ ByteBuf byteBuf = PulsarByteBufAllocator.DEFAULT.buffer(data.length,
data.length);
+ byteBuf.writeBytes(data);
+ if (byteBuf.readableBytes() > 0 && byteBuf.readShort() ==
MAGIC_MANAGED_LEDGER_INFO_METADATA) {
+ try {
+ int metadataSize = byteBuf.readInt();
+ byte[] metadataBytes = new byte[metadataSize];
+ byteBuf.readBytes(metadataBytes);
+ MLDataFormats.ManagedLedgerInfoMetadata metadata =
+
MLDataFormats.ManagedLedgerInfoMetadata.parseFrom(metadataBytes);
+
+ long unpressedSize = metadata.getUnpressedSize();
+ ByteBuf decodeByteBuf =
CompressionCodecProvider.getCompressionCodec(
Review comment:
Do we need a release here ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
