emkornfield commented on a change in pull request #7326: URL: https://github.com/apache/arrow/pull/7326#discussion_r467356218
########## File path: java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionUtil.java ########## @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.compression; + +import org.apache.arrow.flatbuf.BodyCompressionMethod; +import org.apache.arrow.flatbuf.CompressionType; +import org.apache.arrow.vector.ipc.message.ArrowBodyCompression; + +/** + * Utilities for data compression/decompression. + */ +public class CompressionUtil { + + private CompressionUtil() { + } + + /** + * Creates the {@link ArrowBodyCompression} object, given the {@link CompressionCodec}. + * The implementation of this method should depend on the values of {@link CompressionType#names}. + */ + public static ArrowBodyCompression createBodyCompression(CompressionCodec codec) { + switch (codec.getCodecName()) { + case "default": + return DefaultCompressionCodec.DEFAULT_BODY_COMPRESSION; + case "LZ4_FRAME": + return new ArrowBodyCompression((byte) 0, BodyCompressionMethod.BUFFER); Review comment: these isn't an enum of 0 and 1 below? ########## File path: java/vector/src/main/java/org/apache/arrow/vector/compression/DefaultCompressionCodec.java ########## @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.compression; + +import org.apache.arrow.flatbuf.BodyCompressionMethod; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.message.ArrowBodyCompression; + +/** + * The default compression codec that does no compression. + */ +public class DefaultCompressionCodec implements CompressionCodec { Review comment: nit: I think NoCompressionCodec or something similar to indicate that no compression/decompression happens might this easier to understand for future readers. ########## File path: java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java ########## @@ -76,6 +97,10 @@ private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<Ar "wrong number of buffers for field %s in vector %s. found: %s", vector.getField(), vector.getClass().getSimpleName(), fieldBuffers)); } + + fieldBuffers = fieldBuffers.stream().map( Review comment: small nit: if we aren't using streams elsewhere we should avoid using them here as well. they can add some unncessary overhead for serialization/deserialization path. ########## File path: java/vector/src/main/java/org/apache/arrow/vector/compression/DefaultCompressionCodec.java ########## @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector.compression; + +import org.apache.arrow.flatbuf.BodyCompressionMethod; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.message.ArrowBodyCompression; + +/** + * The default compression codec that does no compression. + */ +public class DefaultCompressionCodec implements CompressionCodec { + + public static final DefaultCompressionCodec INSTANCE = new DefaultCompressionCodec(); + + public static final byte COMPRESSION_TYPE = -1; + + public static final ArrowBodyCompression DEFAULT_BODY_COMPRESSION = + new ArrowBodyCompression(COMPRESSION_TYPE, BodyCompressionMethod.BUFFER); + + private DefaultCompressionCodec() { + } + + @Override + public ArrowBuf compress(BufferAllocator allocator, ArrowBuf unCompressedBuffer) { Review comment: we should clarify in the contract of the interface that implementations take ownership of unCompressedBuffer (i.e. I think in order for this to work properly for real codecs is to free uncompressedBuffer here). Does that make sense or were you imagining something else? ########## File path: java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java ########## @@ -408,11 +408,15 @@ public static ArrowRecordBatch deserializeRecordBatch(RecordBatch recordBatchFB, ArrowBuf vectorBuffer = body.slice(bufferFB.offset(), bufferFB.length()); buffers.add(vectorBuffer); } + + ArrowBodyCompression bodyCompression = + new ArrowBodyCompression(recordBatchFB.compression().codec(), recordBatchFB.compression().method()); Review comment: does compression need to be checked for nullness? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
