Github user kevinjmh commented on a diff in the pull request: https://github.com/apache/carbondata/pull/2628#discussion_r209543118 --- Diff: core/src/main/java/org/apache/carbondata/core/datastore/compression/ZstdCompressor.java --- @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.datastore.compression; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.ByteBuffer; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; + +import com.github.luben.zstd.Zstd; + +public class ZstdCompressor implements Compressor, Serializable { + private static final long serialVersionUID = 8181578747306832771L; + private static final LogService LOGGER = + LogServiceFactory.getLogService(ZstdCompressor.class.getName()); + private static final int COMPRESS_LEVEL = 3; + + public ZstdCompressor() { + } + + @Override + public String getName() { + return "zstd"; + } + + @Override + public byte[] compressByte(byte[] unCompInput) { + return Zstd.compress(unCompInput, 3); + } + + @Override + public byte[] compressByte(byte[] unCompInput, int byteSize) { + return Zstd.compress(unCompInput, COMPRESS_LEVEL); + } + + @Override + public byte[] unCompressByte(byte[] compInput) { + long estimatedUncompressLength = Zstd.decompressedSize(compInput); + return Zstd.decompress(compInput, (int) estimatedUncompressLength); + } + + @Override + public byte[] unCompressByte(byte[] compInput, int offset, int length) { + // todo: how to avoid memory copy + byte[] dstBytes = new byte[length]; + System.arraycopy(compInput, offset, dstBytes, 0, length); + return unCompressByte(dstBytes); + } + + @Override + public byte[] compressShort(short[] unCompInput) { + // short use 2 bytes + byte[] unCompArray = new byte[unCompInput.length * 2]; + ByteBuffer unCompBuffer = ByteBuffer.wrap(unCompArray); + for (short input : unCompInput) { + unCompBuffer.putShort(input); + } + return Zstd.compress(unCompBuffer.array(), COMPRESS_LEVEL); + } + + @Override + public short[] unCompressShort(byte[] compInput, int offset, int length) { + byte[] unCompArray = unCompressByte(compInput, offset, length); + ByteBuffer unCompBuffer = ByteBuffer.wrap(unCompArray); + short[] shorts = new short[unCompArray.length / 2]; + for (int i = 0; i < shorts.length; i++) { + shorts[i] = unCompBuffer.getShort(); + } + return shorts; + } + + @Override + public byte[] compressInt(int[] unCompInput) { + // int use 4 bytes + byte[] unCompArray = new byte[unCompInput.length * 4]; + ByteBuffer unCompBuffer = ByteBuffer.wrap(unCompArray); + for (int input : unCompInput) { + unCompBuffer.putInt(input); + } + return Zstd.compress(unCompBuffer.array(), COMPRESS_LEVEL); + } + + @Override + public int[] unCompressInt(byte[] compInput, int offset, int length) { + byte[] unCompArray = unCompressByte(compInput, offset, length); + ByteBuffer unCompBuffer = ByteBuffer.wrap(unCompArray); + int[] ints = new int[unCompArray.length / 4]; + for (int i = 0; i < ints.length; i++) { + ints[i] = unCompBuffer.getInt(); + } + return ints; + } --- End diff -- can try following code style to convert unCompress byte result to target datatype: (take Int for example) ``` byte[] unCompArray = unCompressByte(compInput, offset, length); IntBuffer buf = ByteBuffer.wrap(unCompArray).asIntBuffer(); int[] dest = new int[buf.remaining()]; buf.get(dest); return dest; ```
---