This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new c4f4eb9 [IOTDB-2068] GZIP compressor meets
ArrayIndexOutOfBoundsException (#4488)
c4f4eb9 is described below
commit c4f4eb948644e221fd6b549fdcbdd7fcc8339156
Author: Alan Choo <[email protected]>
AuthorDate: Sun Nov 28 01:10:46 2021 +0800
[IOTDB-2068] GZIP compressor meets ArrayIndexOutOfBoundsException (#4488)
---
.../apache/iotdb/tsfile/compress/ICompressor.java | 58 +++++++++++++++++++++-
.../compress/GZIPCompressOverflowException.java | 26 ++++++++++
.../apache/iotdb/tsfile/write/page/PageWriter.java | 4 ++
.../iotdb/tsfile/write/page/TimePageWriter.java | 4 ++
.../iotdb/tsfile/write/page/ValuePageWriter.java | 4 ++
.../org/apache/iotdb/tsfile/compress/GZIPTest.java | 17 ++++++-
.../org/apache/iotdb/tsfile/compress/LZ4Test.java | 17 ++++++-
.../apache/iotdb/tsfile/compress/SnappyTest.java | 17 ++++++-
8 files changed, 142 insertions(+), 5 deletions(-)
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
index 66d18f9..9243e53 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/ICompressor.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.tsfile.compress;
import
org.apache.iotdb.tsfile.exception.compress.CompressionTypeNotSupportedException;
+import
org.apache.iotdb.tsfile.exception.compress.GZIPCompressOverflowException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import net.jpountz.lz4.LZ4Compressor;
@@ -72,6 +73,15 @@ public interface ICompressor extends Serializable {
byte[] compress(byte[] data) throws IOException;
/**
+ * abstract method of compress. this method has an important overhead due to
the fact that it
+ * needs to allocate a byte array to compress into, and then needs to resize
this buffer to the
+ * actual compressed length.
+ *
+ * @return byte array of compressed data.
+ */
+ byte[] compress(byte[] data, int offset, int length) throws IOException;
+
+ /**
* abstract method of compress.
*
* @return byte length of compressed data.
@@ -87,6 +97,13 @@ public interface ICompressor extends Serializable {
*/
int compress(ByteBuffer data, ByteBuffer compressed) throws IOException;
+ /**
+ * Get the maximum byte size needed for compressing data of the given byte
size. For GZIP, this
+ * method is insecure and may cause {@code GZIPCompressOverflowException}
+ *
+ * @param uncompressedDataSize byte size of the data to compress
+ * @return maximum byte size of the compressed data
+ */
int getMaxBytesForCompression(int uncompressedDataSize);
CompressionType getType();
@@ -100,6 +117,11 @@ public interface ICompressor extends Serializable {
}
@Override
+ public byte[] compress(byte[] data, int offset, int length) throws
IOException {
+ throw new IOException("No Compressor does not support compression
function");
+ }
+
+ @Override
public int compress(byte[] data, int offset, int length, byte[]
compressed) throws IOException {
throw new IOException("No Compressor does not support compression
function");
}
@@ -131,6 +153,20 @@ public interface ICompressor extends Serializable {
}
@Override
+ public byte[] compress(byte[] data, int offset, int length) throws
IOException {
+ byte[] maxCompressed = new byte[getMaxBytesForCompression(length)];
+ int compressedSize = Snappy.compress(data, offset, length,
maxCompressed, 0);
+ byte[] compressed = null;
+ if (compressedSize < maxCompressed.length) {
+ compressed = new byte[compressedSize];
+ System.arraycopy(maxCompressed, 0, compressed, 0, compressedSize);
+ } else {
+ compressed = maxCompressed;
+ }
+ return compressed;
+ }
+
+ @Override
public int compress(byte[] data, int offset, int length, byte[]
compressed) throws IOException {
return Snappy.compress(data, offset, length, compressed, 0);
}
@@ -169,6 +205,11 @@ public interface ICompressor extends Serializable {
}
@Override
+ public byte[] compress(byte[] data, int offset, int length) throws
IOException {
+ return compressor.compress(data, offset, length);
+ }
+
+ @Override
public int compress(byte[] data, int offset, int length, byte[]
compressed) {
return compressor.compress(data, offset, length, compressed, 0);
}
@@ -226,23 +267,36 @@ public interface ICompressor extends Serializable {
}
@Override
+ public byte[] compress(byte[] data, int offset, int length) throws
IOException {
+ byte[] dataBefore = new byte[length];
+ System.arraycopy(data, offset, dataBefore, 0, length);
+ return GZIPCompress.compress(dataBefore);
+ }
+
+ /** @exception GZIPCompressOverflowException if compressed byte array is
too small. */
+ @Override
public int compress(byte[] data, int offset, int length, byte[]
compressed) throws IOException {
byte[] dataBefore = new byte[length];
System.arraycopy(data, offset, dataBefore, 0, length);
byte[] res = GZIPCompress.compress(dataBefore);
+ if (res.length > compressed.length) {
+ throw new GZIPCompressOverflowException();
+ }
System.arraycopy(res, 0, compressed, 0, res.length);
return res.length;
}
+ /** @exception GZIPCompressOverflowException if compressed ByteBuffer is
too small. */
@Override
public int compress(ByteBuffer data, ByteBuffer compressed) throws
IOException {
int length = data.remaining();
byte[] dataBefore = new byte[length];
data.get(dataBefore, 0, length);
-
byte[] res = GZIPCompress.compress(dataBefore);
+ if (res.length > compressed.capacity()) {
+ throw new GZIPCompressOverflowException();
+ }
compressed.put(res);
-
return res.length;
}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/compress/GZIPCompressOverflowException.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/compress/GZIPCompressOverflowException.java
new file mode 100644
index 0000000..9e54818
--- /dev/null
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/compress/GZIPCompressOverflowException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.exception.compress;
+
+public class GZIPCompressOverflowException extends RuntimeException {
+
+ public GZIPCompressOverflowException() {
+ super("compressed data is larger than the given byte container.");
+ }
+}
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
index f003825..0401908 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/PageWriter.java
@@ -216,6 +216,10 @@ public class PageWriter {
if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) {
compressedSize = uncompressedSize;
+ } else if (compressor.getType().equals(CompressionType.GZIP)) {
+ compressedBytes =
+ compressor.compress(pageData.array(), pageData.position(),
uncompressedSize);
+ compressedSize = compressedBytes.length;
} else {
compressedBytes = new
byte[compressor.getMaxBytesForCompression(uncompressedSize)];
// data is never a directByteBuffer now, so we can use data.array()
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java
index 5223d18..1c668fc 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/TimePageWriter.java
@@ -107,6 +107,10 @@ public class TimePageWriter {
if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) {
compressedSize = uncompressedSize;
+ } else if (compressor.getType().equals(CompressionType.GZIP)) {
+ compressedBytes =
+ compressor.compress(pageData.array(), pageData.position(),
uncompressedSize);
+ compressedSize = compressedBytes.length;
} else {
compressedBytes = new
byte[compressor.getMaxBytesForCompression(uncompressedSize)];
// data is never a directByteBuffer now, so we can use data.array()
diff --git
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java
index d707886..0328a56 100644
---
a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java
+++
b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/page/ValuePageWriter.java
@@ -237,6 +237,10 @@ public class ValuePageWriter {
if (compressor.getType().equals(CompressionType.UNCOMPRESSED)) {
compressedSize = uncompressedSize;
+ } else if (compressor.getType().equals(CompressionType.GZIP)) {
+ compressedBytes =
+ compressor.compress(pageData.array(), pageData.position(),
uncompressedSize);
+ compressedSize = compressedBytes.length;
} else {
compressedBytes = new
byte[compressor.getMaxBytesForCompression(uncompressedSize)];
// data is never a directByteBuffer now, so we can use data.array()
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/GZIPTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/GZIPTest.java
index bd16ad7..04d14b6 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/GZIPTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/GZIPTest.java
@@ -52,7 +52,7 @@ public class GZIPTest {
public void tearDown() {}
@Test
- public void testBytes() throws IOException {
+ public void testBytes1() throws IOException {
int n = 500000;
String input = randomString(n);
byte[] uncom = input.getBytes(StandardCharsets.UTF_8);
@@ -63,6 +63,21 @@ public class GZIPTest {
}
@Test
+ public void testBytes2() throws IOException {
+ ICompressor.GZIPCompressor compressor = new ICompressor.GZIPCompressor();
+ IUnCompressor.GZIPUnCompressor unCompressor = new
IUnCompressor.GZIPUnCompressor();
+
+ int n = 500000;
+ String input = randomString(n);
+ byte[] uncom = input.getBytes(StandardCharsets.UTF_8);
+ byte[] compressed = compressor.compress(uncom, 0, uncom.length);
+ // length should be same
+ Assert.assertEquals(compressor.compress(uncom).length, compressed.length);
+ byte[] uncompressed = unCompressor.uncompress(compressed);
+ Assert.assertArrayEquals(uncom, uncompressed);
+ }
+
+ @Test
public void testByteBuffer() throws IOException {
for (int i = 1; i < 500000; i += 100000) {
String input = randomString(i);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/LZ4Test.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/LZ4Test.java
index 005cfe8..8e3aea1 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/LZ4Test.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/LZ4Test.java
@@ -47,7 +47,7 @@ public class LZ4Test {
public void tearDown() {}
@Test
- public void testBytes() throws IOException {
+ public void testBytes1() throws IOException {
String input = randomString(2000000);
byte[] uncom = input.getBytes(StandardCharsets.UTF_8);
long time = System.currentTimeMillis();
@@ -65,4 +65,19 @@ public class LZ4Test {
Assert.assertArrayEquals(uncom, uncompressed);
}
+
+ @Test
+ public void testBytes2() throws IOException {
+ ICompressor.IOTDBLZ4Compressor compressor = new
ICompressor.IOTDBLZ4Compressor();
+ IUnCompressor.LZ4UnCompressor unCompressor = new
IUnCompressor.LZ4UnCompressor();
+
+ int n = 500000;
+ String input = randomString(n);
+ byte[] uncom = input.getBytes(StandardCharsets.UTF_8);
+ byte[] compressed = compressor.compress(uncom, 0, uncom.length);
+ // length should be same
+ Assert.assertEquals(compressor.compress(uncom).length, compressed.length);
+ byte[] uncompressed = unCompressor.uncompress(compressed);
+ Assert.assertArrayEquals(uncom, uncompressed);
+ }
}
diff --git
a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/SnappyTest.java
b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/SnappyTest.java
index 57f7e50..3bd26b6 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/SnappyTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/SnappyTest.java
@@ -48,7 +48,7 @@ public class SnappyTest {
public void tearDown() {}
@Test
- public void testBytes() throws IOException {
+ public void testBytes1() throws IOException {
int n = 500000;
String input = randomString(n);
byte[] uncom = input.getBytes(StandardCharsets.UTF_8);
@@ -64,6 +64,21 @@ public class SnappyTest {
}
@Test
+ public void testBytes2() throws IOException {
+ ICompressor.SnappyCompressor compressor = new
ICompressor.SnappyCompressor();
+ IUnCompressor.SnappyUnCompressor unCompressor = new
IUnCompressor.SnappyUnCompressor();
+
+ int n = 500000;
+ String input = randomString(n);
+ byte[] uncom = input.getBytes(StandardCharsets.UTF_8);
+ byte[] compressed = compressor.compress(uncom, 0, uncom.length);
+ // length should be same
+ Assert.assertEquals(compressor.compress(uncom).length, compressed.length);
+ byte[] uncompressed = unCompressor.uncompress(compressed);
+ Assert.assertArrayEquals(uncom, uncompressed);
+ }
+
+ @Test
public void testByteBuffer() throws IOException {
String input = randomString(5000);
ByteBuffer source = ByteBuffer.allocateDirect(input.getBytes().length);
