This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch add_jni_lz4_option_1.1 in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit c90b92cdf6bb5aa612c535ff49573705501b1b93 Author: Jiang Tian <[email protected]> AuthorDate: Tue Apr 29 09:50:51 2025 +0800 add switch to disable native lz4 (#480) (cherry picked from commit c731ba0fe86812c4e658f1b98ca090d0f300ea48) --- .../main/java/org/apache/tsfile/common/conf/TSFileConfig.java | 10 ++++++++++ .../java/org/apache/tsfile/common/conf/TSFileDescriptor.java | 5 +++++ .../src/main/java/org/apache/tsfile/compress/ICompressor.java | 6 +++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java index 5c0b719a..72cf59be 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java @@ -203,6 +203,8 @@ public class TSFileConfig implements Serializable { private String objectStorageTsFileOutput = "org.apache.iotdb.os.fileSystem.OSTsFileOutput"; + private boolean lz4UseJni = true; + /** customizedProperties, this should be empty by default. */ private Properties customizedProperties = new Properties(); @@ -558,4 +560,12 @@ public class TSFileConfig implements Serializable { public void setObjectStorageTsFileOutput(String objectStorageTsFileOutput) { this.objectStorageTsFileOutput = objectStorageTsFileOutput; } + + public boolean isLz4UseJni() { + return lz4UseJni; + } + + public void setLz4UseJni(boolean lz4UseJni) { + this.lz4UseJni = lz4UseJni; + } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileDescriptor.java b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileDescriptor.java index 3df22502..8f2e2c78 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileDescriptor.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileDescriptor.java @@ -82,6 +82,7 @@ public class TSFileDescriptor { writer.setString(conf::setValueEncoder, "value_encoder"); writer.setString(conf::setCompressor, "compressor"); writer.setInt(conf::setBatchSize, "batch_size"); + writer.setBoolean(conf::setLz4UseJni, "lz4_use_jni"); } private static class PropertiesOverWriter { @@ -107,6 +108,10 @@ public class TSFileDescriptor { set(setter, propertyKey, Function.identity()); } + public void setBoolean(Consumer<Boolean> setter, String propertyKey) { + set(setter, propertyKey, Boolean::parseBoolean); + } + private <T> void set( Consumer<T> setter, String propertyKey, Function<String, T> propertyValueConverter) { String value = this.properties.getProperty(propertyKey); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/compress/ICompressor.java b/java/tsfile/src/main/java/org/apache/tsfile/compress/ICompressor.java index e4579d2b..c86c84b6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/compress/ICompressor.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/compress/ICompressor.java @@ -19,6 +19,7 @@ package org.apache.tsfile.compress; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.exception.compress.CompressionTypeNotSupportedException; import org.apache.tsfile.exception.compress.GZIPCompressOverflowException; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -201,7 +202,10 @@ public interface ICompressor extends Serializable { * This instance should be cached to avoid performance problem. See: * https://github.com/lz4/lz4-java/issues/152 and https://github.com/apache/spark/pull/24905 */ - private static final LZ4Factory factory = LZ4Factory.fastestInstance(); + private static final LZ4Factory factory = + TSFileDescriptor.getInstance().getConfig().isLz4UseJni() + ? LZ4Factory.fastestInstance() + : LZ4Factory.safeInstance(); private static final net.jpountz.lz4.LZ4Compressor compressor = factory.fastCompressor();
