michaelsembwever commented on code in PR #2254: URL: https://github.com/apache/cassandra/pull/2254#discussion_r1184989965
########## test/unit/org/apache/cassandra/schema/CompressionParamsTest.java: ########## @@ -0,0 +1,537 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.schema; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.junit.Test; + +import org.apache.cassandra.config.ParameterizedClass; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.compress.BufferType; +import org.apache.cassandra.io.compress.DeflateCompressor; +import org.apache.cassandra.io.compress.ICompressor; +import org.apache.cassandra.io.compress.LZ4Compressor; +import org.apache.cassandra.io.compress.NoopCompressor; +import org.apache.cassandra.io.compress.SnappyCompressor; +import org.apache.cassandra.io.compress.ZstdCompressor; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.junit.Assert.assertEquals; + +public class CompressionParamsTest +{ + //private ParameterizedClass options; + //private CompressionParams params; + + + private static ParameterizedClass emptyParameterizedClass() { + return new ParameterizedClass(null, new HashMap<>()); + } + + @Test + public void additionalParamsTest() { + // no map + ParameterizedClass options = new ParameterizedClass(); + CompressionParams params = CompressionParams.fromParameterizedClass(options); + assertThat( params.getOtherOptions()).isNotNull(); + assertThat( params.getOtherOptions().isEmpty()).isTrue(); + + options = emptyParameterizedClass(); + params = CompressionParams.fromParameterizedClass(options); + assertThat( params.getOtherOptions()).isNotNull(); + assertThat( params.getOtherOptions().isEmpty()).isTrue(); + + options.parameters.put( "foo", "bar"); + params = CompressionParams.fromParameterizedClass(options); + params = CompressionParams.fromParameterizedClass(options); + assertThat( params.getOtherOptions()).isNotNull(); + assertThat( params.getOtherOptions().get("foo")).isEqualTo("bar"); + } + + // Tests chunklength settings for both Options and Map. + private static <T> void chunkLengthTest(BiConsumer<String,String> put, Consumer<String> remove, Function<T,CompressionParams> func, T instance) + { + // CHUNK_LENGTH + + // test empty string + put.accept(CompressionParams.CHUNK_LENGTH, ""); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length' value"); + + // text zero string + put.accept(CompressionParams.CHUNK_LENGTH, "0MiB"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length' value"); + + + // test properly formated value + put.accept(CompressionParams.CHUNK_LENGTH, "1MiB"); + CompressionParams params = func.apply(instance); + assertEquals(1024, params.chunkLength()); + + // test bad string + put.accept(CompressionParams.CHUNK_LENGTH, "badvalue"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length' value"); + + // test not power of 2 + put.accept(CompressionParams.CHUNK_LENGTH, "3MiB"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length' value") + .withMessageContaining("Must be a power of 2"); + + remove.accept(CompressionParams.CHUNK_LENGTH); + + + // CHUNK_LENGTH_IN_KB + // same tests as above + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, ""); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value"); + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "0"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value"); + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "1"); + params = func.apply(instance); + assertEquals(1024, params.chunkLength()); + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "badvalue"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value"); + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "3"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value") + .withMessageContaining("Must be a power of 2"); + + // test negative value + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "-1"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value") + .withMessageContaining("May not be <= 0"); + + remove.accept(CompressionParams.CHUNK_LENGTH_IN_KB); + + + + + // TEST COMBINATIONS + put.accept(CompressionParams.CHUNK_LENGTH, "3MiB"); + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "2"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessage(CompressionParams.TOO_MANY_CHUNK_LENGTH); + } + + @Test + public void chunkLengthTest() + { + ParameterizedClass options = emptyParameterizedClass(); + chunkLengthTest( options.parameters::put, options.parameters::remove, CompressionParams::fromParameterizedClass, options ); + + Map<String,String> map = new HashMap<String,String>(); + map.put( CompressionParams.CLASS, "lz4"); + Consumer<String> remove = (s) -> map.remove(s); + chunkLengthTest( map::put,remove, CompressionParams::fromMap, map ); + } + + private static <T> void minCompressRatioTest(BiConsumer<String,String> put, Function<T,CompressionParams> func, T instance) + { + + CompressionParams params = func.apply(instance); + assertEquals(CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, params.minCompressRatio(), Double.MIN_VALUE); + assertEquals(Integer.MAX_VALUE, params.maxCompressedLength()); + + put.accept( CompressionParams.MIN_COMPRESS_RATIO, "0.0" ); //CompressionParams.DEFAULT_MIN_COMPRESS_RATIO + params =func.apply(instance); + assertEquals(CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, params.minCompressRatio(), Double.MIN_VALUE); + assertEquals(Integer.MAX_VALUE, params.maxCompressedLength()); + + put.accept( CompressionParams.MIN_COMPRESS_RATIO, "0.3"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining( "Invalid 'min_compress_ratio' value") + .withMessageContaining("Can either be 0 or greater than or equal to 1"); + + put.accept( CompressionParams.MIN_COMPRESS_RATIO, "1.3"); + params = func.apply(instance); + assertEquals(1.3, params.minCompressRatio(), Double.MIN_VALUE); + assertEquals( (int) Math.ceil(CompressionParams.DEFAULT_CHUNK_LENGTH / 1.3), params.maxCompressedLength()); + + put.accept( CompressionParams.MIN_COMPRESS_RATIO, "-1.0"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining( "Invalid 'min_compress_ratio' value") + .withMessageContaining("Can either be 0 or greater than or equal to 1"); + } + + @Test + public void minCompressRatioTest() + { + ParameterizedClass options = emptyParameterizedClass(); + minCompressRatioTest( options.parameters::put, CompressionParams::fromParameterizedClass, options ); + + Map<String,String> map = new HashMap<String,String>(); + map.put( CompressionParams.CLASS, "lz4"); + minCompressRatioTest( map::put, CompressionParams::fromMap, map ); + } + + private static <T> void maxCompressedLengthTest(BiConsumer<String,String> put, Function<T,CompressionParams> func, T instance) + { + CompressionParams params = func.apply(instance); + assertEquals(Integer.MAX_VALUE, params.maxCompressedLength()); + assertEquals(CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, params.minCompressRatio(), Double.MIN_VALUE); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,""); + params = func.apply(instance); + assertEquals(Integer.MAX_VALUE, params.maxCompressedLength()); + assertEquals(CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, params.minCompressRatio(), Double.MIN_VALUE); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,"4MiB"); + params = func.apply(instance); + assertEquals(4*1024, params.maxCompressedLength()); + assertEquals(CompressionParams.DEFAULT_CHUNK_LENGTH / (4.0 * 1024), params.minCompressRatio(), Double.MIN_VALUE); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,"badvalue"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'max_compressed_length' value") + .withMessageContaining("Invalid data storage"); + } + + @Test + public void maxCompressedLengthTest() + { + ParameterizedClass options = emptyParameterizedClass(); + maxCompressedLengthTest(options.parameters::put, CompressionParams::fromParameterizedClass, options); + + Map<String, String> map = new HashMap<String, String>(); + map.put(CompressionParams.CLASS, "lz4"); + maxCompressedLengthTest(map::put, CompressionParams::fromMap, map); + } + + @Test + public void maxCompressionLengthAndMinCompressRatioTest() { + ParameterizedClass options = emptyParameterizedClass(); + options.parameters.put( CompressionParams.MIN_COMPRESS_RATIO, "1.0"); + options.parameters.put( CompressionParams.MAX_COMPRESSED_LENGTH, "4Gib"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> CompressionParams.fromParameterizedClass(options)) + .withMessage("Can not specify both 'min_compress_ratio' and 'max_compressed_length' for the compressor parameters."); + + Map<String,String> map = new HashMap<>(); + map.put( CompressionParams.CLASS, "lz4"); + map.put( CompressionParams.MIN_COMPRESS_RATIO, "1.0"); + map.put( CompressionParams.MAX_COMPRESSED_LENGTH, "4Gib"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> CompressionParams.fromMap(map)) + .withMessage("Can not specify both 'min_compress_ratio' and 'max_compressed_length' for the compressor parameters."); + } + + private static void assertParams(CompressionParams params, boolean enabled, int chunkLength, int maxCompressedLength, double minCompressRatio, Class<?> compressor) + { + assertThat(params.isEnabled()).isEqualTo(enabled); + assertThat(params.chunkLength()).isEqualTo(chunkLength); + assertThat(params.maxCompressedLength()).isEqualTo(maxCompressedLength); + assertThat(params.minCompressRatio()).isEqualTo(minCompressRatio); + if (compressor != null) + { + assertThat(params.getSstableCompressor()).isInstanceOf(compressor); + } else + { + assertThat(params.getSstableCompressor()).isNull(); + } + } + + + @Test + public void defaultTest() + { + CompressionParams params = CompressionParams.fromParameterizedClass( emptyParameterizedClass() ); + assertParams(params,true, CompressionParams.DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, LZ4Compressor.class); + roundTripMapTest(params); + + params = CompressionParams.fromParameterizedClass( null ); + assertParams(params, true, CompressionParams.DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, LZ4Compressor.class); + roundTripMapTest(params); + + params = CompressionParams.fromMap(Collections.EMPTY_MAP ); + assertParams(params,true, CompressionParams.DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, LZ4Compressor.class); + roundTripMapTest(params); + } + + private static <T> void paramsTest(Class<?> clazz, BiConsumer<String,String> put, Consumer<String> remove, Function<T,CompressionParams> func, T instance) + { + CompressionParams params = func.apply(instance); + assertParams(params, true, CompressionParams.DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, clazz); + roundTripMapTest(params); + + put.accept( CompressionParams.CHUNK_LENGTH,"4MiB"); + params = func.apply(instance); + assertParams(params, true, 4*1024, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, clazz); + roundTripMapTest(params); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,"2MiB"); + params = func.apply(instance); + assertParams(params, true, 4*1024, 2*1024, 2.0, clazz); + roundTripMapTest(params); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,"2097151KiB"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() ->func.apply(instance)) + .withMessageContaining("Invalid 'max_compressed_length' value for the 'compression' option: Must be less than or equal to chunk length"); + assertParams(params, true, 4*1024, 2*1024, 2.0, clazz); + roundTripMapTest(params); + + remove.accept(CompressionParams.MAX_COMPRESSED_LENGTH); + put.accept( CompressionParams.MIN_COMPRESS_RATIO,"1.5"); + params = func.apply(instance); + assertParams(params, true, 4*1024, 2731, 1.5, clazz); + roundTripMapTest(params); + + put.accept( CompressionParams.ENABLED,"false"); + params = func.apply(instance); + assertParams(params, false, 4*1024, 2731, 1.5, null); + // round tripped disabled params return disabled default constructor version + } + + + @Test + public void constructorTest() { + Map<String,String> map = new HashMap<>(); + + // chunk length < 0 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), map, -1, 0.0)) + .withMessage("Invalid 'chunk_length' value for the 'compression' option. May not be <= 0: -1"); + + // chunk length = 0 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), map, -1, 0.0)) + .withMessage("Invalid 'chunk_length' value for the 'compression' option. May not be <= 0: -1"); + + // min compress ratio < 0 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), map, CompressionParams.DEFAULT_CHUNK_LENGTH, -1.0)) + .withMessageContaining("Invalid 'min_compress_ratio' value for the 'compression' option. Can either be 0 or greater than or equal to 1"); + + // 0 < min compress ratio < 1 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), map, CompressionParams.DEFAULT_CHUNK_LENGTH, 0.5)) + .withMessageContaining("Invalid 'min_compress_ratio' value for the 'compression' option. Can either be 0 or greater than or equal to 1"); + + // max compressed length > chunk length + int len = 1 << 30; + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), len, len+1, map )) + .withMessageContaining("Invalid 'max_compressed_length' value for the 'compression' option: Must be less than or equal to chunk length"); + + // max compressed length < 0 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), CompressionParams.DEFAULT_CHUNK_LENGTH, -1, map )) + .withMessageContaining("Invalid 'max_compressed_length' value for the 'compression' option. May not be less than zero: -1"); + } + + private static void roundTripMapTest(CompressionParams params) { + CompressionParams other = CompressionParams.fromMap( params.asMap() ); + assertThat( params.getOtherOptions() ).isEqualTo( other.getOtherOptions() ); + assertThat( params.maxCompressedLength()).isEqualTo( other.maxCompressedLength()); + assertThat( params.minCompressRatio()).isEqualTo( other.minCompressRatio()); + assertThat(params.chunkLength()).isEqualTo( other.chunkLength() ); + assertThat(params.isEnabled()).isEqualTo( other.isEnabled()); + assertThat(params.getCrcCheckChance()).isEqualTo( other.getCrcCheckChance()); + assertThat(params.klass()).isEqualTo( other.klass()); + } + + @Test + public void lz4Test() { Review Comment: nit: brace on newline. ########## test/unit/org/apache/cassandra/schema/CompressionParamsTest.java: ########## @@ -0,0 +1,537 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.schema; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.junit.Test; + +import org.apache.cassandra.config.ParameterizedClass; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.compress.BufferType; +import org.apache.cassandra.io.compress.DeflateCompressor; +import org.apache.cassandra.io.compress.ICompressor; +import org.apache.cassandra.io.compress.LZ4Compressor; +import org.apache.cassandra.io.compress.NoopCompressor; +import org.apache.cassandra.io.compress.SnappyCompressor; +import org.apache.cassandra.io.compress.ZstdCompressor; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.junit.Assert.assertEquals; + +public class CompressionParamsTest +{ + //private ParameterizedClass options; + //private CompressionParams params; + + + private static ParameterizedClass emptyParameterizedClass() { + return new ParameterizedClass(null, new HashMap<>()); + } + + @Test + public void additionalParamsTest() { + // no map + ParameterizedClass options = new ParameterizedClass(); + CompressionParams params = CompressionParams.fromParameterizedClass(options); + assertThat( params.getOtherOptions()).isNotNull(); + assertThat( params.getOtherOptions().isEmpty()).isTrue(); + + options = emptyParameterizedClass(); + params = CompressionParams.fromParameterizedClass(options); + assertThat( params.getOtherOptions()).isNotNull(); + assertThat( params.getOtherOptions().isEmpty()).isTrue(); + + options.parameters.put( "foo", "bar"); + params = CompressionParams.fromParameterizedClass(options); + params = CompressionParams.fromParameterizedClass(options); + assertThat( params.getOtherOptions()).isNotNull(); + assertThat( params.getOtherOptions().get("foo")).isEqualTo("bar"); + } + + // Tests chunklength settings for both Options and Map. + private static <T> void chunkLengthTest(BiConsumer<String,String> put, Consumer<String> remove, Function<T,CompressionParams> func, T instance) + { + // CHUNK_LENGTH + + // test empty string + put.accept(CompressionParams.CHUNK_LENGTH, ""); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length' value"); + + // text zero string + put.accept(CompressionParams.CHUNK_LENGTH, "0MiB"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length' value"); + + + // test properly formated value + put.accept(CompressionParams.CHUNK_LENGTH, "1MiB"); + CompressionParams params = func.apply(instance); + assertEquals(1024, params.chunkLength()); + + // test bad string + put.accept(CompressionParams.CHUNK_LENGTH, "badvalue"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length' value"); + + // test not power of 2 + put.accept(CompressionParams.CHUNK_LENGTH, "3MiB"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length' value") + .withMessageContaining("Must be a power of 2"); + + remove.accept(CompressionParams.CHUNK_LENGTH); + + + // CHUNK_LENGTH_IN_KB + // same tests as above + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, ""); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value"); + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "0"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value"); + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "1"); + params = func.apply(instance); + assertEquals(1024, params.chunkLength()); + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "badvalue"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value"); + + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "3"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value") + .withMessageContaining("Must be a power of 2"); + + // test negative value + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "-1"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'chunk_length_in_kb' value") + .withMessageContaining("May not be <= 0"); + + remove.accept(CompressionParams.CHUNK_LENGTH_IN_KB); + + + + + // TEST COMBINATIONS + put.accept(CompressionParams.CHUNK_LENGTH, "3MiB"); + put.accept(CompressionParams.CHUNK_LENGTH_IN_KB, "2"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessage(CompressionParams.TOO_MANY_CHUNK_LENGTH); + } + + @Test + public void chunkLengthTest() + { + ParameterizedClass options = emptyParameterizedClass(); + chunkLengthTest( options.parameters::put, options.parameters::remove, CompressionParams::fromParameterizedClass, options ); + + Map<String,String> map = new HashMap<String,String>(); + map.put( CompressionParams.CLASS, "lz4"); + Consumer<String> remove = (s) -> map.remove(s); + chunkLengthTest( map::put,remove, CompressionParams::fromMap, map ); + } + + private static <T> void minCompressRatioTest(BiConsumer<String,String> put, Function<T,CompressionParams> func, T instance) + { + + CompressionParams params = func.apply(instance); + assertEquals(CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, params.minCompressRatio(), Double.MIN_VALUE); + assertEquals(Integer.MAX_VALUE, params.maxCompressedLength()); + + put.accept( CompressionParams.MIN_COMPRESS_RATIO, "0.0" ); //CompressionParams.DEFAULT_MIN_COMPRESS_RATIO + params =func.apply(instance); + assertEquals(CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, params.minCompressRatio(), Double.MIN_VALUE); + assertEquals(Integer.MAX_VALUE, params.maxCompressedLength()); + + put.accept( CompressionParams.MIN_COMPRESS_RATIO, "0.3"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining( "Invalid 'min_compress_ratio' value") + .withMessageContaining("Can either be 0 or greater than or equal to 1"); + + put.accept( CompressionParams.MIN_COMPRESS_RATIO, "1.3"); + params = func.apply(instance); + assertEquals(1.3, params.minCompressRatio(), Double.MIN_VALUE); + assertEquals( (int) Math.ceil(CompressionParams.DEFAULT_CHUNK_LENGTH / 1.3), params.maxCompressedLength()); + + put.accept( CompressionParams.MIN_COMPRESS_RATIO, "-1.0"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining( "Invalid 'min_compress_ratio' value") + .withMessageContaining("Can either be 0 or greater than or equal to 1"); + } + + @Test + public void minCompressRatioTest() + { + ParameterizedClass options = emptyParameterizedClass(); + minCompressRatioTest( options.parameters::put, CompressionParams::fromParameterizedClass, options ); + + Map<String,String> map = new HashMap<String,String>(); + map.put( CompressionParams.CLASS, "lz4"); + minCompressRatioTest( map::put, CompressionParams::fromMap, map ); + } + + private static <T> void maxCompressedLengthTest(BiConsumer<String,String> put, Function<T,CompressionParams> func, T instance) + { + CompressionParams params = func.apply(instance); + assertEquals(Integer.MAX_VALUE, params.maxCompressedLength()); + assertEquals(CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, params.minCompressRatio(), Double.MIN_VALUE); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,""); + params = func.apply(instance); + assertEquals(Integer.MAX_VALUE, params.maxCompressedLength()); + assertEquals(CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, params.minCompressRatio(), Double.MIN_VALUE); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,"4MiB"); + params = func.apply(instance); + assertEquals(4*1024, params.maxCompressedLength()); + assertEquals(CompressionParams.DEFAULT_CHUNK_LENGTH / (4.0 * 1024), params.minCompressRatio(), Double.MIN_VALUE); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,"badvalue"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> func.apply(instance)) + .withMessageContaining("Invalid 'max_compressed_length' value") + .withMessageContaining("Invalid data storage"); + } + + @Test + public void maxCompressedLengthTest() + { + ParameterizedClass options = emptyParameterizedClass(); + maxCompressedLengthTest(options.parameters::put, CompressionParams::fromParameterizedClass, options); + + Map<String, String> map = new HashMap<String, String>(); + map.put(CompressionParams.CLASS, "lz4"); + maxCompressedLengthTest(map::put, CompressionParams::fromMap, map); + } + + @Test + public void maxCompressionLengthAndMinCompressRatioTest() { + ParameterizedClass options = emptyParameterizedClass(); + options.parameters.put( CompressionParams.MIN_COMPRESS_RATIO, "1.0"); + options.parameters.put( CompressionParams.MAX_COMPRESSED_LENGTH, "4Gib"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> CompressionParams.fromParameterizedClass(options)) + .withMessage("Can not specify both 'min_compress_ratio' and 'max_compressed_length' for the compressor parameters."); + + Map<String,String> map = new HashMap<>(); + map.put( CompressionParams.CLASS, "lz4"); + map.put( CompressionParams.MIN_COMPRESS_RATIO, "1.0"); + map.put( CompressionParams.MAX_COMPRESSED_LENGTH, "4Gib"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> CompressionParams.fromMap(map)) + .withMessage("Can not specify both 'min_compress_ratio' and 'max_compressed_length' for the compressor parameters."); + } + + private static void assertParams(CompressionParams params, boolean enabled, int chunkLength, int maxCompressedLength, double minCompressRatio, Class<?> compressor) + { + assertThat(params.isEnabled()).isEqualTo(enabled); + assertThat(params.chunkLength()).isEqualTo(chunkLength); + assertThat(params.maxCompressedLength()).isEqualTo(maxCompressedLength); + assertThat(params.minCompressRatio()).isEqualTo(minCompressRatio); + if (compressor != null) + { + assertThat(params.getSstableCompressor()).isInstanceOf(compressor); + } else + { + assertThat(params.getSstableCompressor()).isNull(); + } + } + + + @Test + public void defaultTest() + { + CompressionParams params = CompressionParams.fromParameterizedClass( emptyParameterizedClass() ); + assertParams(params,true, CompressionParams.DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, LZ4Compressor.class); + roundTripMapTest(params); + + params = CompressionParams.fromParameterizedClass( null ); + assertParams(params, true, CompressionParams.DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, LZ4Compressor.class); + roundTripMapTest(params); + + params = CompressionParams.fromMap(Collections.EMPTY_MAP ); + assertParams(params,true, CompressionParams.DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, LZ4Compressor.class); + roundTripMapTest(params); + } + + private static <T> void paramsTest(Class<?> clazz, BiConsumer<String,String> put, Consumer<String> remove, Function<T,CompressionParams> func, T instance) + { + CompressionParams params = func.apply(instance); + assertParams(params, true, CompressionParams.DEFAULT_CHUNK_LENGTH, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, clazz); + roundTripMapTest(params); + + put.accept( CompressionParams.CHUNK_LENGTH,"4MiB"); + params = func.apply(instance); + assertParams(params, true, 4*1024, Integer.MAX_VALUE, CompressionParams.DEFAULT_MIN_COMPRESS_RATIO, clazz); + roundTripMapTest(params); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,"2MiB"); + params = func.apply(instance); + assertParams(params, true, 4*1024, 2*1024, 2.0, clazz); + roundTripMapTest(params); + + put.accept( CompressionParams.MAX_COMPRESSED_LENGTH,"2097151KiB"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() ->func.apply(instance)) + .withMessageContaining("Invalid 'max_compressed_length' value for the 'compression' option: Must be less than or equal to chunk length"); + assertParams(params, true, 4*1024, 2*1024, 2.0, clazz); + roundTripMapTest(params); + + remove.accept(CompressionParams.MAX_COMPRESSED_LENGTH); + put.accept( CompressionParams.MIN_COMPRESS_RATIO,"1.5"); + params = func.apply(instance); + assertParams(params, true, 4*1024, 2731, 1.5, clazz); + roundTripMapTest(params); + + put.accept( CompressionParams.ENABLED,"false"); + params = func.apply(instance); + assertParams(params, false, 4*1024, 2731, 1.5, null); + // round tripped disabled params return disabled default constructor version + } + + + @Test + public void constructorTest() { + Map<String,String> map = new HashMap<>(); + + // chunk length < 0 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), map, -1, 0.0)) + .withMessage("Invalid 'chunk_length' value for the 'compression' option. May not be <= 0: -1"); + + // chunk length = 0 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), map, -1, 0.0)) + .withMessage("Invalid 'chunk_length' value for the 'compression' option. May not be <= 0: -1"); + + // min compress ratio < 0 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), map, CompressionParams.DEFAULT_CHUNK_LENGTH, -1.0)) + .withMessageContaining("Invalid 'min_compress_ratio' value for the 'compression' option. Can either be 0 or greater than or equal to 1"); + + // 0 < min compress ratio < 1 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), map, CompressionParams.DEFAULT_CHUNK_LENGTH, 0.5)) + .withMessageContaining("Invalid 'min_compress_ratio' value for the 'compression' option. Can either be 0 or greater than or equal to 1"); + + // max compressed length > chunk length + int len = 1 << 30; + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), len, len+1, map )) + .withMessageContaining("Invalid 'max_compressed_length' value for the 'compression' option: Must be less than or equal to chunk length"); + + // max compressed length < 0 + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> new CompressionParams( TestCompressor.class.getName(), CompressionParams.DEFAULT_CHUNK_LENGTH, -1, map )) + .withMessageContaining("Invalid 'max_compressed_length' value for the 'compression' option. May not be less than zero: -1"); + } + + private static void roundTripMapTest(CompressionParams params) { + CompressionParams other = CompressionParams.fromMap( params.asMap() ); + assertThat( params.getOtherOptions() ).isEqualTo( other.getOtherOptions() ); + assertThat( params.maxCompressedLength()).isEqualTo( other.maxCompressedLength()); + assertThat( params.minCompressRatio()).isEqualTo( other.minCompressRatio()); + assertThat(params.chunkLength()).isEqualTo( other.chunkLength() ); + assertThat(params.isEnabled()).isEqualTo( other.isEnabled()); + assertThat(params.getCrcCheckChance()).isEqualTo( other.getCrcCheckChance()); + assertThat(params.klass()).isEqualTo( other.klass()); + } + + @Test + public void lz4Test() { + ParameterizedClass options = emptyParameterizedClass(); + options.class_name = CompressionParams.CompressorType.lz4.name(); + paramsTest(LZ4Compressor.class, options.parameters::put, options.parameters::remove, CompressionParams::fromParameterizedClass, options ); + + options.parameters.clear(); + options.class_name = "LZ4Compressor"; + paramsTest(LZ4Compressor.class, options.parameters::put, options.parameters::remove, CompressionParams::fromParameterizedClass, options ); + + options.parameters.clear(); + options.class_name = LZ4Compressor.class.getName(); + paramsTest(LZ4Compressor.class, options.parameters::put, options.parameters::remove, CompressionParams::fromParameterizedClass, options ); + } + + @Test + public void noneTest() { Review Comment: nit: brace on newline. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
