vinooganesh commented on code in PR #3397: URL: https://github.com/apache/parquet-java/pull/3397#discussion_r3575349278
########## parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java: ########## @@ -0,0 +1,635 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.alp; + +import static org.apache.parquet.column.values.alp.AlpConstants.*; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.CapacityByteArrayOutputStream; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.bitpacking.BytePacker; +import org.apache.parquet.column.values.bitpacking.BytePackerForLong; +import org.apache.parquet.column.values.bitpacking.Packer; + +/** + * ALP (Adaptive Lossless floating-Point) values writer. + * + * <p>ALP encoding converts floating-point values to integers using decimal scaling, + * then applies Frame of Reference encoding and bit-packing. + * Values that cannot be losslessly converted are stored as exceptions. + * + * <p>Writing is incremental: values are buffered in a fixed-size vector buffer, + * and each full vector is encoded and flushed to the output stream immediately. + * On {@link #getBytes()}, any remaining partial vector is flushed, and the + * final page bytes are assembled. + * + * <p>Interleaved Page Layout: + * <pre> + * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐ + * │ Header │ Offset Array │ Vector 0 │ Vector 1 │ ... │ + * │ 7 bytes │ 4B × numVectors │ (interleaved)│ (interleaved)│ │ + * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘ + * </pre> + * + * <p>Each vector contains interleaved: + * AlpInfo(4B) + ForInfo(5B/9B) + PackedValues + ExceptionPositions + ExceptionValues + */ +public abstract class AlpValuesWriter extends ValuesWriter { + + protected final int initialCapacity; + protected final int pageSize; + protected final ByteBufferAllocator allocator; + protected final int vectorSize; + protected final int logVectorSize; + + AlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + AlpConstants.validateVectorSize(vectorSize); + this.initialCapacity = initialCapacity; + this.pageSize = pageSize; + this.allocator = allocator; + this.vectorSize = vectorSize; + this.logVectorSize = Integer.numberOfTrailingZeros(vectorSize); + } + + @Override + public Encoding getEncoding() { + return Encoding.ALP; + } + + /** Float writer. Buffers one vector at a time, encodes and flushes when full. */ + public static class FloatAlpValuesWriter extends AlpValuesWriter { + private final float[] vectorBuffer; + private int bufferCount; + private int totalCount; + private CapacityByteArrayOutputStream encodedVectors; + private final List<Integer> vectorByteSizes; + + // Preset caching: collect evenly-spaced sample vectors across the rowgroup, + // then build presets using estimated compressed size (matching C++ AlpSampler). + private int vectorsProcessed; + private int[][] cachedPresets; + private final List<float[]> rowgroupSamples; + private final int rowgroupSampleJump; + + // Reusable per-vector buffers + private final int[] encodedBuffer; + private final short[] excPosBuffer; + private final float[] excValBuffer; + private final byte[] metadataBuf; + private final byte[] packBuf; + private final int[] packPadBuf; + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { + this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE); + } + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + super(initialCapacity, pageSize, allocator, vectorSize); + this.vectorBuffer = new float[vectorSize]; + this.bufferCount = 0; + this.totalCount = 0; + this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator); + this.vectorByteSizes = new ArrayList<>(); + this.vectorsProcessed = 0; + this.cachedPresets = null; + this.rowgroupSamples = new ArrayList<>(); + // Space samples evenly: one sample every jump vectors across the rowgroup. + // Math.max(1, ...) guards against very small rowgroups or large vector sizes. + this.rowgroupSampleJump = Math.max(1, SAMPLER_ROWGROUP_SIZE / SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP / vectorSize); + // Pre-allocate reusable buffers + this.encodedBuffer = new int[vectorSize]; + this.excPosBuffer = new short[vectorSize]; + this.excValBuffer = new float[vectorSize]; + this.metadataBuf = new byte[Math.max(ALP_INFO_SIZE, FLOAT_FOR_INFO_SIZE)]; + this.packBuf = new byte[Integer.SIZE]; // max bit width for int + this.packPadBuf = new int[8]; + } + + @Override + public void writeFloat(float v) { + vectorBuffer[bufferCount++] = v; + totalCount++; + if (bufferCount == vectorSize) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + } + + private void encodeAndFlushVector(int vectorLen) { + // Use cached presets after the sampling phase, full search before + AlpEncoderDecoder.EncodingParams params; + if (cachedPresets != null) { + params = AlpEncoderDecoder.findBestFloatParamsWithPresets(vectorBuffer, 0, vectorLen, cachedPresets); + } else { + params = AlpEncoderDecoder.findBestFloatParams(vectorBuffer, 0, vectorLen); + // Collect one sample every rowgroupSampleJump vectors so that samples are + // evenly distributed across the rowgroup (matching C++ AlpSampler spacing). + if (vectorsProcessed % rowgroupSampleJump == 0 + && rowgroupSamples.size() < SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + rowgroupSamples.add(Arrays.copyOf(vectorBuffer, vectorLen)); + } + } + + vectorsProcessed++; + if (cachedPresets == null && rowgroupSamples.size() >= SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + buildPresetCache(); + } + + int excIdx = 0; + + // We need a valid encoded value to fill exception slots (placeholder). + // Any non-exception value works; it gets overwritten on decode. + int placeholder = 0; + for (int i = 0; i < vectorLen; i++) { + if (!AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + placeholder = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + break; + } + } + + int minValue = Integer.MAX_VALUE; + for (int i = 0; i < vectorLen; i++) { + if (AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + excPosBuffer[excIdx] = (short) i; + excValBuffer[excIdx] = vectorBuffer[i]; + excIdx++; + encodedBuffer[i] = placeholder; + } else { + encodedBuffer[i] = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + } + if (encodedBuffer[i] < minValue) { + minValue = encodedBuffer[i]; + } + } + + // Subtract min so deltas start at 0, reducing bit width. + // The subtraction may wrap for large ranges but unsigned bits stay correct. + int maxDelta = 0; + for (int i = 0; i < vectorLen; i++) { + encodedBuffer[i] = encodedBuffer[i] - minValue; + if (Integer.compareUnsigned(encodedBuffer[i], maxDelta) > 0) { + maxDelta = encodedBuffer[i]; + } + } + + int bitWidth = AlpEncoderDecoder.bitWidthForInt(maxDelta); Review Comment: Using BytesUtils.getWidthFromMaxInt here now. ########## parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java: ########## @@ -0,0 +1,635 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.alp; + +import static org.apache.parquet.column.values.alp.AlpConstants.*; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.CapacityByteArrayOutputStream; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.bitpacking.BytePacker; +import org.apache.parquet.column.values.bitpacking.BytePackerForLong; +import org.apache.parquet.column.values.bitpacking.Packer; + +/** + * ALP (Adaptive Lossless floating-Point) values writer. + * + * <p>ALP encoding converts floating-point values to integers using decimal scaling, + * then applies Frame of Reference encoding and bit-packing. + * Values that cannot be losslessly converted are stored as exceptions. + * + * <p>Writing is incremental: values are buffered in a fixed-size vector buffer, + * and each full vector is encoded and flushed to the output stream immediately. + * On {@link #getBytes()}, any remaining partial vector is flushed, and the + * final page bytes are assembled. + * + * <p>Interleaved Page Layout: + * <pre> + * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐ + * │ Header │ Offset Array │ Vector 0 │ Vector 1 │ ... │ + * │ 7 bytes │ 4B × numVectors │ (interleaved)│ (interleaved)│ │ + * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘ + * </pre> + * + * <p>Each vector contains interleaved: + * AlpInfo(4B) + ForInfo(5B/9B) + PackedValues + ExceptionPositions + ExceptionValues + */ +public abstract class AlpValuesWriter extends ValuesWriter { + + protected final int initialCapacity; + protected final int pageSize; + protected final ByteBufferAllocator allocator; + protected final int vectorSize; + protected final int logVectorSize; + + AlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + AlpConstants.validateVectorSize(vectorSize); + this.initialCapacity = initialCapacity; + this.pageSize = pageSize; + this.allocator = allocator; + this.vectorSize = vectorSize; + this.logVectorSize = Integer.numberOfTrailingZeros(vectorSize); + } + + @Override + public Encoding getEncoding() { + return Encoding.ALP; + } + + /** Float writer. Buffers one vector at a time, encodes and flushes when full. */ + public static class FloatAlpValuesWriter extends AlpValuesWriter { + private final float[] vectorBuffer; + private int bufferCount; + private int totalCount; + private CapacityByteArrayOutputStream encodedVectors; + private final List<Integer> vectorByteSizes; + + // Preset caching: collect evenly-spaced sample vectors across the rowgroup, + // then build presets using estimated compressed size (matching C++ AlpSampler). + private int vectorsProcessed; + private int[][] cachedPresets; + private final List<float[]> rowgroupSamples; + private final int rowgroupSampleJump; + + // Reusable per-vector buffers + private final int[] encodedBuffer; + private final short[] excPosBuffer; + private final float[] excValBuffer; + private final byte[] metadataBuf; + private final byte[] packBuf; + private final int[] packPadBuf; + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { + this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE); + } + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + super(initialCapacity, pageSize, allocator, vectorSize); + this.vectorBuffer = new float[vectorSize]; + this.bufferCount = 0; + this.totalCount = 0; + this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator); + this.vectorByteSizes = new ArrayList<>(); + this.vectorsProcessed = 0; + this.cachedPresets = null; + this.rowgroupSamples = new ArrayList<>(); + // Space samples evenly: one sample every jump vectors across the rowgroup. + // Math.max(1, ...) guards against very small rowgroups or large vector sizes. + this.rowgroupSampleJump = Math.max(1, SAMPLER_ROWGROUP_SIZE / SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP / vectorSize); + // Pre-allocate reusable buffers + this.encodedBuffer = new int[vectorSize]; + this.excPosBuffer = new short[vectorSize]; + this.excValBuffer = new float[vectorSize]; + this.metadataBuf = new byte[Math.max(ALP_INFO_SIZE, FLOAT_FOR_INFO_SIZE)]; + this.packBuf = new byte[Integer.SIZE]; // max bit width for int + this.packPadBuf = new int[8]; + } + + @Override + public void writeFloat(float v) { + vectorBuffer[bufferCount++] = v; + totalCount++; + if (bufferCount == vectorSize) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + } + + private void encodeAndFlushVector(int vectorLen) { + // Use cached presets after the sampling phase, full search before + AlpEncoderDecoder.EncodingParams params; + if (cachedPresets != null) { + params = AlpEncoderDecoder.findBestFloatParamsWithPresets(vectorBuffer, 0, vectorLen, cachedPresets); + } else { + params = AlpEncoderDecoder.findBestFloatParams(vectorBuffer, 0, vectorLen); + // Collect one sample every rowgroupSampleJump vectors so that samples are + // evenly distributed across the rowgroup (matching C++ AlpSampler spacing). + if (vectorsProcessed % rowgroupSampleJump == 0 + && rowgroupSamples.size() < SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + rowgroupSamples.add(Arrays.copyOf(vectorBuffer, vectorLen)); + } + } + + vectorsProcessed++; + if (cachedPresets == null && rowgroupSamples.size() >= SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + buildPresetCache(); + } + + int excIdx = 0; + + // We need a valid encoded value to fill exception slots (placeholder). + // Any non-exception value works; it gets overwritten on decode. + int placeholder = 0; + for (int i = 0; i < vectorLen; i++) { + if (!AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + placeholder = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + break; + } + } + + int minValue = Integer.MAX_VALUE; + for (int i = 0; i < vectorLen; i++) { + if (AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + excPosBuffer[excIdx] = (short) i; + excValBuffer[excIdx] = vectorBuffer[i]; + excIdx++; + encodedBuffer[i] = placeholder; + } else { + encodedBuffer[i] = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + } + if (encodedBuffer[i] < minValue) { + minValue = encodedBuffer[i]; + } + } + + // Subtract min so deltas start at 0, reducing bit width. + // The subtraction may wrap for large ranges but unsigned bits stay correct. + int maxDelta = 0; + for (int i = 0; i < vectorLen; i++) { + encodedBuffer[i] = encodedBuffer[i] - minValue; + if (Integer.compareUnsigned(encodedBuffer[i], maxDelta) > 0) { + maxDelta = encodedBuffer[i]; + } + } + + int bitWidth = AlpEncoderDecoder.bitWidthForInt(maxDelta); + + long startSize = encodedVectors.size(); + + // AlpInfo: exponent(1) + factor(1) + numExceptions(2) — little-endian + metadataBuf[0] = (byte) params.exponent; + metadataBuf[1] = (byte) params.factor; + metadataBuf[2] = (byte) (params.numExceptions & 0xFF); + metadataBuf[3] = (byte) ((params.numExceptions >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, ALP_INFO_SIZE); + + // ForInfo: frameOfReference(4) + bitWidth(1) — little-endian + metadataBuf[0] = (byte) (minValue & 0xFF); + metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF); + metadataBuf[4] = (byte) bitWidth; + encodedVectors.write(metadataBuf, 0, FLOAT_FOR_INFO_SIZE); + + if (bitWidth > 0) { + packIntsWithBytePacker(encodedBuffer, vectorLen, bitWidth); + } + + // Exception positions then values, written as separate blocks + if (params.numExceptions > 0) { + for (int i = 0; i < params.numExceptions; i++) { + int pos = excPosBuffer[i] & 0xFFFF; + metadataBuf[0] = (byte) (pos & 0xFF); + metadataBuf[1] = (byte) ((pos >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, Short.BYTES); + } + + for (int i = 0; i < params.numExceptions; i++) { + int bits = Float.floatToRawIntBits(excValBuffer[i]); + metadataBuf[0] = (byte) (bits & 0xFF); + metadataBuf[1] = (byte) ((bits >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((bits >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((bits >>> 24) & 0xFF); + encodedVectors.write(metadataBuf, 0, Float.BYTES); + } + } + + vectorByteSizes.add((int) (encodedVectors.size() - startSize)); + } + + private void packIntsWithBytePacker(int[] values, int count, int bitWidth) { + BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth); + int numFullGroups = count / 8; + int remaining = count % 8; + + for (int g = 0; g < numFullGroups; g++) { + packer.pack8Values(values, g * 8, packBuf, 0); + encodedVectors.write(packBuf, 0, bitWidth); + } + + // Partial last group: pack 8 values (zero-padded), but only write + // ceil(count * bitWidth / 8) - alreadyWritten bytes per spec. + if (remaining > 0) { + System.arraycopy(values, numFullGroups * 8, packPadBuf, 0, remaining); + for (int i = remaining; i < 8; i++) { + packPadBuf[i] = 0; + } + packer.pack8Values(packPadBuf, 0, packBuf, 0); + int totalPackedBytes = (count * bitWidth + 7) / 8; Review Comment: Using BytesUtils.paddedByteCountFromBits now. ########## parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java: ########## @@ -0,0 +1,635 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.alp; + +import static org.apache.parquet.column.values.alp.AlpConstants.*; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.CapacityByteArrayOutputStream; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.bitpacking.BytePacker; +import org.apache.parquet.column.values.bitpacking.BytePackerForLong; +import org.apache.parquet.column.values.bitpacking.Packer; + +/** + * ALP (Adaptive Lossless floating-Point) values writer. + * + * <p>ALP encoding converts floating-point values to integers using decimal scaling, + * then applies Frame of Reference encoding and bit-packing. + * Values that cannot be losslessly converted are stored as exceptions. + * + * <p>Writing is incremental: values are buffered in a fixed-size vector buffer, + * and each full vector is encoded and flushed to the output stream immediately. + * On {@link #getBytes()}, any remaining partial vector is flushed, and the + * final page bytes are assembled. + * + * <p>Interleaved Page Layout: + * <pre> + * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐ + * │ Header │ Offset Array │ Vector 0 │ Vector 1 │ ... │ + * │ 7 bytes │ 4B × numVectors │ (interleaved)│ (interleaved)│ │ + * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘ + * </pre> + * + * <p>Each vector contains interleaved: + * AlpInfo(4B) + ForInfo(5B/9B) + PackedValues + ExceptionPositions + ExceptionValues + */ +public abstract class AlpValuesWriter extends ValuesWriter { + + protected final int initialCapacity; + protected final int pageSize; + protected final ByteBufferAllocator allocator; + protected final int vectorSize; + protected final int logVectorSize; + + AlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + AlpConstants.validateVectorSize(vectorSize); + this.initialCapacity = initialCapacity; + this.pageSize = pageSize; + this.allocator = allocator; + this.vectorSize = vectorSize; + this.logVectorSize = Integer.numberOfTrailingZeros(vectorSize); + } + + @Override + public Encoding getEncoding() { + return Encoding.ALP; + } + + /** Float writer. Buffers one vector at a time, encodes and flushes when full. */ + public static class FloatAlpValuesWriter extends AlpValuesWriter { + private final float[] vectorBuffer; + private int bufferCount; + private int totalCount; + private CapacityByteArrayOutputStream encodedVectors; + private final List<Integer> vectorByteSizes; + + // Preset caching: collect evenly-spaced sample vectors across the rowgroup, + // then build presets using estimated compressed size (matching C++ AlpSampler). + private int vectorsProcessed; + private int[][] cachedPresets; + private final List<float[]> rowgroupSamples; + private final int rowgroupSampleJump; + + // Reusable per-vector buffers + private final int[] encodedBuffer; + private final short[] excPosBuffer; + private final float[] excValBuffer; + private final byte[] metadataBuf; + private final byte[] packBuf; + private final int[] packPadBuf; + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { + this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE); + } + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + super(initialCapacity, pageSize, allocator, vectorSize); + this.vectorBuffer = new float[vectorSize]; + this.bufferCount = 0; + this.totalCount = 0; + this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator); + this.vectorByteSizes = new ArrayList<>(); + this.vectorsProcessed = 0; + this.cachedPresets = null; + this.rowgroupSamples = new ArrayList<>(); + // Space samples evenly: one sample every jump vectors across the rowgroup. + // Math.max(1, ...) guards against very small rowgroups or large vector sizes. + this.rowgroupSampleJump = Math.max(1, SAMPLER_ROWGROUP_SIZE / SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP / vectorSize); + // Pre-allocate reusable buffers + this.encodedBuffer = new int[vectorSize]; + this.excPosBuffer = new short[vectorSize]; + this.excValBuffer = new float[vectorSize]; + this.metadataBuf = new byte[Math.max(ALP_INFO_SIZE, FLOAT_FOR_INFO_SIZE)]; + this.packBuf = new byte[Integer.SIZE]; // max bit width for int + this.packPadBuf = new int[8]; + } + + @Override + public void writeFloat(float v) { + vectorBuffer[bufferCount++] = v; + totalCount++; + if (bufferCount == vectorSize) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + } + + private void encodeAndFlushVector(int vectorLen) { + // Use cached presets after the sampling phase, full search before + AlpEncoderDecoder.EncodingParams params; + if (cachedPresets != null) { + params = AlpEncoderDecoder.findBestFloatParamsWithPresets(vectorBuffer, 0, vectorLen, cachedPresets); + } else { + params = AlpEncoderDecoder.findBestFloatParams(vectorBuffer, 0, vectorLen); + // Collect one sample every rowgroupSampleJump vectors so that samples are + // evenly distributed across the rowgroup (matching C++ AlpSampler spacing). + if (vectorsProcessed % rowgroupSampleJump == 0 + && rowgroupSamples.size() < SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + rowgroupSamples.add(Arrays.copyOf(vectorBuffer, vectorLen)); + } + } + + vectorsProcessed++; + if (cachedPresets == null && rowgroupSamples.size() >= SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + buildPresetCache(); + } + + int excIdx = 0; + + // We need a valid encoded value to fill exception slots (placeholder). + // Any non-exception value works; it gets overwritten on decode. + int placeholder = 0; + for (int i = 0; i < vectorLen; i++) { + if (!AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + placeholder = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + break; + } + } + + int minValue = Integer.MAX_VALUE; + for (int i = 0; i < vectorLen; i++) { + if (AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + excPosBuffer[excIdx] = (short) i; + excValBuffer[excIdx] = vectorBuffer[i]; + excIdx++; + encodedBuffer[i] = placeholder; + } else { + encodedBuffer[i] = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + } + if (encodedBuffer[i] < minValue) { + minValue = encodedBuffer[i]; + } + } + + // Subtract min so deltas start at 0, reducing bit width. + // The subtraction may wrap for large ranges but unsigned bits stay correct. + int maxDelta = 0; + for (int i = 0; i < vectorLen; i++) { + encodedBuffer[i] = encodedBuffer[i] - minValue; + if (Integer.compareUnsigned(encodedBuffer[i], maxDelta) > 0) { + maxDelta = encodedBuffer[i]; + } + } + + int bitWidth = AlpEncoderDecoder.bitWidthForInt(maxDelta); + + long startSize = encodedVectors.size(); + + // AlpInfo: exponent(1) + factor(1) + numExceptions(2) — little-endian + metadataBuf[0] = (byte) params.exponent; + metadataBuf[1] = (byte) params.factor; + metadataBuf[2] = (byte) (params.numExceptions & 0xFF); + metadataBuf[3] = (byte) ((params.numExceptions >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, ALP_INFO_SIZE); + + // ForInfo: frameOfReference(4) + bitWidth(1) — little-endian + metadataBuf[0] = (byte) (minValue & 0xFF); + metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF); + metadataBuf[4] = (byte) bitWidth; + encodedVectors.write(metadataBuf, 0, FLOAT_FOR_INFO_SIZE); + + if (bitWidth > 0) { + packIntsWithBytePacker(encodedBuffer, vectorLen, bitWidth); + } + + // Exception positions then values, written as separate blocks + if (params.numExceptions > 0) { + for (int i = 0; i < params.numExceptions; i++) { + int pos = excPosBuffer[i] & 0xFFFF; + metadataBuf[0] = (byte) (pos & 0xFF); + metadataBuf[1] = (byte) ((pos >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, Short.BYTES); + } + + for (int i = 0; i < params.numExceptions; i++) { + int bits = Float.floatToRawIntBits(excValBuffer[i]); + metadataBuf[0] = (byte) (bits & 0xFF); + metadataBuf[1] = (byte) ((bits >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((bits >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((bits >>> 24) & 0xFF); + encodedVectors.write(metadataBuf, 0, Float.BYTES); + } + } + + vectorByteSizes.add((int) (encodedVectors.size() - startSize)); + } + + private void packIntsWithBytePacker(int[] values, int count, int bitWidth) { + BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth); + int numFullGroups = count / 8; + int remaining = count % 8; + + for (int g = 0; g < numFullGroups; g++) { + packer.pack8Values(values, g * 8, packBuf, 0); + encodedVectors.write(packBuf, 0, bitWidth); + } + + // Partial last group: pack 8 values (zero-padded), but only write + // ceil(count * bitWidth / 8) - alreadyWritten bytes per spec. + if (remaining > 0) { + System.arraycopy(values, numFullGroups * 8, packPadBuf, 0, remaining); + for (int i = remaining; i < 8; i++) { + packPadBuf[i] = 0; + } + packer.pack8Values(packPadBuf, 0, packBuf, 0); + int totalPackedBytes = (count * bitWidth + 7) / 8; + int alreadyWritten = numFullGroups * bitWidth; + encodedVectors.write(packBuf, 0, totalPackedBytes - alreadyWritten); + } + } + + private void buildPresetCache() { + // For each sampled vector, find the best (e,f) using estimated compressed size, + // then rank combos by how many samples they win. Take the top MAX_PRESET_COMBINATIONS. Review Comment: The formatting pass added spacing between the blocks here. Shout if a specific spot still reads cramped. ########## parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java: ########## @@ -0,0 +1,635 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.alp; + +import static org.apache.parquet.column.values.alp.AlpConstants.*; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.CapacityByteArrayOutputStream; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.bitpacking.BytePacker; +import org.apache.parquet.column.values.bitpacking.BytePackerForLong; +import org.apache.parquet.column.values.bitpacking.Packer; + +/** + * ALP (Adaptive Lossless floating-Point) values writer. + * + * <p>ALP encoding converts floating-point values to integers using decimal scaling, + * then applies Frame of Reference encoding and bit-packing. + * Values that cannot be losslessly converted are stored as exceptions. + * + * <p>Writing is incremental: values are buffered in a fixed-size vector buffer, + * and each full vector is encoded and flushed to the output stream immediately. + * On {@link #getBytes()}, any remaining partial vector is flushed, and the + * final page bytes are assembled. + * + * <p>Interleaved Page Layout: + * <pre> + * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐ + * │ Header │ Offset Array │ Vector 0 │ Vector 1 │ ... │ + * │ 7 bytes │ 4B × numVectors │ (interleaved)│ (interleaved)│ │ + * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘ + * </pre> + * + * <p>Each vector contains interleaved: + * AlpInfo(4B) + ForInfo(5B/9B) + PackedValues + ExceptionPositions + ExceptionValues + */ +public abstract class AlpValuesWriter extends ValuesWriter { + + protected final int initialCapacity; + protected final int pageSize; + protected final ByteBufferAllocator allocator; + protected final int vectorSize; + protected final int logVectorSize; + + AlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + AlpConstants.validateVectorSize(vectorSize); + this.initialCapacity = initialCapacity; + this.pageSize = pageSize; + this.allocator = allocator; + this.vectorSize = vectorSize; + this.logVectorSize = Integer.numberOfTrailingZeros(vectorSize); + } + + @Override + public Encoding getEncoding() { + return Encoding.ALP; + } + + /** Float writer. Buffers one vector at a time, encodes and flushes when full. */ + public static class FloatAlpValuesWriter extends AlpValuesWriter { + private final float[] vectorBuffer; + private int bufferCount; + private int totalCount; + private CapacityByteArrayOutputStream encodedVectors; + private final List<Integer> vectorByteSizes; + + // Preset caching: collect evenly-spaced sample vectors across the rowgroup, + // then build presets using estimated compressed size (matching C++ AlpSampler). + private int vectorsProcessed; + private int[][] cachedPresets; + private final List<float[]> rowgroupSamples; + private final int rowgroupSampleJump; + + // Reusable per-vector buffers + private final int[] encodedBuffer; + private final short[] excPosBuffer; + private final float[] excValBuffer; + private final byte[] metadataBuf; + private final byte[] packBuf; + private final int[] packPadBuf; + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { + this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE); + } + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + super(initialCapacity, pageSize, allocator, vectorSize); + this.vectorBuffer = new float[vectorSize]; + this.bufferCount = 0; + this.totalCount = 0; + this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator); + this.vectorByteSizes = new ArrayList<>(); + this.vectorsProcessed = 0; + this.cachedPresets = null; + this.rowgroupSamples = new ArrayList<>(); + // Space samples evenly: one sample every jump vectors across the rowgroup. + // Math.max(1, ...) guards against very small rowgroups or large vector sizes. + this.rowgroupSampleJump = Math.max(1, SAMPLER_ROWGROUP_SIZE / SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP / vectorSize); + // Pre-allocate reusable buffers + this.encodedBuffer = new int[vectorSize]; + this.excPosBuffer = new short[vectorSize]; + this.excValBuffer = new float[vectorSize]; + this.metadataBuf = new byte[Math.max(ALP_INFO_SIZE, FLOAT_FOR_INFO_SIZE)]; + this.packBuf = new byte[Integer.SIZE]; // max bit width for int + this.packPadBuf = new int[8]; + } + + @Override + public void writeFloat(float v) { + vectorBuffer[bufferCount++] = v; + totalCount++; + if (bufferCount == vectorSize) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + } + + private void encodeAndFlushVector(int vectorLen) { + // Use cached presets after the sampling phase, full search before + AlpEncoderDecoder.EncodingParams params; + if (cachedPresets != null) { + params = AlpEncoderDecoder.findBestFloatParamsWithPresets(vectorBuffer, 0, vectorLen, cachedPresets); + } else { + params = AlpEncoderDecoder.findBestFloatParams(vectorBuffer, 0, vectorLen); + // Collect one sample every rowgroupSampleJump vectors so that samples are + // evenly distributed across the rowgroup (matching C++ AlpSampler spacing). + if (vectorsProcessed % rowgroupSampleJump == 0 + && rowgroupSamples.size() < SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + rowgroupSamples.add(Arrays.copyOf(vectorBuffer, vectorLen)); + } + } + + vectorsProcessed++; + if (cachedPresets == null && rowgroupSamples.size() >= SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + buildPresetCache(); + } + + int excIdx = 0; + + // We need a valid encoded value to fill exception slots (placeholder). + // Any non-exception value works; it gets overwritten on decode. + int placeholder = 0; + for (int i = 0; i < vectorLen; i++) { + if (!AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + placeholder = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + break; + } + } + + int minValue = Integer.MAX_VALUE; + for (int i = 0; i < vectorLen; i++) { + if (AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + excPosBuffer[excIdx] = (short) i; + excValBuffer[excIdx] = vectorBuffer[i]; + excIdx++; + encodedBuffer[i] = placeholder; + } else { + encodedBuffer[i] = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + } + if (encodedBuffer[i] < minValue) { + minValue = encodedBuffer[i]; + } + } + + // Subtract min so deltas start at 0, reducing bit width. + // The subtraction may wrap for large ranges but unsigned bits stay correct. + int maxDelta = 0; + for (int i = 0; i < vectorLen; i++) { + encodedBuffer[i] = encodedBuffer[i] - minValue; + if (Integer.compareUnsigned(encodedBuffer[i], maxDelta) > 0) { + maxDelta = encodedBuffer[i]; + } + } + + int bitWidth = AlpEncoderDecoder.bitWidthForInt(maxDelta); + + long startSize = encodedVectors.size(); + + // AlpInfo: exponent(1) + factor(1) + numExceptions(2) — little-endian + metadataBuf[0] = (byte) params.exponent; + metadataBuf[1] = (byte) params.factor; + metadataBuf[2] = (byte) (params.numExceptions & 0xFF); + metadataBuf[3] = (byte) ((params.numExceptions >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, ALP_INFO_SIZE); + + // ForInfo: frameOfReference(4) + bitWidth(1) — little-endian + metadataBuf[0] = (byte) (minValue & 0xFF); + metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF); + metadataBuf[4] = (byte) bitWidth; + encodedVectors.write(metadataBuf, 0, FLOAT_FOR_INFO_SIZE); + + if (bitWidth > 0) { + packIntsWithBytePacker(encodedBuffer, vectorLen, bitWidth); + } + + // Exception positions then values, written as separate blocks + if (params.numExceptions > 0) { + for (int i = 0; i < params.numExceptions; i++) { + int pos = excPosBuffer[i] & 0xFFFF; + metadataBuf[0] = (byte) (pos & 0xFF); + metadataBuf[1] = (byte) ((pos >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, Short.BYTES); + } + + for (int i = 0; i < params.numExceptions; i++) { + int bits = Float.floatToRawIntBits(excValBuffer[i]); + metadataBuf[0] = (byte) (bits & 0xFF); + metadataBuf[1] = (byte) ((bits >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((bits >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((bits >>> 24) & 0xFF); + encodedVectors.write(metadataBuf, 0, Float.BYTES); + } + } + + vectorByteSizes.add((int) (encodedVectors.size() - startSize)); + } + + private void packIntsWithBytePacker(int[] values, int count, int bitWidth) { + BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth); + int numFullGroups = count / 8; + int remaining = count % 8; + + for (int g = 0; g < numFullGroups; g++) { + packer.pack8Values(values, g * 8, packBuf, 0); + encodedVectors.write(packBuf, 0, bitWidth); + } + + // Partial last group: pack 8 values (zero-padded), but only write + // ceil(count * bitWidth / 8) - alreadyWritten bytes per spec. + if (remaining > 0) { + System.arraycopy(values, numFullGroups * 8, packPadBuf, 0, remaining); + for (int i = remaining; i < 8; i++) { + packPadBuf[i] = 0; + } + packer.pack8Values(packPadBuf, 0, packBuf, 0); + int totalPackedBytes = (count * bitWidth + 7) / 8; + int alreadyWritten = numFullGroups * bitWidth; + encodedVectors.write(packBuf, 0, totalPackedBytes - alreadyWritten); + } + } + + private void buildPresetCache() { Review Comment: Storing the sampled (e,f) winners as we sample now, and buildPresetCache just does the frequency count over them, no re-search. ########## parquet-column/src/main/java/org/apache/parquet/column/values/alp/AlpValuesWriter.java: ########## @@ -0,0 +1,635 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.alp; + +import static org.apache.parquet.column.values.alp.AlpConstants.*; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.CapacityByteArrayOutputStream; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.bitpacking.BytePacker; +import org.apache.parquet.column.values.bitpacking.BytePackerForLong; +import org.apache.parquet.column.values.bitpacking.Packer; + +/** + * ALP (Adaptive Lossless floating-Point) values writer. + * + * <p>ALP encoding converts floating-point values to integers using decimal scaling, + * then applies Frame of Reference encoding and bit-packing. + * Values that cannot be losslessly converted are stored as exceptions. + * + * <p>Writing is incremental: values are buffered in a fixed-size vector buffer, + * and each full vector is encoded and flushed to the output stream immediately. + * On {@link #getBytes()}, any remaining partial vector is flushed, and the + * final page bytes are assembled. + * + * <p>Interleaved Page Layout: + * <pre> + * ┌─────────┬──────────────────────┬──────────────┬──────────────┬─────┐ + * │ Header │ Offset Array │ Vector 0 │ Vector 1 │ ... │ + * │ 7 bytes │ 4B × numVectors │ (interleaved)│ (interleaved)│ │ + * └─────────┴──────────────────────┴──────────────┴──────────────┴─────┘ + * </pre> + * + * <p>Each vector contains interleaved: + * AlpInfo(4B) + ForInfo(5B/9B) + PackedValues + ExceptionPositions + ExceptionValues + */ +public abstract class AlpValuesWriter extends ValuesWriter { + + protected final int initialCapacity; + protected final int pageSize; + protected final ByteBufferAllocator allocator; + protected final int vectorSize; + protected final int logVectorSize; + + AlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + AlpConstants.validateVectorSize(vectorSize); + this.initialCapacity = initialCapacity; + this.pageSize = pageSize; + this.allocator = allocator; + this.vectorSize = vectorSize; + this.logVectorSize = Integer.numberOfTrailingZeros(vectorSize); + } + + @Override + public Encoding getEncoding() { + return Encoding.ALP; + } + + /** Float writer. Buffers one vector at a time, encodes and flushes when full. */ + public static class FloatAlpValuesWriter extends AlpValuesWriter { + private final float[] vectorBuffer; + private int bufferCount; + private int totalCount; + private CapacityByteArrayOutputStream encodedVectors; + private final List<Integer> vectorByteSizes; + + // Preset caching: collect evenly-spaced sample vectors across the rowgroup, + // then build presets using estimated compressed size (matching C++ AlpSampler). + private int vectorsProcessed; + private int[][] cachedPresets; + private final List<float[]> rowgroupSamples; + private final int rowgroupSampleJump; + + // Reusable per-vector buffers + private final int[] encodedBuffer; + private final short[] excPosBuffer; + private final float[] excValBuffer; + private final byte[] metadataBuf; + private final byte[] packBuf; + private final int[] packPadBuf; + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { + this(initialCapacity, pageSize, allocator, DEFAULT_VECTOR_SIZE); + } + + public FloatAlpValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator, int vectorSize) { + super(initialCapacity, pageSize, allocator, vectorSize); + this.vectorBuffer = new float[vectorSize]; + this.bufferCount = 0; + this.totalCount = 0; + this.encodedVectors = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator); + this.vectorByteSizes = new ArrayList<>(); + this.vectorsProcessed = 0; + this.cachedPresets = null; + this.rowgroupSamples = new ArrayList<>(); + // Space samples evenly: one sample every jump vectors across the rowgroup. + // Math.max(1, ...) guards against very small rowgroups or large vector sizes. + this.rowgroupSampleJump = Math.max(1, SAMPLER_ROWGROUP_SIZE / SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP / vectorSize); + // Pre-allocate reusable buffers + this.encodedBuffer = new int[vectorSize]; + this.excPosBuffer = new short[vectorSize]; + this.excValBuffer = new float[vectorSize]; + this.metadataBuf = new byte[Math.max(ALP_INFO_SIZE, FLOAT_FOR_INFO_SIZE)]; + this.packBuf = new byte[Integer.SIZE]; // max bit width for int + this.packPadBuf = new int[8]; + } + + @Override + public void writeFloat(float v) { + vectorBuffer[bufferCount++] = v; + totalCount++; + if (bufferCount == vectorSize) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + } + + private void encodeAndFlushVector(int vectorLen) { + // Use cached presets after the sampling phase, full search before + AlpEncoderDecoder.EncodingParams params; + if (cachedPresets != null) { + params = AlpEncoderDecoder.findBestFloatParamsWithPresets(vectorBuffer, 0, vectorLen, cachedPresets); + } else { + params = AlpEncoderDecoder.findBestFloatParams(vectorBuffer, 0, vectorLen); + // Collect one sample every rowgroupSampleJump vectors so that samples are + // evenly distributed across the rowgroup (matching C++ AlpSampler spacing). + if (vectorsProcessed % rowgroupSampleJump == 0 + && rowgroupSamples.size() < SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + rowgroupSamples.add(Arrays.copyOf(vectorBuffer, vectorLen)); + } + } + + vectorsProcessed++; + if (cachedPresets == null && rowgroupSamples.size() >= SAMPLER_SAMPLE_VECTORS_PER_ROWGROUP) { + buildPresetCache(); + } + + int excIdx = 0; + + // We need a valid encoded value to fill exception slots (placeholder). + // Any non-exception value works; it gets overwritten on decode. + int placeholder = 0; + for (int i = 0; i < vectorLen; i++) { + if (!AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + placeholder = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + break; + } + } + + int minValue = Integer.MAX_VALUE; + for (int i = 0; i < vectorLen; i++) { + if (AlpEncoderDecoder.isFloatException(vectorBuffer[i], params.exponent, params.factor)) { + excPosBuffer[excIdx] = (short) i; + excValBuffer[excIdx] = vectorBuffer[i]; + excIdx++; + encodedBuffer[i] = placeholder; + } else { + encodedBuffer[i] = AlpEncoderDecoder.encodeFloat(vectorBuffer[i], params.exponent, params.factor); + } + if (encodedBuffer[i] < minValue) { + minValue = encodedBuffer[i]; + } + } + + // Subtract min so deltas start at 0, reducing bit width. + // The subtraction may wrap for large ranges but unsigned bits stay correct. + int maxDelta = 0; + for (int i = 0; i < vectorLen; i++) { + encodedBuffer[i] = encodedBuffer[i] - minValue; + if (Integer.compareUnsigned(encodedBuffer[i], maxDelta) > 0) { + maxDelta = encodedBuffer[i]; + } + } + + int bitWidth = AlpEncoderDecoder.bitWidthForInt(maxDelta); + + long startSize = encodedVectors.size(); + + // AlpInfo: exponent(1) + factor(1) + numExceptions(2) — little-endian + metadataBuf[0] = (byte) params.exponent; + metadataBuf[1] = (byte) params.factor; + metadataBuf[2] = (byte) (params.numExceptions & 0xFF); + metadataBuf[3] = (byte) ((params.numExceptions >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, ALP_INFO_SIZE); + + // ForInfo: frameOfReference(4) + bitWidth(1) — little-endian + metadataBuf[0] = (byte) (minValue & 0xFF); + metadataBuf[1] = (byte) ((minValue >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((minValue >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((minValue >>> 24) & 0xFF); + metadataBuf[4] = (byte) bitWidth; + encodedVectors.write(metadataBuf, 0, FLOAT_FOR_INFO_SIZE); + + if (bitWidth > 0) { + packIntsWithBytePacker(encodedBuffer, vectorLen, bitWidth); + } + + // Exception positions then values, written as separate blocks + if (params.numExceptions > 0) { + for (int i = 0; i < params.numExceptions; i++) { + int pos = excPosBuffer[i] & 0xFFFF; + metadataBuf[0] = (byte) (pos & 0xFF); + metadataBuf[1] = (byte) ((pos >>> 8) & 0xFF); + encodedVectors.write(metadataBuf, 0, Short.BYTES); + } + + for (int i = 0; i < params.numExceptions; i++) { + int bits = Float.floatToRawIntBits(excValBuffer[i]); + metadataBuf[0] = (byte) (bits & 0xFF); + metadataBuf[1] = (byte) ((bits >>> 8) & 0xFF); + metadataBuf[2] = (byte) ((bits >>> 16) & 0xFF); + metadataBuf[3] = (byte) ((bits >>> 24) & 0xFF); + encodedVectors.write(metadataBuf, 0, Float.BYTES); + } + } + + vectorByteSizes.add((int) (encodedVectors.size() - startSize)); + } + + private void packIntsWithBytePacker(int[] values, int count, int bitWidth) { + BytePacker packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth); + int numFullGroups = count / 8; + int remaining = count % 8; + + for (int g = 0; g < numFullGroups; g++) { + packer.pack8Values(values, g * 8, packBuf, 0); + encodedVectors.write(packBuf, 0, bitWidth); + } + + // Partial last group: pack 8 values (zero-padded), but only write + // ceil(count * bitWidth / 8) - alreadyWritten bytes per spec. + if (remaining > 0) { + System.arraycopy(values, numFullGroups * 8, packPadBuf, 0, remaining); + for (int i = remaining; i < 8; i++) { + packPadBuf[i] = 0; + } + packer.pack8Values(packPadBuf, 0, packBuf, 0); + int totalPackedBytes = (count * bitWidth + 7) / 8; + int alreadyWritten = numFullGroups * bitWidth; + encodedVectors.write(packBuf, 0, totalPackedBytes - alreadyWritten); + } + } + + private void buildPresetCache() { + // For each sampled vector, find the best (e,f) using estimated compressed size, + // then rank combos by how many samples they win. Take the top MAX_PRESET_COMBINATIONS. + Map<Long, Integer> comboCounts = new HashMap<>(); + for (float[] sample : rowgroupSamples) { + AlpEncoderDecoder.EncodingParams best = + AlpEncoderDecoder.findBestFloatParams(sample, 0, sample.length); + long key = ((long) best.exponent << Integer.SIZE) | best.factor; + comboCounts.merge(key, 1, Integer::sum); + } + List<Map.Entry<Long, Integer>> sorted = new ArrayList<>(comboCounts.entrySet()); + sorted.sort(Comparator.<Map.Entry<Long, Integer>, Integer>comparing(Map.Entry::getValue) + .reversed()); + int numPresets = Math.min(sorted.size(), MAX_PRESET_COMBINATIONS); + cachedPresets = new int[numPresets][2]; + for (int i = 0; i < numPresets; i++) { + long key = sorted.get(i).getKey(); + cachedPresets[i][0] = (int) (key >>> Integer.SIZE); // exponent + cachedPresets[i][1] = (int) key; // factor + } + rowgroupSamples.clear(); + } + + @Override + public long getBufferedSize() { + // Encoded vectors so far + estimate for buffered values. + // 3 bytes/value ≈ ALP_INFO_SIZE(4) + FLOAT_FOR_INFO_SIZE(5) amortized over vectorSize(1024), + // plus ~1 bit/value for packed data at typical bit widths — rounds up to ~3 bytes. + return encodedVectors.size() + (long) bufferCount * 3; + } + + @Override + public BytesInput getBytes() { + if (bufferCount > 0) { + encodeAndFlushVector(bufferCount); + bufferCount = 0; + } + + int numVectors = vectorByteSizes.size(); + + // Assemble: [header 7B] [offset array 4B*N] [vector data...] + ByteBuffer header = ByteBuffer.allocate(ALP_HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + header.put((byte) ALP_COMPRESSION_MODE); + header.put((byte) ALP_INTEGER_ENCODING_FOR); + header.put((byte) logVectorSize); + header.putInt(totalCount); + + if (totalCount == 0) { + return BytesInput.from(header.array()); + } + + int offsetArraySize = numVectors * Integer.BYTES; + ByteBuffer offsets = ByteBuffer.allocate(offsetArraySize).order(ByteOrder.LITTLE_ENDIAN); + int currentOffset = offsetArraySize; + for (int v = 0; v < numVectors; v++) { + offsets.putInt(currentOffset); + currentOffset += vectorByteSizes.get(v); + } + + return BytesInput.concat( + BytesInput.from(header.array()), BytesInput.from(offsets.array()), BytesInput.from(encodedVectors)); + } + + @Override + public void reset() { + bufferCount = 0; + totalCount = 0; + encodedVectors.reset(); + vectorByteSizes.clear(); + vectorsProcessed = 0; + // Preserve cachedPresets across page boundaries: the preset (exponent, factor) pairs + // reflect the column's data distribution and remain valid for subsequent pages. + // Clearing them on every reset forces a full parameter search from scratch on each page, + // which is the main source of write overhead. Only reset the sampling state. + rowgroupSamples.clear(); + } + + @Override + public void close() { + encodedVectors.close(); + } + + @Override + public long getAllocatedSize() { + return (long) vectorBuffer.length * Float.BYTES + encodedVectors.getCapacity(); + } + + @Override + public String memUsageString(String prefix) { + return String.format( + "%s FloatAlpValuesWriter %d values, %d bytes allocated", prefix, totalCount, getAllocatedSize()); + } + } + + /** Double writer. Same structure as FloatAlpValuesWriter but uses longs. */ Review Comment: Applied all of these to the double writer path too. ########## parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java: ########## @@ -132,6 +135,8 @@ public static WriterVersion fromString(String name) { private final int pageRowCountLimit; private final boolean pageWriteChecksumEnabled; private final ColumnProperty<ByteStreamSplitMode> byteStreamSplitEnabled; + private final ColumnProperty<Boolean> alpEnabled; Review Comment: Bundled into an AlpConfig, so ParquetProperties now carries one ColumnProperty<AlpConfig> instead of the two separate properties. The public setters and getters are unchanged. ########## parquet-column/src/test/java/org/apache/parquet/column/values/alp/AlpEncoderDecoderTest.java: ########## @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.alp; + +import static org.junit.Assert.*; + +import org.junit.Test; + +/** + * Tests for the core ALP encoder/decoder logic. + */ +public class AlpEncoderDecoderTest { + + // ========== Float Encoding/Decoding Tests ========== + + @Test + public void testFloatRoundTrip() { + float[] testValues = {0.0f, 1.0f, -1.0f, 3.14159f, 100.5f, 0.001f, 1234567.0f}; + + for (float value : testValues) { + for (int exponent = 0; exponent <= AlpConstants.FLOAT_MAX_EXPONENT; exponent++) { + for (int factor = 0; factor <= exponent; factor++) { + if (!AlpEncoderDecoder.isFloatException(value, exponent, factor)) { + int encoded = AlpEncoderDecoder.encodeFloat(value, exponent, factor); + float decoded = AlpEncoderDecoder.decodeFloat(encoded, exponent, factor); + assertEquals( + "Round-trip failed for value=" + value + ", exponent=" + exponent + ", factor=" + + factor, + Float.floatToRawIntBits(value), + Float.floatToRawIntBits(decoded)); + } + } + } + } + } + + @Test + public void testFloatExceptionDetection() { + assertTrue("NaN should be an exception", AlpEncoderDecoder.isFloatException(Float.NaN)); + assertTrue( + "Positive infinity should be an exception", + AlpEncoderDecoder.isFloatException(Float.POSITIVE_INFINITY)); + assertTrue( + "Negative infinity should be an exception", + AlpEncoderDecoder.isFloatException(Float.NEGATIVE_INFINITY)); + assertTrue("Negative zero should be an exception", AlpEncoderDecoder.isFloatException(-0.0f)); + + assertFalse("1.0f should not be a basic exception", AlpEncoderDecoder.isFloatException(1.0f)); + assertFalse("0.0f should not be a basic exception", AlpEncoderDecoder.isFloatException(0.0f)); + } + + @Test + public void testFloatEncoding() { + assertEquals(123, AlpEncoderDecoder.encodeFloat(1.23f, 2, 0)); + assertEquals(123, AlpEncoderDecoder.encodeFloat(12.3f, 2, 1)); + assertEquals(0, AlpEncoderDecoder.encodeFloat(0.0f, 5, 0)); + } + + @Test + public void testFloatDecoding() { + assertEquals(1.23f, AlpEncoderDecoder.decodeFloat(123, 2, 0), 1e-6f); + assertEquals(12.3f, AlpEncoderDecoder.decodeFloat(123, 2, 1), 1e-6f); + assertEquals(0.0f, AlpEncoderDecoder.decodeFloat(0, 5, 0), 0.0f); + } + + @Test + public void testFloatEncodeRounding() { + // Verify rounding behavior (magic number trick rounds to nearest) + assertEquals(5, AlpEncoderDecoder.encodeFloat(5.4f, 0, 0)); + assertEquals(6, AlpEncoderDecoder.encodeFloat(5.6f, 0, 0)); + assertEquals(-5, AlpEncoderDecoder.encodeFloat(-5.4f, 0, 0)); + assertEquals(-6, AlpEncoderDecoder.encodeFloat(-5.6f, 0, 0)); + assertEquals(0, AlpEncoderDecoder.encodeFloat(0.0f, 0, 0)); + } + + @Test + public void testFloatEncodeDecodeWithFactor() { + // Verify that encode/decode with non-zero factor works correctly. + // The key correctness property: encode uses value * POW10[e] * POW10_NEGATIVE[f], + // and decode uses encoded * POW10[f] * POW10_NEGATIVE[e]. + float value = 12.3f; + int encoded = AlpEncoderDecoder.encodeFloat(value, 2, 1); + assertEquals(123, encoded); // 12.3 * 100 * 0.1 = 123 + float decoded = AlpEncoderDecoder.decodeFloat(encoded, 2, 1); + assertEquals(Float.floatToRawIntBits(value), Float.floatToRawIntBits(decoded)); + } + + // ========== Double Encoding/Decoding Tests ========== + + @Test + public void testDoubleRoundTrip() { + double[] testValues = {0.0, 1.0, -1.0, 3.14159265358979, 100.5, 0.001, 12345678901234.0}; + + for (double value : testValues) { + for (int exponent = 0; exponent <= Math.min(AlpConstants.DOUBLE_MAX_EXPONENT, 10); exponent++) { + for (int factor = 0; factor <= exponent; factor++) { + if (!AlpEncoderDecoder.isDoubleException(value, exponent, factor)) { + long encoded = AlpEncoderDecoder.encodeDouble(value, exponent, factor); + double decoded = AlpEncoderDecoder.decodeDouble(encoded, exponent, factor); + assertEquals( + "Round-trip failed for value=" + value + ", exponent=" + exponent + ", factor=" + + factor, + Double.doubleToRawLongBits(value), + Double.doubleToRawLongBits(decoded)); + } + } + } + } + } + + @Test + public void testDoubleExceptionDetection() { + assertTrue("NaN should be an exception", AlpEncoderDecoder.isDoubleException(Double.NaN)); + assertTrue( + "Positive infinity should be an exception", + AlpEncoderDecoder.isDoubleException(Double.POSITIVE_INFINITY)); + assertTrue( + "Negative infinity should be an exception", + AlpEncoderDecoder.isDoubleException(Double.NEGATIVE_INFINITY)); + assertTrue("Negative zero should be an exception", AlpEncoderDecoder.isDoubleException(-0.0)); + + assertFalse("1.0 should not be a basic exception", AlpEncoderDecoder.isDoubleException(1.0)); + assertFalse("0.0 should not be a basic exception", AlpEncoderDecoder.isDoubleException(0.0)); + } + + @Test + public void testDoubleEncoding() { + assertEquals(123L, AlpEncoderDecoder.encodeDouble(1.23, 2, 0)); + assertEquals(123L, AlpEncoderDecoder.encodeDouble(12.3, 2, 1)); + assertEquals(0L, AlpEncoderDecoder.encodeDouble(0.0, 5, 0)); + } + + @Test + public void testDoubleDecoding() { + assertEquals(1.23, AlpEncoderDecoder.decodeDouble(123, 2, 0), 1e-10); + assertEquals(12.3, AlpEncoderDecoder.decodeDouble(123, 2, 1), 1e-10); + assertEquals(0.0, AlpEncoderDecoder.decodeDouble(0, 5, 0), 0.0); + } + + @Test + public void testDoubleEncodeRounding() { + // Verify rounding behavior (magic number trick rounds to nearest) + assertEquals(5L, AlpEncoderDecoder.encodeDouble(5.4, 0, 0)); + assertEquals(6L, AlpEncoderDecoder.encodeDouble(5.6, 0, 0)); + assertEquals(-5L, AlpEncoderDecoder.encodeDouble(-5.4, 0, 0)); + assertEquals(-6L, AlpEncoderDecoder.encodeDouble(-5.6, 0, 0)); + assertEquals(0L, AlpEncoderDecoder.encodeDouble(0.0, 0, 0)); + } + + @Test + public void testDoubleEncodeDecodeWithFactor() { + // Verify that encode/decode with non-zero factor works correctly. + // The key correctness property: encode uses value * POW10[e] * POW10_NEGATIVE[f], + // and decode uses encoded * POW10[f] * POW10_NEGATIVE[e]. + double value = 12.3; + long encoded = AlpEncoderDecoder.encodeDouble(value, 2, 1); + assertEquals(123L, encoded); // 12.3 * 100 * 0.1 = 123 + double decoded = AlpEncoderDecoder.decodeDouble(encoded, 2, 1); + assertEquals(Double.doubleToRawLongBits(value), Double.doubleToRawLongBits(decoded)); + } + + @Test + public void testDoubleEncodeDecodeArithmeticOrder() { + // This test verifies that the exact order of operations in encode/decode + // is critical for IEEE 754 correctness. The encode uses + // fastRound(value * POW10[e] * POW10_NEGATIVE[f]) and decode uses + // (encoded * POW10[f] * POW10_NEGATIVE[e]), both as single expressions. + // Splitting the multiplies or reordering changes rounding. + double[] testValues = {0.123456789, 1.23456789, 12.3456789, 123.456789, 1234.56789}; + for (double value : testValues) { + for (int e = 0; e <= 10; e++) { + for (int f = 0; f <= e; f++) { + if (!AlpEncoderDecoder.isDoubleException(value, e, f)) { + long encoded = AlpEncoderDecoder.encodeDouble(value, e, f); + double decoded = AlpEncoderDecoder.decodeDouble(encoded, e, f); + assertEquals( + "Roundtrip failed for " + value + " (e=" + e + ", f=" + f + ")", + Double.doubleToRawLongBits(value), + Double.doubleToRawLongBits(decoded)); + } + } + } + } + } + + // ========== Bit Width Tests (renamed methods) ========== + + @Test + public void testBitWidthForInt() { + assertEquals(0, AlpEncoderDecoder.bitWidthForInt(0)); + assertEquals(1, AlpEncoderDecoder.bitWidthForInt(1)); + assertEquals(2, AlpEncoderDecoder.bitWidthForInt(2)); + assertEquals(2, AlpEncoderDecoder.bitWidthForInt(3)); + assertEquals(3, AlpEncoderDecoder.bitWidthForInt(4)); + assertEquals(8, AlpEncoderDecoder.bitWidthForInt(255)); + assertEquals(9, AlpEncoderDecoder.bitWidthForInt(256)); + assertEquals(16, AlpEncoderDecoder.bitWidthForInt(65535)); + assertEquals(31, AlpEncoderDecoder.bitWidthForInt(Integer.MAX_VALUE)); + } + + @Test + public void testBitWidthForLong() { + assertEquals(0, AlpEncoderDecoder.bitWidthForLong(0L)); + assertEquals(1, AlpEncoderDecoder.bitWidthForLong(1L)); + assertEquals(2, AlpEncoderDecoder.bitWidthForLong(2L)); + assertEquals(2, AlpEncoderDecoder.bitWidthForLong(3L)); + assertEquals(3, AlpEncoderDecoder.bitWidthForLong(4L)); + assertEquals(8, AlpEncoderDecoder.bitWidthForLong(255L)); + assertEquals(9, AlpEncoderDecoder.bitWidthForLong(256L)); + assertEquals(16, AlpEncoderDecoder.bitWidthForLong(65535L)); + assertEquals(31, AlpEncoderDecoder.bitWidthForLong((long) Integer.MAX_VALUE)); + assertEquals(63, AlpEncoderDecoder.bitWidthForLong(Long.MAX_VALUE)); + } + + // ========== Best Parameters Tests ========== + + @Test + public void testFindBestFloatParams() { + float[] values = {1.23f, 4.56f, 7.89f, 10.11f, 12.13f}; + AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestFloatParams(values, 0, values.length); + + assertNotNull(params); + assertTrue(params.exponent >= 0 && params.exponent <= AlpConstants.FLOAT_MAX_EXPONENT); + assertTrue(params.factor >= 0 && params.factor <= params.exponent); + + for (float v : values) { + if (!AlpEncoderDecoder.isFloatException(v, params.exponent, params.factor)) { + int encoded = AlpEncoderDecoder.encodeFloat(v, params.exponent, params.factor); + float decoded = AlpEncoderDecoder.decodeFloat(encoded, params.exponent, params.factor); + assertEquals(Float.floatToRawIntBits(v), Float.floatToRawIntBits(decoded)); + } + } + } + + @Test + public void testFindBestFloatParamsWithAllExceptions() { + float[] values = {Float.NaN, Float.NaN, Float.NaN}; + AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestFloatParams(values, 0, values.length); + + assertNotNull(params); + assertEquals(values.length, params.numExceptions); + } + + @Test + public void testFindBestDoubleParams() { + double[] values = {1.23, 4.56, 7.89, 10.11, 12.13}; + AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestDoubleParams(values, 0, values.length); + + assertNotNull(params); + assertTrue(params.exponent >= 0 && params.exponent <= AlpConstants.DOUBLE_MAX_EXPONENT); + assertTrue(params.factor >= 0 && params.factor <= params.exponent); + + for (double v : values) { + if (!AlpEncoderDecoder.isDoubleException(v, params.exponent, params.factor)) { + long encoded = AlpEncoderDecoder.encodeDouble(v, params.exponent, params.factor); + double decoded = AlpEncoderDecoder.decodeDouble(encoded, params.exponent, params.factor); + assertEquals(Double.doubleToRawLongBits(v), Double.doubleToRawLongBits(decoded)); + } + } + } + + @Test + public void testFindBestDoubleParamsWithAllExceptions() { + double[] values = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}; + AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestDoubleParams(values, 0, values.length); + + assertNotNull(params); + assertEquals(values.length, params.numExceptions); + } + + @Test + public void testFindBestParamsWithOffset() { + float[] values = {Float.NaN, Float.NaN, 1.23f, 4.56f, 7.89f, Float.NaN}; + AlpEncoderDecoder.EncodingParams params = AlpEncoderDecoder.findBestFloatParams(values, 2, 3); + + assertNotNull(params); + assertEquals(0, params.numExceptions); + } + + // ========== Preset-Based Parameter Search Tests ========== + + @Test + public void testFindBestFloatParamsWithPresets() { + float[] values = {1.23f, 4.56f, 7.89f, 10.11f, 12.13f}; + int[][] presets = {{2, 0}, {3, 0}, {4, 1}}; + AlpEncoderDecoder.EncodingParams params = + AlpEncoderDecoder.findBestFloatParamsWithPresets(values, 0, values.length, presets); + + assertNotNull(params); + // Should select one of the preset combinations + boolean foundMatch = false; + for (int[] preset : presets) { + if (params.exponent == preset[0] && params.factor == preset[1]) { + foundMatch = true; + break; + } + } + assertTrue("Result should be one of the preset combinations", foundMatch); + } + + @Test + public void testFindBestDoubleParamsWithPresets() { + double[] values = {1.23, 4.56, 7.89, 10.11, 12.13}; + int[][] presets = {{2, 0}, {3, 0}, {4, 1}}; + AlpEncoderDecoder.EncodingParams params = + AlpEncoderDecoder.findBestDoubleParamsWithPresets(values, 0, values.length, presets); + + assertNotNull(params); + boolean foundMatch = false; + for (int[] preset : presets) { + if (params.exponent == preset[0] && params.factor == preset[1]) { + foundMatch = true; + break; + } + } + assertTrue("Result should be one of the preset combinations", foundMatch); + } + + @Test + public void testPresetsProduceSameResultAsFullSearch() { + float[] values = {1.23f, 4.56f, 7.89f}; + AlpEncoderDecoder.EncodingParams fullResult = AlpEncoderDecoder.findBestFloatParams(values, 0, values.length); + + // Include the best params in presets + int[][] presets = {{fullResult.exponent, fullResult.factor}, {0, 0}, {1, 0}}; + AlpEncoderDecoder.EncodingParams presetResult = + AlpEncoderDecoder.findBestFloatParamsWithPresets(values, 0, values.length, presets); + + assertTrue( + "Preset result should be at least as good as full search", + presetResult.numExceptions <= fullResult.numExceptions); + } Review Comment: Added findBestFloatParamsMixedSignUsesSignedDeltaRange (and a preset variant) with exactly this case. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
