This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch 9.1.X in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 8256f17ce110ff213a0c0a255a3838332f821b72 Author: Lee Rhodes <[email protected]> AuthorDate: Thu Oct 29 14:25:29 2026 -0700 CountMin: check the image size in deserialize() - Check that the image holds the preamble. - For a non-empty image, check that it holds the total weight and the whole sketch array. An empty image still allocates the configured sketch array, as designed: a CountMin sketch does not grow, and is sized up front for the accuracy it needs. Adds AllocationTracker, a test helper that measures allocation by the current thread. Co-Authored-By: Claude Opus 5.5 <[email protected]> --- .../apache/datasketches/count/CountMinSketch.java | 8 +++- .../datasketches/common/AllocationTracker.java | 46 ++++++++++++++++++++++ .../datasketches/count/CountMinSketchTest.java | 33 ++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/datasketches/count/CountMinSketch.java b/src/main/java/org/apache/datasketches/count/CountMinSketch.java index 9bf576e13..5354eb702 100644 --- a/src/main/java/org/apache/datasketches/count/CountMinSketch.java +++ b/src/main/java/org/apache/datasketches/count/CountMinSketch.java @@ -441,6 +441,8 @@ public class CountMinSketch { * @return The deserialized CountMinSketch. */ public static CountMinSketch deserialize(final byte[] b, final long seed) { + final int preambleBytes = Family.COUNTMIN.getMinPreLongs() * Long.BYTES; + Util.checkBounds(0, preambleBytes, b.length); final PositionalSegment posSeg = PositionalSegment.wrap(MemorySegment.ofArray(b)); final byte preambleLongs = posSeg.getByte(); @@ -476,8 +478,12 @@ public class CountMinSketch { + Util.computeSeedHash(seed)); } - final CountMinSketch cms = new CountMinSketch(numHashes, numBuckets, seed); final boolean empty = (flagsByte & Flag.IS_EMPTY.mask()) > 0; + if (!empty) { //the image must hold the total weight and the whole sketch array + final long arrayBytes = (long) numHashes * numBuckets * Long.BYTES; + Util.checkBounds(preambleBytes, Long.BYTES + Math.max(arrayBytes, 0), b.length); + } + final CountMinSketch cms = new CountMinSketch(numHashes, numBuckets, seed); if (empty) { return cms; } diff --git a/src/test/java/org/apache/datasketches/common/AllocationTracker.java b/src/test/java/org/apache/datasketches/common/AllocationTracker.java new file mode 100644 index 000000000..7dd8f1b44 --- /dev/null +++ b/src/test/java/org/apache/datasketches/common/AllocationTracker.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.common; + +import java.lang.management.ManagementFactory; + +/** + * Test utility that measures how many bytes the current thread allocates while running a task. + * Used to check that deserializing a small, corrupt image does not cause a large allocation. + */ +public final class AllocationTracker { + private static final com.sun.management.ThreadMXBean TMX = + (com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + + private AllocationTracker() { } + + /** + * Runs the given task and returns the number of bytes the current thread allocated while running it. + * Any exception thrown by the task is ignored. + * @param task the task to run + * @return the number of bytes allocated + */ + public static long allocatedBytes(final Runnable task) { + final long tid = Thread.currentThread().threadId(); + final long before = TMX.getThreadAllocatedBytes(tid); + try { task.run(); } catch (final RuntimeException e) { /* ignored */ } + return TMX.getThreadAllocatedBytes(tid) - before; + } +} diff --git a/src/test/java/org/apache/datasketches/count/CountMinSketchTest.java b/src/test/java/org/apache/datasketches/count/CountMinSketchTest.java index 0f37c1a71..59590d9a6 100644 --- a/src/test/java/org/apache/datasketches/count/CountMinSketchTest.java +++ b/src/test/java/org/apache/datasketches/count/CountMinSketchTest.java @@ -24,6 +24,8 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; +import org.apache.datasketches.common.AllocationTracker; +import org.apache.datasketches.common.ByteArrayUtil; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesException; import org.testng.annotations.Test; @@ -238,4 +240,35 @@ public class CountMinSketchTest { assertEquals(d.getEstimate(i), c.getEstimate(i)); } } + + @Test + public void emptyImageIsUsable() { + final long seed = 123L; + final CountMinSketch cms = CountMinSketch.deserialize(new CountMinSketch((byte) 3, 5, seed).toByteArray(), seed); + final CountMinSketch other = new CountMinSketch((byte) 3, 5, seed); + cms.merge(other); //both empty + assertTrue(cms.isEmpty()); + other.update(1L, 2); + cms.merge(other); + assertEquals(cms.getEstimate(1L), 2); + cms.update(1L, 3); + assertEquals(cms.getEstimate(1L), 5); + final CountMinSketch cms2 = CountMinSketch.deserialize(cms.toByteArray(), seed); + assertEquals(cms2.getEstimate(1L), 5); + } + + @Test + public void truncatedImageThrows() { + final long seed = 123L; + final CountMinSketch cms = new CountMinSketch((byte) 3, 5, seed); + cms.update(1L, 1); + final byte[] bytes = cms.toByteArray(); + ByteArrayUtil.putIntLE(bytes, 8, 1 << 28); //numBuckets larger than the image holds + final long allocated = AllocationTracker.allocatedBytes(() -> CountMinSketch.deserialize(bytes, seed)); + assertTrue(allocated < (1 << 20), "allocated " + allocated); + assertThrows(SketchesArgumentException.class, () -> CountMinSketch.deserialize(bytes, seed)); + final byte[] shortBytes = java.util.Arrays.copyOf(bytes, 8); + assertThrows(SketchesArgumentException.class, () -> CountMinSketch.deserialize(shortBytes, seed)); + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
