This is an automated email from the ASF dual-hosted git repository. alsay pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/datasketches-go.git
commit 1d9b6dfb1d13c9c6e96cc10fb772a313c705e78b Author: Pierre Lacave <[email protected]> AuthorDate: Wed Dec 20 22:54:58 2023 +0100 Add slice memory corruption test --- frequencies/long_sketch.go | 12 ++-- frequencies/long_sketch_test.go | 156 ++++++++++++++++++++++++---------------- frequencies/preable_utils.go | 1 + 3 files changed, 102 insertions(+), 67 deletions(-) diff --git a/frequencies/long_sketch.go b/frequencies/long_sketch.go index 0fcc9db..996ac6a 100644 --- a/frequencies/long_sketch.go +++ b/frequencies/long_sketch.go @@ -164,9 +164,9 @@ func NewLongSketchFromSlice(slc []byte) (*LongSketch, error) { itemArray[i] = int64(binary.LittleEndian.Uint64(slc[itemsOffset+(i<<3):])) } - // Update the sketch + // UpdateMany the sketch for i := 0; i < activeItems; i++ { - fls.Update(itemArray[i], countArray[i]) + fls.UpdateMany(itemArray[i], countArray[i]) } fls.streamWeight = preArr[2] //override streamWeight due to updating @@ -304,7 +304,11 @@ func (s *LongSketch) isEmpty() bool { return s.getNumActiveItems() == 0 } -func (s *LongSketch) Update(item int64, count int64) error { +func (s *LongSketch) Update(item int64) error { + return s.UpdateMany(item, 1) +} + +func (s *LongSketch) UpdateMany(item int64, count int64) error { if count == 0 { return nil } @@ -341,7 +345,7 @@ func (s *LongSketch) merge(other *LongSketch) (*LongSketch, error) { streamWt := s.streamWeight + other.streamWeight //capture before merge iter := other.hashMap.iterator() for iter.next() { - err := s.Update(iter.getKey(), iter.getValue()) + err := s.UpdateMany(iter.getKey(), iter.getValue()) if err != nil { return nil, err } diff --git a/frequencies/long_sketch_test.go b/frequencies/long_sketch_test.go index 37eea96..d370994 100644 --- a/frequencies/long_sketch_test.go +++ b/frequencies/long_sketch_test.go @@ -18,6 +18,7 @@ package frequencies import ( + "encoding/binary" "github.com/stretchr/testify/assert" "testing" ) @@ -27,11 +28,11 @@ func TestFrequentItemsStringSerial(t *testing.T) { assert.NoError(t, err) sketch2, err := NewLongSketchWithMaxMapSize(128) assert.NoError(t, err) - sketch.Update(10, 100) - sketch.Update(10, 100) - sketch.Update(15, 3443) - sketch.Update(1000001, 1010230) - sketch.Update(1000002, 1010230) + sketch.UpdateMany(10, 100) + sketch.UpdateMany(10, 100) + sketch.UpdateMany(15, 3443) + sketch.UpdateMany(1000001, 1010230) + sketch.UpdateMany(1000002, 1010230) ser, err := sketch.serializeToString() assert.NoError(t, err) @@ -43,25 +44,25 @@ func TestFrequentItemsStringSerial(t *testing.T) { assert.Equal(t, sketch.getMaximumMapCapacity(), newSk0.getMaximumMapCapacity()) assert.Equal(t, sketch.getCurrentMapCapacity(), newSk0.getCurrentMapCapacity()) - sketch2.Update(190, 12902390) - sketch2.Update(191, 12902390) - sketch2.Update(192, 12902390) - sketch2.Update(193, 12902390) - sketch2.Update(194, 12902390) - sketch2.Update(195, 12902390) - sketch2.Update(196, 12902390) - sketch2.Update(197, 12902390) - sketch2.Update(198, 12902390) - sketch2.Update(199, 12902390) - sketch2.Update(200, 12902390) - sketch2.Update(201, 12902390) - sketch2.Update(202, 12902390) - sketch2.Update(203, 12902390) - sketch2.Update(204, 12902390) - sketch2.Update(205, 12902390) - sketch2.Update(206, 12902390) - sketch2.Update(207, 12902390) - sketch2.Update(208, 12902390) + sketch2.UpdateMany(190, 12902390) + sketch2.UpdateMany(191, 12902390) + sketch2.UpdateMany(192, 12902390) + sketch2.UpdateMany(193, 12902390) + sketch2.UpdateMany(194, 12902390) + sketch2.UpdateMany(195, 12902390) + sketch2.UpdateMany(196, 12902390) + sketch2.UpdateMany(197, 12902390) + sketch2.UpdateMany(198, 12902390) + sketch2.UpdateMany(199, 12902390) + sketch2.UpdateMany(200, 12902390) + sketch2.UpdateMany(201, 12902390) + sketch2.UpdateMany(202, 12902390) + sketch2.UpdateMany(203, 12902390) + sketch2.UpdateMany(204, 12902390) + sketch2.UpdateMany(205, 12902390) + sketch2.UpdateMany(206, 12902390) + sketch2.UpdateMany(207, 12902390) + sketch2.UpdateMany(208, 12902390) s2, err := sketch2.serializeToString() assert.NoError(t, err) @@ -102,11 +103,11 @@ func TestFrequentItemsByteSerial(t *testing.T) { sketch2, err := NewLongSketchWithMaxMapSize(128) assert.NoError(t, err) - sketch.Update(10, 100) - sketch.Update(10, 100) - sketch.Update(15, 3443) - sketch.Update(1000001, 1010230) - sketch.Update(1000002, 1010230) + sketch.UpdateMany(10, 100) + sketch.UpdateMany(10, 100) + sketch.UpdateMany(15, 3443) + sketch.UpdateMany(1000001, 1010230) + sketch.UpdateMany(1000002, 1010230) byteArray1, err := sketch.toSlice() assert.NoError(t, err) @@ -119,25 +120,25 @@ func TestFrequentItemsByteSerial(t *testing.T) { assert.Equal(t, sketch.getMaximumMapCapacity(), newSk1.getMaximumMapCapacity()) assert.Equal(t, sketch.getCurrentMapCapacity(), newSk1.getCurrentMapCapacity()) - sketch2.Update(190, 12902390) - sketch2.Update(191, 12902390) - sketch2.Update(192, 12902390) - sketch2.Update(193, 12902390) - sketch2.Update(194, 12902390) - sketch2.Update(195, 12902390) - sketch2.Update(196, 12902390) - sketch2.Update(197, 12902390) - sketch2.Update(198, 12902390) - sketch2.Update(199, 12902390) - sketch2.Update(200, 12902390) - sketch2.Update(201, 12902390) - sketch2.Update(202, 12902390) - sketch2.Update(203, 12902390) - sketch2.Update(204, 12902390) - sketch2.Update(205, 12902390) - sketch2.Update(206, 12902390) - sketch2.Update(207, 12902390) - sketch2.Update(208, 12902390) + sketch2.UpdateMany(190, 12902390) + sketch2.UpdateMany(191, 12902390) + sketch2.UpdateMany(192, 12902390) + sketch2.UpdateMany(193, 12902390) + sketch2.UpdateMany(194, 12902390) + sketch2.UpdateMany(195, 12902390) + sketch2.UpdateMany(196, 12902390) + sketch2.UpdateMany(197, 12902390) + sketch2.UpdateMany(198, 12902390) + sketch2.UpdateMany(199, 12902390) + sketch2.UpdateMany(200, 12902390) + sketch2.UpdateMany(201, 12902390) + sketch2.UpdateMany(202, 12902390) + sketch2.UpdateMany(203, 12902390) + sketch2.UpdateMany(204, 12902390) + sketch2.UpdateMany(205, 12902390) + sketch2.UpdateMany(206, 12902390) + sketch2.UpdateMany(207, 12902390) + sketch2.UpdateMany(208, 12902390) byteArray2, err := sketch2.toSlice() assert.NoError(t, err) @@ -171,11 +172,11 @@ func TestFrequentItemsByteSerial(t *testing.T) { func TestFrequentItemsByteResetAndEmptySerial(t *testing.T) { sketch, err := NewLongSketchWithMaxMapSize(16) assert.NoError(t, err) - sketch.Update(10, 100) - sketch.Update(10, 100) - sketch.Update(15, 3443) - sketch.Update(1000001, 1010230) - sketch.Update(1000002, 1010230) + sketch.UpdateMany(10, 100) + sketch.UpdateMany(10, 100) + sketch.UpdateMany(15, 3443) + sketch.UpdateMany(1000001, 1010230) + sketch.UpdateMany(1000002, 1010230) sketch.Reset() byteArray0, err := sketch.toSlice() @@ -195,11 +196,11 @@ func TestFreqLongSliceSerDe(t *testing.T) { minSize := 1 << _LG_MIN_MAP_SIZE sk1, err := NewLongSketchWithMaxMapSize(minSize) assert.NoError(t, err) - sk1.Update(10, 100) - sk1.Update(10, 100) - sk1.Update(15, 3443) - sk1.Update(1000001, 1010230) - sk1.Update(1000002, 1010230) + sk1.UpdateMany(10, 100) + sk1.UpdateMany(10, 100) + sk1.UpdateMany(15, 3443) + sk1.UpdateMany(1000001, 1010230) + sk1.UpdateMany(1000002, 1010230) byteArray0, err := sk1.toSlice() assert.NoError(t, err) @@ -213,11 +214,11 @@ func TestFreqLongStringSerDe(t *testing.T) { minSize := 1 << _LG_MIN_MAP_SIZE sk1, err := NewLongSketchWithMaxMapSize(minSize) assert.NoError(t, err) - sk1.Update(10, 100) - sk1.Update(10, 100) - sk1.Update(15, 3443) - sk1.Update(1000001, 1010230) - sk1.Update(1000002, 1010230) + sk1.UpdateMany(10, 100) + sk1.UpdateMany(10, 100) + sk1.UpdateMany(15, 3443) + sk1.UpdateMany(1000001, 1010230) + sk1.UpdateMany(1000002, 1010230) str1, err := sk1.serializeToString() assert.NoError(t, err) @@ -261,3 +262,32 @@ func checkEquality(t *testing.T, sk1, sk2 *LongSketch) { assert.Equal(t, s1, s2) } } + +func TestFreqLongSliceSerDeError(t *testing.T) { + minSize := 1 << _LG_MIN_MAP_SIZE + sk1, err := NewLongSketchWithMaxMapSize(minSize) + assert.NoError(t, err) + sk1.Update(1) + + byteArray0, err := sk1.toSlice() + pre0 := binary.LittleEndian.Uint64(byteArray0) + + tryBadMem(t, byteArray0, _PREAMBLE_LONGS_BYTE, 2) //Corrupt + binary.LittleEndian.PutUint64(byteArray0, pre0) + + tryBadMem(t, byteArray0, _SER_VER_BYTE, 2) //Corrupt + binary.LittleEndian.PutUint64(byteArray0, pre0) + + tryBadMem(t, byteArray0, _FAMILY_BYTE, 2) //Corrupt + binary.LittleEndian.PutUint64(byteArray0, pre0) + + tryBadMem(t, byteArray0, _FLAGS_BYTE, 4) //Corrupt + binary.LittleEndian.PutUint64(byteArray0, pre0) + +} + +func tryBadMem(t *testing.T, mem []byte, byteOffset, byteValue int) { + binary.LittleEndian.PutUint64(mem[byteOffset:], uint64(byteValue)) + _, err := NewLongSketchFromSlice(mem) + assert.Error(t, err) +} diff --git a/frequencies/preable_utils.go b/frequencies/preable_utils.go index 7180832..ca3854c 100644 --- a/frequencies/preable_utils.go +++ b/frequencies/preable_utils.go @@ -25,6 +25,7 @@ import ( const ( // ###### DO NOT MESS WITH THIS FROM HERE ... // Preamble byte Addresses + _PREAMBLE_LONGS_BYTE = 0 _SER_VER_BYTE = 1 _FAMILY_BYTE = 2 _LG_MAX_MAP_SIZE_BYTE = 3 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
