tisonkun commented on code in PR #44: URL: https://github.com/apache/datasketches-rust/pull/44#discussion_r2651244165
########## datasketches/src/frequencies/sketch.rs: ########## @@ -0,0 +1,408 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Frequent items sketch implementations. + +use std::hash::Hash; + +use crate::error::SerdeError; +use crate::frequencies::reverse_purge_item_hash_map::ReversePurgeItemHashMap; +use crate::frequencies::serde::ItemsSerde; +use crate::frequencies::serialization::*; + +const LG_MIN_MAP_SIZE: u8 = 3; +const SAMPLE_SIZE: usize = 1024; +const EPSILON_FACTOR: f64 = 3.5; +const LOAD_FACTOR_NUMERATOR: usize = 3; +const LOAD_FACTOR_DENOMINATOR: usize = 4; + +/// Error guarantees for frequent item queries. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ErrorType { + /// Include items if upper bound exceeds threshold (no false negatives). + NoFalseNegatives, + /// Include items if lower bound exceeds threshold (no false positives). + NoFalsePositives, +} + +/// Result row for frequent item queries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Row<T> { + item: T, + estimate: i64, + upper_bound: i64, + lower_bound: i64, +} + +impl<T> Row<T> { + /// Returns the item value. + pub fn item(&self) -> &T { + &self.item + } + + /// Returns the estimated frequency. + pub fn estimate(&self) -> i64 { + self.estimate + } + + /// Returns the upper bound for the frequency. + pub fn upper_bound(&self) -> i64 { + self.upper_bound + } + + /// Returns the lower bound for the frequency. + pub fn lower_bound(&self) -> i64 { + self.lower_bound + } +} + +/// Frequent items sketch for generic item types. +#[derive(Debug, Clone)] +pub struct FrequentItemsSketch<T> { + lg_max_map_size: u8, + cur_map_cap: usize, + offset: i64, + stream_weight: i64, + sample_size: usize, + hash_map: ReversePurgeItemHashMap<T>, +} + +impl<T: Eq + Hash> FrequentItemsSketch<T> { + /// Creates a new sketch with the given maximum map size (power of two). + pub fn new(max_map_size: usize) -> Self { + let lg_max_map_size = exact_log2(max_map_size); + Self::with_lg_map_sizes(lg_max_map_size, LG_MIN_MAP_SIZE) + } + + /// Returns true if the sketch is empty. + pub fn is_empty(&self) -> bool { + self.hash_map.get_num_active() == 0 + } + + /// Returns the number of active items being tracked. + pub fn get_num_active_items(&self) -> usize { + self.hash_map.get_num_active() + } + + /// Returns the total weight of the stream. + pub fn get_total_weight(&self) -> i64 { + self.stream_weight + } + + /// Returns the estimated frequency for an item. + pub fn get_estimate(&self, item: &T) -> i64 { + let value = self.hash_map.get(item); + if value > 0 { value + self.offset } else { 0 } + } + + /// Returns the lower bound for an item's frequency. + pub fn get_lower_bound(&self, item: &T) -> i64 { + self.hash_map.get(item) + } + + /// Returns the upper bound for an item's frequency. + pub fn get_upper_bound(&self, item: &T) -> i64 { + self.hash_map.get(item) + self.offset + } + + /// Returns the maximum error across all items. + pub fn get_maximum_error(&self) -> i64 { + self.offset + } + + /// Returns epsilon for this sketch. + pub fn get_epsilon(&self) -> f64 { + Self::get_epsilon_for_lg(self.lg_max_map_size) + } + + /// Returns epsilon for a sketch configured with `lg_max_map_size`. + pub fn get_epsilon_for_lg(lg_max_map_size: u8) -> f64 { + EPSILON_FACTOR / (1u64 << lg_max_map_size) as f64 + } + + /// Returns the a priori error estimate. + pub fn get_apriori_error(lg_max_map_size: u8, estimated_total_weight: i64) -> f64 { + Self::get_epsilon_for_lg(lg_max_map_size) * estimated_total_weight as f64 + } + + /// Returns the maximum map capacity for this sketch. + pub fn get_maximum_map_capacity(&self) -> usize { + (1usize << self.lg_max_map_size) * LOAD_FACTOR_NUMERATOR / LOAD_FACTOR_DENOMINATOR + } + + /// Returns the current map capacity. + pub fn get_current_map_capacity(&self) -> usize { + self.cur_map_cap + } + + /// Returns the configured lg_max_map_size. + pub fn get_lg_max_map_size(&self) -> u8 { + self.lg_max_map_size + } + + /// Returns the current map size in log2. + pub fn get_lg_cur_map_size(&self) -> u8 { + self.hash_map.get_lg_length() + } + + /// Updates the sketch with a count of one. + pub fn update(&mut self, item: T) { + self.update_with_count(item, 1); + } + + /// Updates the sketch with an item and count. + pub fn update_with_count(&mut self, item: T, count: i64) { + if count == 0 { + return; + } + assert!(count > 0, "count may not be negative"); + self.stream_weight += count; + self.hash_map.adjust_or_put_value(item, count); + self.maybe_resize_or_purge(); + } + + /// Merges another sketch into this one. + pub fn merge(&mut self, other: &Self) + where + T: Clone, + { + if other.is_empty() { + return; + } + let merged_total = self.stream_weight + other.stream_weight; + for (item, count) in other.hash_map.iter() { + self.update_with_count(item.clone(), count); + } + self.offset += other.offset; + self.stream_weight = merged_total; + } + + /// Resets the sketch to an empty state. + pub fn reset(&mut self) { + *self = Self::with_lg_map_sizes(self.lg_max_map_size, LG_MIN_MAP_SIZE); + } + + /// Returns frequent items using the sketch maximum error as threshold. + pub fn get_frequent_items(&self, error_type: ErrorType) -> Vec<Row<T>> + where + T: Clone, + { + self.get_frequent_items_with_threshold(error_type, self.offset) + } + + /// Returns frequent items using a custom threshold. + pub fn get_frequent_items_with_threshold( + &self, + error_type: ErrorType, + threshold: i64, + ) -> Vec<Row<T>> + where + T: Clone, + { + let threshold = threshold.max(self.offset); + let mut rows = Vec::new(); + for (item, count) in self.hash_map.iter() { + let lower = count; + let upper = count + self.offset; + let include = match error_type { + ErrorType::NoFalseNegatives => upper > threshold, + ErrorType::NoFalsePositives => lower > threshold, + }; + if include { + rows.push(Row { + item: item.clone(), + estimate: upper, + upper_bound: upper, + lower_bound: lower, + }); + } + } + rows.sort_by(|a, b| b.estimate.cmp(&a.estimate)); + rows + } + + /// Serializes this sketch into a byte vector using the provided serializer. + pub fn serialize_with<S: ItemsSerde<T>>(&self, serde: &S) -> Vec<u8> Review Comment: ```suggestion pub fn serialize<S: ItemsSerde<T>>(&self, serde: &S) -> Vec<u8> ``` Other sketches use `serialize` with context. Let's follow this pattern now. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
