martinzink commented on code in PR #2258: URL: https://github.com/apache/nifi-minifi-cpp/pull/2258#discussion_r4144615934
########## minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs: ########## @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use minifi_native::{GetAttribute, MinifiError}; + +/// The exact placement of an aspect-preserving resize inside a target canvas. +/// +/// `ImageToTensor` applies this when resizing, and `FilterBoundingBoxes` inverts +/// it when un-mapping model coordinates back to the original image. Both must +/// agree down to the pixel, so the arithmetic lives here and nowhere else: +/// deriving the padding from the *unrounded* scaled size instead of `new_w`/ +/// `new_h` drifts by up to half a target pixel, which is several pixels once +/// divided back through `scale`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct LetterboxGeometry { + pub(crate) scale: f32, + pub(crate) new_width: u32, + pub(crate) new_height: u32, + pub(crate) pad_x: u32, + pub(crate) pad_y: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Dimensions { + pub(crate) width: f32, + pub(crate) height: f32, +} Review Comment: https://github.com/apache/nifi-minifi-cpp/pull/2258/changes/568f895725eb77ef470121155e99e91f91f6a621 ########## minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs: ########## @@ -0,0 +1,590 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod filter_bounding_boxes_def; + +use crate::low_level_processors::image_to_tensor::ResizeMode; +use crate::utils::bounding_box::BoundingBox; +use crate::utils::dimensions::Dimensions; +use crate::utils::score_activation::{ScoreActivation, SoftmaxTerms}; +use crate::utils::tensor_helpers::{deserialize_tensors, tensor_as_f32}; +use filter_bounding_boxes_def::SUCCESS; +pub(crate) use filter_bounding_boxes_def::{ + BACKGROUND_CLASS_INDEX, BOX_FORMAT, BOX_OUTPUT_INDEX, CLASS_OUTPUT_INDEX, CONFIDENCE_THRESHOLD, + IOU_THRESHOLD, MIME_TYPE_ATTR, OUTPUT_ATTRIBUTE_NAME, SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, +}; +use minifi_native::macros::{ComponentIdentifier, PropertyType}; +use minifi_native::{ + Content, FlowFileTransform, GetAttribute, GetId, GetProperty, InputStream, Logger, MinifiError, + ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, debug, trace, +}; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; +use tract::Tensor; + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum BoxFormat { + /// `[x_min, y_min, x_max, y_max]` — SSD, MobileNet-SSD, most PyTorch models. + Xyxy, + /// `[y_min, x_min, y_max, x_max]` — TensorFlow Object Detection API. + Yxyx, + /// `[cx, cy, w, h]` — YOLOv3/5/8 raw output (center + size). + Cxcywh, +} + +/// Convert the four floats at `box_floats[offset..offset+4]` into a canonical +/// `(x_min, y_min, x_max, y_max)` tuple, regardless of the source layout. +fn decode_box(box_floats: &[f32], offset: usize, format: BoxFormat) -> (f32, f32, f32, f32) { + let a = box_floats[offset]; + let b = box_floats[offset + 1]; + let c = box_floats[offset + 2]; + let d = box_floats[offset + 3]; + match format { + BoxFormat::Xyxy => (a, b, c, d), + BoxFormat::Yxyx => (b, a, d, c), + BoxFormat::Cxcywh => { + let (cx, cy, w, h) = (a, b, c, d); + (cx - w / 2.0, cy - h / 2.0, cx + w / 2.0, cy + h / 2.0) + } + } +} + +struct ScoredClass { + class_id: usize, + confidence: f32, +} + +fn score_box( + logits: &[f32], + activation: ScoreActivation, + background_class_index: Option<usize>, +) -> ScoredClass { + let num_classes = logits.len(); + + let best_valid = logits + .iter() + .enumerate() + .filter(|&(_, &logit)| logit.is_finite()) + .filter(|&(id, _)| match background_class_index { + Some(bg_idx) => !(num_classes > 1 && id == bg_idx), + None => true, + }) + .max_by(|a, b| a.1.total_cmp(b.1)); + + let (class_id, &best_logit) = match best_valid { + Some(val) => val, + None => { + return ScoredClass { + class_id: 0, + confidence: f32::NEG_INFINITY, + }; + } + }; + + let confidence = activation.confidence(best_logit, SoftmaxTerms::over(logits.iter().copied())); + + ScoredClass { + class_id, + confidence, + } +} + +#[derive(ComponentIdentifier)] +pub(crate) struct FilterBoundingBoxes { + confidence_threshold: f32, + iou_threshold: f32, + score_output_index: usize, + box_output_index: usize, + box_format: BoxFormat, + score_activation: ScoreActivation, + background_class_index: Option<usize>, + class_output_index: Option<usize>, +} + +impl Schedule for FilterBoundingBoxes { + fn schedule<Ctx: GetProperty, L: Logger>( Review Comment: https://github.com/apache/nifi-minifi-cpp/pull/2258/changes/568f895725eb77ef470121155e99e91f91f6a621 ########## minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs: ########## @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::utils::bounding_box::{BoundingBox, BoundingBoxes}; +use image::Rgb; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, OutputAttribute, ProcessError, ProcessorDefinition, ProcessorInputRequirement, + Property, PropertyConstraints, PropertyType, Relationship, RouteErrorExt, Schedule, + TransformedFlowFile, +}; +use minifi_native::{PropertyDefinition, PropertySchema, property_definitions}; +use std::io::Cursor; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Flowfiles are routed here after drawing the bounding boxes", +}; + +pub(crate) const FAILURE: Relationship = Relationship { + name: "failure", + description: "Invalid FlowFiles are routed here", +}; + +pub(crate) const BOUNDING_BOXES: Property<BoundingBoxes> = Property::new( + "Bounding boxes", + "JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, \ + y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the \ + attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.", +) +.with_default("${enrichment.value}") +.supports_expression_language(); + +const LINE_THICKNESS: Property<u32> = Property::new( + "Line thickness", + "Thickness in pixels of the drawn box outline.", +) +.with_default("5"); + +const LINE_COLOR: Property<LineColor> = Property::new( + "Line color", + "Outline color as a hex string (e.g., '#ff00ff' or '#f0f')", +) +.with_default("#00FF00"); + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DrawBoundingBox {} + +impl Schedule for DrawBoundingBox { + fn schedule<Ctx: GetProperty, L: Logger>( + _context: &Ctx, + _logger: &L, + ) -> Result<Self, MinifiError> + where + Self: Sized, + { + Ok(Self {}) + } +} + +struct LineColor {} + +impl PropertySchema for LineColor { + const CONSTRAINT: Option<PropertyConstraints> = None; + const IS_REQUIRED: bool = false; +} + +impl PropertyType for LineColor { + type Output = Rgb<u8>; + + fn parse(s: &str) -> Result<Self::Output, MinifiError> { + let Some(hex) = s.trim().strip_prefix('#') else { + return Err(MinifiError::validation("Line color must start with #")); + }; + + let (r, g, b) = match hex.len() { + 6 => ( + u8::from_str_radix(&hex[0..2], 16).map_err(MinifiError::from)?, + u8::from_str_radix(&hex[2..4], 16).map_err(MinifiError::from)?, + u8::from_str_radix(&hex[4..6], 16).map_err(MinifiError::from)?, + ), + 3 => ( + u8::from_str_radix(&hex[0..1], 16).map_err(MinifiError::from)? * 17, + u8::from_str_radix(&hex[1..2], 16).map_err(MinifiError::from)? * 17, + u8::from_str_radix(&hex[2..3], 16).map_err(MinifiError::from)? * 17, + ), + _ => return Err(MinifiError::validation("expected 3 or 6 digit hex color")), + }; + Ok(Rgb::<u8>([r, g, b])) + } +} + +impl FlowFileTransform for DrawBoundingBox { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + _logger: &LoggerImpl, + ) -> Result<TransformedFlowFile<'a>, ProcessError> { + let line_thickness = context + .get_property(&LINE_THICKNESS) + .route_err_to_failure()?; + let line_color = context.get_property(&LINE_COLOR).route_err_to_failure()?; + let boxes: Vec<BoundingBox> = context + .get_property(&BOUNDING_BOXES) + .route_err_to_failure()?; + + let mut image_bytes = Vec::new(); + input_stream.read_to_end(&mut image_bytes)?; + + let format = image::guess_format(&image_bytes).route_err_to_failure()?; + + let mut img = image::load_from_memory_with_format(&image_bytes, format) + .map(|dyn_img| dyn_img.to_rgb8()) + .route_err_to_failure()?; + + boxes + .iter() + .for_each(|bbox| bbox.draw_onto(&mut img, line_thickness, line_color)); + + let mut output_bytes = Vec::new(); + img.write_to(&mut Cursor::new(&mut output_bytes), format) + .route_err_to_failure()?; + + Ok(TransformedFlowFile::new( + &SUCCESS, + Some(output_bytes.into()), + )) + } +} + +impl ProcessorDefinition for DrawBoundingBox { + const DESCRIPTION: &'static str = "Decodes the image from the flow file content, draws each bounding box supplied via the \ + 'Bounding boxes' property onto it, and re-encodes the annotated image as PNG. Pair with an \ + upstream DetectObject / FilterBoundingBoxes to visualise detections."; Review Comment: https://github.com/apache/nifi-minifi-cpp/pull/2258/changes/568f895725eb77ef470121155e99e91f91f6a621 ########## minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs: ########## @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use minifi_native::{GetAttribute, MinifiError}; + +/// The exact placement of an aspect-preserving resize inside a target canvas. +/// +/// `ImageToTensor` applies this when resizing, and `FilterBoundingBoxes` inverts +/// it when un-mapping model coordinates back to the original image. Both must +/// agree down to the pixel, so the arithmetic lives here and nowhere else: +/// deriving the padding from the *unrounded* scaled size instead of `new_w`/ +/// `new_h` drifts by up to half a target pixel, which is several pixels once +/// divided back through `scale`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct LetterboxGeometry { + pub(crate) scale: f32, + pub(crate) new_width: u32, + pub(crate) new_height: u32, + pub(crate) pad_x: u32, + pub(crate) pad_y: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Dimensions { + pub(crate) width: f32, + pub(crate) height: f32, +} + +impl Dimensions { + /// Fit `self` into `target` preserving aspect ratio, centring the result. + /// + /// Assumes both dimensions are non-zero; `ImageToTensor::schedule` rejects a + /// zero 'Target width'/'Target height', and a decoded image always has at + /// least one pixel per axis. + pub(crate) fn letterbox_into(&self, target: Dimensions) -> LetterboxGeometry { + let scale = (target.width / self.width).min(target.height / self.height); + let new_width = (self.width * scale).round().max(1.0) as u32; + let new_height = (self.height * scale).round().max(1.0) as u32; + LetterboxGeometry { + scale, + new_width, + new_height, + // Saturating: `new_*` is clamped up to 1, so it can exceed a target + // axis of 0. Callers reject that config, but wrapping here would + // turn a misconfiguration into a panic or a garbage offset. + pad_x: (target.width as u32).saturating_sub(new_width) / 2, + pad_y: (target.height as u32).saturating_sub(new_height) / 2, + } + } + + pub(crate) fn from_image(img: &image::DynamicImage) -> Self { + Self { + width: img.width() as f32, + height: img.height() as f32, + } + } + + pub(crate) fn original_from_attributes<Context: GetAttribute>( + context: &Context, + ) -> Result<Dimensions, MinifiError> { + let orig_w = context + .get_required_attribute("image.original.width")? + .parse::<f32>()?; + + let orig_h = context + .get_required_attribute("image.original.height")? + .parse::<f32>()?; + Review Comment: https://github.com/apache/nifi-minifi-cpp/pull/2258/changes/568f895725eb77ef470121155e99e91f91f6a621 ########## minifi_rust/extensions/minifi_tensor/features/detection.feature: ########## @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@SUPPORTS_WINDOWS +Feature: Face detection with UltraFace Single Shot MultiBox Detector (SSD) + + # Based on https://github.com/sonos/tract/blob/main/examples/face_detection_yolov8onnx_example/src/main.rs + Scenario: Grace Hopper image yields at least one face detection (ImageToTensor + InvokeTract + FilterBoundingBoxes) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "version-RFB-320.onnx" is copied to the "/tmp/models/ultraface.onnx" path in the MiNiFi container + + And a TractModelService controller service named "UltraFace" is set up and the "Model File Path" property set to "/tmp/models/ultraface.onnx" + And the "Model format" property of the UltraFace controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And an ImageToTensor processor with the "Target width" property set to "320" + And the "Target height" property of the ImageToTensor processor is set to "240" + And the "Resize filter" property of the ImageToTensor processor is set to "Bilinear" + And the "Resize mode" property of the ImageToTensor processor is set to "Letterbox" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Tensor shape format" property of the ImageToTensor processor is set to "CHW" + And the "Mean" property of the ImageToTensor processor is set to "127.0" + And the "Standard Deviation" property of the ImageToTensor processor is set to "128.0" + + And an InvokeTractModel processor with the "Tract model service" property set to "UltraFace" + + # UltraFace: output 0 = scores [1, N, 2] (softmax over background/face), + # output 1 = boxes [1, N, 4] in Xyxy normalised to 0..1. Class 0 is + # background so leave the defaults. + And a FilterBoundingBoxes processor with the "Confidence Threshold" property set to "0.5" + And the "IoU Threshold" property of the FilterBoundingBoxes processor is set to "0.45" + And the "Score output index" property of the FilterBoundingBoxes processor is set to "0" + And the "Box output index" property of the FilterBoundingBoxes processor is set to "1" + And the "Box format" property of the FilterBoundingBoxes processor is set to "Xyxy" + And the "Score activation" property of the FilterBoundingBoxes processor is set to "Softmax" + And the "Background class index" property of the FilterBoundingBoxes processor is set to "0" + + And a PutFile processor with the "Directory" property set to "/tmp/output" + + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the InvokeTractModel + And the "success" relationship of the InvokeTractModel processor is connected to the FilterBoundingBoxes + And the "success" relationship of the FilterBoundingBoxes processor is connected to the LogAttribute + And the "success" relationship of the LogAttribute processor is connected to the PutFile + And ImageToTensor's failure relationship is auto-terminated + And InvokeTractModel's failure relationship is auto-terminated + And FilterBoundingBoxes's failure relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs match the following regex: "key:object.count value:[1-9][0-9]*" in less than 60 seconds + And the Minifi logs contain the following message: "key:mime.type value:application/json" in less than 1 seconds + And at least one file in "/tmp/output" content match the following regex: "\"class_id\":1" in less than 30 seconds + And at least one file in "/tmp/output" content match the following regex: "\"confidence\":0\.[5-9][0-9]*" in less than 30 seconds + And the Minifi logs do not contain errors + + # Based on https://github.com/sonos/tract/blob/main/examples/face_detection_yolov8onnx_example/src/main.rs + Scenario: Grace Hopper image yields at least one face detection (DetectObject) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "version-RFB-320.onnx" is copied to the "/tmp/models/ultraface.onnx" path in the MiNiFi container + + And a TractModelService controller service named "UltraFace" is set up and the "Model File Path" property set to "/tmp/models/ultraface.onnx" + And the "Model format" property of the UltraFace controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And a DetectObject processor with the "Target width" property set to "320" + And the "Target height" property of the DetectObject processor is set to "240" + And the "Resize filter" property of the DetectObject processor is set to "Bilinear" + And the "Resize mode" property of the DetectObject processor is set to "Letterbox" + And the "Color format" property of the DetectObject processor is set to "RGB" + And the "Tensor shape format" property of the DetectObject processor is set to "CHW" + And the "Mean" property of the DetectObject processor is set to "127.0" + And the "Standard Deviation" property of the DetectObject processor is set to "128.0" + And the "Tract model service" property of the DetectObject processor is set to "UltraFace" + # UltraFace: output 0 = scores [1, N, 2] (softmax over background/face), + # output 1 = boxes [1, N, 4] in Xyxy normalised to 0..1. Class 0 is + # background so leave the defaults. + And the "Confidence Threshold" property of the DetectObject processor is set to "0.5" + And the "IoU Threshold" property of the DetectObject processor is set to "0.45" + And the "Score output index" property of the DetectObject processor is set to "0" + And the "Box output index" property of the DetectObject processor is set to "1" + And the "Box format" property of the DetectObject processor is set to "Xyxy" + And the "Score activation" property of the DetectObject processor is set to "Softmax" + And the "Background class index" property of the DetectObject processor is set to "0" + And the "Output attribute name" property of the DetectObject processor is set to "detected_objects" + + And a DrawBoundingBox processor with the "Bounding boxes" property set to "${detected_objects}" + And the "Line color" property of the DrawBoundingBox processor is set to "0, 255, 0" Review Comment: it turns out we dont assert anything based on the DrawBoundingBox, we only check the LogAttribute's result which is before the DrawBoundingBox, I've added an assertion at least that the PutFile (after dbb) succeeds, ive checked and that would fail with the previous line color format `14:32:07 - INFO - [2026-09-30 12:31:47.948] [minifi_tensor::processors::draw_bounding_box::DrawBoundingBox] [warning] Routing flow file to 'failure': Line color must start with # (cc91be23-380a-424e-8fd8-b6108500f8ea)` https://github.com/apache/nifi-minifi-cpp/pull/2258/commits/041c432c6f192715f9d5ee3b998e2439a0e1f233 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
