use serde::{Deserialize, Serialize}; // Composition targets are deliberately expressed relative to the crop rather // than the source. A person's visual centre sits below the middle of a hero, // leaving their head near the upper third while retaining more of their body. const PERSON_CENTER_TARGET: f32 = 0.66; const FACE_CENTER_TARGET: f32 = 0.44; const FACE_REFINEMENT_WEIGHT: f32 = 0.28; const PERSON_BOTTOM_COVERAGE: f32 = 0.93; const PERSON_TOP_MARGIN: f32 = 0.07; /// Headroom kept above the highest detected face, as a fraction of that face /// box and as a floor relative to the crop. A face box starts at the brow, not /// the crown, so roughly a third of a face height of hair sits above it. const FACE_HEADROOM_OF_FACE: f32 = 0.38; const FACE_HEADROOM_OF_CROP: f32 = 0.03; /// Headroom above the topmost PERSON box when no face was associated. A person /// box top is already the crown, so this only needs to avoid a hairline shave. const PERSON_CROWN_HEADROOM: f32 = 0.01; /// Pull a proposed crop start so every detected face stays inside the window, /// with hair headroom above the highest one. Returns `desired_y` unchanged /// when there are no faces to protect. /// /// When the faces span MORE than the crop height (a group photographed at very /// different depths), no placement can hold them all; the span is centred /// instead, which cuts the least from the people at both extremes. fn protect_faces( desired_y: f32, faces: &[SanitizedBox], people: &[SanitizedBox], crop_height: u32, ) -> f32 { let Some(highest) = faces .iter() .min_by(|left, right| left.top.total_cmp(&right.top)) else { // No face survived association — common for small, distant or // turned-away subjects. The person box's top edge IS the crown, so it // still protects the head from the body-coverage guards, which use a // MEDIAN top margin and therefore happily over-cut the topmost person // of a group. Only ever raises the window; never pushes it down. let Some(highest_person) = people .iter() .min_by(|left, right| left.top.total_cmp(&right.top)) else { return desired_y; }; let headroom = crop_height as f32 * PERSON_CROWN_HEADROOM; return desired_y.min((highest_person.top - headroom).max(0.0)); }; let lowest_bottom = faces .iter() .map(|face| face.bottom) .fold(f32::MIN, f32::max); let headroom = (highest.height() * FACE_HEADROOM_OF_FACE).max(crop_height as f32 * FACE_HEADROOM_OF_CROP); let top_limit = (highest.top - headroom).max(0.0); let bottom_need = lowest_bottom + crop_height as f32 * FACE_HEADROOM_OF_CROP - crop_height as f32; if bottom_need <= top_limit { desired_y.clamp(bottom_need.max(0.0), top_limit) } else { (highest.top + lowest_bottom) / 2.0 - crop_height as f32 / 2.0 } } /// Placement tracing for crop audits (`SOULSTREAM_CROP_TRACE=1`). The chosen /// `y` is the product of four competing terms, so a bad composition is only /// diagnosable with the intermediates in hand. fn crop_trace(message: impl FnOnce() -> String) { static ON: std::sync::OnceLock = std::sync::OnceLock::new(); if *ON.get_or_init(|| std::env::var("SOULSTREAM_CROP_TRACE").is_ok_and(|value| value == "1")) { eprintln!("CROP\t{}", message()); } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct HeroRect { pub x: u32, pub y: u32, pub width: u32, pub height: u32, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct HeroCropPlan { pub source_width: u32, pub source_height: u32, pub crop: HeroRect, /// True when valid person detections affected the vertical placement. pub content_aware: bool, /// Robust vertical focus in source pixels. Useful for diagnostics only; /// consumers should apply `crop` verbatim for preview and final renders. pub focal_y: Option, pub subject_count: u32, pub associated_face_count: u32, } #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct SubjectBox { pub x: f32, pub y: f32, pub width: f32, pub height: f32, pub confidence: f32, } #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct FaceBox { pub x: f32, pub y: f32, pub width: f32, pub height: f32, pub confidence: f32, } #[derive(Debug, Clone, Copy)] struct WeightedValue { value: f32, weight: f32, } #[derive(Debug, Clone, Copy)] struct SanitizedBox { left: f32, top: f32, right: f32, bottom: f32, confidence: f32, } impl SanitizedBox { fn width(self) -> f32 { self.right - self.left } fn height(self) -> f32 { self.bottom - self.top } fn area(self) -> f32 { self.width() * self.height() } fn center_x(self) -> f32 { (self.left + self.right) * 0.5 } fn center_y(self) -> f32 { (self.top + self.bottom) * 0.5 } } /// Returns the largest pixel crop that fits the source's 16:9 frame. /// /// Tall and square sources retain every horizontal source pixel (`x = 0`), so /// content awareness can move only on the Y axis. Wide sources retain the full /// source height and center the crop on X. Integer rounding can leave the raw /// crop less than one pixel away from 16:9; preview/final rendering normalizes /// that same rectangle to exact 16:9 output dimensions. pub(crate) fn maximal_hero_rect(source_width: u32, source_height: u32) -> Option { if source_width == 0 || source_height == 0 { return None; } if source_width as u64 * 9 <= source_height as u64 * 16 { let height = (source_width as u64 * 9 / 16) as u32; (height > 0).then_some(HeroRect { x: 0, y: (source_height - height) / 2, width: source_width, height, }) } else { let width = (source_height as u64 * 16 / 9) as u32; (width > 0).then_some(HeroRect { x: (source_width - width) / 2, y: 0, width, height: source_height, }) } } /// Builds a deterministic hero crop plan from optional person and face boxes. /// /// Person boxes are the primary signal. Faces only refine the vertical /// composition when they can be associated with a person, which prevents faces /// in murals, posters, and album art from pulling the crop away from the artist. /// Horizontal placement never follows a detection: wide sources stay centered, /// and tall sources retain the maximum possible source width. pub(crate) fn plan_hero_crop( source_width: u32, source_height: u32, subjects: &[SubjectBox], faces: &[FaceBox], ) -> Option { let mut crop = maximal_hero_rect(source_width, source_height)?; let fallback_y = crop.y; let max_y = source_height - crop.height; let valid_subjects: Vec<_> = subjects .iter() .filter_map(|subject| { sanitize_box( subject.x, subject.y, subject.width, subject.height, subject.confidence, source_width, source_height, ) }) .collect(); if max_y == 0 { return Some(HeroCropPlan { source_width, source_height, crop, content_aware: false, focal_y: None, subject_count: valid_subjects.len() as u32, associated_face_count: 0, }); } if valid_subjects.is_empty() { // Face-only is a secondary fallback for portraits where the general // person model misses a close or partially visible subject. Provider // hardening already rejects most posters/text cards; cap implausibly // huge boxes and use a robust median so one false positive cannot // dominate a group. let source_area = source_width as f32 * source_height as f32; let valid_faces: Vec<_> = faces .iter() .filter_map(|face| { sanitize_box( face.x, face.y, face.width, face.height, face.confidence, source_width, source_height, ) }) .filter(|face| face.area() <= source_area * 0.25) .collect(); let face_values: Vec<_> = valid_faces .iter() .map(|face| WeightedValue { value: face.center_y() - crop.height as f32 * FACE_CENTER_TARGET, weight: detection_weight(*face), }) .collect(); if let Some(desired_y) = weighted_median(&face_values) { // Same inviolable rule as the person path: the composition target // may place the window, but never past a face. let desired_y = protect_faces(desired_y, &valid_faces, &[], crop.height); crop.y = desired_y.round().clamp(0.0, max_y as f32) as u32; return Some(HeroCropPlan { source_width, source_height, crop, content_aware: true, focal_y: weighted_median( &valid_faces .iter() .map(|face| WeightedValue { value: face.center_y(), weight: detection_weight(*face), }) .collect::>(), ), subject_count: 0, associated_face_count: valid_faces.len() as u32, }); } return Some(HeroCropPlan { source_width, source_height, crop, content_aware: false, focal_y: None, subject_count: 0, associated_face_count: 0, }); } let person_values: Vec<_> = valid_subjects .iter() .map(|subject| WeightedValue { value: subject.center_y() - crop.height as f32 * PERSON_CENTER_TARGET, weight: detection_weight(*subject), }) .collect(); let person_y = weighted_median(&person_values).unwrap_or(fallback_y as f32); // These robust bounds keep nearly all of a typical detected body visible, // without allowing one outlying/oversized box to dictate the result. let lower_values: Vec<_> = valid_subjects .iter() .map(|subject| WeightedValue { value: subject.top + subject.height() * PERSON_BOTTOM_COVERAGE - crop.height as f32, weight: detection_weight(*subject), }) .collect(); let upper_values: Vec<_> = valid_subjects .iter() .map(|subject| WeightedValue { value: subject.top + subject.height() * PERSON_TOP_MARGIN, weight: detection_weight(*subject), }) .collect(); let lower_y = weighted_median(&lower_values).unwrap_or(0.0); let upper_y = weighted_median(&upper_values).unwrap_or(max_y as f32); let valid_faces: Vec<_> = faces .iter() .filter_map(|face| { sanitize_box( face.x, face.y, face.width, face.height, face.confidence, source_width, source_height, ) }) .filter(|face| { valid_subjects .iter() .any(|subject| face_belongs_to(*face, *subject)) }) .collect(); let face_values: Vec<_> = valid_faces .iter() .map(|face| WeightedValue { value: face.center_y() - crop.height as f32 * FACE_CENTER_TARGET, weight: detection_weight(*face), }) .collect(); let mut desired_y = person_y; if let Some(face_y) = weighted_median(&face_values) { desired_y = person_y * (1.0 - FACE_REFINEMENT_WEIGHT) + face_y * FACE_REFINEMENT_WEIGHT; } // Use body-coverage guards when both can be satisfied. If a close subject // fills more than the crop height, the robust composition targets decide // which portion to retain rather than sacrificing the head or lower body by // blindly snapping to one incompatible edge. let unclamped_y = desired_y; if upper_y >= lower_y { desired_y = desired_y.clamp(lower_y, upper_y); } // Faces outrank body coverage, always. A standing subject is usually // TALLER than the 16:9 window, so the coverage guards above ask the window // to slide down until 93% of the body fits — and when the body runs to the // bottom edge, the only thing left to sacrifice is the head. Measured on // Crazy P's MusicRadar photo: composition wanted y=-3.6 (every head in // frame) and the coverage clamp forced y=113, decapitating all three. // Legs may be cut; faces may not. desired_y = protect_faces(desired_y, &valid_faces, &valid_subjects, crop.height); crop.y = desired_y.round().clamp(0.0, max_y as f32) as u32; crop_trace(|| { let faces: Vec = valid_faces .iter() .map(|face| format!("[y{:.0}+{:.0}]", face.top, face.height())) .collect(); let people: Vec = valid_subjects .iter() .map(|subject| format!("[y{:.0}+{:.0}]", subject.top, subject.height())) .collect(); format!( "src={source_width}x{source_height} croph={} max_y={max_y} people={} faces={} \ person_y={person_y:.1} face_y={:?} desired={unclamped_y:.1} \ lower={lower_y:.1} upper={upper_y:.1} clamped={} final_y={}", crop.height, people.join(","), faces.join(","), weighted_median(&face_values).map(|value| (value * 10.0).round() / 10.0), upper_y >= lower_y, crop.y, ) }); let focal_y = if valid_faces.is_empty() { weighted_median( &valid_subjects .iter() .map(|subject| WeightedValue { value: subject.center_y(), weight: detection_weight(*subject), }) .collect::>(), ) } else { weighted_median( &valid_faces .iter() .map(|face| WeightedValue { value: face.center_y(), weight: detection_weight(*face), }) .collect::>(), ) }; Some(HeroCropPlan { source_width, source_height, crop, content_aware: true, focal_y, subject_count: valid_subjects.len() as u32, associated_face_count: valid_faces.len() as u32, }) } fn sanitize_box( x: f32, y: f32, width: f32, height: f32, confidence: f32, source_width: u32, source_height: u32, ) -> Option { if !x.is_finite() || !y.is_finite() || !width.is_finite() || !height.is_finite() || !confidence.is_finite() || width <= 0.0 || height <= 0.0 || confidence <= 0.0 { return None; } let left = x.clamp(0.0, source_width as f32); let top = y.clamp(0.0, source_height as f32); let right = (x + width).clamp(0.0, source_width as f32); let bottom = (y + height).clamp(0.0, source_height as f32); if right - left < 1.0 || bottom - top < 1.0 { return None; } Some(SanitizedBox { left, top, right, bottom, confidence: confidence.clamp(0.0, 1.0), }) } fn detection_weight(detection: SanitizedBox) -> f32 { // Square-root area weighting lets prominent subjects matter more while // avoiding the domination that raw pixel area would cause. detection.confidence.max(0.05) * detection.area().sqrt().max(1.0) } fn face_belongs_to(face: SanitizedBox, subject: SanitizedBox) -> bool { // A face can sit slightly outside an imperfect person box, especially at // the top, but it must remain in the upper portion of that subject. let horizontal_padding = subject.width() * 0.10; let top_padding = subject.height() * 0.18; let center_inside = face.center_x() >= subject.left - horizontal_padding && face.center_x() <= subject.right + horizontal_padding && face.center_y() >= subject.top - top_padding && face.center_y() <= subject.top + subject.height() * 0.62; let overlap_left = face.left.max(subject.left); let overlap_top = face.top.max(subject.top - top_padding); let overlap_right = face.right.min(subject.right); let overlap_bottom = face.bottom.min(subject.top + subject.height() * 0.62); let overlap_area = (overlap_right - overlap_left).max(0.0) * (overlap_bottom - overlap_top).max(0.0); let mostly_overlaps = overlap_area / face.area() >= 0.35; // Reject implausibly large "faces" such as portrait murals that happen to // overlap a small person box. let plausible_size = face.area() <= subject.area() * 0.60; plausible_size && (center_inside || mostly_overlaps) } fn weighted_median(values: &[WeightedValue]) -> Option { let mut values: Vec<_> = values .iter() .copied() .filter(|entry| entry.value.is_finite() && entry.weight.is_finite() && entry.weight > 0.0) .collect(); if values.is_empty() { return None; } values.sort_by(|a, b| a.value.total_cmp(&b.value)); let total_weight: f32 = values.iter().map(|entry| entry.weight).sum(); let midpoint = total_weight * 0.5; let mut accumulated = 0.0; for entry in &values { accumulated += entry.weight; if accumulated >= midpoint { return Some(entry.value); } } values.last().map(|entry| entry.value) } #[cfg(test)] mod tests { use super::*; fn subject(x: f32, y: f32, width: f32, height: f32) -> SubjectBox { SubjectBox { x, y, width, height, confidence: 0.95, } } fn face(x: f32, y: f32, width: f32, height: f32) -> FaceBox { FaceBox { x, y, width, height, confidence: 0.95, } } /// Real detector output for Crazy P's MusicRadar photo (850x638, three /// standing subjects running to the bottom edge). The body-coverage guards /// wanted y=113, which cut every head off; faces must win. #[test] fn standing_group_keeps_heads_when_bodies_cannot_fit() { let subjects = [ subject(0.0, 83.0, 200.0, 547.0), subject(250.0, 26.0, 200.0, 610.0), subject(500.0, 102.0, 200.0, 486.0), ]; let faces = [ face(60.0, 47.0, 70.0, 81.0), face(300.0, 87.0, 62.0, 72.0), face(560.0, 127.0, 50.0, 57.0), ]; let plan = plan_hero_crop(850, 638, &subjects, &faces).expect("plan"); assert!(plan.content_aware); // Every face, and hair above the highest one, stays inside the window. let highest_face_top = 47.0; assert!( (plan.crop.y as f32) < highest_face_top, "crop started at {} which cuts the highest face at {highest_face_top}", plan.crop.y, ); let lowest_face_bottom = 127.0 + 57.0; assert!( (plan.crop.y + plan.crop.height) as f32 >= lowest_face_bottom, "crop bottom {} clipped the lowest face at {lowest_face_bottom}", plan.crop.y + plan.crop.height, ); } /// A face span taller than the window cannot be fully covered; the crop /// centres it rather than sacrificing one extreme entirely. #[test] fn face_span_taller_than_crop_is_centred() { // Two subjects at very different depths, each with a face at its own // head: the face span (20..930) exceeds the 900px window. let subjects = [ subject(0.0, 0.0, 300.0, 700.0), subject(400.0, 760.0, 300.0, 700.0), ]; let faces = [ face(50.0, 20.0, 120.0, 140.0), face(450.0, 790.0, 120.0, 140.0), ]; let plan = plan_hero_crop(1600, 1500, &subjects, &faces).expect("plan"); let span_center = (20.0 + 930.0) / 2.0; let crop_center = plan.crop.y as f32 + plan.crop.height as f32 / 2.0; assert!( (crop_center - span_center).abs() <= plan.crop.height as f32 * 0.25, "crop centre {crop_center} strayed from the face span centre {span_center}", ); } #[test] fn maximal_crop_retains_full_width_or_height() { let square = maximal_hero_rect(2048, 2048).unwrap(); assert_eq!( square, HeroRect { x: 0, y: 448, width: 2048, height: 1152 } ); assert_eq!(square.x, 0); assert_eq!(square.width, 2048); let wide = maximal_hero_rect(2560, 1080).unwrap(); assert_eq!( wide, HeroRect { x: 320, y: 0, width: 1920, height: 1080 } ); assert_eq!(wide.y, 0); assert_eq!(wide.height, 1080); } #[test] fn odd_tall_source_dimensions_keep_every_horizontal_pixel() { let crop = maximal_hero_rect(1001, 1001).unwrap(); assert_eq!( crop, HeroRect { x: 0, y: 219, width: 1001, height: 563 } ); assert_eq!(crop.width, 1001); assert!((crop.width as f64 / crop.height as f64 - 16.0 / 9.0).abs() < 0.002); } #[test] fn zero_or_subpixel_crop_dimensions_return_none() { assert_eq!(maximal_hero_rect(1, 100), None); assert_eq!(maximal_hero_rect(100, 0), None); assert_eq!(maximal_hero_rect(0, 0), None); } #[test] fn camping_dimensions_have_expected_centered_fallback() { let plan = plan_hero_crop(2560, 1920, &[], &[]).unwrap(); assert_eq!( plan.crop, HeroRect { x: 0, y: 240, width: 2560, height: 1440 } ); assert!(!plan.content_aware); assert_eq!(plan.focal_y, None); } #[test] fn face_only_portrait_still_gets_vertical_composition() { let faces = [face(900.0, 120.0, 420.0, 480.0)]; let plan = plan_hero_crop(2560, 1920, &[], &faces).unwrap(); assert!(plan.content_aware); assert_eq!(plan.subject_count, 0); assert_eq!(plan.associated_face_count, 1); assert_eq!(plan.crop.y, 0); } #[test] fn bottom_heavy_group_moves_camping_crop_down() { let subjects = [ subject(80.0, 790.0, 520.0, 1130.0), subject(610.0, 800.0, 520.0, 1120.0), subject(1210.0, 760.0, 510.0, 1160.0), subject(1810.0, 650.0, 650.0, 1270.0), ]; let faces = [ face(210.0, 820.0, 180.0, 190.0), face(760.0, 810.0, 170.0, 190.0), face(1370.0, 760.0, 170.0, 190.0), face(2030.0, 650.0, 190.0, 205.0), ]; let plan = plan_hero_crop(2560, 1920, &subjects, &faces).unwrap(); assert_eq!(plan.crop.x, 0); assert_eq!((plan.crop.width, plan.crop.height), (2560, 1440)); assert!( (390..=430).contains(&plan.crop.y), "unexpected y={}", plan.crop.y ); assert!(plan.content_aware); assert_eq!(plan.subject_count, 4); assert_eq!(plan.associated_face_count, 4); } #[test] fn top_heavy_subjects_move_crop_to_top() { let subjects = [ subject(250.0, 0.0, 700.0, 920.0), subject(1250.0, 20.0, 700.0, 900.0), ]; let faces = [ face(440.0, 80.0, 190.0, 210.0), face(1450.0, 100.0, 190.0, 210.0), ]; let plan = plan_hero_crop(2560, 1920, &subjects, &faces).unwrap(); assert_eq!(plan.crop.y, 0); assert_eq!(plan.associated_face_count, 2); } #[test] fn unassociated_mural_face_cannot_pull_crop_up() { let subjects = [ subject(150.0, 790.0, 500.0, 1130.0), subject(750.0, 800.0, 500.0, 1120.0), subject(1350.0, 760.0, 500.0, 1160.0), subject(1950.0, 700.0, 500.0, 1220.0), ]; let people_faces = [ face(300.0, 810.0, 170.0, 190.0), face(900.0, 820.0, 170.0, 190.0), face(1500.0, 780.0, 170.0, 190.0), face(2100.0, 720.0, 170.0, 190.0), ]; let mut with_mural = people_faces.to_vec(); with_mural.push(face(900.0, 70.0, 500.0, 560.0)); let without_mural = plan_hero_crop(2560, 1920, &subjects, &people_faces).unwrap(); let with_mural = plan_hero_crop(2560, 1920, &subjects, &with_mural).unwrap(); assert_eq!(with_mural.crop.y, without_mural.crop.y); assert_eq!(with_mural.associated_face_count, 4); } #[test] fn malformed_detections_produce_centered_fallback() { let subjects = [ subject(0.0, 0.0, -20.0, 100.0), SubjectBox { confidence: f32::NAN, ..subject(10.0, 10.0, 100.0, 100.0) }, subject(3000.0, 3000.0, 100.0, 100.0), ]; let plan = plan_hero_crop(2560, 1920, &subjects, &[]).unwrap(); assert_eq!(plan.crop.y, 240); assert!(!plan.content_aware); assert_eq!(plan.subject_count, 0); } #[test] fn wide_sources_always_keep_horizontal_crop_centered() { let subjects = [subject(0.0, 0.0, 500.0, 1080.0)]; let faces = [face(20.0, 40.0, 200.0, 220.0)]; let plan = plan_hero_crop(2560, 1080, &subjects, &faces).unwrap(); assert_eq!( plan.crop, HeroRect { x: 320, y: 0, width: 1920, height: 1080 } ); } #[test] fn horizontal_position_never_tracks_an_off_center_subject() { let subjects = [subject(2200.0, 700.0, 340.0, 1220.0)]; let faces = [face(2300.0, 760.0, 130.0, 150.0)]; let plan = plan_hero_crop(2560, 1920, &subjects, &faces).unwrap(); assert_eq!(plan.crop.x, 0); assert_eq!(plan.crop.width, 2560); assert!(plan.crop.y > 240); } }