Baseline: full tree snapshot after git repository loss

The previous .git directory was found gutted (only info/ remained), so
history is unrecoverable locally. This commit restores version control
with the tree exactly as it stands: the working app, the preview-tracks
launcher on the artist hero, the scrollable-track-previews design bundle
(docs + approved mockups), and the Phase 0 validation artifacts.
This commit is contained in:
KeelrLeek committed 2026-07-22 17:57:00 -05:00
commit a667b4b28e
161 files changed
+53170

No files matched your search

File diff suppressed because it is too large. Load diff
+311
View File
@@ -0,0 +1,311 @@
//! App config. Non-secret settings live in `config.json`; the Soulseek
//! password lives in the Windows Credential Manager (via `keyring`).
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use tauri::Manager;
const KEYRING_SERVICE: &str = "com.soulstream.app";
/// Pre-rename service name; stored passwords migrate on first read.
const LEGACY_KEYRING_SERVICE: &str = "com.trifecta.soulseek-plus";
/// Pre-rename app-data folder; migrated by `migrate_app_data` at startup.
const LEGACY_APP_DATA_DIR: &str = "com.trifecta.soulseek-plus";
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ArtistHeroUpscaler {
/// Fixed 3840x2160 GPU master, mip-filtered to the visible window.
#[default]
#[serde(alias = "fsr")]
LiveGpu,
#[serde(rename = "fsr_legacy")]
Fsr,
Lanczos3,
}
/// Ceiling for peer-source formats. `LossyOnly` skips lossless files —
/// the "don't stream multi-hundred-MB FLACs on this connection" cap.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceQualityCap {
#[default]
Any,
LossyOnly,
}
/// Floor for peer-source quality. Lossless formats satisfy every floor; a
/// lossy file must carry a known bitrate at or above the named floor.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceQualityFloor {
#[default]
Any,
Kbps192,
Kbps256,
Kbps320,
Lossless,
}
/// What the resolver optimizes among sources that satisfy the cap/floor:
/// best format (current behavior) or fastest transfer (free slot, upload
/// speed, short queue — format only breaks ties).
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SourcePriority {
#[default]
Quality,
Speed,
}
/// Per-mode source-selection preferences; streaming and downloading each
/// carry their own copy.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourcePrefs {
#[serde(default)]
pub max: SourceQualityCap,
#[serde(default)]
pub min: SourceQualityFloor,
#[serde(default)]
pub priority: SourcePriority,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AppConfig {
#[serde(default)]
pub username: String,
#[serde(default)]
pub listen_port: Option<u16>,
/// Where downloads are saved. None → default `<Music>\SoulSeek+`.
#[serde(default)]
pub download_dir: Option<String>,
/// Album-folder pattern below the managed library/download root.
/// Available placeholders: {artist}, {album}, {date}, {year}.
/// `\` or `/` in the pattern create subfolders (separators coming from
/// tag values are flattened to `_`).
#[serde(default)]
pub library_layout: Option<String>,
/// Extra folders to share (the downloads folder is always shared).
#[serde(default)]
pub shared_folders: Vec<String>,
/// Concurrent upload slots offered to peers.
#[serde(default)]
pub upload_slots: Option<u32>,
/// Named platform output endpoint. None follows Windows' default device.
#[serde(default)]
pub audio_output_device: Option<String>,
/// Spatial upscaler used for provider artist heroes below native 4K.
#[serde(default)]
pub artist_hero_upscaler: ArtistHeroUpscaler,
/// Source-selection preferences for streaming.
#[serde(default)]
pub stream_source_prefs: SourcePrefs,
/// Source-selection preferences for downloads/saves.
#[serde(default)]
pub download_source_prefs: SourcePrefs,
/// Replace whatever picture a download arrives with by the canonical
/// high-resolution cover. None → true. Startup-scanned library files are
/// never rewritten by this; it applies to fresh downloads only.
#[serde(default)]
pub replace_embedded_art: Option<bool>,
/// Longest edge for covers embedded into audio files. None → 1500.
/// Some(0) → embed the original untouched. The full-resolution original
/// is always kept in the art cache and the album folder regardless.
#[serde(default)]
pub embed_art_max_edge: Option<u32>,
/// Write the canonical cover as cover.jpg + folder.jpg in the organized
/// album folder so Explorer and other players show the same artwork.
/// None → true.
#[serde(default)]
pub write_folder_cover: Option<bool>,
/// Save every fully streamed track into the library. None → false:
/// streams stay only in the bounded stream cache below.
#[serde(default)]
pub stream_keep_forever: Option<bool>,
/// Days a streamed track stays in the stream cache after it was last
/// played. None → 30. Ignored when `stream_keep_forever` is on.
#[serde(default)]
pub stream_retention_days: Option<u32>,
/// Stream cache size cap in MB, oldest evicted first. None → 2048.
/// Some(0) → unlimited.
#[serde(default)]
pub stream_cache_max_mb: Option<u64>,
}
/// Default longest edge for embedded covers when the user hasn't chosen.
pub const DEFAULT_EMBED_ART_MAX_EDGE: u32 = 1500;
pub fn replace_embedded_art(cfg: &AppConfig) -> bool {
cfg.replace_embedded_art.unwrap_or(true)
}
/// 0 means "embed the original at full resolution".
pub fn embed_art_max_edge(cfg: &AppConfig) -> u32 {
cfg.embed_art_max_edge.unwrap_or(DEFAULT_EMBED_ART_MAX_EDGE)
}
pub fn write_folder_cover(cfg: &AppConfig) -> bool {
cfg.write_folder_cover.unwrap_or(true)
}
pub const DEFAULT_STREAM_RETENTION_DAYS: u32 = 30;
pub const DEFAULT_STREAM_CACHE_MAX_MB: u64 = 2048;
pub fn stream_keep_forever(cfg: &AppConfig) -> bool {
cfg.stream_keep_forever.unwrap_or(false)
}
pub fn stream_retention_days(cfg: &AppConfig) -> u32 {
cfg.stream_retention_days
.unwrap_or(DEFAULT_STREAM_RETENTION_DAYS)
.max(1)
}
/// 0 = unlimited.
pub fn stream_cache_max_mb(cfg: &AppConfig) -> u64 {
cfg.stream_cache_max_mb
.unwrap_or(DEFAULT_STREAM_CACHE_MAX_MB)
}
/// One-time migration after the identifier rename: the app-data dir moved
/// from `com.trifecta.soulseek-plus` to the new identifier's folder. Rename
/// the old dir into place so the library DB, config and stream cache
/// survive. Never overwrites existing data at the new location.
pub fn migrate_app_data(app: &tauri::AppHandle) {
let Ok(new_dir) = app.path().app_data_dir() else {
return;
};
let Some(parent) = new_dir.parent() else {
return;
};
let old_dir = parent.join(LEGACY_APP_DATA_DIR);
if !old_dir.is_dir() {
return;
}
let occupied = new_dir.exists()
&& std::fs::read_dir(&new_dir)
.map(|mut d| d.next().is_some())
.unwrap_or(true);
if occupied {
return;
}
if new_dir.exists() {
let _ = std::fs::remove_dir(&new_dir);
}
match std::fs::rename(&old_dir, &new_dir) {
Ok(()) => log::info!("migrated app data from {}", old_dir.display()),
Err(e) => log::warn!(
"app data migration failed ({e}); old data stays in {}",
old_dir.display()
),
}
}
fn config_path(app: &tauri::AppHandle) -> Option<PathBuf> {
let dir = app.path().app_data_dir().ok()?;
std::fs::create_dir_all(&dir).ok()?;
Some(dir.join("config.json"))
}
pub fn load(app: &tauri::AppHandle) -> AppConfig {
config_path(app)
.and_then(|p| std::fs::read_to_string(p).ok())
.and_then(|s| serde_json::from_str(&s).ok())
.unwrap_or_default()
}
pub fn save(app: &tauri::AppHandle, cfg: &AppConfig) {
if let Some(p) = config_path(app) {
if let Ok(json) = serde_json::to_string_pretty(cfg) {
if let Err(e) = std::fs::write(&p, json) {
log::warn!("failed to save config to {}: {e}", p.display());
}
}
}
}
/// The effective downloads directory: configured, or `<Music>\SoulStream`.
/// A pre-rename `<Music>\SoulSeek+` folder keeps being used if it exists,
/// so an existing library doesn't fork into a new empty root.
pub fn download_dir(app: &tauri::AppHandle, cfg: &AppConfig) -> PathBuf {
if let Some(d) = &cfg.download_dir {
if !d.is_empty() {
return PathBuf::from(d);
}
}
let music = app
.path()
.audio_dir()
.or_else(|_| app.path().home_dir())
.unwrap_or_else(|_| PathBuf::from("."));
let legacy = music.join("SoulSeek+");
if legacy.is_dir() {
return legacy;
}
music.join("SoulStream")
}
pub const DEFAULT_LIBRARY_LAYOUT: &str = r"{artist}\{year} - {album}";
pub fn library_layout(cfg: &AppConfig) -> &str {
cfg.library_layout
.as_deref()
.map(str::trim)
.filter(|layout| !layout.is_empty())
.unwrap_or(DEFAULT_LIBRARY_LAYOUT)
}
// ---- password storage (Windows Credential Manager) ----
fn entry(username: &str) -> Option<keyring::Entry> {
keyring::Entry::new(KEYRING_SERVICE, username).ok()
}
pub fn save_password(username: &str, password: &str) {
if let Some(e) = entry(username) {
if let Err(err) = e.set_password(password) {
log::warn!("failed to store password in credential manager: {err}");
}
}
}
pub fn load_password(username: &str) -> Option<String> {
if let Some(pw) = entry(username).and_then(|e| e.get_password().ok()) {
return Some(pw);
}
// Fall back to the pre-rename service entry and migrate it over.
let legacy = keyring::Entry::new(LEGACY_KEYRING_SERVICE, username).ok()?;
let pw = legacy.get_password().ok()?;
save_password(username, &pw);
let _ = legacy.delete_credential();
Some(pw)
}
pub fn has_password(username: &str) -> bool {
!username.is_empty() && load_password(username).is_some()
}
pub fn delete_password(username: &str) {
if let Some(e) = entry(username) {
let _ = e.delete_credential();
}
}
#[cfg(test)]
mod tests {
use super::ArtistHeroUpscaler;
#[test]
fn legacy_fsr_setting_migrates_to_live_gpu() {
let migrated: ArtistHeroUpscaler = serde_json::from_str("\"fsr\"").unwrap();
assert_eq!(migrated, ArtistHeroUpscaler::LiveGpu);
assert_eq!(ArtistHeroUpscaler::default(), ArtistHeroUpscaler::LiveGpu);
assert_eq!(
serde_json::to_string(&ArtistHeroUpscaler::LiveGpu).unwrap(),
"\"live_gpu\""
);
assert_eq!(
serde_json::to_string(&ArtistHeroUpscaler::Fsr).unwrap(),
"\"fsr_legacy\""
);
}
}
+655
View File
@@ -0,0 +1,655 @@
//! Derive an artist wordmark from album art for artists that have no provider
//! (Fanart.tv / TheAudioDB) logo. Many covers print the band name as clean text;
//! on-device OCR (Windows.Media.Ocr) locates that line, and (increment 2) it is
//! segmented off a near-uniform background into a transparent PNG that the
//! existing logo tracer vectorizes and tints like any provider logo.
//!
//! Fallible by design: any failure (non-Windows build, no OCR, no matching text)
//! leaves the caller on the plain text-name fallback.
/// Axis-aligned box in source-image pixels.
#[derive(Clone, Copy, Debug)]
pub(crate) struct Rect {
pub x: f32,
pub y: f32,
pub w: f32,
pub h: f32,
}
/// One recognized word with its own bounding box.
#[derive(Clone, Debug)]
pub(crate) struct OcrWord {
pub text: String,
pub rect: Rect,
}
/// One recognized line of text with its bounding box (union of its words) plus
/// the individual words — so a wordmark can be boxed to JUST the words that spell
/// the artist name, excluding a track number / remix credit sharing the line.
#[derive(Clone, Debug)]
pub(crate) struct OcrLine {
pub text: String,
pub rect: Rect,
pub words: Vec<OcrWord>,
}
/// Lowercase alphanumerics only — matches the normalization used elsewhere so a
/// cover's "CAMPING IN ALASKA" lines up with the artist name "Camping In Alaska".
fn normalize(s: &str) -> String {
s.to_lowercase()
.chars()
.filter(|c| c.is_alphanumeric())
.collect()
}
// Segmentation tuning (RGB distance space, 0..~441).
const BG_NEAR_DIST: f32 = 46.0; // a pixel this close to the dominant colour counts as background
const BG_MIN_DOMINANCE: f32 = 0.5; // background must fill at least this share of the crop, else it's a photo → reject
const ALPHA_LO: f32 = 46.0; // distance-from-bg below which a pixel is background (transparent)
const ALPHA_HI: f32 = 120.0; // distance-from-bg above which a pixel is full ink (opaque)
fn dist(a: [f32; 3], b: [f32; 3]) -> f32 {
((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2) + (a[2] - b[2]).powi(2)).sqrt()
}
fn smoothstep(lo: f32, hi: f32, x: f32) -> f32 {
let t = ((x - lo) / (hi - lo)).clamp(0.0, 1.0);
t * t * (3.0 - 2.0 * t)
}
/// A wordmark lifted off a cover, plus a quality score for picking the best cover.
pub(crate) struct Wordmark {
pub image: image::RgbaImage,
pub score: f32,
}
/// The band-name line on a cover plus the vertical band it may occupy. `clamp_top`
/// / `clamp_bottom` are the midpoints to the nearest OTHER text lines (or image
/// edges), so the extraction crop can NEVER reach another line — the album title
/// and any other cover text are structurally excluded from the logo.
#[derive(Clone, Copy, Debug)]
pub(crate) struct BandLocation {
pub rect: Rect,
pub clamp_top: f32,
pub clamp_bottom: f32,
}
impl BandLocation {
/// Scale to another resolution of the same cover (locate at 1000px where OCR
/// is reliable, extract at 1800px for quality).
pub fn scaled(&self, sx: f32, sy: f32) -> BandLocation {
BandLocation {
rect: Rect {
x: self.rect.x * sx,
y: self.rect.y * sy,
w: self.rect.w * sx,
h: self.rect.h * sy,
},
clamp_top: self.clamp_top * sy,
clamp_bottom: self.clamp_bottom * sy,
}
}
}
/// Segment the band-name text at `loc` off its (near-uniform) background into a
/// transparent RGBA image. The crop is hard-clamped to `loc`'s band so no other
/// text line can enter. Returns `None` when the background is too busy to segment
/// (text over a photo) or the result is empty — those stay on the text fallback.
pub(crate) fn extract_wordmark(
image: &image::DynamicImage,
loc: &BandLocation,
) -> Option<Wordmark> {
use image::GenericImageView as _;
let rect = loc.rect;
let (iw, ih) = image.dimensions();
// Pad the located word box — vertical for ascenders/descenders + tracer room —
// but never past the clamp to a neighbouring line.
let pad_x = (rect.w * 0.05).max(6.0);
let pad_y = (rect.h * 0.45).max(6.0);
let x0 = (rect.x - pad_x).max(0.0) as u32;
let y0 = (rect.y - pad_y).max(loc.clamp_top).max(0.0) as u32;
let x1 = ((rect.x + rect.w + pad_x).min(iw as f32)) as u32;
let y1 = ((rect.y + rect.h + pad_y)
.min(loc.clamp_bottom)
.min(ih as f32)) as u32;
if x1 <= x0 + 4 || y1 <= y0 + 4 {
return None;
}
let crop = image.crop_imm(x0, y0, x1 - x0, y1 - y0).to_rgba8();
let (w, h) = crop.dimensions();
// Background = the dominant colour, via a coarse 16³ RGB histogram over the
// whole crop. Robust to sparse texture (faint line-art on black) in a way a
// ring mean is not — the text is a minority, so the mode is the background.
let mut hist = [0u32; 16 * 16 * 16];
for p in crop.pixels() {
let (r, g, b) = (p.0[0] >> 4, p.0[1] >> 4, p.0[2] >> 4);
hist[(r as usize) * 256 + (g as usize) * 16 + b as usize] += 1;
}
let mode = hist
.iter()
.enumerate()
.max_by_key(|(_, c)| **c)
.map(|(i, _)| i)
.unwrap_or(0);
let bg = [
((mode / 256) as f32) * 16.0 + 8.0,
(((mode / 16) % 16) as f32) * 16.0 + 8.0,
((mode % 16) as f32) * 16.0 + 8.0,
];
// Dominance guard: what share of the crop is near that background colour?
// A clean wordmark has a dominant background (text is the minority); a photo
// has no single dominant colour, so this share is low → reject.
let total = (w * h) as f32;
let near = crop
.pixels()
.filter(|p| dist([p.0[0] as f32, p.0[1] as f32, p.0[2] as f32], bg) < BG_NEAR_DIST)
.count() as f32;
let dominance = near / total;
if dominance < BG_MIN_DOMINANCE {
return None; // busy background — text over a photo, not segmentable
}
// Alpha ramp by distance from the background colour, tracking per-row ink so
// we can trim adjacent text lines the padding caught (the album title above /
// below the band name), which sit in their own ink band across an empty gap.
let mut full = image::RgbaImage::new(w, h);
let mut row_ink = vec![0u32; h as usize];
for y in 0..h {
for x in 0..w {
let p = crop.get_pixel(x, y).0;
let d = dist([p[0] as f32, p[1] as f32, p[2] as f32], bg);
let a = smoothstep(ALPHA_LO, ALPHA_HI, d);
let alpha = (a * 255.0).round() as u8;
// Flat white ink (not the cover's own colour). The frontend logo pass
// only traces the ALPHA silhouette and tints it to the accent, and it
// only does so when the mark is monochrome — a cover's varied/print
// colour scored below that bar and fell back to a small untinted PNG.
// A single flat colour guarantees the traced, accent-tinted SVG path.
full.put_pixel(x, y, image::Rgba([255, 255, 255, alpha]));
if alpha > 128 {
row_ink[y as usize] += 1;
}
}
}
// The wordmark is CENTRED on the located box. A single line of text never has a
// fully transparent row running through it — some glyph always spans every row
// (an i-dot's little gap is filled by neighbouring letters). So the first fully
// transparent row above/below the centre is the wordmark's TRUE edge; whatever
// lies past that transparent gap is separated content — a rule bar above, the
// album-title ghost below — and is dropped. We crop to the centre content, then
// re-add clean transparent padding.
let empty = |r: usize| (row_ink[r] as f32 / w as f32) < 0.004;
// A near-fully-inked row is a solid rule/bar touching the text with no gap —
// also a boundary (glyph rows always leave inter-letter gaps, never ~full).
let solid = |r: usize| row_ink[r] as f32 / w as f32 > 0.85;
let center = (((rect.y + rect.h / 2.0) as u32).saturating_sub(y0)).min(h - 1) as usize;
// 2 transparent rows = a real gap (bridges a 1px anti-aliasing fluke).
let gap_stop = 2usize;
// Safety cap so a pathological run can't swallow the whole cover.
let max_half = (rect.h * 1.5).round().max(6.0) as usize;
let lo = center.saturating_sub(max_half);
let hi = (center + max_half).min(h as usize - 1);
let mut top = center;
{
let (mut r, mut gap) = (center, 0usize);
while r > lo {
r -= 1;
if solid(r) {
break;
}
if empty(r) {
gap += 1;
if gap >= gap_stop {
break;
}
} else {
gap = 0;
top = r;
}
}
}
let mut bottom = center;
{
let (mut r, mut gap) = (center, 0usize);
while r + 1 <= hi {
r += 1;
if solid(r) {
break;
}
if empty(r) {
gap += 1;
if gap >= gap_stop {
break;
}
} else {
gap = 0;
bottom = r;
}
}
}
// Re-add transparent breathing room (~6% of content height) for the tracer/blur.
let pad = (((bottom - top + 1) as f32) * 0.06)
.round()
.clamp(3.0, 40.0) as usize;
let band_top = top.saturating_sub(pad) as u32;
let band_bottom = (bottom + pad).min(h as usize - 1) as u32;
let band_h = band_bottom - band_top + 1;
if band_h < 6 {
return None;
}
// Crop to the wordmark's row band and tally ink/partial for scoring.
let mut out = image::RgbaImage::new(w, band_h);
let mut ink = 0u32;
let mut partial = 0u32;
for (dy, y) in (band_top..=band_bottom).enumerate() {
for x in 0..w {
let px = *full.get_pixel(x, y);
out.put_pixel(x, dy as u32, px);
if px.0[3] > 200 {
ink += 1;
} else if px.0[3] > 20 {
partial += 1;
}
}
}
let coverage = ink as f32 / (w * band_h) as f32;
if coverage < 0.008 || coverage > 0.6 {
return None;
}
// Recipe D: median despeckle removes the ragged spurs a lo-fi print leaves on
// the letter edges before the frontend supersamples/blurs/traces it.
let out = median_alpha(&out);
// Score: reward anti-aliasing (edge quality), a title-like wide aspect, and
// a healthy ink coverage; higher wins across candidate covers.
let aa = (partial as f32 / (ink as f32 + 1.0)).min(0.6);
let aspect = ((w as f32 / band_h as f32) / 10.0).min(0.6);
let cover_score = 1.0 - (coverage - 0.15).abs();
Some(Wordmark {
image: out,
score: aa + aspect + cover_score,
})
}
// ---- Experimental alpha post-processing (for the quality comparison) ----
/// 3×3 median on the alpha channel — despeckles the ragged spurs a noisy print
/// leaves on the letter edges, without rounding real corners much.
fn median_alpha(src: &image::RgbaImage) -> image::RgbaImage {
let (w, h) = src.dimensions();
let mut out = src.clone();
for y in 0..h {
for x in 0..w {
// The previous implementation allocated a Vec for every output
// pixel. A fixed nine-byte buffer computes the identical median.
let mut n = [0u8; 9];
let mut len = 0usize;
for dy in -1..=1 {
for dx in -1..=1 {
let nx = x as i32 + dx;
let ny = y as i32 + dy;
if nx >= 0 && ny >= 0 && (nx as u32) < w && (ny as u32) < h {
n[len] = src.get_pixel(nx as u32, ny as u32).0[3];
len += 1;
}
}
}
n[..len].sort_unstable();
let mut p = *src.get_pixel(x, y);
p.0[3] = n[len / 2];
out.put_pixel(x, y, p);
}
}
out
}
/// Otsu-binarized copy (black text on white / white on black, auto-thresholded on
/// luma). Experimental preprocessing to test whether the OCR reads stylized
/// display type better with the colour and gradients flattened away.
pub(crate) fn binarize(image: &image::DynamicImage) -> image::DynamicImage {
use image::GenericImageView as _;
let gray = image.to_luma8();
// Otsu threshold.
let mut hist = [0u32; 256];
for p in gray.pixels() {
hist[p.0[0] as usize] += 1;
}
let total: u32 = hist.iter().sum();
let sum: f64 = hist
.iter()
.enumerate()
.map(|(i, &c)| i as f64 * c as f64)
.sum();
let (mut sum_b, mut w_b, mut max, mut thresh) = (0.0f64, 0u32, 0.0f64, 128u8);
for (t, &c) in hist.iter().enumerate() {
w_b += c;
if w_b == 0 {
continue;
}
let w_f = total - w_b;
if w_f == 0 {
break;
}
sum_b += t as f64 * c as f64;
let m_b = sum_b / w_b as f64;
let m_f = (sum - sum_b) / w_f as f64;
let between = w_b as f64 * w_f as f64 * (m_b - m_f).powi(2);
if between > max {
max = between;
thresh = t as u8;
}
}
// Decide polarity so text is dark on light (OCR's expectation) by majority.
let (w, h) = image.dimensions();
let mut out = image::RgbaImage::new(w, h);
for (x, y, p) in gray.enumerate_pixels().map(|(x, y, p)| (x, y, p.0[0])) {
let v = if p > thresh { 255 } else { 0 };
out.put_pixel(x, y, image::Rgba([v, v, v, 255]));
}
image::DynamicImage::ImageRgba8(out)
}
/// OCR the cover twice — once as-is and once Otsu-binarized — and merge the line
/// sets. Colour/gradient display type (the cyan "CRAZY P" on Paper Years) is
/// often unreadable raw but recognized once flattened to black-on-white; the raw
/// pass still catches anything the threshold destroys. Both passes share pixel
/// coordinates (the binarized copy is the same dimensions), so word boxes line up.
pub(crate) fn ocr_lines_multi(image: &image::DynamicImage) -> Result<Vec<OcrLine>, String> {
let mut lines = ocr_lines(image)?;
if let Ok(mut bin) = ocr_lines(&binarize(image)) {
lines.append(&mut bin);
}
Ok(lines)
}
/// All text lines OCR finds on the image, in source-pixel coordinates.
pub(crate) fn ocr_lines(image: &image::DynamicImage) -> Result<Vec<OcrLine>, String> {
#[cfg(windows)]
{
windows_ocr::ocr_lines(image)
}
#[cfg(not(windows))]
{
let _ = image;
Err("cover-logo OCR is only available on Windows".into())
}
}
/// A substring match must be at least this share of its OCR line's letters, or
/// the line is a tracklist / credit that merely MENTIONS the name (e.g.
/// "A1. Disc Odyssey Crazy P remix" — "crazyp" is 25% of it) rather than a
/// wordmark that IS the name. Precision-first: only "just the band name" lines
/// (optionally with a hyphen/initial) clear this bar, never a whole credit line.
const NAME_LINE_MIN_RATIO: f32 = 0.6;
/// Locate the band-name line and the vertical band it may occupy — clamped at
/// the midpoint to the nearest OTHER text line above/below (or the image edge),
/// so extraction can never include the album title or any other cover text.
/// Scores every line so a bare-name line beats one that merely embeds the name,
/// and rejects lines where the name is a minority (a tracklist / remix credit).
pub(crate) fn locate_wordmark(
image: &image::DynamicImage,
artist: &str,
) -> Result<Option<BandLocation>, String> {
Ok(locate_wordmark_named(image, artist)?.map(|(loc, _, _)| loc))
}
/// Within one OCR line, find the consecutive run of WORDS whose normalized text
/// spells the artist name exactly, and return the union box of just those words.
/// This lifts "Crazy P" cleanly out of "A1. Disc Odyssey Crazy P remix" or
/// "CRAZY P THE PAPER YEARS" — the track number, remix credit and album title
/// are never inside the returned box. `None` if no run spells the name.
fn word_run_box(line: &OcrLine, want: &str) -> Option<Rect> {
// Only real word tokens: a "///", "A1." or "-" normalizes to empty and must
// NOT be allowed into the run's box, or the decorative mark next to the name
// gets extracted with it (Turnover's "/// TURNOVER").
let toks: Vec<&OcrWord> = line
.words
.iter()
.filter(|w| !normalize(&w.text).is_empty())
.collect();
for i in 0..toks.len() {
let mut acc = String::new();
for j in i..toks.len() {
acc.push_str(&normalize(&toks[j].text));
if acc.len() > want.len() {
break;
}
if acc == want {
let (mut x0, mut y0, mut x1, mut y1) = (f32::MAX, f32::MAX, f32::MIN, f32::MIN);
for w in &toks[i..=j] {
x0 = x0.min(w.rect.x);
y0 = y0.min(w.rect.y);
x1 = x1.max(w.rect.x + w.rect.w);
y1 = y1.max(w.rect.y + w.rect.h);
}
return Some(Rect {
x: x0,
y: y0,
w: x1 - x0,
h: y1 - y0,
});
}
}
}
None
}
/// Like [`locate_wordmark`] but also returns the matched OCR text and its score
/// (1.0 = a clean word-run of the name; <1 = a line the name only partly fills).
/// For diagnostics — "which line on which cover did we lift?".
pub(crate) fn locate_wordmark_named(
image: &image::DynamicImage,
artist: &str,
) -> Result<Option<(BandLocation, String, f32)>, String> {
let want = normalize(artist);
if want.is_empty() {
return Ok(None);
}
// Two-pass OCR (raw + binarized) so stylized colour wordmarks are read.
let lines = ocr_lines_multi(image)?;
// Prefer a precise word-run match (the box is JUST the name); among those,
// take the widest — a cover's big printed wordmark over a tiny corner label.
// Fall back to a whole-line substring only if no cover word-run matches, and
// only when the name fills most of the line (else it's a tracklist/credit).
let mut word_best: Option<(usize, Rect)> = None;
let mut line_best: Option<(usize, f32)> = None;
for (i, line) in lines.iter().enumerate() {
if let Some(rect) = word_run_box(line, &want) {
if word_best.is_none_or(|(_, r)| rect.w > r.w) {
word_best = Some((i, rect));
}
continue;
}
let norm = normalize(&line.text);
if norm.is_empty() {
continue;
}
let score = if norm == want {
1.0
} else if want.len() >= 4 && norm.contains(&want) {
let ratio = want.len() as f32 / norm.len() as f32;
if ratio >= NAME_LINE_MIN_RATIO {
ratio
} else {
continue;
}
} else {
continue;
};
if line_best.is_none_or(|(_, s)| score > s) {
line_best = Some((i, score));
}
}
let (bi, band, matched_score) = if let Some((i, rect)) = word_best {
(i, rect, 1.0)
} else if let Some((i, s)) = line_best {
(i, lines[i].rect, s)
} else {
return Ok(None);
};
let matched_text = lines[bi].text.clone();
let mut clamp_top = 0.0f32;
let mut clamp_bottom = image.height() as f32;
for (i, line) in lines.iter().enumerate() {
if i == bi {
continue;
}
let r = line.rect;
let r_bottom = r.y + r.h;
if r_bottom <= band.y {
// A line above — stop halfway up to it.
clamp_top = clamp_top.max((r_bottom + band.y) / 2.0);
} else if r.y >= band.y + band.h {
// A line below — stop halfway down to it.
clamp_bottom = clamp_bottom.min((band.y + band.h + r.y) / 2.0);
}
}
Ok(Some((
BandLocation {
rect: band,
clamp_top,
clamp_bottom,
},
matched_text,
matched_score,
)))
}
#[cfg(windows)]
mod windows_ocr {
use image::{DynamicImage, ImageEncoder as _};
use windows::{
Graphics::Imaging::{BitmapDecoder, BitmapPixelFormat, SoftwareBitmap},
Media::Ocr::OcrEngine,
Storage::Streams::{DataWriter, InMemoryRandomAccessStream},
Win32::System::WinRT::{RoInitialize, RoUninitialize, RO_INIT_MULTITHREADED},
};
use super::{OcrLine, OcrWord, Rect};
struct WinRtGuard;
impl Drop for WinRtGuard {
fn drop(&mut self) {
unsafe { RoUninitialize() };
}
}
fn initialize_winrt() -> Result<Option<WinRtGuard>, String> {
match unsafe { RoInitialize(RO_INIT_MULTITHREADED) } {
Ok(()) => Ok(Some(WinRtGuard)),
// Thread already in a different apartment: WinRT is usable, we just
// must not uninitialize it.
Err(error) if error.code().0 as u32 == 0x8001_0106 => Ok(None),
Err(error) => Err(format!("Windows Runtime initialization failed: {error}")),
}
}
/// Load a DynamicImage into a Bgra8 SoftwareBitmap (the format OCR wants),
/// reusing the same PNG→stream→decoder path the face detector uses.
fn software_bitmap(image: &DynamicImage) -> Result<SoftwareBitmap, String> {
let rgba = image.to_rgba8();
let (width, height) = rgba.dimensions();
let mut png = Vec::new();
image::codecs::png::PngEncoder::new(&mut png)
.write_image(
rgba.as_raw(),
width,
height,
image::ExtendedColorType::Rgba8,
)
.map_err(|error| format!("could not encode OCR raster: {error}"))?;
let stream = InMemoryRandomAccessStream::new()
.map_err(|error| format!("could not allocate OCR stream: {error}"))?;
let writer = DataWriter::CreateDataWriter(&stream)
.map_err(|error| format!("could not create OCR writer: {error}"))?;
writer
.WriteBytes(&png)
.map_err(|error| format!("could not write OCR raster: {error}"))?;
writer
.StoreAsync()
.and_then(|op| op.get())
.map_err(|error| format!("could not store OCR raster: {error}"))?;
writer
.FlushAsync()
.and_then(|op| op.get())
.map_err(|error| format!("could not flush OCR raster: {error}"))?;
let _ = writer.DetachStream();
stream
.Seek(0)
.map_err(|error| format!("could not rewind OCR raster: {error}"))?;
let decoder = BitmapDecoder::CreateAsync(&stream)
.and_then(|op| op.get())
.map_err(|error| format!("could not decode OCR raster: {error}"))?;
let decoded = decoder
.GetSoftwareBitmapAsync()
.and_then(|op| op.get())
.map_err(|error| format!("could not prepare OCR bitmap: {error}"))?;
SoftwareBitmap::Convert(&decoded, BitmapPixelFormat::Bgra8)
.map_err(|error| format!("could not convert OCR bitmap: {error}"))
}
pub(crate) fn ocr_lines(image: &DynamicImage) -> Result<Vec<OcrLine>, String> {
let _winrt = initialize_winrt()?;
let bitmap = software_bitmap(image)?;
let engine = OcrEngine::TryCreateFromUserProfileLanguages()
.map_err(|error| format!("could not create OCR engine: {error}"))?;
let result = engine
.RecognizeAsync(&bitmap)
.and_then(|op| op.get())
.map_err(|error| format!("OCR failed: {error}"))?;
let mut lines = Vec::new();
for line in &result
.Lines()
.map_err(|error| format!("could not read OCR lines: {error}"))?
{
let text = line.Text().map(|t| t.to_string_lossy()).unwrap_or_default();
if text.trim().is_empty() {
continue;
}
// The line has no box of its own — union its words' boxes, and keep
// each word's box so a wordmark can be located word-precisely.
let (mut min_x, mut min_y, mut max_x, mut max_y) =
(f32::MAX, f32::MAX, f32::MIN, f32::MIN);
let mut word_boxes = Vec::new();
if let Ok(words) = line.Words() {
for word in &words {
if let Ok(r) = word.BoundingRect() {
min_x = min_x.min(r.X);
min_y = min_y.min(r.Y);
max_x = max_x.max(r.X + r.Width);
max_y = max_y.max(r.Y + r.Height);
let wtext = word.Text().map(|t| t.to_string_lossy()).unwrap_or_default();
word_boxes.push(OcrWord {
text: wtext,
rect: Rect {
x: r.X,
y: r.Y,
w: r.Width,
h: r.Height,
},
});
}
}
}
if word_boxes.is_empty() {
continue;
}
lines.push(OcrLine {
text: text.trim().to_string(),
rect: Rect {
x: min_x,
y: min_y,
w: max_x - min_x,
h: max_y - min_y,
},
words: word_boxes,
});
}
Ok(lines)
}
}
+787
View File
@@ -0,0 +1,787 @@
use serde::{Deserialize, Serialize};
// Composition targets are deliberately expressed relative to the crop rather
// than the source. A person's visual centre sits below the middle of a hero,
// leaving their head near the upper third while retaining more of their body.
const PERSON_CENTER_TARGET: f32 = 0.66;
const FACE_CENTER_TARGET: f32 = 0.44;
const FACE_REFINEMENT_WEIGHT: f32 = 0.28;
const PERSON_BOTTOM_COVERAGE: f32 = 0.93;
const PERSON_TOP_MARGIN: f32 = 0.07;
/// Headroom kept above the highest detected face, as a fraction of that face
/// box and as a floor relative to the crop. A face box starts at the brow, not
/// the crown, so roughly a third of a face height of hair sits above it.
const FACE_HEADROOM_OF_FACE: f32 = 0.38;
const FACE_HEADROOM_OF_CROP: f32 = 0.03;
/// Headroom above the topmost PERSON box when no face was associated. A person
/// box top is already the crown, so this only needs to avoid a hairline shave.
const PERSON_CROWN_HEADROOM: f32 = 0.01;
/// Pull a proposed crop start so every detected face stays inside the window,
/// with hair headroom above the highest one. Returns `desired_y` unchanged
/// when there are no faces to protect.
///
/// When the faces span MORE than the crop height (a group photographed at very
/// different depths), no placement can hold them all; the span is centred
/// instead, which cuts the least from the people at both extremes.
fn protect_faces(
desired_y: f32,
faces: &[SanitizedBox],
people: &[SanitizedBox],
crop_height: u32,
) -> f32 {
let Some(highest) = faces
.iter()
.min_by(|left, right| left.top.total_cmp(&right.top))
else {
// No face survived association — common for small, distant or
// turned-away subjects. The person box's top edge IS the crown, so it
// still protects the head from the body-coverage guards, which use a
// MEDIAN top margin and therefore happily over-cut the topmost person
// of a group. Only ever raises the window; never pushes it down.
let Some(highest_person) = people
.iter()
.min_by(|left, right| left.top.total_cmp(&right.top))
else {
return desired_y;
};
let headroom = crop_height as f32 * PERSON_CROWN_HEADROOM;
return desired_y.min((highest_person.top - headroom).max(0.0));
};
let lowest_bottom = faces
.iter()
.map(|face| face.bottom)
.fold(f32::MIN, f32::max);
let headroom =
(highest.height() * FACE_HEADROOM_OF_FACE).max(crop_height as f32 * FACE_HEADROOM_OF_CROP);
let top_limit = (highest.top - headroom).max(0.0);
let bottom_need =
lowest_bottom + crop_height as f32 * FACE_HEADROOM_OF_CROP - crop_height as f32;
if bottom_need <= top_limit {
desired_y.clamp(bottom_need.max(0.0), top_limit)
} else {
(highest.top + lowest_bottom) / 2.0 - crop_height as f32 / 2.0
}
}
/// Placement tracing for crop audits (`SOULSTREAM_CROP_TRACE=1`). The chosen
/// `y` is the product of four competing terms, so a bad composition is only
/// diagnosable with the intermediates in hand.
fn crop_trace(message: impl FnOnce() -> String) {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
if *ON.get_or_init(|| std::env::var("SOULSTREAM_CROP_TRACE").is_ok_and(|value| value == "1")) {
eprintln!("CROP\t{}", message());
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct HeroRect {
pub x: u32,
pub y: u32,
pub width: u32,
pub height: u32,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct HeroCropPlan {
pub source_width: u32,
pub source_height: u32,
pub crop: HeroRect,
/// True when valid person detections affected the vertical placement.
pub content_aware: bool,
/// Robust vertical focus in source pixels. Useful for diagnostics only;
/// consumers should apply `crop` verbatim for preview and final renders.
pub focal_y: Option<f32>,
pub subject_count: u32,
pub associated_face_count: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct SubjectBox {
pub x: f32,
pub y: f32,
pub width: f32,
pub height: f32,
pub confidence: f32,
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct FaceBox {
pub x: f32,
pub y: f32,
pub width: f32,
pub height: f32,
pub confidence: f32,
}
#[derive(Debug, Clone, Copy)]
struct WeightedValue {
value: f32,
weight: f32,
}
#[derive(Debug, Clone, Copy)]
struct SanitizedBox {
left: f32,
top: f32,
right: f32,
bottom: f32,
confidence: f32,
}
impl SanitizedBox {
fn width(self) -> f32 {
self.right - self.left
}
fn height(self) -> f32 {
self.bottom - self.top
}
fn area(self) -> f32 {
self.width() * self.height()
}
fn center_x(self) -> f32 {
(self.left + self.right) * 0.5
}
fn center_y(self) -> f32 {
(self.top + self.bottom) * 0.5
}
}
/// Returns the largest pixel crop that fits the source's 16:9 frame.
///
/// Tall and square sources retain every horizontal source pixel (`x = 0`), so
/// content awareness can move only on the Y axis. Wide sources retain the full
/// source height and center the crop on X. Integer rounding can leave the raw
/// crop less than one pixel away from 16:9; preview/final rendering normalizes
/// that same rectangle to exact 16:9 output dimensions.
pub(crate) fn maximal_hero_rect(source_width: u32, source_height: u32) -> Option<HeroRect> {
if source_width == 0 || source_height == 0 {
return None;
}
if source_width as u64 * 9 <= source_height as u64 * 16 {
let height = (source_width as u64 * 9 / 16) as u32;
(height > 0).then_some(HeroRect {
x: 0,
y: (source_height - height) / 2,
width: source_width,
height,
})
} else {
let width = (source_height as u64 * 16 / 9) as u32;
(width > 0).then_some(HeroRect {
x: (source_width - width) / 2,
y: 0,
width,
height: source_height,
})
}
}
/// Builds a deterministic hero crop plan from optional person and face boxes.
///
/// Person boxes are the primary signal. Faces only refine the vertical
/// composition when they can be associated with a person, which prevents faces
/// in murals, posters, and album art from pulling the crop away from the artist.
/// Horizontal placement never follows a detection: wide sources stay centered,
/// and tall sources retain the maximum possible source width.
pub(crate) fn plan_hero_crop(
source_width: u32,
source_height: u32,
subjects: &[SubjectBox],
faces: &[FaceBox],
) -> Option<HeroCropPlan> {
let mut crop = maximal_hero_rect(source_width, source_height)?;
let fallback_y = crop.y;
let max_y = source_height - crop.height;
let valid_subjects: Vec<_> = subjects
.iter()
.filter_map(|subject| {
sanitize_box(
subject.x,
subject.y,
subject.width,
subject.height,
subject.confidence,
source_width,
source_height,
)
})
.collect();
if max_y == 0 {
return Some(HeroCropPlan {
source_width,
source_height,
crop,
content_aware: false,
focal_y: None,
subject_count: valid_subjects.len() as u32,
associated_face_count: 0,
});
}
if valid_subjects.is_empty() {
// Face-only is a secondary fallback for portraits where the general
// person model misses a close or partially visible subject. Provider
// hardening already rejects most posters/text cards; cap implausibly
// huge boxes and use a robust median so one false positive cannot
// dominate a group.
let source_area = source_width as f32 * source_height as f32;
let valid_faces: Vec<_> = faces
.iter()
.filter_map(|face| {
sanitize_box(
face.x,
face.y,
face.width,
face.height,
face.confidence,
source_width,
source_height,
)
})
.filter(|face| face.area() <= source_area * 0.25)
.collect();
let face_values: Vec<_> = valid_faces
.iter()
.map(|face| WeightedValue {
value: face.center_y() - crop.height as f32 * FACE_CENTER_TARGET,
weight: detection_weight(*face),
})
.collect();
if let Some(desired_y) = weighted_median(&face_values) {
// Same inviolable rule as the person path: the composition target
// may place the window, but never past a face.
let desired_y = protect_faces(desired_y, &valid_faces, &[], crop.height);
crop.y = desired_y.round().clamp(0.0, max_y as f32) as u32;
return Some(HeroCropPlan {
source_width,
source_height,
crop,
content_aware: true,
focal_y: weighted_median(
&valid_faces
.iter()
.map(|face| WeightedValue {
value: face.center_y(),
weight: detection_weight(*face),
})
.collect::<Vec<_>>(),
),
subject_count: 0,
associated_face_count: valid_faces.len() as u32,
});
}
return Some(HeroCropPlan {
source_width,
source_height,
crop,
content_aware: false,
focal_y: None,
subject_count: 0,
associated_face_count: 0,
});
}
let person_values: Vec<_> = valid_subjects
.iter()
.map(|subject| WeightedValue {
value: subject.center_y() - crop.height as f32 * PERSON_CENTER_TARGET,
weight: detection_weight(*subject),
})
.collect();
let person_y = weighted_median(&person_values).unwrap_or(fallback_y as f32);
// These robust bounds keep nearly all of a typical detected body visible,
// without allowing one outlying/oversized box to dictate the result.
let lower_values: Vec<_> = valid_subjects
.iter()
.map(|subject| WeightedValue {
value: subject.top + subject.height() * PERSON_BOTTOM_COVERAGE - crop.height as f32,
weight: detection_weight(*subject),
})
.collect();
let upper_values: Vec<_> = valid_subjects
.iter()
.map(|subject| WeightedValue {
value: subject.top + subject.height() * PERSON_TOP_MARGIN,
weight: detection_weight(*subject),
})
.collect();
let lower_y = weighted_median(&lower_values).unwrap_or(0.0);
let upper_y = weighted_median(&upper_values).unwrap_or(max_y as f32);
let valid_faces: Vec<_> = faces
.iter()
.filter_map(|face| {
sanitize_box(
face.x,
face.y,
face.width,
face.height,
face.confidence,
source_width,
source_height,
)
})
.filter(|face| {
valid_subjects
.iter()
.any(|subject| face_belongs_to(*face, *subject))
})
.collect();
let face_values: Vec<_> = valid_faces
.iter()
.map(|face| WeightedValue {
value: face.center_y() - crop.height as f32 * FACE_CENTER_TARGET,
weight: detection_weight(*face),
})
.collect();
let mut desired_y = person_y;
if let Some(face_y) = weighted_median(&face_values) {
desired_y = person_y * (1.0 - FACE_REFINEMENT_WEIGHT) + face_y * FACE_REFINEMENT_WEIGHT;
}
// Use body-coverage guards when both can be satisfied. If a close subject
// fills more than the crop height, the robust composition targets decide
// which portion to retain rather than sacrificing the head or lower body by
// blindly snapping to one incompatible edge.
let unclamped_y = desired_y;
if upper_y >= lower_y {
desired_y = desired_y.clamp(lower_y, upper_y);
}
// Faces outrank body coverage, always. A standing subject is usually
// TALLER than the 16:9 window, so the coverage guards above ask the window
// to slide down until 93% of the body fits — and when the body runs to the
// bottom edge, the only thing left to sacrifice is the head. Measured on
// Crazy P's MusicRadar photo: composition wanted y=-3.6 (every head in
// frame) and the coverage clamp forced y=113, decapitating all three.
// Legs may be cut; faces may not.
desired_y = protect_faces(desired_y, &valid_faces, &valid_subjects, crop.height);
crop.y = desired_y.round().clamp(0.0, max_y as f32) as u32;
crop_trace(|| {
let faces: Vec<String> = valid_faces
.iter()
.map(|face| format!("[y{:.0}+{:.0}]", face.top, face.height()))
.collect();
let people: Vec<String> = valid_subjects
.iter()
.map(|subject| format!("[y{:.0}+{:.0}]", subject.top, subject.height()))
.collect();
format!(
"src={source_width}x{source_height} croph={} max_y={max_y} people={} faces={} \
person_y={person_y:.1} face_y={:?} desired={unclamped_y:.1} \
lower={lower_y:.1} upper={upper_y:.1} clamped={} final_y={}",
crop.height,
people.join(","),
faces.join(","),
weighted_median(&face_values).map(|value| (value * 10.0).round() / 10.0),
upper_y >= lower_y,
crop.y,
)
});
let focal_y = if valid_faces.is_empty() {
weighted_median(
&valid_subjects
.iter()
.map(|subject| WeightedValue {
value: subject.center_y(),
weight: detection_weight(*subject),
})
.collect::<Vec<_>>(),
)
} else {
weighted_median(
&valid_faces
.iter()
.map(|face| WeightedValue {
value: face.center_y(),
weight: detection_weight(*face),
})
.collect::<Vec<_>>(),
)
};
Some(HeroCropPlan {
source_width,
source_height,
crop,
content_aware: true,
focal_y,
subject_count: valid_subjects.len() as u32,
associated_face_count: valid_faces.len() as u32,
})
}
fn sanitize_box(
x: f32,
y: f32,
width: f32,
height: f32,
confidence: f32,
source_width: u32,
source_height: u32,
) -> Option<SanitizedBox> {
if !x.is_finite()
|| !y.is_finite()
|| !width.is_finite()
|| !height.is_finite()
|| !confidence.is_finite()
|| width <= 0.0
|| height <= 0.0
|| confidence <= 0.0
{
return None;
}
let left = x.clamp(0.0, source_width as f32);
let top = y.clamp(0.0, source_height as f32);
let right = (x + width).clamp(0.0, source_width as f32);
let bottom = (y + height).clamp(0.0, source_height as f32);
if right - left < 1.0 || bottom - top < 1.0 {
return None;
}
Some(SanitizedBox {
left,
top,
right,
bottom,
confidence: confidence.clamp(0.0, 1.0),
})
}
fn detection_weight(detection: SanitizedBox) -> f32 {
// Square-root area weighting lets prominent subjects matter more while
// avoiding the domination that raw pixel area would cause.
detection.confidence.max(0.05) * detection.area().sqrt().max(1.0)
}
fn face_belongs_to(face: SanitizedBox, subject: SanitizedBox) -> bool {
// A face can sit slightly outside an imperfect person box, especially at
// the top, but it must remain in the upper portion of that subject.
let horizontal_padding = subject.width() * 0.10;
let top_padding = subject.height() * 0.18;
let center_inside = face.center_x() >= subject.left - horizontal_padding
&& face.center_x() <= subject.right + horizontal_padding
&& face.center_y() >= subject.top - top_padding
&& face.center_y() <= subject.top + subject.height() * 0.62;
let overlap_left = face.left.max(subject.left);
let overlap_top = face.top.max(subject.top - top_padding);
let overlap_right = face.right.min(subject.right);
let overlap_bottom = face.bottom.min(subject.top + subject.height() * 0.62);
let overlap_area =
(overlap_right - overlap_left).max(0.0) * (overlap_bottom - overlap_top).max(0.0);
let mostly_overlaps = overlap_area / face.area() >= 0.35;
// Reject implausibly large "faces" such as portrait murals that happen to
// overlap a small person box.
let plausible_size = face.area() <= subject.area() * 0.60;
plausible_size && (center_inside || mostly_overlaps)
}
fn weighted_median(values: &[WeightedValue]) -> Option<f32> {
let mut values: Vec<_> = values
.iter()
.copied()
.filter(|entry| entry.value.is_finite() && entry.weight.is_finite() && entry.weight > 0.0)
.collect();
if values.is_empty() {
return None;
}
values.sort_by(|a, b| a.value.total_cmp(&b.value));
let total_weight: f32 = values.iter().map(|entry| entry.weight).sum();
let midpoint = total_weight * 0.5;
let mut accumulated = 0.0;
for entry in &values {
accumulated += entry.weight;
if accumulated >= midpoint {
return Some(entry.value);
}
}
values.last().map(|entry| entry.value)
}
#[cfg(test)]
mod tests {
use super::*;
fn subject(x: f32, y: f32, width: f32, height: f32) -> SubjectBox {
SubjectBox {
x,
y,
width,
height,
confidence: 0.95,
}
}
fn face(x: f32, y: f32, width: f32, height: f32) -> FaceBox {
FaceBox {
x,
y,
width,
height,
confidence: 0.95,
}
}
/// Real detector output for Crazy P's MusicRadar photo (850x638, three
/// standing subjects running to the bottom edge). The body-coverage guards
/// wanted y=113, which cut every head off; faces must win.
#[test]
fn standing_group_keeps_heads_when_bodies_cannot_fit() {
let subjects = [
subject(0.0, 83.0, 200.0, 547.0),
subject(250.0, 26.0, 200.0, 610.0),
subject(500.0, 102.0, 200.0, 486.0),
];
let faces = [
face(60.0, 47.0, 70.0, 81.0),
face(300.0, 87.0, 62.0, 72.0),
face(560.0, 127.0, 50.0, 57.0),
];
let plan = plan_hero_crop(850, 638, &subjects, &faces).expect("plan");
assert!(plan.content_aware);
// Every face, and hair above the highest one, stays inside the window.
let highest_face_top = 47.0;
assert!(
(plan.crop.y as f32) < highest_face_top,
"crop started at {} which cuts the highest face at {highest_face_top}",
plan.crop.y,
);
let lowest_face_bottom = 127.0 + 57.0;
assert!(
(plan.crop.y + plan.crop.height) as f32 >= lowest_face_bottom,
"crop bottom {} clipped the lowest face at {lowest_face_bottom}",
plan.crop.y + plan.crop.height,
);
}
/// A face span taller than the window cannot be fully covered; the crop
/// centres it rather than sacrificing one extreme entirely.
#[test]
fn face_span_taller_than_crop_is_centred() {
// Two subjects at very different depths, each with a face at its own
// head: the face span (20..930) exceeds the 900px window.
let subjects = [
subject(0.0, 0.0, 300.0, 700.0),
subject(400.0, 760.0, 300.0, 700.0),
];
let faces = [
face(50.0, 20.0, 120.0, 140.0),
face(450.0, 790.0, 120.0, 140.0),
];
let plan = plan_hero_crop(1600, 1500, &subjects, &faces).expect("plan");
let span_center = (20.0 + 930.0) / 2.0;
let crop_center = plan.crop.y as f32 + plan.crop.height as f32 / 2.0;
assert!(
(crop_center - span_center).abs() <= plan.crop.height as f32 * 0.25,
"crop centre {crop_center} strayed from the face span centre {span_center}",
);
}
#[test]
fn maximal_crop_retains_full_width_or_height() {
let square = maximal_hero_rect(2048, 2048).unwrap();
assert_eq!(
square,
HeroRect {
x: 0,
y: 448,
width: 2048,
height: 1152
}
);
assert_eq!(square.x, 0);
assert_eq!(square.width, 2048);
let wide = maximal_hero_rect(2560, 1080).unwrap();
assert_eq!(
wide,
HeroRect {
x: 320,
y: 0,
width: 1920,
height: 1080
}
);
assert_eq!(wide.y, 0);
assert_eq!(wide.height, 1080);
}
#[test]
fn odd_tall_source_dimensions_keep_every_horizontal_pixel() {
let crop = maximal_hero_rect(1001, 1001).unwrap();
assert_eq!(
crop,
HeroRect {
x: 0,
y: 219,
width: 1001,
height: 563
}
);
assert_eq!(crop.width, 1001);
assert!((crop.width as f64 / crop.height as f64 - 16.0 / 9.0).abs() < 0.002);
}
#[test]
fn zero_or_subpixel_crop_dimensions_return_none() {
assert_eq!(maximal_hero_rect(1, 100), None);
assert_eq!(maximal_hero_rect(100, 0), None);
assert_eq!(maximal_hero_rect(0, 0), None);
}
#[test]
fn camping_dimensions_have_expected_centered_fallback() {
let plan = plan_hero_crop(2560, 1920, &[], &[]).unwrap();
assert_eq!(
plan.crop,
HeroRect {
x: 0,
y: 240,
width: 2560,
height: 1440
}
);
assert!(!plan.content_aware);
assert_eq!(plan.focal_y, None);
}
#[test]
fn face_only_portrait_still_gets_vertical_composition() {
let faces = [face(900.0, 120.0, 420.0, 480.0)];
let plan = plan_hero_crop(2560, 1920, &[], &faces).unwrap();
assert!(plan.content_aware);
assert_eq!(plan.subject_count, 0);
assert_eq!(plan.associated_face_count, 1);
assert_eq!(plan.crop.y, 0);
}
#[test]
fn bottom_heavy_group_moves_camping_crop_down() {
let subjects = [
subject(80.0, 790.0, 520.0, 1130.0),
subject(610.0, 800.0, 520.0, 1120.0),
subject(1210.0, 760.0, 510.0, 1160.0),
subject(1810.0, 650.0, 650.0, 1270.0),
];
let faces = [
face(210.0, 820.0, 180.0, 190.0),
face(760.0, 810.0, 170.0, 190.0),
face(1370.0, 760.0, 170.0, 190.0),
face(2030.0, 650.0, 190.0, 205.0),
];
let plan = plan_hero_crop(2560, 1920, &subjects, &faces).unwrap();
assert_eq!(plan.crop.x, 0);
assert_eq!((plan.crop.width, plan.crop.height), (2560, 1440));
assert!(
(390..=430).contains(&plan.crop.y),
"unexpected y={}",
plan.crop.y
);
assert!(plan.content_aware);
assert_eq!(plan.subject_count, 4);
assert_eq!(plan.associated_face_count, 4);
}
#[test]
fn top_heavy_subjects_move_crop_to_top() {
let subjects = [
subject(250.0, 0.0, 700.0, 920.0),
subject(1250.0, 20.0, 700.0, 900.0),
];
let faces = [
face(440.0, 80.0, 190.0, 210.0),
face(1450.0, 100.0, 190.0, 210.0),
];
let plan = plan_hero_crop(2560, 1920, &subjects, &faces).unwrap();
assert_eq!(plan.crop.y, 0);
assert_eq!(plan.associated_face_count, 2);
}
#[test]
fn unassociated_mural_face_cannot_pull_crop_up() {
let subjects = [
subject(150.0, 790.0, 500.0, 1130.0),
subject(750.0, 800.0, 500.0, 1120.0),
subject(1350.0, 760.0, 500.0, 1160.0),
subject(1950.0, 700.0, 500.0, 1220.0),
];
let people_faces = [
face(300.0, 810.0, 170.0, 190.0),
face(900.0, 820.0, 170.0, 190.0),
face(1500.0, 780.0, 170.0, 190.0),
face(2100.0, 720.0, 170.0, 190.0),
];
let mut with_mural = people_faces.to_vec();
with_mural.push(face(900.0, 70.0, 500.0, 560.0));
let without_mural = plan_hero_crop(2560, 1920, &subjects, &people_faces).unwrap();
let with_mural = plan_hero_crop(2560, 1920, &subjects, &with_mural).unwrap();
assert_eq!(with_mural.crop.y, without_mural.crop.y);
assert_eq!(with_mural.associated_face_count, 4);
}
#[test]
fn malformed_detections_produce_centered_fallback() {
let subjects = [
subject(0.0, 0.0, -20.0, 100.0),
SubjectBox {
confidence: f32::NAN,
..subject(10.0, 10.0, 100.0, 100.0)
},
subject(3000.0, 3000.0, 100.0, 100.0),
];
let plan = plan_hero_crop(2560, 1920, &subjects, &[]).unwrap();
assert_eq!(plan.crop.y, 240);
assert!(!plan.content_aware);
assert_eq!(plan.subject_count, 0);
}
#[test]
fn wide_sources_always_keep_horizontal_crop_centered() {
let subjects = [subject(0.0, 0.0, 500.0, 1080.0)];
let faces = [face(20.0, 40.0, 200.0, 220.0)];
let plan = plan_hero_crop(2560, 1080, &subjects, &faces).unwrap();
assert_eq!(
plan.crop,
HeroRect {
x: 320,
y: 0,
width: 1920,
height: 1080
}
);
}
#[test]
fn horizontal_position_never_tracks_an_off_center_subject() {
let subjects = [subject(2200.0, 700.0, 340.0, 1220.0)];
let faces = [face(2300.0, 760.0, 130.0, 150.0)];
let plan = plan_hero_crop(2560, 1920, &subjects, &faces).unwrap();
assert_eq!(plan.crop.x, 0);
assert_eq!(plan.crop.width, 2560);
assert!(plan.crop.y > 240);
}
}
+468
View File
@@ -0,0 +1,468 @@
//! Lightweight, local subject analysis for artist-hero composition.
//!
//! Person boxes come from the bundled YOLOX-Nano model through the Windows
//! ML runtime already present in Windows 10/11. Faces come from the native
//! Windows face detector, so no second model or inference runtime is shipped.
//! All failures are recoverable: callers fall back to a deterministic crop.
use crate::hero_crop::{FaceBox, SubjectBox};
#[cfg(windows)]
mod windows_detector {
use std::sync::OnceLock;
use image::{DynamicImage, GenericImageView as _, ImageEncoder as _};
use tauri::Manager as _;
use windows::{
core::{Interface as _, HSTRING},
Graphics::Imaging::{BitmapDecoder, BitmapPixelFormat, BitmapSize, SoftwareBitmap},
Media::FaceAnalysis::FaceDetector,
Storage::Streams::{DataWriter, InMemoryRandomAccessStream},
Win32::System::WinRT::{RoInitialize, RoUninitialize, RO_INIT_MULTITHREADED},
AI::MachineLearning::{
LearningModel, LearningModelBinding, LearningModelDevice, LearningModelDeviceKind,
LearningModelSession, TensorFloat,
},
};
use super::{FaceBox, SubjectBox};
const YOLOX_EDGE: u32 = 416;
const YOLOX_CANDIDATES: usize = 3549;
const YOLOX_VALUES: usize = 85;
const PERSON_SCORE_MIN: f32 = 0.25;
const PERSON_NMS_IOU: f32 = 0.45;
const FACE_ANALYSIS_MAX_EDGE: u32 = 960;
static PERSON_SESSION: OnceLock<Result<LearningModelSession, String>> = OnceLock::new();
static FACE_DETECTOR: OnceLock<Result<FaceDetector, String>> = OnceLock::new();
struct WinRtGuard;
impl Drop for WinRtGuard {
fn drop(&mut self) {
unsafe { RoUninitialize() };
}
}
fn initialize_winrt() -> Result<Option<WinRtGuard>, String> {
match unsafe { RoInitialize(RO_INIT_MULTITHREADED) } {
Ok(()) => Ok(Some(WinRtGuard)),
Err(error) if error.code().0 as u32 == 0x8001_0106 => {
// The current thread already has a different apartment. WinRT
// remains usable there; it simply must not be uninitialized by us.
Ok(None)
}
Err(error) => Err(format!("Windows Runtime initialization failed: {error}")),
}
}
fn model_path(app: &tauri::AppHandle) -> Result<std::path::PathBuf, String> {
let bundled = app
.path()
.resolve(
"resources/yolox_nano.onnx",
tauri::path::BaseDirectory::Resource,
)
.map_err(|error| error.to_string())?;
if bundled.is_file() {
return Ok(bundled);
}
let development = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("resources")
.join("yolox_nano.onnx");
if development.is_file() {
return Ok(development);
}
Err("bundled YOLOX-Nano model is missing".into())
}
fn person_session_from_path(path: &std::path::Path) -> Result<LearningModelSession, String> {
PERSON_SESSION
.get_or_init(|| {
let model = LearningModel::LoadFromFilePath(&HSTRING::from(
path.to_string_lossy().as_ref(),
))
.map_err(|error| format!("could not load YOLOX-Nano: {error}"))?;
let device = LearningModelDevice::Create(LearningModelDeviceKind::Cpu)
.map_err(|error| format!("could not create WinML CPU device: {error}"))?;
LearningModelSession::CreateFromModelOnDevice(&model, &device)
.map_err(|error| format!("could not create YOLOX session: {error}"))
})
.clone()
}
fn person_session(app: &tauri::AppHandle) -> Result<LearningModelSession, String> {
person_session_from_path(&model_path(app)?)
}
fn face_detector() -> Result<FaceDetector, String> {
FACE_DETECTOR
.get_or_init(|| {
if !FaceDetector::IsSupported()
.map_err(|error| format!("could not query face detector: {error}"))?
{
return Err("Windows face detection is unavailable".into());
}
let detector = FaceDetector::CreateAsync()
.map_err(|error| format!("could not create face detector: {error}"))?
.get()
.map_err(|error| format!("could not initialize face detector: {error}"))?;
// Small faces matter in group press photos. Analysis is capped
// at 960 px, so 18 px still avoids tiny texture false positives.
let _ = detector.SetMinDetectableFaceSize(BitmapSize {
Width: 18,
Height: 18,
});
Ok(detector)
})
.clone()
}
pub(super) fn detect(
app: &tauri::AppHandle,
image: &DynamicImage,
) -> Result<(Vec<SubjectBox>, Vec<FaceBox>), String> {
let _winrt = initialize_winrt()?;
let subjects = detect_people(app, image)?;
// Face refinement is optional. A native-detector failure must not throw
// away valid person boxes from WinML.
let faces = match detect_faces(image) {
Ok(faces) => faces,
Err(error) => {
log::warn!("artist hero face refinement unavailable: {error}");
Vec::new()
}
};
Ok((subjects, faces))
}
pub(super) fn warm_up(app: tauri::AppHandle) {
std::thread::Builder::new()
.name("hero-vision-warmup".into())
.spawn(move || {
let Ok(_winrt) = initialize_winrt() else {
return;
};
if let Err(error) = person_session(&app) {
log::warn!("artist hero person detector warm-up failed: {error}");
}
if let Err(error) = face_detector() {
log::warn!("artist hero face detector warm-up failed: {error}");
}
})
.ok();
}
fn detect_people(
app: &tauri::AppHandle,
image: &DynamicImage,
) -> Result<Vec<SubjectBox>, String> {
detect_people_with_model_path(&model_path(app)?, image)
}
fn detect_people_with_model_path(
model_path: &std::path::Path,
image: &DynamicImage,
) -> Result<Vec<SubjectBox>, String> {
let (source_width, source_height) = image.dimensions();
if source_width == 0 || source_height == 0 {
return Ok(Vec::new());
}
let scale =
(YOLOX_EDGE as f32 / source_width as f32).min(YOLOX_EDGE as f32 / source_height as f32);
let resized_width = ((source_width as f32 * scale).floor() as u32).clamp(1, YOLOX_EDGE);
let resized_height = ((source_height as f32 * scale).floor() as u32).clamp(1, YOLOX_EDGE);
let rgb = image.to_rgb8();
let resized = image::imageops::resize(
&rgb,
resized_width,
resized_height,
image::imageops::FilterType::Triangle,
);
let plane_len = (YOLOX_EDGE * YOLOX_EDGE) as usize;
let mut tensor_data = vec![114.0f32; plane_len * 3];
for y in 0..resized_height {
for x in 0..resized_width {
let pixel = resized.get_pixel(x, y).0;
let offset = (y * YOLOX_EDGE + x) as usize;
// Official YOLOX preprocessing consumes OpenCV BGR without
// normalization, transposed directly into NCHW planes.
tensor_data[offset] = f32::from(pixel[2]);
tensor_data[plane_len + offset] = f32::from(pixel[1]);
tensor_data[plane_len * 2 + offset] = f32::from(pixel[0]);
}
}
let input = TensorFloat::CreateFromShapeArrayAndDataArray(
&[1, 3, i64::from(YOLOX_EDGE), i64::from(YOLOX_EDGE)],
&tensor_data,
)
.map_err(|error| format!("could not create YOLOX input: {error}"))?;
let session = person_session_from_path(model_path)?;
let binding = LearningModelBinding::CreateFromSession(&session)
.map_err(|error| format!("could not create YOLOX binding: {error}"))?;
binding
.Bind(&HSTRING::from("images"), &input)
.map_err(|error| format!("could not bind YOLOX input: {error}"))?;
let result = session
.Evaluate(&binding, &HSTRING::from("artist-hero"))
.map_err(|error| format!("YOLOX inference failed: {error}"))?;
if !result
.Succeeded()
.map_err(|error| format!("could not read YOLOX status: {error}"))?
{
return Err(format!(
"YOLOX evaluation failed with status {}",
result.ErrorStatus().unwrap_or_default()
));
}
let output = result
.Outputs()
.and_then(|outputs| outputs.Lookup(&HSTRING::from("output")))
.map_err(|error| format!("YOLOX output is missing: {error}"))?;
let tensor: TensorFloat = output
.cast()
.map_err(|error| format!("YOLOX output has the wrong type: {error}"))?;
let view = tensor
.GetAsVectorView()
.map_err(|error| format!("could not read YOLOX output: {error}"))?;
let values: Vec<f32> = view.into_iter().collect();
if values.len() != YOLOX_CANDIDATES * YOLOX_VALUES {
return Err(format!("unexpected YOLOX output length: {}", values.len()));
}
let mut candidates = Vec::new();
let mut candidate_index = 0usize;
for stride in [8u32, 16, 32] {
let grid = YOLOX_EDGE / stride;
for gy in 0..grid {
for gx in 0..grid {
let row = &values
[candidate_index * YOLOX_VALUES..(candidate_index + 1) * YOLOX_VALUES];
candidate_index += 1;
let confidence = row[4] * row[5];
if !confidence.is_finite() || confidence < PERSON_SCORE_MIN {
continue;
}
let center_x = (row[0] + gx as f32) * stride as f32 / scale;
let center_y = (row[1] + gy as f32) * stride as f32 / scale;
let width = row[2].clamp(-10.0, 10.0).exp() * stride as f32 / scale;
let height = row[3].clamp(-10.0, 10.0).exp() * stride as f32 / scale;
if !center_x.is_finite()
|| !center_y.is_finite()
|| !width.is_finite()
|| !height.is_finite()
{
continue;
}
let left = (center_x - width * 0.5).clamp(0.0, source_width as f32);
let top = (center_y - height * 0.5).clamp(0.0, source_height as f32);
let right = (center_x + width * 0.5).clamp(0.0, source_width as f32);
let bottom = (center_y + height * 0.5).clamp(0.0, source_height as f32);
if right - left < 8.0 || bottom - top < 16.0 {
continue;
}
candidates.push(SubjectBox {
x: left,
y: top,
width: right - left,
height: bottom - top,
confidence,
});
}
}
}
debug_assert_eq!(candidate_index, YOLOX_CANDIDATES);
Ok(non_maximum_suppression(candidates, PERSON_NMS_IOU, 24))
}
fn detect_faces(image: &DynamicImage) -> Result<Vec<FaceBox>, String> {
let (source_width, source_height) = image.dimensions();
if source_width == 0 || source_height == 0 {
return Ok(Vec::new());
}
let detection = image.thumbnail(FACE_ANALYSIS_MAX_EDGE, FACE_ANALYSIS_MAX_EDGE);
let gray = detection.to_luma8();
let (width, height) = gray.dimensions();
let mut png = Vec::new();
image::codecs::png::PngEncoder::new(&mut png)
.write_image(gray.as_raw(), width, height, image::ExtendedColorType::L8)
.map_err(|error| format!("could not encode face-analysis raster: {error}"))?;
let stream = InMemoryRandomAccessStream::new()
.map_err(|error| format!("could not allocate face-analysis stream: {error}"))?;
let writer = DataWriter::CreateDataWriter(&stream)
.map_err(|error| format!("could not create face-analysis writer: {error}"))?;
writer
.WriteBytes(&png)
.map_err(|error| format!("could not write face-analysis raster: {error}"))?;
writer
.StoreAsync()
.and_then(|operation| operation.get())
.map_err(|error| format!("could not store face-analysis raster: {error}"))?;
writer
.FlushAsync()
.and_then(|operation| operation.get())
.map_err(|error| format!("could not flush face-analysis raster: {error}"))?;
let _ = writer.DetachStream();
stream
.Seek(0)
.map_err(|error| format!("could not rewind face-analysis raster: {error}"))?;
let decoder = BitmapDecoder::CreateAsync(&stream)
.and_then(|operation| operation.get())
.map_err(|error| format!("could not decode face-analysis raster: {error}"))?;
let decoded_bitmap = decoder
.GetSoftwareBitmapAsync()
.and_then(|operation| operation.get())
.map_err(|error| format!("could not prepare face-analysis bitmap: {error}"))?;
let bitmap = SoftwareBitmap::Convert(&decoded_bitmap, BitmapPixelFormat::Gray8)
.map_err(|error| format!("could not convert face-analysis bitmap: {error}"))?;
let found = face_detector()?
.DetectFacesAsync(&bitmap)
.and_then(|operation| operation.get())
.map_err(|error| format!("face detection failed: {error}"))?;
let scale_x = source_width as f32 / width as f32;
let scale_y = source_height as f32 / height as f32;
let mut faces = Vec::new();
for detected in &found {
let bounds = detected
.FaceBox()
.map_err(|error| format!("could not read face bounds: {error}"))?;
faces.push(FaceBox {
x: bounds.X as f32 * scale_x,
y: bounds.Y as f32 * scale_y,
width: bounds.Width as f32 * scale_x,
height: bounds.Height as f32 * scale_y,
confidence: 1.0,
});
}
Ok(faces)
}
fn non_maximum_suppression(
mut candidates: Vec<SubjectBox>,
threshold: f32,
limit: usize,
) -> Vec<SubjectBox> {
candidates.sort_by(|a, b| b.confidence.total_cmp(&a.confidence));
let mut kept = Vec::new();
'candidate: for candidate in candidates {
for accepted in &kept {
if intersection_over_union(candidate, *accepted) > threshold {
continue 'candidate;
}
}
kept.push(candidate);
if kept.len() == limit {
break;
}
}
kept
}
fn intersection_over_union(a: SubjectBox, b: SubjectBox) -> f32 {
let left = a.x.max(b.x);
let top = a.y.max(b.y);
let right = (a.x + a.width).min(b.x + b.width);
let bottom = (a.y + a.height).min(b.y + b.height);
let intersection = (right - left).max(0.0) * (bottom - top).max(0.0);
let union = a.width * a.height + b.width * b.height - intersection;
if union > 0.0 {
intersection / union
} else {
0.0
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn nms_keeps_the_best_overlapping_person() {
let kept = non_maximum_suppression(
vec![
SubjectBox {
x: 0.0,
y: 0.0,
width: 100.0,
height: 200.0,
confidence: 0.9,
},
SubjectBox {
x: 2.0,
y: 4.0,
width: 100.0,
height: 200.0,
confidence: 0.8,
},
SubjectBox {
x: 300.0,
y: 0.0,
width: 100.0,
height: 200.0,
confidence: 0.7,
},
],
0.45,
24,
);
assert_eq!(kept.len(), 2);
assert_eq!(kept[0].confidence, 0.9);
}
/// Manual, reusable integration probe. It is ignored in normal CI;
/// provide an image through SOULSTREAM_HERO_FIXTURE when validating a
/// new detector/model or an especially difficult artist photo.
#[test]
#[ignore]
fn fixture_detection_and_crop_probe() {
let fixture = std::env::var("SOULSTREAM_HERO_FIXTURE")
.expect("set SOULSTREAM_HERO_FIXTURE to a source image");
let image = image::open(fixture).expect("fixture must decode");
let _winrt = initialize_winrt().expect("WinRT must initialize");
let model = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("resources")
.join("yolox_nano.onnx");
let subjects = detect_people_with_model_path(&model, &image)
.expect("person inference must succeed");
let faces = detect_faces(&image).expect("face detection must succeed");
let plan =
crate::hero_crop::plan_hero_crop(image.width(), image.height(), &subjects, &faces)
.expect("fixture must plan");
eprintln!("subjects={subjects:#?}");
eprintln!("faces={faces:#?}");
eprintln!("plan={plan:#?}");
assert!(plan.crop.width > 0 && plan.crop.height > 0);
}
}
}
/// Detects people and faces in source-image coordinates. This is deliberately
/// a fallible enhancement: hero processing must continue when vision APIs are
/// unavailable (including non-Windows development builds).
pub(crate) fn detect_subjects(
app: &tauri::AppHandle,
image: &image::DynamicImage,
) -> Result<(Vec<SubjectBox>, Vec<FaceBox>), String> {
#[cfg(windows)]
{
windows_detector::detect(app, image)
}
#[cfg(not(windows))]
{
let _ = (app, image);
Err("content-aware hero detection is only available on Windows".into())
}
}
pub(crate) fn warm_up(app: tauri::AppHandle) {
#[cfg(windows)]
windows_detector::warm_up(app);
#[cfg(not(windows))]
let _ = app;
}
+127
View File
@@ -0,0 +1,127 @@
mod commands;
mod config;
mod cover_logo;
mod hero_crop;
mod hero_detector;
mod pipeline;
mod playback;
mod state;
#[cfg_attr(mobile, tauri::mobile_entry_point)]
pub fn run() {
tauri::Builder::default()
.manage(state::AppState::default())
.setup(|app| {
app.handle().plugin(
tauri_plugin_log::Builder::default()
.level(log::LevelFilter::Info)
.clear_targets()
.target(tauri_plugin_log::Target::new(
tauri_plugin_log::TargetKind::Stdout,
))
.build(),
)?;
app.handle().plugin(tauri_plugin_dialog::init())?;
// Must run before anything touches the app-data dir.
config::migrate_app_data(app.handle());
// Prepare the small local vision model while the user is still on
// the app shell, so a first artist visit pays only warm inference.
hero_detector::warm_up(app.handle().clone());
playback::init(app.handle());
playback::prune_cache(app.handle());
if let Err(e) = pipeline::init(app.handle()) {
log::error!("library init failed: {e}");
}
Ok(())
})
.invoke_handler(tauri::generate_handler![
commands::app_status,
commands::excluded_search_phrases,
commands::get_credentials,
commands::connect_and_login,
commands::connect_stored,
commands::disconnect,
commands::start_search,
commands::stop_search,
commands::download_file,
commands::copy_stream_to_folder,
commands::save_to_library,
commands::cancel_download,
commands::show_in_folder,
commands::refresh_artist_art,
commands::play_stream,
commands::play_local,
commands::preload_local,
commands::player_control,
commands::waveform,
commands::library_albums,
commands::album_tracks,
commands::album_art,
commands::refresh_album_art,
commands::refresh_all_album_art,
commands::get_settings,
commands::set_settings,
commands::forget_credentials,
commands::pick_folder,
commands::open_external,
commands::record_play,
commands::add_play_seconds,
commands::artist_play_stats,
commands::list_playlists,
commands::create_playlist,
commands::delete_playlist,
commands::playlist_tracks,
commands::add_to_playlist,
commands::remove_from_playlist,
commands::catalog_search,
commands::catalog_artist,
commands::catalog_artist_art,
commands::catalog_artist_logos,
commands::catalog_artist_identity,
commands::catalog_artist_socials,
commands::catalog_artist_bio,
commands::catalog_cached_cover_logo,
commands::catalog_cover_logo,
commands::catalog_cover_logo_debug,
commands::catalog_ocr_experiment,
commands::catalog_artist_hero_seed,
commands::catalog_artist_overview,
commands::pick_artist_hero,
commands::clear_artist_hero,
commands::catalog_related,
commands::catalog_album,
commands::catalog_album_info,
commands::catalog_art,
commands::catalog_cover,
commands::invalidate_catalog_cover,
commands::catalog_hero_preview,
commands::catalog_hero_gpu_source,
commands::catalog_cached_hero_art,
commands::catalog_hero_art,
commands::preload_stream,
commands::preload_cached_stream,
commands::stream_waveform,
])
.run({
#[allow(unused_mut)]
let mut context = tauri::generate_context!();
// Debug builds expose CDP on 9222 for instrumentation (see HANDOFF.md).
// wry always sets AdditionalBrowserArguments programmatically, so the
// WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS env var is ignored; the flag has
// to go through the window config. Explicit args replace wry's defaults,
// so those are restated here.
#[cfg(debug_assertions)]
{
for window in &mut context.config_mut().app.windows {
window.additional_browser_args = Some(
"--disable-features=msWebOOUI,msPdfOOUI,msSmartScreenProtection \
--autoplay-policy=no-user-gesture-required \
--remote-debugging-port=9222"
.into(),
);
}
}
context
})
.expect("error while running tauri application");
}
+6
View File
@@ -0,0 +1,6 @@
// Prevents additional console window on Windows in release, DO NOT REMOVE!!
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
fn main() {
app_lib::run();
}
File diff suppressed because it is too large. Load diff
+445
View File
@@ -0,0 +1,445 @@
//! Playback orchestration: bridges the audio thread, the streaming cache,
//! and the webview.
use serde::Serialize;
use std::collections::{HashMap, VecDeque};
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
static CACHE_PRUNE_LOCK: Mutex<()> = Mutex::new(());
use std::time::{Duration, Instant};
use tauri::{Emitter, Manager};
use player::{Player, PlayerEvent, StreamProgress};
// The audio thread keeps its responsive 60 ms command/end-detection loop, but
// the webview does not need 16-17 IPC/reactivity updates per second to move a
// progress indicator. Roughly four to five updates per second remain visually
// smooth while letting Chromium spend most playback time asleep.
const UI_POSITION_INTERVAL: Duration = Duration::from_millis(200);
const WAVEFORM_CACHE_MAX_ENTRIES: usize = 256;
const PROGRESSIVE_WAVEFORM_CACHE_MAX_ENTRIES: usize = 64;
const WAVEFORM_JOB_MAX_IDLE_ENTRIES: usize = 256;
/// Tiny insertion-order LRU used for decoded waveform data. Waveforms are
/// cheap individually, but a path-keyed session cache must not grow with every
/// track ever visited.
struct WaveformLru<T> {
entries: HashMap<String, T>,
order: VecDeque<String>,
capacity: usize,
}
impl<T: Clone> WaveformLru<T> {
fn new(capacity: usize) -> Self {
Self {
entries: HashMap::new(),
order: VecDeque::new(),
capacity,
}
}
fn get(&mut self, key: &str) -> Option<T> {
let value = self.entries.get(key)?.clone();
self.touch(key);
Some(value)
}
fn insert(&mut self, key: String, value: T) {
self.entries.insert(key.clone(), value);
self.touch(&key);
while self.entries.len() > self.capacity {
let Some(oldest) = self.order.pop_front() else {
break;
};
self.entries.remove(&oldest);
}
}
fn remove(&mut self, key: &str) {
self.entries.remove(key);
if let Some(index) = self.order.iter().position(|candidate| candidate == key) {
self.order.remove(index);
}
}
fn touch(&mut self, key: &str) {
if let Some(index) = self.order.iter().position(|candidate| candidate == key) {
self.order.remove(index);
}
self.order.push_back(key.to_string());
}
}
#[derive(Clone)]
pub(crate) struct ProgressiveWaveform {
decoded_bytes: u64,
next_decode_bytes: u64,
peaks: Vec<f32>,
}
impl ProgressiveWaveform {
pub(crate) fn new(decoded_bytes: u64, total_bytes: u64, peaks: Vec<f32>) -> Self {
Self {
decoded_bytes,
next_decode_bytes: next_progressive_waveform_threshold(decoded_bytes, total_bytes),
peaks,
}
}
/// Recompute only after the growing file has at least doubled. If a
/// transfer restarted and the file shrank, force a fresh decode instead of
/// returning peaks from bytes that no longer exist.
pub(crate) fn should_recompute(&self, available_bytes: u64) -> bool {
available_bytes < self.decoded_bytes
|| (available_bytes > self.decoded_bytes && available_bytes >= self.next_decode_bytes)
}
pub(crate) fn peaks(&self) -> Vec<f32> {
self.peaks.clone()
}
}
/// When the next partial re-decode is due. Early on the file doubles between
/// decodes — they're cheap and the shape fills in fast. But pure doubling made
/// the LAST gap span half the transfer (50% → 100%), so the drawn wave sat up
/// to 2× behind the buffer and the playhead visibly walked off its end
/// (decoded audio is always ahead of playback, the DRAWING wasn't). Past the
/// eighth-of-file mark the schedule switches to fixed total/8 steps: the wave
/// is never more than 12.5% behind the transfer, for at most ~7 extra decodes
/// spread across the download.
fn next_progressive_waveform_threshold(decoded_bytes: u64, total_bytes: u64) -> u64 {
let doubled = decoded_bytes
.saturating_mul(2)
.max(decoded_bytes.saturating_add(1));
let linear_step = decoded_bytes.saturating_add((total_bytes / 8).max(1));
doubled
.min(linear_step)
.min(total_bytes.max(decoded_bytes.saturating_add(1)))
}
pub struct PlayerState {
pub player: Player,
/// Active stream downloads: download id → progress shared with the player.
pub streams: Mutex<HashMap<u64, Arc<StreamProgress>>>,
/// Cache target for each active stream. This lets a library save adopt an
/// in-flight stream instead of requesting the same peer/path again.
pub stream_caches: Mutex<HashMap<u64, PathBuf>>,
/// Bounded completed-waveform LRU, keyed by file path + bucket count.
waveforms: Mutex<WaveformLru<Vec<f32>>>,
/// Last useful partial decode per active/recent stream. A growing file is
/// decoded again only after its byte length doubles.
progressive_waveforms: Mutex<WaveformLru<ProgressiveWaveform>>,
/// Per-key single-flight locks. The owned guard moves into the blocking
/// decoder so cancellation of the IPC future cannot expose overlapping
/// work while that decoder is still running.
waveform_jobs: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
/// Pre-buffered next-track downloads: download id → cache target.
/// Their events are swallowed (no UI, no library pipeline); play_stream
/// attaches to them instead of restarting the transfer.
pub preloads: Mutex<HashMap<u64, PathBuf>>,
/// Transfers whose completed cache file should also be organized into the
/// library. A save can subscribe to a stream/preload already in progress.
pub deferred_saves: Mutex<HashMap<u64, PathBuf>>,
}
#[derive(Serialize, Clone)]
#[serde(tag = "kind", rename_all = "kebab-case")]
enum PlayerUiEvent {
Buffering,
Playing,
Paused,
Stopped,
TrackEnded { gapless: bool },
Position { secs: f64, duration: Option<f64> },
Error { message: String },
}
/// Create the audio thread and forward its events to the webview.
pub fn init(app: &tauri::AppHandle) {
let output_device = crate::config::load(app).audio_output_device;
let (player, events) = Player::spawn_with_device(output_device);
app.manage(PlayerState {
player,
streams: Mutex::new(HashMap::new()),
stream_caches: Mutex::new(HashMap::new()),
waveforms: Mutex::new(WaveformLru::new(WAVEFORM_CACHE_MAX_ENTRIES)),
progressive_waveforms: Mutex::new(WaveformLru::new(PROGRESSIVE_WAVEFORM_CACHE_MAX_ENTRIES)),
waveform_jobs: Mutex::new(HashMap::new()),
preloads: Mutex::new(HashMap::new()),
deferred_saves: Mutex::new(HashMap::new()),
});
let handle = app.clone();
std::thread::Builder::new()
.name("player-events".into())
.spawn(move || {
let mut last_position_emit = Instant::now() - UI_POSITION_INTERVAL;
for ev in events {
let ui = match ev {
PlayerEvent::Buffering => {
last_position_emit = Instant::now() - UI_POSITION_INTERVAL;
PlayerUiEvent::Buffering
}
PlayerEvent::Playing => {
last_position_emit = Instant::now() - UI_POSITION_INTERVAL;
PlayerUiEvent::Playing
}
PlayerEvent::Paused => PlayerUiEvent::Paused,
PlayerEvent::Stopped => PlayerUiEvent::Stopped,
PlayerEvent::TrackEnded { gapless } => {
last_position_emit = Instant::now() - UI_POSITION_INTERVAL;
PlayerUiEvent::TrackEnded { gapless }
}
PlayerEvent::Position { secs, duration } => {
let now = Instant::now();
if now.duration_since(last_position_emit) < UI_POSITION_INTERVAL {
continue;
}
last_position_emit = now;
PlayerUiEvent::Position { secs, duration }
}
PlayerEvent::Error(message) => PlayerUiEvent::Error { message },
};
let _ = handle.emit("soulseek:player", ui);
}
})
.expect("spawn player event forwarder");
}
impl PlayerState {
pub(crate) fn cached_waveform(&self, key: &str) -> Option<Vec<f32>> {
self.waveforms.lock().unwrap().get(key)
}
pub(crate) fn cache_waveform(&self, key: String, peaks: Vec<f32>) {
self.waveforms.lock().unwrap().insert(key, peaks);
}
pub(crate) fn remove_cached_waveform(&self, key: &str) {
self.waveforms.lock().unwrap().remove(key);
}
pub(crate) fn progressive_waveform(&self, key: &str) -> Option<ProgressiveWaveform> {
self.progressive_waveforms.lock().unwrap().get(key)
}
pub(crate) fn cache_progressive_waveform(&self, key: String, waveform: ProgressiveWaveform) {
self.progressive_waveforms
.lock()
.unwrap()
.insert(key, waveform);
}
pub(crate) fn remove_progressive_waveform(&self, key: &str) {
self.progressive_waveforms.lock().unwrap().remove(key);
}
pub(crate) fn waveform_job(&self, key: &str) -> Arc<tokio::sync::Mutex<()>> {
let mut jobs = self.waveform_jobs.lock().unwrap();
if jobs.len() >= WAVEFORM_JOB_MAX_IDLE_ENTRIES {
// An entry with only the map's Arc is idle and safe to forget.
// Active/waiting jobs may temporarily exceed the soft bound, but
// they must keep their shared lock until their decoder exits.
let idle: Vec<String> = jobs
.iter()
.filter(|(_, job)| Arc::strong_count(job) == 1)
.map(|(key, _)| key.clone())
.collect();
for idle_key in idle {
if jobs.len() < WAVEFORM_JOB_MAX_IDLE_ENTRIES {
break;
}
jobs.remove(&idle_key);
}
}
jobs.entry(key.to_string())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
pub(crate) fn release_waveform_job(&self, key: &str, job: &Arc<tokio::sync::Mutex<()>>) {
let mut jobs = self.waveform_jobs.lock().unwrap();
if jobs
.get(key)
.is_some_and(|current| Arc::ptr_eq(current, job) && Arc::strong_count(current) == 2)
{
jobs.remove(key);
}
}
}
/// Stable cache filename for a remote file: short hash + original name.
pub fn cache_path(app: &tauri::AppHandle, username: &str, remote_path: &str) -> Option<PathBuf> {
let dir = app.path().app_data_dir().ok()?.join("stream-cache");
std::fs::create_dir_all(&dir).ok()?;
let mut hash: u64 = 0xcbf29ce484222325;
for b in username.bytes().chain(remote_path.bytes()) {
hash ^= b as u64;
hash = hash.wrapping_mul(0x100000001b3);
}
let name = remote_path.rsplit('\\').next().unwrap_or(remote_path);
let safe: String = name
.chars()
.map(|c| match c {
'<' | '>' | ':' | '"' | '/' | '\\' | '|' | '?' | '*' => '_',
c if (c as u32) < 0x20 => '_',
c => c,
})
.collect();
Some(dir.join(format!("{hash:016x} {safe}")))
}
/// The in-flight download target for a cache file: `<name>.<ext>.partial`.
pub fn partial_path(cache: &std::path::Path) -> PathBuf {
cache.with_extension(format!(
"{}partial",
cache
.extension()
.map(|e| format!("{}.", e.to_string_lossy()))
.unwrap_or_default()
))
}
/// Enforce the configured streamed-track retention on the cache: drop files
/// not played within the retention window, then oldest-first beyond the size
/// cap. The playing track's file is held open by the OS, so its delete fails
/// and it is simply kept.
pub fn prune_cache(app: &tauri::AppHandle) {
// Completion/settings paths can request pruning concurrently. Serialize
// their directory snapshots so a stale running total cannot over-delete.
let _prune_guard = CACHE_PRUNE_LOCK
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let cfg = crate::config::load(app);
let max_bytes = crate::config::stream_cache_max_mb(&cfg) * 1024 * 1024; // 0 = unlimited
let retention = std::time::Duration::from_secs(
crate::config::stream_retention_days(&cfg) as u64 * 24 * 60 * 60,
);
// In keep-forever mode streams land in the library, so the cache is only
// a transfer buffer — age expiry off, size cap still applies.
let age_prune = !crate::config::stream_keep_forever(&cfg);
let Ok(dir) = app.path().app_data_dir().map(|d| d.join("stream-cache")) else {
return;
};
let Ok(entries) = std::fs::read_dir(&dir) else {
return;
};
let mut files: Vec<(std::time::SystemTime, u64, PathBuf)> = entries
.flatten()
.filter_map(|e| {
let path = e.path();
if path
.extension()
.is_some_and(|ext| ext.to_string_lossy().eq_ignore_ascii_case("partial"))
{
// Active downloads are transfer buffers, not evictable cache
// entries. Deleting one can make its final rename fail.
return None;
}
let md = e.metadata().ok()?;
if !md.is_file() {
return None;
}
Some((md.modified().ok()?, md.len(), path))
})
.collect();
files.sort_by_key(|(t, ..)| *t);
let now = std::time::SystemTime::now();
let mut total: u64 = files.iter().map(|(_, len, _)| len).sum();
let mut kept = Vec::new();
for (modified, len, path) in files {
let expired = age_prune
&& now
.duration_since(modified)
.map(|age| age > retention)
.unwrap_or(false);
if expired && std::fs::remove_file(&path).is_ok() {
total -= len;
continue;
}
kept.push((len, path));
}
if max_bytes == 0 {
return;
}
for (len, path) in kept {
if total <= max_bytes {
break;
}
if std::fs::remove_file(&path).is_ok() {
total -= len;
}
}
}
/// Mark a cache file as freshly used so retention counts from the last play,
/// not the first stream.
pub fn touch(path: &std::path::Path) {
if let Ok(f) = std::fs::File::options().write(true).open(path) {
let _ = f.set_modified(std::time::SystemTime::now());
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn waveform_lru_evicts_oldest_and_refreshes_hits() {
let mut cache = WaveformLru::new(2);
cache.insert("a".into(), vec![1.0]);
cache.insert("b".into(), vec![2.0]);
assert_eq!(cache.get("a"), Some(vec![1.0]));
cache.insert("c".into(), vec![3.0]);
assert_eq!(cache.get("b"), None);
assert_eq!(cache.get("a"), Some(vec![1.0]));
assert_eq!(cache.get("c"), Some(vec![3.0]));
assert_eq!(cache.entries.len(), 2);
}
#[test]
fn progressive_waveform_waits_for_doubling_but_restarts_after_shrink() {
let waveform = ProgressiveWaveform::new(10, 100, vec![1.0]);
assert!(!waveform.should_recompute(10));
assert!(!waveform.should_recompute(19));
assert!(waveform.should_recompute(20));
assert!(waveform.should_recompute(9));
}
#[test]
fn progressive_thresholds_bound_lag_and_total_decode_cost() {
let total: u64 = 256 * 1024 * 1024;
let mut decoded: u64 = 1024 * 1024;
let mut cumulative = 0u64;
let mut steps = 0;
while decoded < total {
cumulative += decoded;
let next = next_progressive_waveform_threshold(decoded, total);
assert!(next > decoded, "thresholds must advance");
// The drawn wave may never fall more than an eighth of the file
// behind the transfer — the doubling-only schedule's half-file
// final gap is the bug this replaced (playhead walking off the
// stale wave's end mid-song).
assert!(
next - decoded <= total / 8,
"gap {} exceeds total/8",
next - decoded
);
decoded = next;
steps += 1;
}
assert_eq!(decoded, total);
// Early doubling + eighth-of-file steps: the whole progressive series
// costs a small constant number of full-file decodes, spread across
// the download.
assert!(steps <= 16, "too many recomputes: {steps}");
assert!(cumulative <= total * 4, "cumulative redecode too large");
}
}
+12
View File
@@ -0,0 +1,12 @@
use tokio::sync::Mutex;
#[derive(Default)]
pub struct AppState {
pub client: Mutex<Option<soulseek_client::Client>>,
pub connected_as: Mutex<Option<String>>,
/// Server-distributed excluded search phrases (message 160). A query
/// containing one returns ZERO responses network-wide — the server simply
/// never routes it — so the resolver strips these terms up front instead
/// of discovering the silence by timeout. Lowercased on receipt.
pub excluded_search_phrases: Mutex<Vec<String>>,
}