//! Word-timing PREPARATION worker + artifact cache. //! //! The research's standing rule (future-implementation-guidelines §7): heavy //! lyric analysis never runs against the active track — it is a background //! job whose artifacts land in a cache, and playback never depends on the //! worker existing. This module is the first tier of that design: //! //! - Input: a track's LOCAL file + its synced line-LRC (the sourced line //! timing stays immutable — §10 calls it "an excellent intermediate asset"). //! - Work: ONE vocal-band envelope pass over the file (player::vocals — the //! same mid-band physics the reducer and rough-sync use), then per-line //! word-onset distribution by syllable weight across the ACTIVE audio //! inside each line's bounded window. No models, no network. //! - Output: an immutable JSON artifact keyed by (file identity, lyric doc //! hash, aligner version), `estimated: true`, per-line confidence. A real //! forced aligner later is just a new aligner version — artifacts never //! mutate, they get siblings. //! //! Honesty rules carried from the docs: word times are distributed only //! WITHIN each line's own sourced window (never across arbitrary time), a //! window with too little vocal evidence falls back to proportional layout at //! floor confidence, and everything is labeled estimated so the UI can say so. use std::collections::{HashSet, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; use tauri::Emitter; const ALIGNER: &str = "envelope-est-v3"; /// v2: real self-loop trellis (word spans instead of 20ms point emissions) /// + margin-to-free-decoding confidence. v3: the acoustic model is MMS-300m /// forced-aligner (multilingual, alignment-tuned) fed the band-passed mid /// channel instead of the raw downmix. v4: lines align in PASSAGE windows /// (cues are hints, not walls — words may land before a late cue) with /// per-line margin confidence. Bumping this re-keys every artifact so /// tracks re-prepare under the new tier. pub(crate) const ALIGNER_CTC: &str = "ctc-v4"; /// v5: same v4 alignment, but listening to a LEARNED vocal stem /// (vocal_sep.rs, UVR MDX Voc_FT) instead of the band-passed mixture. This /// tag is only ever active while BOTH models are on disk, so machines /// without the separator keep their v4 artifacts untouched — and deleting /// the separator resurrects them. pub(crate) const ALIGNER_CTC_STEM: &str = "ctc-v5"; /// Passage bounds: break on a big inter-cue gap, and cap the window — CTC /// attention cost grows quadratically with window length, and a bounded /// passage still frees words from their own line's cue. const PASSAGE_GAP_MS: i64 = 12_000; const PASSAGE_SPAN_MS: i64 = 24_000; const PASSAGE_MAX_LINES: usize = 8; /// How far before its line's cue a word may be found: the pad the passage /// window opens ahead of its first cue (late-cue tolerance). const PASSAGE_LEAD_MS: i64 = 1_200; /// The aligner this machine would use RIGHT NOW — part of the artifact key, /// so downloading the model naturally re-prepares tracks under the new tier. fn model_ready(app: &tauri::AppHandle) -> bool { crate::lyric_align::model_present(app) } pub(crate) fn active_aligner(app: &tauri::AppHandle) -> &'static str { if model_ready(app) { if crate::vocal_sep::model_present(app) { ALIGNER_CTC_STEM } else { ALIGNER_CTC } } else { ALIGNER } } /// A group of consecutive lines through CTC forced alignment in ONE window, /// words mapped back to absolute track time. Each entry is (text, cue_ms); /// the cue floors its line's emissions at cue − PASSAGE_LEAD_MS, so a /// repeated chorus cannot lock onto an earlier instance of the same words. /// None = the model could not carry the passage (caller ladders down: /// per-line, then the estimate). fn align_passage_ctc( app: &tauri::AppHandle, pcm16k: &[f32], group: &[(&str, i64)], window_start: i64, window_end: i64, ) -> Option, f32)>> { let words: Vec> = group .iter() .map(|(t, _)| t.split_whitespace().map(|w| w.to_string()).collect()) .collect(); if words.iter().any(|w: &Vec| w.is_empty()) { return None; } let s = ((window_start.max(0) * 16) as usize).min(pcm16k.len()); let e = (((window_end + 300) * 16) as usize).min(pcm16k.len()); if e <= s { return None; } let offset = (s / 16) as i64; let floors: Vec = group .iter() .map(|(_, cue)| (cue - PASSAGE_LEAD_MS).max(window_start).max(0) - offset) .collect(); // The floors' mirror: line l's tokens may not emit past the NEXT line's // cue + the same lead (the last line gets the window tail). Without the // ceiling, sustained vocal-ish audio BETWEEN lines — a separated stem // keeps the reverb/delay tails the mixture's instruments used to mask — // lets the self-loop smear one word for SECONDS at high margin // confidence, and every later line in the passage dominos late // (Mr Noah "Don't want to get out of bed", 2026-08-19: a 4.3 s "Don't", // successors +5.6 s and +4.5 s). let ceils: Vec = group .iter() .enumerate() .map(|(i, _)| { let raw = match group.get(i + 1) { Some((_, next_cue)) => *next_cue + PASSAGE_LEAD_MS, None => window_end + 300, }; raw.max(0) - offset }) .collect(); match crate::lyric_align::align_passage(app, &pcm16k[s..e], &words, &floors, &ceils) { Ok(aligned) => Some( words .into_iter() .zip(aligned) .map(|(texts, line)| { let spans = texts .into_iter() .zip(line.words) .map(|(text, w)| WordSpan { text, start_ms: offset + w.start_ms, end_ms: offset + w.end_ms, }) .collect(); (spans, line.confidence) }) .collect(), ), Err(_) => None, } } /// One line alone — the middle rung of the ladder when a passage fails /// (e.g. one line's words cannot map): the OTHER lines still get real CTC. /// The lone window opens only the classic ±300ms breathing room. fn align_line_ctc( app: &tauri::AppHandle, pcm16k: &[f32], text: &str, cue_ms: i64, window_end: i64, ) -> Option<(Vec, f32)> { let mut out = align_passage_ctc( app, pcm16k, &[(text, cue_ms)], (cue_ms - 300).max(0), window_end, )?; Some(out.remove(0)) } // ---- CTC span refinement (the within-line honesty pass) -------------------- // // The floor/ceiling bounds keep every LINE in its place; these fix the three // shapes the trellis still gets wrong INSIDE a line, all reported by Kyle as // the fill "going too quick or slow": // tail-fat ends, sliver words, and a fat word packing its followers late. /// Envelope frame for the refinement pass — matches the aligner's 20 ms /// frame, 320 samples at 16 kHz. const ENV_FRAME_MS: i64 = 20; /// A word's end is pulled back to the last frame still carrying this /// fraction of the word's own peak (plus the hangover below). A genuinely /// HELD note keeps its energy and survives; a reverb/delay tail decays and /// is trimmed — the distinction no amount of boundary math can make. const TRIM_GATE: f32 = 0.30; const TRIM_HANGOVER_FRAMES: i64 = 3; /// Never churn spans for small corrections — only trims this big apply. const MIN_TRIM_MS: i64 = 160; /// Minimum visible word width (the karaoke fill must not BLINK), extended /// into the following gap only — never into the next word. const MIN_WIDTH_MS: i64 = 120; /// The relay fires on either of two evidences, both requiring the followers /// to hold under PACK_RATIO of their syllable-fair width (the signature of /// a fat word having stacked them against the line's end): /// - a freed gap this big after an energy-trimmed word, or /// - a NON-FINAL word at FAT_RATIO× its syllable share with FAT_EXCESS_MS /// of absolute excess — the loud-echo case (a feedback delay repeats the /// word at near-full volume, so the energy gate never fires; Mr Noah's /// hook). The fat word is capped to a generous syllable budget first. /// A line-final word is never fat here — it squeezes nobody, and held /// line-final notes are real and good karaoke. const RELAY_GAP_MS: i64 = 800; const PACK_RATIO: f64 = 0.55; const FAT_RATIO: f64 = 2.5; const FAT_EXCESS_MS: f64 = 800.0; /// Cap applied to a fat word when the relay fires on shape evidence. const FAT_CAP_RATIO: f64 = 1.5; const FAT_CAP_MIN_MS: f64 = 400.0; /// Packed followers keep this much of their CTC position, the rest comes /// from the syllable-proportional layout — their CTC starts are mechanical /// stacking after the fat word, so proportion carries most of the truth. const CTC_BLEND: f64 = 0.35; /// 20 ms RMS envelope of the aligner's own input PCM — the separated stem /// under the v5 tier, the band-passed mixture under v4. Whatever the /// aligner heard is what referees its spans. fn rms_envelope(pcm16k: &[f32]) -> Vec { let n = (16 * ENV_FRAME_MS) as usize; pcm16k .chunks(n) .map(|c| (c.iter().map(|s| s * s).sum::() / c.len().max(1) as f32).sqrt()) .collect() } /// Post-pass over one CTC line's word spans. Order matters: the energy trim /// frees the room the relay redistributes, and the width floor runs last so /// it sees final neighbors. fn refine_ctc_spans(words: &mut [WordSpan], env: &[f32]) { if words.is_empty() || env.is_empty() { return; } // 1) Tail-fat ends: trim each word to its own sustained energy. for w in words.iter_mut() { let f0 = (w.start_ms / ENV_FRAME_MS).max(0) as usize; let f1 = ((w.end_ms + ENV_FRAME_MS - 1) / ENV_FRAME_MS).max(0) as usize; let (f0, f1) = (f0.min(env.len()), f1.min(env.len())); if f1 <= f0 + 1 { continue; } let span = &env[f0..f1]; let peak = span.iter().cloned().fold(0.0f32, f32::max); if peak <= 0.0 { continue; } let gate = peak * TRIM_GATE; let mut last = 0usize; for (k, &e) in span.iter().enumerate() { if e >= gate { last = k; } } let trimmed = ((f0 as i64 + last as i64 + 1 + TRIM_HANGOVER_FRAMES) * ENV_FRAME_MS).min(w.end_ms); if w.end_ms - trimmed >= MIN_TRIM_MS && trimmed > w.start_ms { w.end_ms = trimmed; } } // 2) Packed lines: one relay per line, anchored at the (trimmed) fat // word's end and the line's own final end. let line_end = words.last().map(|w| w.end_ms).unwrap_or(0); let line_span = (line_end - words[0].start_ms).max(1) as f64; let syl: Vec = words .iter() .map(|w| syllable_count(&w.text) as f64) .collect(); let rate = line_span / syl.iter().sum::().max(1.0); for i in 0..words.len().saturating_sub(2) { // ≥2 followers required: a single naturally-short final word after // a mid-line breath must never be dragged forward. let width = (words[i].end_ms - words[i].start_ms) as f64; let expect = syl[i] * rate; let gap_evidence = words[i + 1].start_ms - words[i].end_ms >= RELAY_GAP_MS; let fat_evidence = width > expect * FAT_RATIO && width - expect > FAT_EXCESS_MS; if !gap_evidence && !fat_evidence { continue; } let follow_syl: f64 = syl[i + 1..].iter().sum(); let follow_width: f64 = words[i + 1..] .iter() .map(|w| (w.end_ms - w.start_ms) as f64) .sum(); if follow_syl <= 0.0 || follow_width >= follow_syl * rate * PACK_RATIO { // Followers are healthy — a real pause, or a real hold. continue; } if fat_evidence && !gap_evidence { // Loud-echo fat word: the energy gate saw no decay, so cap by // syllable budget before relaying into the room this frees. let cap = (expect * FAT_CAP_RATIO).max(FAT_CAP_MIN_MS) as i64; words[i].end_ms = words[i].end_ms.min(words[i].start_ms + cap); } let base = words[i].end_ms as f64; let room = (line_end as f64 - base).max(1.0); let mut acc = 0.0f64; for k in i + 1..words.len() { let prop_start = base + room * (acc / follow_syl); acc += syl[k]; let prop_end = base + room * (acc / follow_syl); let s = (CTC_BLEND * words[k].start_ms as f64 + (1.0 - CTC_BLEND) * prop_start) as i64; let e = (CTC_BLEND * words[k].end_ms as f64 + (1.0 - CTC_BLEND) * prop_end) as i64; words[k].start_ms = s; words[k].end_ms = e.max(s + ENV_FRAME_MS); } break; } // 3) Width floor: a word may grow into the gap after it, never into its // neighbor. for i in 0..words.len() { let next_start = words.get(i + 1).map(|w| w.start_ms).unwrap_or(i64::MAX); let want = words[i].start_ms + MIN_WIDTH_MS; if words[i].end_ms < want { words[i].end_ms = want.min(next_start).max(words[i].start_ms + ENV_FRAME_MS); } } } const FRAME_MS: u32 = 50; /// Singing-rate prior, ms per syllable-weight unit. v1 stretched words to /// fill whatever window the next cue left, so pacing tracked the LAYOUT /// instead of the voice — a chorus crawled and a tight verse sprinted. Words /// now move at a bounded singing rate and surplus window time is simply not /// singing time. const RATE_MIN_MS: f32 = 130.0; const RATE_MAX_MS: f32 = 420.0; /// Bridge sub-gaps inside a sung phrase (breaths, melisma dips). const BRIDGE_MS: i64 = 150; /// A line's search window never runs past this, no matter how far away the /// next cue is — the sweep-crawl lesson: unmarked instrumental stretches are /// not singing time. const MAX_WINDOW_MS: i64 = 12_000; const MIN_WINDOW_MS: i64 = 500; /// A sustained final syllable may cross the next cue by this much. const OVERLAP_MS: i64 = 350; #[derive(serde::Deserialize, Clone)] pub struct PrepLine { pub index: u32, pub time_ms: i64, pub text: String, } #[derive(serde::Serialize, serde::Deserialize, Clone)] pub struct WordSpan { pub text: String, pub start_ms: i64, pub end_ms: i64, } #[derive(serde::Serialize, serde::Deserialize, Clone)] pub struct ArtifactLine { pub index: u32, pub confidence: f32, /// True = these words came from real forced alignment: they are ABSOLUTE /// track times, independent of the source doc's cues, so display must /// NOT shift them by doc offsets. False = estimate laid out from the /// cues; it moves with them. #[serde(default)] pub ctc: bool, pub words: Vec, } #[derive(serde::Serialize, serde::Deserialize, Clone)] pub struct WordTimingArtifact { pub version: u32, pub aligner: String, pub estimated: bool, pub lines: Vec, } struct Job { key: String, path: PathBuf, lines: Vec, duration_ms: Option, app: tauri::AppHandle, } struct WorkerState { queue: VecDeque, inflight: HashSet, running: bool, } static WORKER: OnceLock> = OnceLock::new(); fn worker() -> &'static Mutex { WORKER.get_or_init(|| { Mutex::new(WorkerState { queue: VecDeque::new(), inflight: HashSet::new(), running: false, }) }) } fn wordtime_dir(app: &tauri::AppHandle) -> Option { let cfg = crate::config::load(app); let dir = crate::config::ensure_library_data_dir(app, &cfg) .ok()? .join("lyrics") .join("wordtime"); std::fs::create_dir_all(&dir).ok()?; Some(dir) } /// FNV-1a — stable, dependency-free; these are cache keys, not security. /// Shared with the setlist listener's artifact keys. pub(crate) fn fnv(bytes: &[u8]) -> u64 { let mut hash: u64 = 0xcbf29ce484222325; for b in bytes { hash ^= *b as u64; hash = hash.wrapping_mul(0x100000001b3); } hash } /// Artifact identity: the FILE (path + size + mtime — a re-rip re-prepares), /// the LYRIC DOC (every cue + every word feeds the hash), and the aligner /// version. Any of the three changing is a different artifact. fn artifact_key(path: &Path, lines: &[PrepLine], aligner: &str) -> String { let meta = std::fs::metadata(path).ok(); let size = meta.as_ref().map(|m| m.len()).unwrap_or(0); let mtime = meta .and_then(|m| m.modified().ok()) .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) .map(|d| d.as_secs()) .unwrap_or(0); let mut doc = String::new(); for l in lines { doc.push_str(&format!("{}|{}|{}\n", l.index, l.time_ms, l.text)); } format!( "{:016x}-{:016x}-{}", fnv(path.to_string_lossy().to_lowercase().as_bytes()) ^ size ^ (mtime << 20), fnv(doc.as_bytes()), aligner, ) } fn artifact_path(app: &tauri::AppHandle, key: &str) -> Option { Some(wordtime_dir(app)?.join(format!("{key}.json"))) } fn read_artifact(path: &Path) -> Option { let bytes = std::fs::read(path).ok()?; serde_json::from_slice(&bytes).ok() } /// Aligner evidence for the doc-level auto offset: when an EXISTING artifact /// (never computed here — this is a cache read on the lyrics-get path) shows /// the sung words consistently starting N ms away from their lines' cues, /// that median IS the doc's timing error. Only confident forced-aligned /// lines vote; a scattered spread (IQR) means verse confusion, not a shift, /// and yields nothing. Returns the shift to ADD to the doc (negative = /// lyrics should display earlier). pub(crate) fn cue_shift_evidence( app: &tauri::AppHandle, path: &Path, lines: &[PrepLine], ) -> Option { if lines.len() < 5 { return None; } // The tier this machine would prepare TODAY — a v5 (stem) machine reads // v5 evidence; only ctc-flagged lines vote, so the est tier never can. let key = artifact_key(path, lines, active_aligner(app)); let artifact = read_artifact(&artifact_path(app, &key)?)?; let cue_of: std::collections::HashMap = lines.iter().map(|l| (l.index, l.time_ms)).collect(); let mut shifts: Vec = artifact .lines .iter() .filter(|l| l.ctc && l.confidence >= 0.6 && !l.words.is_empty()) .filter_map(|l| Some(l.words[0].start_ms - cue_of.get(&l.index)?)) .collect(); if shifts.len() < 5 { return None; } shifts.sort_unstable(); let median = shifts[shifts.len() / 2]; let iqr = shifts[(shifts.len() * 3) / 4] - shifts[shifts.len() / 4]; if median.abs() < 250 || median.abs() > 3_000 || iqr > 500 { return None; } Some(median) } /// Plain syllable count (vowel groups, min 1) for nuclei matching. fn syllable_count(word: &str) -> usize { let mut groups = 0usize; let mut in_group = false; for c in word.to_lowercase().chars() { let vowel = matches!(c, 'a' | 'e' | 'i' | 'o' | 'u' | 'y'); if vowel && !in_group { groups += 1; } in_group = vowel; } groups.max(1) } /// Crude syllable weight: vowel groups (min 1) plus a consonant floor, so /// "extraordinary" outlasts "a" without pretending to be phonetics. fn syllable_weight(word: &str) -> f32 { let mut groups = 0u32; let mut in_group = false; for c in word.to_lowercase().chars() { let vowel = matches!(c, 'a' | 'e' | 'i' | 'o' | 'u' | 'y'); if vowel && !in_group { groups += 1; } in_group = vowel; } groups.max(1) as f32 + 0.6 } /// Distribute one line's words inside its window. v2 rules: /// - a global activity floor (frames are already track-normalized) plus a /// soft window-relative term, so quiet passages neither light up on noise /// nor vanish; /// - the layout is anchored at the CUE unless the voice audibly starts /// later, and spans the SINGING-RATE estimate — never the whole window /// (that is how v1 smeared words across instrumental tails); /// - words walk active audio at a bounded per-syllable rate, bridging /// breath-sized dips; anything the evidence cannot carry falls back to the /// rate-prior layout from the cue at floor confidence. fn lay_out_line( text: &str, window_start: i64, window_end: i64, frames: &[f32], voiced_frames: &[bool], frame_ms: i64, ) -> (Vec, f32) { let words: Vec<&str> = text.split_whitespace().collect(); if words.is_empty() || window_end <= window_start { return (Vec::new(), 0.0); } let weights: Vec = words.iter().map(|w| syllable_weight(w)).collect(); let total: f32 = weights.iter().sum(); let window_ms = window_end - window_start; let first = ((window_start / frame_ms).max(0) as usize).min(frames.len()); let last = ((window_end / frame_ms).max(0) as usize).min(frames.len()); let slice = &frames[first..last]; let vslice: Vec = (first..last) .map(|i| voiced_frames.get(i).copied().unwrap_or(false)) .collect(); let window_peak = slice.iter().cloned().fold(0.0f32, f32::max); // v3: VOICING carries the verdict — energy only needs to clear a floor, // so drums and noise stop counting as singing. let threshold = (0.10f32).max(window_peak * 0.22); let mut active: Vec = slice .iter() .zip(&vslice) .map(|(&v, &vo)| vo && v >= threshold) .collect(); // Bridge breath-sized dips so a sung phrase reads as one span. let bridge = (BRIDGE_MS / frame_ms).max(1) as usize; let mut i = 0; while i < active.len() { if !active[i] { let run_start = i; while i < active.len() && !active[i] { i += 1; } if run_start > 0 && i < active.len() && i - run_start <= bridge { for a in &mut active[run_start..i] { *a = true; } } } else { i += 1; } } let active_ms = active.iter().filter(|&&a| a).count() as i64 * frame_ms; let first_active = active .iter() .position(|&a| a) .map(|i| window_start + i as i64 * frame_ms); // The rate prior: how long this many syllables plausibly takes. let est_ms = ((total * 260.0) as i64).clamp(700, window_ms); let rate = if active_ms > 0 { (active_ms as f32 / total).clamp(RATE_MIN_MS, RATE_MAX_MS) } else { (est_ms as f32 / total).clamp(RATE_MIN_MS, RATE_MAX_MS) }; // Start at the cue unless the voice audibly arrives later (capped: a // wildly late "first active" is evidence noise, not a 2s+ dead cue). let start = match first_active { Some(fa) if fa > window_start + 400 => fa.min(window_start + 2_000), _ => window_start, }; // v3 first choice: SNAP syllables to detected nuclei. A nucleus is an // energy peak inside a voiced region, ≥150ms from its neighbour — a sung // syllable's centre. When the detected count roughly agrees with the // text's syllable count, words land on real syllables instead of // proportional guesses (the literature's single biggest model-free win). let mut nuclei: Vec = Vec::new(); let min_sep = ((150 / frame_ms.max(1)) as usize).max(2); for i in 1..active.len().saturating_sub(1) { if active[i] && slice[i] >= slice[i - 1] && slice[i] > slice[i + 1] { if let Some(&back) = nuclei.last() { if i - back < min_sep { if slice[i] > slice[back] { *nuclei.last_mut().unwrap() = i; } continue; } } nuclei.push(i); } } let sylls: Vec = words.iter().map(|w| syllable_count(w)).collect(); let s_total: usize = sylls.iter().sum::().max(1); let n = nuclei.len(); if n >= (s_total / 2).max(2) && n <= s_total * 2 { let nucleus_at = |syl: usize| -> i64 { let idx = if s_total <= 1 { 0 } else { (syl.min(s_total - 1) * (n - 1) + (s_total - 1) / 2) / (s_total - 1) }; window_start + nuclei[idx.min(n - 1)] as i64 * frame_ms }; let mut out = Vec::with_capacity(words.len()); let mut syl = 0usize; let mut prev_end = window_start; for (wi, w) in words.iter().enumerate() { let start = (nucleus_at(syl) - 60).max(prev_end).max(window_start); syl += sylls[wi]; let mut end = if wi == words.len() - 1 { (nucleus_at(s_total.saturating_sub(1)) + 250).min(window_end) } else { nucleus_at(syl) - 60 }; end = end.max(start + 80); out.push(WordSpan { text: (*w).into(), start_ms: start, end_ms: end, }); prev_end = end; } let ratio = (n as f32 - s_total as f32).abs() / s_total as f32; let confidence = (0.55 + 0.30 * (1.0 - ratio.min(1.0))).min(0.85); return (out, confidence); } let mut out = Vec::with_capacity(words.len()); let evidence_ok = active_ms >= 240; if evidence_ok { // Walk active frames from `start`, spending each word's rate-share of // sung time; inactive frames advance the clock but cost nothing. let start_idx = (((start - window_start) / frame_ms).max(0) as usize).min(active.len()); let mut wi = 0usize; let mut need = weights[0] * rate; let mut word_start = start; for (i, &is_active) in active.iter().enumerate().skip(start_idx) { if wi >= words.len() { break; } if !is_active { continue; } let t = window_start + i as i64 * frame_ms; if out.len() == wi && word_start < window_start { word_start = t; } need -= frame_ms as f32; while need <= 0.0 && wi < words.len() { out.push(WordSpan { text: words[wi].into(), start_ms: word_start, end_ms: t + frame_ms, }); wi += 1; word_start = t + frame_ms; if wi < words.len() { need += weights[wi] * rate; } } } // Words the active audio could not carry continue at the prior rate. let mut cursor = out.last().map(|w| w.end_ms).unwrap_or(start); while out.len() < words.len() { let wi = out.len(); let len = (weights[wi] * rate) as i64; let end = (cursor + len).min(window_end); out.push(WordSpan { text: words[wi].into(), start_ms: cursor, end_ms: end.max(cursor + 80), }); cursor = out.last().unwrap().end_ms; } let span_end = start + est_ms; let covered = active .iter() .enumerate() .filter(|(i, &a)| { let t = window_start + *i as i64 * frame_ms; a && t >= start && t < span_end }) .count() as i64 * frame_ms; let quality = (covered as f32 / est_ms as f32).clamp(0.0, 1.0); (out, (0.4 + quality * 0.45).min(0.85)) } else { // No usable evidence: the rate-prior layout from the cue — bounded by // the singing estimate, NEVER stretched across the window. let mut cursor = start as f32; let per = est_ms as f32 / total; for (w, weight) in words.iter().zip(&weights) { let len = per * weight; out.push(WordSpan { text: (*w).into(), start_ms: cursor as i64, end_ms: (cursor + len) as i64, }); cursor += len; } (out, 0.25) } } fn run_job(job: &Job) -> Result { // Tier B: with the downloaded model present, lines go through CTC forced // alignment against the real audio; each line that the model cannot // carry falls back PER LINE to the envelope estimate below. Without the // model this is exactly the est-v3 worker. With the SEPARATOR model also // present (the v5 tier), the audio the aligner hears is a learned vocal // stem — and a per-track separation failure quietly falls back to the // band-passed mixture, because nothing may ever depend on that module. // The tier comes from the KEY the job was queued under (its last dash // segment), so a model download landing mid-queue cannot make the // artifact's stamp and its filename disagree. let tag = if job.key.ends_with(ALIGNER_CTC_STEM) { ALIGNER_CTC_STEM } else if job.key.ends_with(ALIGNER_CTC) { ALIGNER_CTC } else { ALIGNER }; let ctc = tag != ALIGNER; let pcm16k = if ctc { let stem = if tag == ALIGNER_CTC_STEM { match crate::vocal_sep::separate_vocals_16k(&job.app, &job.path) { Ok(pcm) => Some(pcm), Err(e) => { log::warn!("vocal_sep fell back to the mixture: {e}"); None } } } else { None }; stem.or_else(|| player::vocals::vocal_pcm_16k(&job.path).ok()) } else { None }; // Referee for the span refinement: the same audio the aligner hears. let env16k: Option> = pcm16k.as_deref().map(rms_envelope); let feat = player::vocals::vocal_frames(&job.path, FRAME_MS)?; let frame_ms = feat.frame_ms as i64; let frames = feat.energy; let voiced = feat.voiced; let mut sorted: Vec<&PrepLine> = job.lines.iter().collect(); sorted.sort_by_key(|l| l.time_ms); sorted.retain(|l| !l.text.trim().is_empty()); // Per-line search windows (the est fallback's bounds, and each // passage's tail). let window_end: Vec = sorted .iter() .enumerate() .map(|(i, line)| { let next_start = sorted .get(i + 1) .map(|n| n.time_ms) .or(job.duration_ms) .unwrap_or(line.time_ms + MAX_WINDOW_MS); (next_start + OVERLAP_MS) .min(line.time_ms + MAX_WINDOW_MS) .max(line.time_ms + MIN_WINDOW_MS) }) .collect(); // Group consecutive lines into passages: inside one, the cues are only // hints — a word may land up to PASSAGE_LEAD_MS before its own cue, // which is how words pull free of a slightly-late source doc. let mut passages: Vec<(usize, usize)> = Vec::new(); // [from, to) let mut from = 0usize; for i in 1..=sorted.len() { let split = i == sorted.len() || i - from >= PASSAGE_MAX_LINES || sorted[i].time_ms - sorted[i - 1].time_ms > PASSAGE_GAP_MS || sorted[i].time_ms - sorted[from].time_ms > PASSAGE_SPAN_MS; if split { passages.push((from, i)); from = i; } } let mut lines = Vec::with_capacity(sorted.len()); for &(from, to) in &passages { let group = &sorted[from..to]; let passage_result = pcm16k.as_ref().and_then(|pcm| { let lines: Vec<(&str, i64)> = group.iter().map(|l| (l.text.as_str(), l.time_ms)).collect(); align_passage_ctc( &job.app, pcm, &lines, group[0].time_ms - PASSAGE_LEAD_MS, window_end[to - 1], ) }); match passage_result { Some(aligned) => { for (line, (mut words, confidence)) in group.iter().zip(aligned) { if let Some(env) = env16k.as_deref() { refine_ctc_spans(&mut words, env); } if !words.is_empty() { lines.push(ArtifactLine { index: line.index, confidence, ctc: true, words, }); } } } None => { // Ladder: each line alone through CTC, then the estimate. for (i, line) in group.iter().enumerate() { let w_end = window_end[from + i]; let (words, confidence, ctc) = match pcm16k.as_ref().and_then(|pcm| { align_line_ctc(&job.app, pcm, &line.text, line.time_ms, w_end) }) { Some((mut words, confidence)) => { if let Some(env) = env16k.as_deref() { refine_ctc_spans(&mut words, env); } (words, confidence, true) } None => { let (words, confidence) = lay_out_line( &line.text, line.time_ms, w_end, &frames, &voiced, frame_ms, ); (words, confidence, false) } }; if !words.is_empty() { lines.push(ArtifactLine { index: line.index, confidence, ctc, words, }); } std::thread::sleep(std::time::Duration::from_millis(2)); } } } // Politeness between passages: this thread must never contend with // the audio path for sustained stretches. std::thread::sleep(std::time::Duration::from_millis(3)); } Ok(WordTimingArtifact { version: 1, // The tag the job was KEYED under: v5 stem, v4 mixture, else est. aligner: tag.into(), // CTC lines are real alignment; a model-less run is all estimate. // (Per-line fallbacks under CTC keep their floor confidences.) estimated: !ctc, lines, }) } fn pump() { loop { let job = { let mut w = worker().lock().unwrap(); match w.queue.pop_front() { Some(j) => j, None => { w.running = false; return; } } }; let result = run_job(&job); let ok = result.is_ok(); if let Ok(artifact) = result { // Atomic publish: a partial artifact must never look ready. if let Some(dest) = artifact_path(&job.app, &job.key) { let tmp = dest.with_extension("json.writing"); if std::fs::write(&tmp, serde_json::to_vec(&artifact).unwrap_or_default()) .is_ok() { let _ = std::fs::rename(&tmp, &dest); } } } { let mut w = worker().lock().unwrap(); w.inflight.remove(&job.key); } if ok { let _ = job.app.emit("lyrics:wordtime-ready", &job.key); } } } #[derive(serde::Deserialize)] pub struct StreamRef { pub username: String, pub path: String, pub size: u64, } /// Request word timing for a track + its synced lines. The audio is either a /// local file or a COMPLETE stream-cache file (a partial download has no /// tail to measure — the caller's poll simply asks again once the transfer /// finishes). Returns the artifact immediately on a cache hit; otherwise /// queues the job (single-flight per key) and returns null — /// `lyrics:wordtime-ready` carries the key when it lands. #[tauri::command] pub async fn lyric_prep_request( app: tauri::AppHandle, path: Option, stream: Option, duration_secs: Option, lines: Vec, ) -> Result { let file = match (&path, &stream) { (Some(p), _) => PathBuf::from(p), (None, Some(s)) => { let cache = crate::playback::cache_path(&app, &s.username, &s.path) .ok_or("no stream cache dir")?; let complete = s.size > 0 && matches!(std::fs::metadata(&cache), Ok(md) if md.len() == s.size); if !complete { return Err("stream not fully cached yet".into()); } cache } (None, None) => return Err("no source".into()), }; if !file.is_file() { return Err("not a local file".into()); } if lines.is_empty() { return Err("no lines".into()); } let key = artifact_key(&file, &lines, active_aligner(&app)); let dest = artifact_path(&app, &key).ok_or("no library data dir")?; // The blocking stat/read rides a blocking task, never the async runtime. let cached = tauri::async_runtime::spawn_blocking({ let dest = dest.clone(); move || read_artifact(&dest) }) .await .map_err(|e| e.to_string())?; if let Some(artifact) = cached { return Ok(serde_json::json!({ "key": key, "artifact": artifact })); } let mut w = worker().lock().unwrap(); if !w.inflight.contains(&key) { w.inflight.insert(key.clone()); w.queue.push_back(Job { key: key.clone(), path: file, lines, duration_ms: duration_secs.map(|d| d as i64 * 1000), app: app.clone(), }); if !w.running { w.running = true; std::thread::Builder::new() .name("lyric-prep".into()) .spawn(pump) .map_err(|e| e.to_string())?; } } Ok(serde_json::json!({ "key": key, "artifact": null })) } #[cfg(test)] mod tests { use super::*; fn word(text: &str, start_ms: i64, end_ms: i64) -> WordSpan { WordSpan { text: text.into(), start_ms, end_ms, } } /// Envelope builder: (level, ms) segments at the 20 ms frame. fn env(segments: &[(f32, i64)]) -> Vec { let mut out = Vec::new(); for &(level, ms) in segments { for _ in 0..(ms / ENV_FRAME_MS) { out.push(level); } } out } /// A reverb tail decays and is trimmed; a genuinely held note keeps its /// energy and keeps its span — the pass must tell them apart. #[test] fn refine_trims_tails_but_keeps_held_notes() { // "day": sung 0..400ms, then a -26dB tail to 1400ms. // "hooold": sung strong through 1600..3400ms. let mut words = vec![word("day", 0, 1400), word("hooold", 1600, 3400)]; let e = env(&[(1.0, 400), (0.05, 1200), (0.9, 1800), (0.0, 400)]); refine_ctc_spans(&mut words, &e); assert!( words[0].end_ms <= 520, "tail must be trimmed near the voice's end, got {}", words[0].end_ms ); assert_eq!( words[1].end_ms, 3400, "a held note's sustained energy must survive untrimmed" ); } /// A packed line relays: the fat first word is trimmed by energy, and /// the followers spread across the freed room instead of staying /// stacked at the back. #[test] fn refine_relays_packed_followers() { // "Don't" fat to 2600ms but only sung to ~500ms; followers jammed // into the last 800ms of a 3400ms line. let mut words = vec![ word("Don't", 0, 2600), word("want", 2600, 2800), word("out", 2800, 3000), word("bed", 3000, 3400), ]; let e = env(&[(1.0, 500), (0.04, 2100), (0.8, 800)]); refine_ctc_spans(&mut words, &e); assert!( words[0].end_ms <= 620, "fat word must trim to its sung span, got {}", words[0].end_ms ); assert!( words[1].start_ms < 1800, "first follower must relay forward into the freed room, got {}", words[1].start_ms ); for pair in words.windows(2) { assert!( pair[1].start_ms >= pair[0].start_ms, "relay must stay monotone" ); } assert_eq!(words[3].end_ms, 3400, "the line's final end is the anchor"); } /// The loud-echo case: a feedback delay repeats the hook word at /// near-full volume, so energy never decays and the gate cannot trim — /// the SHAPE evidence (fat non-final word + packed followers) must cap /// and relay anyway. #[test] fn refine_caps_loud_echo_fat_word() { let mut words = vec![ word("Don't", 0, 2600), word("want", 2600, 2800), word("out", 2800, 3000), word("bed", 3000, 3400), ]; // Energy loud the whole way — no decay for the gate to find. let e = env(&[(0.9, 3400)]); refine_ctc_spans(&mut words, &e); // rate = 3400/4 = 850; cap = 850*1.5 = 1275. assert!( words[0].end_ms <= 1275, "echo-fat word must be capped by syllable budget, got {}", words[0].end_ms ); assert!( words[1].start_ms < 2200, "followers must relay into the capped room, got {}", words[1].start_ms ); assert_eq!(words[3].end_ms, 3400); } /// A real held note with HEALTHY followers is never capped: the packed /// guard, not the energy, is what protects it. #[test] fn refine_keeps_held_word_when_followers_are_healthy() { // "Dread" held 1.9s of a 4s line, but the followers own their fair // share — no relay, no cap. let mut words = vec![ word("Dread", 0, 1900), word("follows", 2000, 2900), word("that", 2950, 3300), word("lead", 3350, 4000), ]; let before: Vec<(i64, i64)> = words.iter().map(|w| (w.start_ms, w.end_ms)).collect(); let e = env(&[(0.8, 4000)]); refine_ctc_spans(&mut words, &e); let after: Vec<(i64, i64)> = words.iter().map(|w| (w.start_ms, w.end_ms)).collect(); assert_eq!(before, after); } /// Healthy lines pass through untouched — no churn on good CTC output. #[test] fn refine_leaves_healthy_lines_alone() { let mut words = vec![ word("near", 0, 350), word("as", 380, 520), word("far", 540, 980), ]; let before: Vec<(i64, i64)> = words.iter().map(|w| (w.start_ms, w.end_ms)).collect(); let e = env(&[(0.8, 1000)]); refine_ctc_spans(&mut words, &e); let after: Vec<(i64, i64)> = words.iter().map(|w| (w.start_ms, w.end_ms)).collect(); assert_eq!(before, after); } /// Sliver words widen into their following gap, never into a neighbor. #[test] fn refine_floors_sliver_widths() { let mut words = vec![word("a", 1000, 1020), word("sender", 1400, 1900)]; let e = env(&[(0.7, 2000)]); refine_ctc_spans(&mut words, &e); assert_eq!( words[0].end_ms, 1120, "sliver must widen to the visible floor" ); let mut tight = vec![word("a", 1000, 1020), word("sender", 1060, 1900)]; refine_ctc_spans(&mut tight, &e); assert_eq!( tight[0].end_ms, 1060, "the floor clamps at the neighbor's start" ); } }