From 1dd8573808268f4c6d15f53d635ece14955eac65 Mon Sep 17 00:00:00 2001 From: KeelrLeek <156356974+KeelrLeek@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:49:10 -0500 Subject: [PATCH] dj sets + lyrics: a mix that names its own tracks, and words pulled off the separated voice --- Cargo.lock | 60 ++ crates/library/src/lib.rs | 136 ++- crates/metadata/src/lib.rs | 21 + crates/player/src/lib.rs | 2 +- crates/player/src/vocals.rs | 406 ++++++++ crates/player/src/waveform.rs | 74 +- src-tauri/Cargo.toml | 8 + src-tauri/src/commands.rs | 46 +- src-tauri/src/config.rs | 9 + src-tauri/src/lib.rs | 21 + src-tauri/src/lyric_align.rs | 525 ++++++++++ src-tauri/src/lyric_prep.rs | 1141 +++++++++++++++++++++ src-tauri/src/lyrics.rs | 33 + src-tauri/src/playlist_formats.rs | 2 +- src-tauri/src/playlists.rs | 191 +++- src-tauri/src/setlist_scan.rs | 1005 ++++++++++++++++++ src-tauri/src/shazam.rs | 478 +++++++++ src-tauri/src/vocal_sep.rs | 499 +++++++++ src-tauri/src/youtube_setlist.rs | 652 ++++++++++++ src-tauri/src/ytdlp.rs | 493 +++++++++ src/App.svelte | 16 +- src/components/LyricsOverlay.svelte | 104 -- src/components/LyricsPane.svelte | 70 +- src/components/NowPlaying.svelte | 12 +- src/components/PlayerBar.svelte | 16 +- src/components/PlaylistImportModal.svelte | 604 ++++++++++- src/components/PlaylistMenu.svelte | 149 ++- src/components/SetlistScanChip.svelte | 185 ++++ src/lib/api.js | 46 +- src/lib/lyrics.svelte.js | Bin 12062 -> 18586 bytes src/lib/navigation.svelte.js | 13 + src/lib/setlistScan.svelte.js | 101 ++ src/lib/state.svelte.js | 302 +++++- src/views/CatalogAlbum.svelte | 106 ++ src/views/CatalogArtist.svelte | 121 +++ src/views/CatalogSearch.svelte | 40 +- src/views/Downloads.svelte | 20 + src/views/Lyrics.svelte | 791 ++++++++++++++ src/views/Playlist.svelte | 868 +++++++++++----- src/views/Playlists.svelte | 386 +++++-- src/views/Settings.svelte | 183 ++++ 41 files changed, 9288 insertions(+), 647 deletions(-) create mode 100644 src-tauri/src/lyric_align.rs create mode 100644 src-tauri/src/lyric_prep.rs create mode 100644 src-tauri/src/setlist_scan.rs create mode 100644 src-tauri/src/shazam.rs create mode 100644 src-tauri/src/vocal_sep.rs create mode 100644 src-tauri/src/youtube_setlist.rs create mode 100644 src-tauri/src/ytdlp.rs delete mode 100644 src/components/LyricsOverlay.svelte create mode 100644 src/components/SetlistScanChip.svelte create mode 100644 src/lib/setlistScan.svelte.js create mode 100644 src/views/Lyrics.svelte diff --git a/Cargo.lock b/Cargo.lock index 16b671d..f92dac4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,6 +165,8 @@ version = "0.1.0-alpha.4" dependencies = [ "audio-formats", "base64 0.22.1", + "crc32fast", + "flate2", "image", "keyring", "library", @@ -172,6 +174,7 @@ dependencies = [ "metadata", "player", "quick-xml 0.37.5", + "realfft", "reqwest 0.12.28", "serde", "serde_json", @@ -2579,6 +2582,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -3181,6 +3193,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -3376,6 +3397,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3653,6 +3683,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.4" @@ -4150,6 +4194,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.9.0" @@ -5209,6 +5259,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.24.1" diff --git a/crates/library/src/lib.rs b/crates/library/src/lib.rs index a9212ce..4696894 100644 --- a/crates/library/src/lib.rs +++ b/crates/library/src/lib.rs @@ -93,11 +93,12 @@ pub struct PlaylistItem { pub artist: String, pub album: Option, pub duration_secs: Option, - /// The playable local file, when resolution found one. An unresolved - /// import row keeps its metadata and order with no path at all. + /// The playable local file, when resolution found one. A stream or + /// unresolved row keeps its metadata and order with no path at all. pub path: Option, pub art_path: Option, - /// "local" (has a playable path) | "unresolved" (import placeholder). + /// "local" (a file on disk) | "stream" (full identity, plays through the + /// resolver) | "unresolved" (import placeholder awaiting repair). pub kind: String, /// Sanitized provenance: the raw locator or pasted line this row came /// from. Repair tools read it; nothing ever executes it. @@ -106,6 +107,11 @@ pub struct PlaylistItem { /// begins, milliseconds. pub start_ms: Option, pub note: Option, + /// A stream row's identity anchor: the catalog track id (exact id, never + /// a name match — the same idiom as plays.catalog_album_id). + pub track_id: Option, + /// A stream row's cover, which has no local file to be read from. + pub art_url: Option, } /// Insert payload for one playlist row (`add_playlist_items` / `import_playlist`). @@ -117,11 +123,34 @@ pub struct NewPlaylistItem { pub duration_secs: Option, pub path: Option, pub art_path: Option, - /// "local" | "unresolved"; empty defaults to "local" when a path exists. + /// "local" | "stream" | "unresolved"; empty lets the insert classify: + /// a path is local, a full identity streams, anything weaker is a + /// placeholder (see `pathless_kind`). pub kind: String, pub source_line: Option, pub start_ms: Option, pub note: Option, + pub track_id: Option, + pub art_url: Option, +} + +/// Classify a row that names no local file. A full identity — an artist and +/// a real title, never the DJ unknown-token "ID" — plays through the normal +/// resolver exactly like a catalog track, so it is a first-class 'stream' +/// row. Anything weaker stays an 'unresolved' placeholder for the repair +/// tools. (The same gate the playlist page always used to decide whether a +/// click streams or opens repair.) +fn pathless_kind(artist: &str, title: &str) -> &'static str { + let t = title.trim(); + if !artist.trim().is_empty() + && !t.is_empty() + && !t.eq_ignore_ascii_case("id") + && !t.eq_ignore_ascii_case("i.d.") + { + "stream" + } else { + "unresolved" + } } /// One row of the whole-library matching index (playlist import resolution: @@ -292,7 +321,9 @@ impl Library { source_line TEXT, start_ms INTEGER, note TEXT, - added_at INTEGER NOT NULL DEFAULT 0 + added_at INTEGER NOT NULL DEFAULT 0, + track_id TEXT, + art_url TEXT ); CREATE INDEX IF NOT EXISTS idx_playlist_items ON playlist_items(playlist_id, position); @@ -392,6 +423,22 @@ impl Library { "ALTER TABLE playlists ADD COLUMN revision INTEGER NOT NULL DEFAULT 0", [], ); + // Playlist rows split into local vs stream (2026-08-18): a row with a + // full identity and no file is a STREAMING track, not an error state. + // Its anchor is the catalog track id (exact id, never a name match — + // the plays.catalog_album_id idiom) with the cover URL riding along, + // since a stream row has no local file to read artwork from. + let _ = conn.execute("ALTER TABLE playlist_items ADD COLUMN track_id TEXT", []); + let _ = conn.execute("ALTER TABLE playlist_items ADD COLUMN art_url TEXT", []); + // Reclassify rows stored before the split: anything 'unresolved' that + // carries an artist and a real title already streamed on click — the + // kind now says so. Idempotent, so it simply runs on every open. + let _ = conn.execute( + "UPDATE playlist_items SET kind = 'stream' + WHERE kind = 'unresolved' AND TRIM(artist) <> '' AND TRIM(title) <> '' + AND LOWER(TRIM(title)) NOT IN ('id', 'i.d.')", + [], + ); migrate_playlist_rows(&mut conn)?; Ok(Library { conn: Mutex::new(conn), @@ -1126,7 +1173,7 @@ impl Library { let conn = self.conn.lock().unwrap(); let mut stmt = conn.prepare( "SELECT id, position, title, artist, album, duration_secs, path, art_path, - kind, source_line, start_ms, note + kind, source_line, start_ms, note, track_id, art_url FROM playlist_items WHERE playlist_id = ?1 ORDER BY position", )?; let rows = stmt.query_map([playlist_id], |r| { @@ -1143,6 +1190,8 @@ impl Library { source_line: r.get(9)?, start_ms: r.get(10)?, note: r.get(11)?, + track_id: r.get(12)?, + art_url: r.get(13)?, }) })?; rows.collect::>>().map_err(Into::into) @@ -1198,12 +1247,17 @@ impl Library { let mut ins = tx.prepare( "INSERT INTO playlist_items (playlist_id, position, title, artist, album, duration_secs, - path, art_path, kind, source_line, start_ms, note, added_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + path, art_path, kind, source_line, start_ms, note, added_at, + track_id, art_url) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)", )?; for (offset, item) in items.iter().enumerate() { let kind = if item.kind.is_empty() { - if item.path.is_some() { "local" } else { "unresolved" } + if item.path.is_some() { + "local" + } else { + pathless_kind(&item.artist, &item.title) + } } else { &item.kind }; @@ -1221,6 +1275,8 @@ impl Library { item.start_ms, item.note, now, + item.track_id, + item.art_url, ])?; } } @@ -1286,7 +1342,8 @@ impl Library { } /// Repair one row: attach a playable local file (or correct its display - /// metadata). Passing `path` Some flips the row to "local". + /// metadata). Passing `path` Some flips the row to "local". `track_id` + /// and `art_url` enrich a stream row's catalog anchor and cover. #[allow(clippy::too_many_arguments)] pub fn relink_playlist_item( &self, @@ -1298,6 +1355,8 @@ impl Library { duration_secs: Option, path: Option<&str>, art_path: Option<&str>, + track_id: Option<&str>, + art_url: Option<&str>, now: i64, ) -> Result<()> { let mut conn = self.conn.lock().unwrap(); @@ -1310,9 +1369,11 @@ impl Library { duration_secs = COALESCE(?6, duration_secs), path = COALESCE(?7, path), art_path = COALESCE(?8, art_path), + track_id = COALESCE(?9, track_id), + art_url = COALESCE(?10, art_url), kind = CASE WHEN ?7 IS NOT NULL THEN 'local' ELSE kind END WHERE playlist_id = ?1 AND id = ?2", - rusqlite::params![playlist_id, item_id, title, artist, album, duration_secs, path, art_path], + rusqlite::params![playlist_id, item_id, title, artist, album, duration_secs, path, art_path, track_id, art_url], )?; Self::touch_playlist(&tx, playlist_id, now)?; tx.commit()?; @@ -1337,9 +1398,11 @@ impl Library { tx.execute( "INSERT INTO playlist_items (playlist_id, position, title, artist, album, duration_secs, - path, art_path, kind, source_line, start_ms, note, added_at) + path, art_path, kind, source_line, start_ms, note, added_at, + track_id, art_url) SELECT ?2, position, title, artist, album, duration_secs, - path, art_path, kind, source_line, start_ms, note, ?3 + path, art_path, kind, source_line, start_ms, note, ?3, + track_id, art_url FROM playlist_items WHERE playlist_id = ?1", rusqlite::params![id, new_id, now], )?; @@ -1369,12 +1432,17 @@ impl Library { let mut ins = tx.prepare( "INSERT INTO playlist_items (playlist_id, position, title, artist, album, duration_secs, - path, art_path, kind, source_line, start_ms, note, added_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + path, art_path, kind, source_line, start_ms, note, added_at, + track_id, art_url) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)", )?; for (position, item) in items.iter().enumerate() { let kind = if item.kind.is_empty() { - if item.path.is_some() { "local" } else { "unresolved" } + if item.path.is_some() { + "local" + } else { + pathless_kind(&item.artist, &item.title) + } } else { &item.kind }; @@ -1392,6 +1460,8 @@ impl Library { item.start_ms, item.note, now, + item.track_id, + item.art_url, ])?; } } @@ -2261,7 +2331,9 @@ mod tests { art_path: art.map(String::from), ..Default::default() }; - // Two copies of the same file, on purpose, plus an unresolved import row. + // Two copies of the same file, on purpose, plus a pathless row with a + // full identity (a streaming track, first-class) and a genuinely + // unknown one (the DJ token "ID" stays a placeholder for repair). lib.add_playlist_items( id, &[ @@ -2274,10 +2346,31 @@ mod tests { 110, ) .unwrap(); + lib.add_playlist_items( + id, + &[NewPlaylistItem { + title: "ID".into(), + artist: "Artist".into(), + ..Default::default() + }], + None, + 111, + ) + .unwrap(); + let items = lib.playlist_items(id).unwrap(); + assert_eq!(items.len(), 5); + assert_eq!( + items[3].kind, "stream", + "a pathless row with full identity streams — never dropped, never an error" + ); + assert_eq!( + items[4].kind, "unresolved", + "the DJ unknown-token stays a placeholder for repair" + ); + lib.remove_playlist_items(id, &[items[4].id], 112).unwrap(); let items = lib.playlist_items(id).unwrap(); assert_eq!(items.len(), 4); - assert_eq!(items[3].kind, "unresolved", "no path means placeholder, never dropped"); // Reorder: move the SECOND "One" first; the two copies stay distinct. let ids: Vec = items.iter().map(|i| i.id).collect(); @@ -2308,9 +2401,10 @@ mod tests { assert_eq!(items.len(), 4); // Aggregates: unresolved count, duration sum, distinct art seeds. + // "Ghost" is a stream row (full identity), so nothing is unresolved. let meta = lib.playlist_meta(id).unwrap().unwrap(); assert_eq!(meta.track_count, 4); - assert_eq!(meta.unresolved_count, 1); + assert_eq!(meta.unresolved_count, 0); assert_eq!(meta.duration_secs, 400, "all four rows carry 100s"); assert_eq!(meta.art_seeds, vec!["C:\\m\\a.jpg".to_string(), "C:\\m\\b.jpg".into()]); assert!(meta.revision > 0); @@ -2349,7 +2443,7 @@ mod tests { .unwrap(); let rows = lib.playlist_items(imported).unwrap(); assert_eq!(rows.len(), 2); - assert_eq!(rows[0].kind, "unresolved"); + assert_eq!(rows[0].kind, "stream", "full identity, no file: streams"); assert_eq!(rows[1].kind, "local"); assert_eq!(rows[1].start_ms, Some(3_600_000)); let meta = lib.playlist_meta(imported).unwrap().unwrap(); @@ -2365,6 +2459,8 @@ mod tests { Some(240), Some("C:\\m\\opener.mp3"), None, + None, + None, 170, ) .unwrap(); diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index e7e960b..31fe129 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -4500,6 +4500,27 @@ impl MetadataClient { } } + /// One catalog track by ISRC — the anchor-exact lookup for rows whose + /// identity came from AUDIO RECOGNITION (the recognizer reports the + /// matched release's ISRC). An id-grade identity, never a name guess; + /// `None` is an honest "that release isn't in the catalog". + pub async fn catalog_track_by_isrc(&self, isrc: &str) -> Option { + let clean: String = isrc + .chars() + .filter(|c| c.is_ascii_alphanumeric()) + .map(|c| c.to_ascii_uppercase()) + .collect(); + // ISRCs are exactly 12 characters (CC-XXX-YY-NNNNN without the + // dashes); anything else would only ever be a malformed URL. + if clean.len() != 12 { + return None; + } + let body = self + .deezer(&format!("https://api.deezer.com/track/isrc:{clean}")) + .await?; + parse_track(&body) + } + /// The two things the search hero can say about an artist that Deezer /// cannot: what they sound like, and how big they are. /// diff --git a/crates/player/src/lib.rs b/crates/player/src/lib.rs index 0673697..57d760c 100644 --- a/crates/player/src/lib.rs +++ b/crates/player/src/lib.rs @@ -13,7 +13,7 @@ mod waveform; pub use loudness::{analyze_loudness, Analysis, AnalysisSource, Incomplete, LoudnessMeter}; pub use source::{GrowingFile, StreamProgress, Terminal}; -pub use waveform::compute_waveform; +pub use waveform::{compute_waveform, compute_waveform_range}; use std::collections::VecDeque; use std::path::{Path, PathBuf}; diff --git a/crates/player/src/vocals.rs b/crates/player/src/vocals.rs index 191d1f6..5980788 100644 --- a/crates/player/src/vocals.rs +++ b/crates/player/src/vocals.rs @@ -115,6 +115,138 @@ fn decode(path: &Path, limit_secs: Option) -> Result { }) } +// ---- streaming long-form decode --------------------------------------------- + +/// Streaming 16 kHz mono decode for LONG sources (DJ sets, hour-plus mixes): +/// packets are downmixed and resampled as they arrive, so memory holds only +/// the 16 kHz mono result (~230 MB per hour as f32) and never the source-rate +/// PCM. The whole-file `decode` above deliberately caps at 20 minutes; this +/// is the path past that cap, bounded by `max_secs` of OUTPUT. +pub fn mono_pcm_16k_streaming(path: &Path, max_secs: f64) -> Result, String> { + let file = std::fs::File::open(path).map_err(|e| e.to_string())?; + let mss = MediaSourceStream::new(Box::new(file), Default::default()); + let mut hint = Hint::new(); + if let Some(ext) = path.extension().and_then(|e| e.to_str()) { + hint.with_extension(ext); + } + let probed = symphonia::default::get_probe() + .format( + &hint, + mss, + &FormatOptions::default(), + &MetadataOptions::default(), + ) + .map_err(|e| e.to_string())?; + let mut format = probed.format; + let track = format + .default_track() + .ok_or("no default audio track")? + .clone(); + let mut decoder = symphonia::default::get_codecs() + .make(&track.codec_params, &DecoderOptions::default()) + .map_err(|e| e.to_string())?; + + let mut out: Vec = Vec::new(); + let cap = (max_secs * 16_000.0) as usize; + let mut resampler: Option = None; + let mut buffer: Option> = None; + loop { + if out.len() >= cap { + break; + } + let packet = match format.next_packet() { + Ok(p) => p, + Err(_) => break, + }; + if packet.track_id() != track.id { + continue; + } + let decoded = match decoder.decode(&packet) { + Ok(d) => d, + Err(symphonia::core::errors::Error::DecodeError(_)) => continue, + Err(_) => break, + }; + let spec = *decoded.spec(); + let channels = spec.channels.count().max(1); + let buf = buffer.get_or_insert_with(|| { + SampleBuffer::::new(decoded.capacity() as u64, spec) + }); + if buf.capacity() < decoded.capacity() * channels { + *buf = SampleBuffer::::new(decoded.capacity() as u64, spec); + } + buf.copy_interleaved_ref(decoded); + let rs = resampler.get_or_insert_with(|| StreamResampler::new(spec.rate, 16_000)); + rs.push_interleaved(buf.samples(), channels, &mut out); + } + if out.is_empty() { + return Err("nothing decodable".into()); + } + out.truncate(cap); + Ok(out) +} + +/// Linear resampler with carry across pushes, so packet boundaries do not +/// click or drift: keeps the previous source sample and a fractional read +/// position between calls. +struct StreamResampler { + ratio: f64, + /// Fractional position into the CURRENT push, may start negative into + /// the carried sample. + pos: f64, + carry: f32, + has_carry: bool, + passthrough: bool, +} + +impl StreamResampler { + fn new(src_rate: u32, dst_rate: u32) -> Self { + StreamResampler { + ratio: src_rate as f64 / dst_rate as f64, + pos: 0.0, + carry: 0.0, + has_carry: false, + passthrough: src_rate == dst_rate, + } + } + + /// Downmix `interleaved` to mono and append the resampled result to + /// `out`. + fn push_interleaved(&mut self, interleaved: &[f32], channels: usize, out: &mut Vec) { + let frames = interleaved.len() / channels.max(1); + if frames == 0 { + return; + } + let mono = |i: usize| -> f32 { + let row = &interleaved[i * channels..(i + 1) * channels]; + row.iter().sum::() / channels as f32 + }; + if self.passthrough { + out.extend((0..frames).map(mono)); + return; + } + // Source timeline for this push: sample -1 is the carried one. + loop { + let i0 = self.pos.floor(); + let frac = (self.pos - i0) as f32; + let i0 = i0 as isize; + if i0 + 1 >= frames as isize { + break; + } + let a = if i0 < 0 { + if self.has_carry { self.carry } else { mono(0) } + } else { + mono(i0 as usize) + }; + let b = mono((i0 + 1) as usize); + out.push(a + (b - a) * frac); + self.pos += self.ratio; + } + self.carry = mono(frames - 1); + self.has_carry = true; + self.pos -= frames as f64; + } +} + // ---- leading silence ------------------------------------------------------- /// Milliseconds of leading silence in `path`, or None when the file cannot be @@ -216,6 +348,216 @@ pub fn vocal_activity(path: &Path, frame_ms: u32) -> Result Result, String> { + let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?; + let ch = decoded.channels.max(1); + let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0); + let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0); + let mono: Vec = decoded + .samples + .chunks(ch) + .map(|f| high.process(low.process(f.iter().sum::() / ch as f32))) + .collect(); + let src = decoded.sample_rate as f64; + if (src - 16_000.0).abs() < 1.0 { + return Ok(mono); + } + let ratio = 16_000.0 / src; + let out_len = (mono.len() as f64 * ratio) as usize; + let mut out = Vec::with_capacity(out_len); + for i in 0..out_len { + let pos = i as f64 / ratio; + let i0 = pos as usize; + let frac = (pos - i0 as f64) as f32; + let a = mono.get(i0).copied().unwrap_or(0.0); + let b = mono.get(i0 + 1).copied().unwrap_or(a); + out.push(a + (b - a) * frac); + } + Ok(out) +} + +/// Stereo 44.1 kHz planar PCM for the learned vocal separator: any layout +/// folds to two channels (mono duplicates its one, extra channels beyond the +/// front pair are ignored), any rate linearly resampled. Linear interpolation +/// is deliberate — the stem feeds ALIGNMENT, not ears, the same tradeoff +/// `vocal_pcm_16k` already makes for its own input. +pub fn separation_input_44k1(path: &Path) -> Result<[Vec; 2], String> { + let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?; + let ch = decoded.channels.max(1); + let frames = decoded.samples.len() / ch; + let mut left = Vec::with_capacity(frames); + let mut right = Vec::with_capacity(frames); + for f in decoded.samples.chunks(ch) { + let l = f[0]; + left.push(l); + right.push(if f.len() > 1 { f[1] } else { l }); + } + Ok([ + resample_linear(left, decoded.sample_rate as f64, 44_100.0), + resample_linear(right, decoded.sample_rate as f64, 44_100.0), + ]) +} + +/// Aligner input from a separated vocal stem (44.1 kHz planar stereo): mid +/// fold, a 60 Hz rumble high-pass, the same 7.5 kHz low-pass `vocal_pcm_16k` +/// uses as its 16 kHz anti-alias guard, then the resample. No 180 Hz floor +/// here — that floor exists to fight INSTRUMENTS sharing the band, and a +/// separated stem has (mostly) none left to fight. +pub fn stem_pcm_16k(left: &[f32], right: &[f32]) -> Vec { + let mut rumble = Biquad::highpass(44_100.0, 60.0); + let mut guard = Biquad::lowpass(44_100.0, 7_500.0); + let n = left.len().min(right.len()); + let mono: Vec = (0..n) + .map(|i| guard.process(rumble.process((left[i] + right[i]) * 0.5))) + .collect(); + resample_linear(mono, 44_100.0, 16_000.0) +} + +/// Linear resampler shared by the separator inputs (see the tradeoff note on +/// `separation_input_44k1`). Within 1 Hz of the target it hands the buffer +/// back untouched — whole-track copies are not free at 44.1 kHz. +fn resample_linear(src_pcm: Vec, src: f64, dst: f64) -> Vec { + if (src - dst).abs() < 1.0 { + return src_pcm; + } + let ratio = dst / src; + let out_len = (src_pcm.len() as f64 * ratio) as usize; + let mut out = Vec::with_capacity(out_len); + for i in 0..out_len { + let pos = i as f64 / ratio; + let i0 = pos as usize; + let frac = (pos - i0 as f64) as f32; + let a = src_pcm.get(i0).copied().unwrap_or(0.0); + let b = src_pcm.get(i0 + 1).copied().unwrap_or(a); + out.push(a + (b - a) * frac); + } + out +} + +// ---- vocal features (word-timing worker) ----------------------------------- + +/// Per-frame vocal evidence for the lyric word-timing worker: the band +/// energy `vocal_activity` measures PLUS a voicing verdict — vocals are +/// PITCHED, so an autocorrelation clarity check on a decimated mid channel +/// separates sung frames from drums and noise that light the band up. +pub struct VocalFrames { + pub frame_ms: u32, + /// 0..1 per frame, normalized like `vocal_activity`. + pub energy: Vec, + /// Frame carries periodic (pitched) content in the vocal F0 range. + pub voiced: Vec, +} + +pub fn vocal_frames(path: &Path, frame_ms: u32) -> Result { + let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?; + let frame_ms = frame_ms.clamp(40, 500); + let frame_len = (decoded.sample_rate as usize * frame_ms as usize) / 1000; + let channels = decoded.channels; + let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0); + let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0); + + // Decimate the mid channel to ~8 kHz for the pitch check — full-rate + // autocorrelation across a song is debug-build poison, and F0 up to + // 500 Hz needs nothing like 44.1 kHz. + let decim = (decoded.sample_rate as usize / 8_000).max(1); + let target_rate = decoded.sample_rate as usize / decim; + let mut energy = Vec::new(); + let mut voiced = Vec::new(); + let mut acc = 0.0f64; + let mut n = 0usize; + let mut pitch_buf: Vec = Vec::with_capacity(frame_len / decim + 2); + let mut decim_acc = 0.0f32; + let mut decim_n = 0usize; + for frame in decoded.samples.chunks(channels) { + let mid = if channels >= 2 { + 0.5 * (frame[0] + frame[1]) + } else { + frame[0] + }; + let banded = high.process(low.process(mid)); + acc += (banded * banded) as f64; + n += 1; + decim_acc += banded; + decim_n += 1; + if decim_n >= decim { + pitch_buf.push(decim_acc / decim as f32); + decim_acc = 0.0; + decim_n = 0; + } + if n >= frame_len { + energy.push((acc / n as f64).sqrt() as f32); + voiced.push(frame_is_voiced(&pitch_buf, target_rate)); + pitch_buf.clear(); + acc = 0.0; + n = 0; + } + } + if n > frame_len / 4 { + energy.push((acc / n.max(1) as f64).sqrt() as f32); + voiced.push(frame_is_voiced(&pitch_buf, target_rate)); + } + if energy.is_empty() { + return Err("nothing decodable".into()); + } + let mut sorted: Vec = energy.clone(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let p95 = sorted[(sorted.len() * 95 / 100).min(sorted.len() - 1)].max(1e-6); + for value in &mut energy { + *value = (*value / p95).min(1.0); + } + Ok(VocalFrames { + frame_ms, + energy, + voiced, + }) +} + +/// Normalized-autocorrelation clarity over the sung F0 range (80–500 Hz). +/// A frame is voiced when some lag in that range correlates strongly — +/// periodic content — rather than merely being loud. +fn frame_is_voiced(buf: &[f32], rate: usize) -> bool { + if buf.len() < 64 || rate == 0 { + return false; + } + let energy: f32 = buf.iter().map(|s| s * s).sum(); + if energy < 1e-7 { + return false; + } + let lag_min = (rate / 500).max(2); + let lag_max = (rate / 80).min(buf.len() / 2); + if lag_min >= lag_max { + return false; + } + let mut best = 0.0f32; + let mut lag = lag_min; + // Stride lags by 2: clarity peaks are broad; halving the work loses + // nothing a boolean cares about. + while lag < lag_max { + let n = buf.len() - lag; + let mut num = 0.0f32; + let mut den_a = 0.0f32; + let mut den_b = 0.0f32; + for i in 0..n { + num += buf[i] * buf[i + lag]; + den_a += buf[i] * buf[i]; + den_b += buf[i + lag] * buf[i + lag]; + } + let den = (den_a * den_b).sqrt().max(1e-9); + best = best.max(num / den); + lag += 2; + } + best >= 0.55 +} + // ---- vocal reduction ------------------------------------------------------- /// What the renderer produced (for the caller's labels and sanity checks). @@ -407,6 +749,70 @@ mod tests { out } + /// The streaming resampler must produce the same stream regardless of + /// how the source is chopped into pushes — no clicks, no drift at + /// packet boundaries. + #[test] + fn stream_resampler_is_chunking_invariant() { + let src_rate = 44_100u32; + let n = 44_100usize; + let stereo: Vec = (0..n) + .flat_map(|i| { + let s = (2.0 * std::f32::consts::PI * 440.0 * i as f32 / 44_100.0).sin(); + [s, s] + }) + .collect(); + let mut one_shot = Vec::new(); + let mut a = StreamResampler::new(src_rate, 16_000); + a.push_interleaved(&stereo, 2, &mut one_shot); + let mut chunked = Vec::new(); + let mut b = StreamResampler::new(src_rate, 16_000); + let mut at = 0usize; + for (k, step) in [431usize, 977, 1153, 64, 2048].iter().cycle().enumerate() { + if at >= stereo.len() { + break; + } + let end = (at + step * 2).min(stereo.len()); + b.push_interleaved(&stereo[at..end], 2, &mut chunked); + at = end; + assert!(k < 200_000, "runaway"); + } + assert!( + (one_shot.len() as i64 - chunked.len() as i64).abs() <= 2, + "length drift: {} vs {}", + one_shot.len(), + chunked.len() + ); + let worst = one_shot + .iter() + .zip(&chunked) + .map(|(x, y)| (x - y).abs()) + .fold(0.0f32, f32::max); + assert!(worst < 1e-4, "boundary artifacts, worst {worst}"); + } + + /// The stem→aligner path: a vocal-band tone passes the 60 Hz / 7.5 kHz + /// hygiene filters and lands at 16 kHz with its energy intact. + #[test] + fn stem_pcm_16k_resamples_and_keeps_vocal_band() { + let frames = 44_100usize; + let mono: Vec = (0..frames) + .map(|i| (2.0 * std::f32::consts::PI * 1_000.0 * i as f32 / 44_100.0).sin() * 0.5) + .collect(); + let out = stem_pcm_16k(&mono, &mono); + assert!( + (15_998..=16_000).contains(&out.len()), + "one second must land at ~16000 samples, got {}", + out.len() + ); + let level = rms(&out); + assert!( + level > 0.3 && level < 0.4, + "1kHz mid tone must pass the stem filters near unity, rms {level}" + ); + assert!(out.iter().all(|s| s.is_finite())); + } + /// The whole point: a center-panned vocal-band tone dies, a center bass /// tone survives, side content survives. #[test] diff --git a/crates/player/src/waveform.rs b/crates/player/src/waveform.rs index e0fadf9..cd4ce45 100644 --- a/crates/player/src/waveform.rs +++ b/crates/player/src/waveform.rs @@ -10,10 +10,11 @@ use std::path::Path; use symphonia::core::audio::SampleBuffer; use symphonia::core::codecs::DecoderOptions; -use symphonia::core::formats::FormatOptions; +use symphonia::core::formats::{FormatOptions, SeekMode, SeekTo}; use symphonia::core::io::MediaSourceStream; use symphonia::core::meta::MetadataOptions; use symphonia::core::probe::Hint; +use symphonia::core::units::Time; /// Peaks folded over this many frames before resampling to the final bucket /// count. Keeps memory bounded regardless of track length. @@ -31,6 +32,25 @@ pub fn compute_waveform(path: &Path, buckets: usize) -> Vec { } } +/// Like [`compute_waveform`], but for one SLICE of the file — the setlist +/// segment rows' lane. `end_secs: None` runs to the file's end. The decode +/// seeks coarsely and then trims by packet timestamps, so a 4-minute slice +/// of a 90-minute set costs a 4-minute decode, not a 90-minute one. +pub fn compute_waveform_range( + path: &Path, + buckets: usize, + start_secs: f64, + end_secs: Option, +) -> Vec { + match coarse_rms_range(path, start_secs, end_secs).map(|c| resample(&c, buckets)) { + Ok(v) => v, + Err(e) => { + log::debug!("range waveform for {} failed: {e}", path.display()); + Vec::new() + } + } +} + fn try_compute(path: &Path, buckets: usize) -> Result, String> { Ok(resample(&coarse_rms(path)?, buckets)) } @@ -41,6 +61,14 @@ fn try_compute(path: &Path, buckets: usize) -> Result, String> { /// against a real decode without re-reading the file per variant — see /// `bucketing_strategies_on_a_real_track`. fn coarse_rms(path: &Path) -> Result, String> { + coarse_rms_range(path, 0.0, None) +} + +fn coarse_rms_range( + path: &Path, + start_secs: f64, + end_secs: Option, +) -> Result, String> { let file = std::fs::File::open(path).map_err(|e| e.to_string())?; let mss = MediaSourceStream::new(Box::new(file), Default::default()); @@ -73,14 +101,34 @@ fn coarse_rms(path: &Path) -> Result, String> { .default_track() .ok_or_else(|| "no default track".to_string())?; let track_id = track.id; + let time_base = track.codec_params.time_base; + let sample_rate = track.codec_params.sample_rate; let mut decoder = symphonia::default::get_codecs() .make(&track.codec_params, &DecoderOptions::default()) .map_err(|e| e.to_string())?; + if start_secs > 0.0 { + // A coarse seek lands at a sync point near (usually before) the + // target; the timestamp gate in the loop trims the difference. A + // format that cannot seek just decodes from the top — slower, same + // result. + let _ = format.seek( + SeekMode::Coarse, + SeekTo::Time { + time: Time::from(start_secs), + track_id: Some(track_id), + }, + ); + decoder.reset(); + } + let mut sample_buf: Option> = None; let mut coarse: Vec = Vec::new(); let mut win_energy = 0f64; let mut win_count = 0usize; + // When the container carries no timestamps, the slice is bounded by + // counted frames from wherever the seek landed instead. + let mut frames_seen: u64 = 0; loop { let packet = match format.next_packet() { @@ -90,6 +138,22 @@ fn coarse_rms(path: &Path) -> Result, String> { if packet.track_id() != track_id { continue; } + // Where this packet sits in the file, when the format can say. + let packet_secs = time_base.map(|tb| { + let t = tb.calc_time(packet.ts()); + t.seconds as f64 + t.frac + }); + if let Some(t) = packet_secs { + if t < start_secs { + // Before the slice: decode and discard, keeping codec state + // warm across the boundary. + let _ = decoder.decode(&packet); + continue; + } + if end_secs.is_some_and(|end| t >= end) { + break; + } + } let decoded = match decoder.decode(&packet) { Ok(d) => d, Err(symphonia::core::errors::Error::DecodeError(_)) => continue, @@ -115,6 +179,14 @@ fn coarse_rms(path: &Path) -> Result, String> { win_count = 0; } } + if packet_secs.is_none() { + frames_seen += (sb.samples().len() / channels) as u64; + if let (Some(end), Some(rate)) = (end_secs, sample_rate) { + if frames_seen as f64 / rate as f64 >= end - start_secs { + break; + } + } + } } if win_count > 0 { coarse.push((win_energy / win_count as f64).sqrt() as f32); diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 4e3ca5a..dafa60e 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -29,6 +29,14 @@ image.workspace = true tokio.workspace = true base64 = "0.22" +# Shazam signature packets (shazam.rs) carry a CRC-32 of their body. +crc32fast = "1" +# Inflate for the deno release zip (ytdlp.rs) — already in the lock via the +# image stack, so this adds no download. +flate2 = "1" +# STFT/iSTFT for the vocal separator (vocal_sep.rs) — pure Rust, so the +# no-child-processes charter holds; wraps rustfft with real transforms. +realfft = "3" # XSPF playlist import (playlist_formats.rs). quick-xml never resolves DTDs # or external entities, which is exactly the safety posture the importer wants. quick-xml = "0.37" diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 43ecd7e..cc58c8c 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -76,6 +76,8 @@ pub struct Settings { pub stream_source_prefs: config::SourcePrefs, pub download_source_prefs: config::SourcePrefs, pub volume_normalization: bool, + /// Word-by-word lyric fill — experimental, default off. + pub word_timing: bool, } #[tauri::command] @@ -114,6 +116,7 @@ pub async fn get_settings( stream_source_prefs: cfg.stream_source_prefs, download_source_prefs: cfg.download_source_prefs, volume_normalization: config::volume_normalization(&cfg), + word_timing: config::word_timing(&cfg), username: cfg.username, }) } @@ -141,6 +144,7 @@ pub async fn set_settings( stream_source_prefs: Option, download_source_prefs: Option, volume_normalization: Option, + word_timing: Option, ) -> Result<(), String> { let mut cfg = config::load(&app); let library_folder_moved = download_dir @@ -209,6 +213,9 @@ pub async fn set_settings( // takes effect from the next track. crate::loudness::set_enabled(&app, enabled); } + if word_timing.is_some() { + cfg.word_timing = word_timing; + } config::save(&app, &cfg); // A new library folder is a new library. Re-base immediately rather than @@ -1550,6 +1557,15 @@ pub async fn catalog_search( Ok(state.metadata.catalog_search(&query).await) } +#[tauri::command] +pub async fn catalog_track_by_isrc( + state: State<'_, LibraryState>, + isrc: String, +) -> Result, String> { + state.touch_catalog_activity(); + Ok(state.metadata.catalog_track_by_isrc(&isrc).await) +} + #[tauri::command] pub async fn catalog_artist( app: tauri::AppHandle, @@ -4051,6 +4067,7 @@ pub fn play_local( app: tauri::AppHandle, path: String, duration: Option, + start_secs: Option, ) -> Result<(), String> { if std::fs::metadata(&path).is_err() { return Err("file not found".into()); @@ -4064,7 +4081,11 @@ pub fn play_local( 0, None, Box::new(move || { - app2.state::().player.play_local(&path2, duration); + // `start_secs` = play a SLICE of the file from there (the + // setlist segment rows); plain tracks pass None. + app2.state::() + .player + .play_local_from(&path2, duration, start_secs); }), ); Ok(()) @@ -4407,6 +4428,8 @@ pub async fn waveform( players: State<'_, PlayerState>, path: String, buckets: usize, + start_secs: Option, + end_secs: Option, ) -> Result, String> { const MAX_WAVEFORM_BUCKETS: usize = 4096; if buckets > MAX_WAVEFORM_BUCKETS { @@ -4417,7 +4440,20 @@ pub async fn waveform( if buckets == 0 { return Ok(Vec::new()); } - let key = format!("{path}#{buckets}"); + // A range = a setlist SLICE's lane; each slice of one file caches + // separately under its own key. + let ranged = start_secs.is_some() || end_secs.is_some(); + let key = if ranged { + format!( + "{path}#{buckets}@{:.3}-{}", + start_secs.unwrap_or(0.0), + end_secs + .map(|e| format!("{e:.3}")) + .unwrap_or_else(|| "end".into()), + ) + } else { + format!("{path}#{buckets}") + }; let job = players.waveform_job(&key); let result = async { let job_guard = job.clone().lock_owned().await; @@ -4426,7 +4462,11 @@ pub async fn waveform( } let p = std::path::PathBuf::from(&path); let (job_guard, peaks) = tauri::async_runtime::spawn_blocking(move || { - let peaks = player::compute_waveform(&p, buckets); + let peaks = if ranged { + player::compute_waveform_range(&p, buckets, start_secs.unwrap_or(0.0), end_secs) + } else { + player::compute_waveform(&p, buckets) + }; (job_guard, peaks) }) .await diff --git a/src-tauri/src/config.rs b/src-tauri/src/config.rs index c74a104..0010c55 100644 --- a/src-tauri/src/config.rs +++ b/src-tauri/src/config.rs @@ -139,6 +139,7 @@ pub struct AppConfig { /// every track plays at roughly the same perceived level. None → true. #[serde(default)] pub volume_normalization: Option, + pub word_timing: Option, } /// Default longest edge for embedded covers when the user hasn't chosen. @@ -180,6 +181,14 @@ pub fn volume_normalization(cfg: &AppConfig) -> bool { cfg.volume_normalization.unwrap_or(true) } +/// Word-by-word lyric timing (the karaoke fill) — EXPERIMENTAL, off by +/// default: generated word spans still drift on dense or echo-heavy mixes, +/// and the solid line-level sync must be what users live with until the +/// feature earns its keep (Kyle, 2026-08-19). +pub fn word_timing(cfg: &AppConfig) -> bool { + cfg.word_timing.unwrap_or(false) +} + /// One-time migration after the identifier rename: the app-data dir moved /// from `com.trifecta.soulseek-plus` to the new identifier's folder. Rename /// the old dir into place so the library DB, config and stream cache diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index ff15b39..8af782f 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -6,15 +6,22 @@ mod hero_crop; mod import; mod hero_detector; mod loudness; +mod lyric_align; +mod lyric_prep; mod lyrics; mod pipeline; mod playback; mod playlist_formats; mod playlists; mod preview; +mod setlist_scan; +mod shazam; mod smtc; mod state; mod thumbbar; +mod vocal_sep; +mod youtube_setlist; +mod ytdlp; #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { @@ -114,7 +121,15 @@ pub fn run() { playlists::playlist_import_probe, playlists::playlist_import_folder, playlists::playlist_import_text, + playlists::playlist_import_youtube, playlists::playlist_import_commit, + setlist_scan::playlist_scan_youtube, + setlist_scan::playlist_pick_set_file, + setlist_scan::playlist_scan_file, + setlist_scan::playlist_scan_cancel, + ytdlp::ytdlp_status, + ytdlp::ytdlp_download, + ytdlp::deno_download, playlists::playlist_export, playlists::library_track_search, lyrics::lyrics_get, @@ -128,6 +143,7 @@ pub fn run() { lyrics::lyrics_save_text, lyrics::lyrics_clear, commands::catalog_search, + commands::catalog_track_by_isrc, import::import_scan, import::import_portable_root, import::import_resolve_root, @@ -187,6 +203,11 @@ pub fn run() { preview::player_fade_resume, diagnostics::debug_stats, smtc::smtc_metadata, + lyric_prep::lyric_prep_request, + lyric_align::lyric_model_status, + lyric_align::lyric_model_download, + vocal_sep::vocal_sep_status, + vocal_sep::vocal_sep_download, ]) .run({ #[allow(unused_mut)] diff --git a/src-tauri/src/lyric_align.rs b/src-tauri/src/lyric_align.rs new file mode 100644 index 0000000..04d7e2b --- /dev/null +++ b/src-tauri/src/lyric_align.rs @@ -0,0 +1,525 @@ +//! Tier B word alignment: CTC forced alignment with a small acoustic model +//! through WinML — the same built-in runtime the hero detector uses, so no +//! new dependency and no child processes (the preparation-worker charter). +//! +//! The model (MMS-300m forced-aligner, ONNX f32 export — Meta's multilingual +//! wav2vec2 fine-tuned specifically FOR forced alignment, the checkpoint +//! torchaudio's alignment API standardized on) emits per-20ms CHARACTER +//! probabilities; a Viterbi pass aligns the KNOWN lyric text against them — +//! no transcription, no guessing, just "when does each known word occur". +//! Its vocab is lowercase romanized characters with NO word separator and +//! blank = ``; the loader reads all of that from the repo's own +//! vocab.json rather than hardcoding (the vocab-never-hardcoded rule). +//! It is downloaded ONLY from the user's explicit click in Settings +//! (`lyric_model_download`) — the charter's consent rule — verified by +//! length, and loaded lazily. When it is absent or fails, the worker's +//! envelope estimate stands; nothing ever depends on this module working. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Mutex; + +use tauri::{Emitter, Manager}; +use windows::{ + core::{Interface, HSTRING}, + AI::MachineLearning::{ + LearningModel, LearningModelBinding, LearningModelDevice, LearningModelDeviceKind, + LearningModelSession, TensorFloat, + }, + Win32::System::WinRT::{RoInitialize, RO_INIT_MULTITHREADED}, +}; + +const MODEL_URL: &str = + "https://huggingface.co/onnx-community/mms-300m-1130-forced-aligner-ONNX/resolve/main/onnx/model.onnx"; +const VOCAB_URL: &str = + "https://huggingface.co/onnx-community/mms-300m-1130-forced-aligner-ONNX/resolve/main/vocab.json"; +/// Exact f32 model size, shown in Settings and sanity-checked after download. +const MODEL_BYTES: u64 = 1_262_529_881; +/// wav2vec2-family conv stride: 320 samples at 16 kHz (MMS keeps it). +const FRAME_MS: i64 = 20; + +struct Engine { + session: LearningModelSession, + /// char → token id, from the repo's own vocab.json (never hardcoded). + chars: HashMap, + vocab_len: usize, + blank: usize, + /// Word-separator token — wav2vec2-960h had `|`, MMS has none. + sep: Option, +} + +static ENGINE: Mutex> = Mutex::new(None); + +pub(crate) fn models_dir(app: &tauri::AppHandle) -> Result { + let dir = app + .path() + .app_data_dir() + .map_err(|e| e.to_string())? + .join("lyric-models"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + Ok(dir) +} + +fn model_file(app: &tauri::AppHandle) -> Result { + Ok(models_dir(app)?.join("mms-300m-fa.onnx")) +} + +fn vocab_file(app: &tauri::AppHandle) -> Result { + Ok(models_dir(app)?.join("mms-300m-fa.vocab.json")) +} + +pub fn model_present(app: &tauri::AppHandle) -> bool { + matches!((model_file(app), vocab_file(app)), (Ok(m), Ok(v)) if m.is_file() && v.is_file()) +} + +#[tauri::command] +pub fn lyric_model_status(app: tauri::AppHandle) -> serde_json::Value { + let size = model_file(&app) + .ok() + .and_then(|p| std::fs::metadata(p).ok()) + .map(|m| m.len()) + .unwrap_or(0); + serde_json::json!({ + "present": model_present(&app), + "bytes": size, + // The tier the machine would actually prepare under right now — + // ctc-v5 once the separator model is ALSO on disk. + "aligner": crate::lyric_prep::active_aligner(&app), + "downloadBytes": MODEL_BYTES, + }) +} + +/// Download the model + vocab. Runs ONLY from the user's explicit Settings +/// click — the consent-before-download charter rule. Streams to a temp file, +/// renames atomically, emits `lyrics:model-progress` along the way. +#[tauri::command] +pub async fn lyric_model_download(app: tauri::AppHandle) -> Result<(), String> { + let client = reqwest::Client::builder() + .user_agent("SoulStream/0.1 (lyric alignment)") + .build() + .map_err(|e| e.to_string())?; + // Vocab first — tiny, and the engine cannot load without it. + let vocab_dest = vocab_file(&app)?; + let vocab = client + .get(VOCAB_URL) + .send() + .await + .map_err(|e| e.to_string())? + .error_for_status() + .map_err(|e| e.to_string())? + .bytes() + .await + .map_err(|e| e.to_string())?; + serde_json::from_slice::>(&vocab) + .map_err(|_| "vocab.json did not parse".to_string())?; + std::fs::write(&vocab_dest, &vocab).map_err(|e| e.to_string())?; + + let dest = model_file(&app)?; + let tmp = dest.with_extension("onnx.downloading"); + let mut response = client + .get(MODEL_URL) + .send() + .await + .map_err(|e| e.to_string())? + .error_for_status() + .map_err(|e| e.to_string())?; + let total = response.content_length().unwrap_or(0); + let mut file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?; + let mut received: u64 = 0; + let mut last_emit: u64 = 0; + use std::io::Write; + while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? { + file.write_all(&chunk).map_err(|e| e.to_string())?; + received += chunk.len() as u64; + if received - last_emit > 4 * 1024 * 1024 { + last_emit = received; + let _ = app.emit( + "lyrics:model-progress", + serde_json::json!({ "received": received, "total": total }), + ); + } + } + drop(file); + if total > 0 && received != total { + let _ = std::fs::remove_file(&tmp); + return Err(format!("download truncated: {received} of {total} bytes")); + } + std::fs::rename(&tmp, &dest).map_err(|e| e.to_string())?; + // The superseded wav2vec2 tier's files are dead weight once this lands. + if let Ok(dir) = models_dir(&app) { + for legacy in ["wav2vec2-base-960h.onnx", "wav2vec2-base-960h.vocab.json"] { + let _ = std::fs::remove_file(dir.join(legacy)); + } + } + let _ = app.emit( + "lyrics:model-progress", + serde_json::json!({ "received": received, "total": total, "done": true }), + ); + Ok(()) +} + +fn ensure_engine(app: &tauri::AppHandle) -> Result<(), String> { + let mut guard = ENGINE.lock().unwrap(); + if guard.is_some() { + return Ok(()); + } + let model_path = model_file(app)?; + if !model_path.is_file() { + return Err("model not downloaded".into()); + } + let raw = std::fs::read(vocab_file(app)?).map_err(|e| e.to_string())?; + let vocab: HashMap = + serde_json::from_slice(&raw).map_err(|_| "vocab.json did not parse".to_string())?; + let vocab_len = vocab.values().copied().max().unwrap_or(0) + 1; + // CTC blank: MMS names it `` (its pad_token_id points there); + // the wav2vec2-960h family used `` itself. Prefer the explicit one. + let blank = *vocab + .get("") + .or_else(|| vocab.get("")) + .ok_or("vocab has no blank token")?; + let mut chars = HashMap::new(); + for (token, id) in &vocab { + let mut it = token.chars(); + if let (Some(c), None) = (it.next(), it.next()) { + chars.insert(c, *id); + } + } + let sep = chars.get(&'|').copied(); + // Match the hero detector's WinRT posture: initialize, tolerate an + // existing apartment, never uninitialize someone else's. + match unsafe { RoInitialize(RO_INIT_MULTITHREADED) } { + Ok(()) => {} + Err(e) if e.code().0 as u32 == 0x8001_0106 => {} + Err(e) => return Err(format!("WinRT init failed: {e}")), + } + let model = + LearningModel::LoadFromFilePath(&HSTRING::from(model_path.to_string_lossy().as_ref())) + .map_err(|e| format!("could not load alignment model: {e}"))?; + let device = LearningModelDevice::Create(LearningModelDeviceKind::Cpu) + .map_err(|e| format!("could not create WinML device: {e}"))?; + let session = LearningModelSession::CreateFromModelOnDevice(&model, &device) + .map_err(|e| format!("could not create alignment session: {e}"))?; + *guard = Some(Engine { + session, + chars, + vocab_len, + blank, + sep, + }); + Ok(()) +} + +/// Bare-ASCII fold for the Latin letters MMS's romanized vocab expects: +/// Western lyrics keep their diacritics, the model's chars do not. Anything +/// outside this stays unmapped and the line falls back to the estimate. +fn fold_latin(c: char) -> char { + match c { + 'à'..='å' | 'À'..='Å' => 'a', + 'è'..='ë' | 'È'..='Ë' => 'e', + 'ì'..='ï' | 'Ì'..='Ï' => 'i', + 'ò'..='ö' | 'Ò'..='Ö' | 'ø' | 'Ø' => 'o', + 'ù'..='ü' | 'Ù'..='Ü' => 'u', + 'ý' | 'ÿ' | 'Ý' => 'y', + 'ñ' | 'Ñ' => 'n', + 'ç' | 'Ç' => 'c', + // Typographic apostrophes down to the vocab's plain one. + '’' | 'ʼ' | '`' => '\'', + _ => c, + } +} + +pub struct AlignedWord { + pub start_ms: i64, + pub end_ms: i64, +} + +pub struct AlignedLine { + pub words: Vec, + pub confidence: f32, +} + +/// Force-align a PASSAGE — one or more consecutive lines' words — against a +/// 16 kHz mono window. Returns per-line word times RELATIVE to the window +/// plus a per-line confidence, or Err when the model cannot carry the +/// passage (the caller retries per line, then falls back to the estimate). +/// +/// Aligning lines TOGETHER is what lets word boundaries pull free of a +/// slightly-late source cue: inside a passage the line cues are only hints, +/// not walls — the research bundle's "whole-song alignment using line +/// starts as hints", bounded to passage windows. `floors_ms[l]` is line +/// l's earliest permitted emission (window-relative): without that floor, +/// a repeating chorus lets the path lock whole lines onto the PREVIOUS +/// acoustic instance of the same words (the charter's "competing repeated +/// sections") — hint means "at most this much early", never "anywhere". +/// `ceils_ms[l]` is the floor's mirror, the latest permitted emission: +/// without it a line may smear arbitrarily LATE across its successors' +/// audio when the space between lines still sounds vocal-ish (a separated +/// stem keeps reverb/delay tails), sustaining one token for seconds at +/// HIGH margin confidence — free decoding hears the same tail, so the +/// score cannot object. Hint means "at most this much late", too. +pub fn align_passage( + app: &tauri::AppHandle, + pcm: &[f32], + lines: &[Vec], + floors_ms: &[i64], + ceils_ms: &[i64], +) -> Result, String> { + let word_total: usize = lines.iter().map(|l| l.len()).sum(); + if pcm.len() < 1600 + || word_total == 0 + || floors_ms.len() != lines.len() + || ceils_ms.len() != lines.len() + { + return Err("window too short".into()); + } + ensure_engine(app)?; + let guard = ENGINE.lock().unwrap(); + let engine = guard.as_ref().ok_or("engine unavailable")?; + + // Zero-mean / unit-variance, the model's expected normalization. + let mean = pcm.iter().sum::() / pcm.len() as f32; + let var = pcm.iter().map(|s| (s - mean) * (s - mean)).sum::() / pcm.len() as f32; + let scale = 1.0 / (var.sqrt() + 1e-7); + let normed: Vec = pcm.iter().map(|s| (s - mean) * scale).collect(); + + let input = TensorFloat::CreateFromShapeArrayAndDataArray(&[1, normed.len() as i64], &normed) + .map_err(|e| format!("could not create input tensor: {e}"))?; + let binding = LearningModelBinding::CreateFromSession(&engine.session) + .map_err(|e| format!("could not create binding: {e}"))?; + binding + .Bind(&HSTRING::from("input_values"), &input) + .map_err(|e| format!("could not bind input: {e}"))?; + let result = engine + .session + .Evaluate(&binding, &HSTRING::from("lyric-align")) + .map_err(|e| format!("inference failed: {e}"))?; + if !result.Succeeded().unwrap_or(false) { + return Err("inference did not succeed".into()); + } + let logits: TensorFloat = result + .Outputs() + .and_then(|o| o.Lookup(&HSTRING::from("logits"))) + .map_err(|e| format!("logits missing: {e}"))? + .cast() + .map_err(|e| format!("logits wrong type: {e}"))?; + let values: Vec = logits + .GetAsVectorView() + .map_err(|e| format!("could not read logits: {e}"))? + .into_iter() + .collect(); + let v = engine.vocab_len; + if v == 0 || values.len() % v != 0 { + return Err(format!("unexpected logits length {}", values.len())); + } + let t_frames = values.len() / v; + + // log-softmax per frame. + let mut logp = vec![0.0f32; values.len()]; + for t in 0..t_frames { + let row = &values[t * v..(t + 1) * v]; + let max = row.iter().cloned().fold(f32::NEG_INFINITY, f32::max); + let sum: f32 = row.iter().map(|x| (x - max).exp()).sum(); + let log_z = max + sum.ln(); + for (i, x) in row.iter().enumerate() { + logp[t * v + i] = x - log_z; + } + } + + // Token sequence: each token knows its WORD (passage-global index), each + // word knows its line; a separator joins words only when the vocab has + // one (wav2vec2-960h did) — MMS aligns bare chars. Mapping is + // vocab-driven and case-agnostic, with a Latin fold so accented lyrics + // reach the romanized vocab. A word that maps to nothing (digits, + // symbols, other scripts) sinks the whole passage to the per-line + // ladder rather than being silently skipped. + let map_char = |c: char| -> Option { + engine + .chars + .get(&c) + .or_else(|| c.to_lowercase().next().and_then(|l| engine.chars.get(&l))) + .or_else(|| c.to_uppercase().next().and_then(|u| engine.chars.get(&u))) + .or_else(|| { + let f = fold_latin(c); + engine + .chars + .get(&f) + .or_else(|| f.to_uppercase().next().and_then(|u| engine.chars.get(&u))) + }) + .copied() + }; + let mut tokens: Vec<(usize, usize)> = Vec::new(); + // Per-token emission bounds, frames — the line's cue-derived floor and + // its successor-derived ceiling. + let mut floor_frame: Vec = Vec::new(); + let mut ceil_frame: Vec = Vec::new(); + let mut next_wi = 0usize; + for (li, line) in lines.iter().enumerate() { + let floor = (floors_ms[li].max(0) / FRAME_MS) as usize; + let ceil = (ceils_ms[li].max(0) / FRAME_MS) as usize; + for word in line { + let wi = next_wi; + next_wi += 1; + if wi > 0 { + if let Some(sep) = engine.sep { + tokens.push((sep, wi)); + floor_frame.push(floor); + ceil_frame.push(ceil); + } + } + let before = tokens.len(); + for c in word.chars() { + if let Some(id) = map_char(c) { + tokens.push((id, wi)); + floor_frame.push(floor); + ceil_frame.push(ceil); + } + } + if tokens.len() == before { + return Err("word has no mappable characters".into()); + } + } + } + let n = tokens.len(); + if t_frames < n { + return Err("window shorter than token sequence".into()); + } + + // Real CTC trellis: at each frame stay on blank, SUSTAIN the current + // token by re-emitting it (the self-loop), or advance by emitting the + // next. The self-loop turns single emission frames into token SPANS — + // without it every word "ended" one frame after it started and sustained + // notes were invisible. Known approximation, kept deliberately: identical + // CONSECUTIVE tokens (double letters, "LL") properly require a mandatory + // blank between them; collapsing that costs ~one frame (20ms). + const NEG: f32 = f32::NEG_INFINITY; + let mut trellis = vec![NEG; (t_frames + 1) * (n + 1)]; + // Backpointers (0 = blank, 1 = self-loop, 2 = advance) make the + // backtrack exact instead of re-deriving float comparisons. + let mut back = vec![0u8; (t_frames + 1) * (n + 1)]; + let idx = |t: usize, j: usize| t * (n + 1) + j; + trellis[idx(0, 0)] = 0.0; + for t in 0..t_frames { + let blank_p = logp[t * v + engine.blank]; + for j in 0..=n.min(t + 1) { + let mut best = trellis[idx(t, j)] + blank_p; + let mut way = 0u8; + // A token may not be emitted before its line's floor (repeated + // choruses would lock onto an earlier acoustic instance) nor at + // or past its ceiling (stem reverb tails would smear a sustain + // across the successors' audio). A line that cannot fit between + // its bounds leaves no path — the caller's ladder handles it. + if j > 0 && t >= floor_frame[j - 1] && t < ceil_frame[j - 1] { + let tok_p = logp[t * v + tokens[j - 1].0]; + let sustain = trellis[idx(t, j)] + tok_p; + if sustain > best { + best = sustain; + way = 1; + } + let advance = trellis[idx(t, j - 1)] + tok_p; + if advance > best { + best = advance; + way = 2; + } + } + trellis[idx(t + 1, j)] = best; + back[idx(t + 1, j)] = way; + } + } + if trellis[idx(t_frames, n)] == NEG { + return Err("no alignment path".into()); + } + // Backtrack the pointers into per-token spans: an advance opens the + // token's span, self-loops extend it. Walking backwards, the first + // emission met is the token's LAST frame. Every backtrack step consumes + // exactly one frame emitting blank or a token — recording that symbol's + // logp gives the path's per-frame score, which per-LINE confidence + // needs below. + let mut first_emit = vec![usize::MAX; n]; + let mut last_emit = vec![usize::MAX; n]; + let mut path_logp = vec![0.0f32; t_frames]; + let mut j = n; + let mut t = t_frames; + while t > 0 { + match back[idx(t, j)] { + 2 => { + first_emit[j - 1] = t - 1; + if last_emit[j - 1] == usize::MAX { + last_emit[j - 1] = t - 1; + } + path_logp[t - 1] = logp[(t - 1) * v + tokens[j - 1].0]; + j -= 1; + } + 1 => { + if last_emit[j - 1] == usize::MAX { + last_emit[j - 1] = t - 1; + } + path_logp[t - 1] = logp[(t - 1) * v + tokens[j - 1].0]; + } + _ => { + path_logp[t - 1] = logp[(t - 1) * v + engine.blank]; + } + } + t -= 1; + } + if j != 0 { + return Err("backtrack failed".into()); + } + + // Per-word spans, then grouped into lines. + let mut word_span: Vec> = vec![None; word_total]; + for (k, (id, w)) in tokens.iter().enumerate() { + if Some(*id) == engine.sep || first_emit[k] == usize::MAX { + continue; + } + word_span[*w] = Some(match word_span[*w] { + None => (first_emit[k], last_emit[k]), + Some((f, _)) => (f, last_emit[k]), + }); + } + + // Confidence = margin to FREE decoding, per LINE: exp of the mean + // per-frame gap between the forced path and the best unconstrained path + // over the line's own sung span — "how close is the known lyric to what + // the model actually hears", the calibration CTC-segmentation systems + // use. Near 1.0 when text matches audio, sinking as it diverges. (v1 + // sampled exp(logprob) at each token's single emission frame, where CTC + // piles mass on blank — every line pinned to the clamp floor.) + let free: Vec = (0..t_frames) + .map(|t| logp[t * v..(t + 1) * v].iter().cloned().fold(NEG, f32::max)) + .collect(); + let mut out = Vec::with_capacity(lines.len()); + let mut wi = 0usize; + for line in lines { + let mut words_out = Vec::with_capacity(line.len()); + let mut span_first = usize::MAX; + let mut span_last = 0usize; + for _ in line { + match word_span[wi] { + Some((f, l)) => { + span_first = span_first.min(f); + span_last = span_last.max(l); + words_out.push(AlignedWord { + start_ms: f as i64 * FRAME_MS, + end_ms: (l as i64 + 1) * FRAME_MS, + }); + } + None => return Err("word lost in backtrack".into()), + } + wi += 1; + } + let confidence = if span_first == usize::MAX { + 0.05 + } else { + let gap: f32 = (span_first..=span_last.min(t_frames - 1)) + .map(|t| path_logp[t] - free[t]) + .sum(); + let frames = (span_last.min(t_frames - 1) - span_first + 1) as f32; + (gap / frames).exp().clamp(0.05, 0.99) + }; + out.push(AlignedLine { + words: words_out, + confidence, + }); + } + Ok(out) +} diff --git a/src-tauri/src/lyric_prep.rs b/src-tauri/src/lyric_prep.rs new file mode 100644 index 0000000..51469a6 --- /dev/null +++ b/src-tauri/src/lyric_prep.rs @@ -0,0 +1,1141 @@ +//! Word-timing PREPARATION worker + artifact cache. +//! +//! The research's standing rule (future-implementation-guidelines §7): heavy +//! lyric analysis never runs against the active track — it is a background +//! job whose artifacts land in a cache, and playback never depends on the +//! worker existing. This module is the first tier of that design: +//! +//! - Input: a track's LOCAL file + its synced line-LRC (the sourced line +//! timing stays immutable — §10 calls it "an excellent intermediate asset"). +//! - Work: ONE vocal-band envelope pass over the file (player::vocals — the +//! same mid-band physics the reducer and rough-sync use), then per-line +//! word-onset distribution by syllable weight across the ACTIVE audio +//! inside each line's bounded window. No models, no network. +//! - Output: an immutable JSON artifact keyed by (file identity, lyric doc +//! hash, aligner version), `estimated: true`, per-line confidence. A real +//! forced aligner later is just a new aligner version — artifacts never +//! mutate, they get siblings. +//! +//! Honesty rules carried from the docs: word times are distributed only +//! WITHIN each line's own sourced window (never across arbitrary time), a +//! window with too little vocal evidence falls back to proportional layout at +//! floor confidence, and everything is labeled estimated so the UI can say so. + +use std::collections::{HashSet, VecDeque}; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock}; + +use tauri::Emitter; + +const ALIGNER: &str = "envelope-est-v3"; +/// v2: real self-loop trellis (word spans instead of 20ms point emissions) +/// + margin-to-free-decoding confidence. v3: the acoustic model is MMS-300m +/// forced-aligner (multilingual, alignment-tuned) fed the band-passed mid +/// channel instead of the raw downmix. v4: lines align in PASSAGE windows +/// (cues are hints, not walls — words may land before a late cue) with +/// per-line margin confidence. Bumping this re-keys every artifact so +/// tracks re-prepare under the new tier. +pub(crate) const ALIGNER_CTC: &str = "ctc-v4"; +/// v5: same v4 alignment, but listening to a LEARNED vocal stem +/// (vocal_sep.rs, UVR MDX Voc_FT) instead of the band-passed mixture. This +/// tag is only ever active while BOTH models are on disk, so machines +/// without the separator keep their v4 artifacts untouched — and deleting +/// the separator resurrects them. +pub(crate) const ALIGNER_CTC_STEM: &str = "ctc-v5"; +/// Passage bounds: break on a big inter-cue gap, and cap the window — CTC +/// attention cost grows quadratically with window length, and a bounded +/// passage still frees words from their own line's cue. +const PASSAGE_GAP_MS: i64 = 12_000; +const PASSAGE_SPAN_MS: i64 = 24_000; +const PASSAGE_MAX_LINES: usize = 8; +/// How far before its line's cue a word may be found: the pad the passage +/// window opens ahead of its first cue (late-cue tolerance). +const PASSAGE_LEAD_MS: i64 = 1_200; + +/// The aligner this machine would use RIGHT NOW — part of the artifact key, +/// so downloading the model naturally re-prepares tracks under the new tier. +fn model_ready(app: &tauri::AppHandle) -> bool { + crate::lyric_align::model_present(app) +} + +pub(crate) fn active_aligner(app: &tauri::AppHandle) -> &'static str { + if model_ready(app) { + if crate::vocal_sep::model_present(app) { + ALIGNER_CTC_STEM + } else { + ALIGNER_CTC + } + } else { + ALIGNER + } +} + +/// A group of consecutive lines through CTC forced alignment in ONE window, +/// words mapped back to absolute track time. Each entry is (text, cue_ms); +/// the cue floors its line's emissions at cue − PASSAGE_LEAD_MS, so a +/// repeated chorus cannot lock onto an earlier instance of the same words. +/// None = the model could not carry the passage (caller ladders down: +/// per-line, then the estimate). +fn align_passage_ctc( + app: &tauri::AppHandle, + pcm16k: &[f32], + group: &[(&str, i64)], + window_start: i64, + window_end: i64, +) -> Option, f32)>> { + let words: Vec> = group + .iter() + .map(|(t, _)| t.split_whitespace().map(|w| w.to_string()).collect()) + .collect(); + if words.iter().any(|w: &Vec| w.is_empty()) { + return None; + } + let s = ((window_start.max(0) * 16) as usize).min(pcm16k.len()); + let e = (((window_end + 300) * 16) as usize).min(pcm16k.len()); + if e <= s { + return None; + } + let offset = (s / 16) as i64; + let floors: Vec = group + .iter() + .map(|(_, cue)| (cue - PASSAGE_LEAD_MS).max(window_start).max(0) - offset) + .collect(); + // The floors' mirror: line l's tokens may not emit past the NEXT line's + // cue + the same lead (the last line gets the window tail). Without the + // ceiling, sustained vocal-ish audio BETWEEN lines — a separated stem + // keeps the reverb/delay tails the mixture's instruments used to mask — + // lets the self-loop smear one word for SECONDS at high margin + // confidence, and every later line in the passage dominos late + // (Mr Noah "Don't want to get out of bed", 2026-08-19: a 4.3 s "Don't", + // successors +5.6 s and +4.5 s). + let ceils: Vec = group + .iter() + .enumerate() + .map(|(i, _)| { + let raw = match group.get(i + 1) { + Some((_, next_cue)) => *next_cue + PASSAGE_LEAD_MS, + None => window_end + 300, + }; + raw.max(0) - offset + }) + .collect(); + match crate::lyric_align::align_passage(app, &pcm16k[s..e], &words, &floors, &ceils) { + Ok(aligned) => Some( + words + .into_iter() + .zip(aligned) + .map(|(texts, line)| { + let spans = texts + .into_iter() + .zip(line.words) + .map(|(text, w)| WordSpan { + text, + start_ms: offset + w.start_ms, + end_ms: offset + w.end_ms, + }) + .collect(); + (spans, line.confidence) + }) + .collect(), + ), + Err(_) => None, + } +} + +/// One line alone — the middle rung of the ladder when a passage fails +/// (e.g. one line's words cannot map): the OTHER lines still get real CTC. +/// The lone window opens only the classic ±300ms breathing room. +fn align_line_ctc( + app: &tauri::AppHandle, + pcm16k: &[f32], + text: &str, + cue_ms: i64, + window_end: i64, +) -> Option<(Vec, f32)> { + let mut out = align_passage_ctc( + app, + pcm16k, + &[(text, cue_ms)], + (cue_ms - 300).max(0), + window_end, + )?; + Some(out.remove(0)) +} + +// ---- CTC span refinement (the within-line honesty pass) -------------------- +// +// The floor/ceiling bounds keep every LINE in its place; these fix the three +// shapes the trellis still gets wrong INSIDE a line, all reported by Kyle as +// the fill "going too quick or slow": +// tail-fat ends, sliver words, and a fat word packing its followers late. + +/// Envelope frame for the refinement pass — matches the aligner's 20 ms +/// frame, 320 samples at 16 kHz. +const ENV_FRAME_MS: i64 = 20; +/// A word's end is pulled back to the last frame still carrying this +/// fraction of the word's own peak (plus the hangover below). A genuinely +/// HELD note keeps its energy and survives; a reverb/delay tail decays and +/// is trimmed — the distinction no amount of boundary math can make. +const TRIM_GATE: f32 = 0.30; +const TRIM_HANGOVER_FRAMES: i64 = 3; +/// Never churn spans for small corrections — only trims this big apply. +const MIN_TRIM_MS: i64 = 160; +/// Minimum visible word width (the karaoke fill must not BLINK), extended +/// into the following gap only — never into the next word. +const MIN_WIDTH_MS: i64 = 120; +/// The relay fires on either of two evidences, both requiring the followers +/// to hold under PACK_RATIO of their syllable-fair width (the signature of +/// a fat word having stacked them against the line's end): +/// - a freed gap this big after an energy-trimmed word, or +/// - a NON-FINAL word at FAT_RATIO× its syllable share with FAT_EXCESS_MS +/// of absolute excess — the loud-echo case (a feedback delay repeats the +/// word at near-full volume, so the energy gate never fires; Mr Noah's +/// hook). The fat word is capped to a generous syllable budget first. +/// A line-final word is never fat here — it squeezes nobody, and held +/// line-final notes are real and good karaoke. +const RELAY_GAP_MS: i64 = 800; +const PACK_RATIO: f64 = 0.55; +const FAT_RATIO: f64 = 2.5; +const FAT_EXCESS_MS: f64 = 800.0; +/// Cap applied to a fat word when the relay fires on shape evidence. +const FAT_CAP_RATIO: f64 = 1.5; +const FAT_CAP_MIN_MS: f64 = 400.0; +/// Packed followers keep this much of their CTC position, the rest comes +/// from the syllable-proportional layout — their CTC starts are mechanical +/// stacking after the fat word, so proportion carries most of the truth. +const CTC_BLEND: f64 = 0.35; + +/// 20 ms RMS envelope of the aligner's own input PCM — the separated stem +/// under the v5 tier, the band-passed mixture under v4. Whatever the +/// aligner heard is what referees its spans. +fn rms_envelope(pcm16k: &[f32]) -> Vec { + let n = (16 * ENV_FRAME_MS) as usize; + pcm16k + .chunks(n) + .map(|c| (c.iter().map(|s| s * s).sum::() / c.len().max(1) as f32).sqrt()) + .collect() +} + +/// Post-pass over one CTC line's word spans. Order matters: the energy trim +/// frees the room the relay redistributes, and the width floor runs last so +/// it sees final neighbors. +fn refine_ctc_spans(words: &mut [WordSpan], env: &[f32]) { + if words.is_empty() || env.is_empty() { + return; + } + // 1) Tail-fat ends: trim each word to its own sustained energy. + for w in words.iter_mut() { + let f0 = (w.start_ms / ENV_FRAME_MS).max(0) as usize; + let f1 = ((w.end_ms + ENV_FRAME_MS - 1) / ENV_FRAME_MS).max(0) as usize; + let (f0, f1) = (f0.min(env.len()), f1.min(env.len())); + if f1 <= f0 + 1 { + continue; + } + let span = &env[f0..f1]; + let peak = span.iter().cloned().fold(0.0f32, f32::max); + if peak <= 0.0 { + continue; + } + let gate = peak * TRIM_GATE; + let mut last = 0usize; + for (k, &e) in span.iter().enumerate() { + if e >= gate { + last = k; + } + } + let trimmed = + ((f0 as i64 + last as i64 + 1 + TRIM_HANGOVER_FRAMES) * ENV_FRAME_MS).min(w.end_ms); + if w.end_ms - trimmed >= MIN_TRIM_MS && trimmed > w.start_ms { + w.end_ms = trimmed; + } + } + // 2) Packed lines: one relay per line, anchored at the (trimmed) fat + // word's end and the line's own final end. + let line_end = words.last().map(|w| w.end_ms).unwrap_or(0); + let line_span = (line_end - words[0].start_ms).max(1) as f64; + let syl: Vec = words + .iter() + .map(|w| syllable_count(&w.text) as f64) + .collect(); + let rate = line_span / syl.iter().sum::().max(1.0); + for i in 0..words.len().saturating_sub(2) { + // ≥2 followers required: a single naturally-short final word after + // a mid-line breath must never be dragged forward. + let width = (words[i].end_ms - words[i].start_ms) as f64; + let expect = syl[i] * rate; + let gap_evidence = words[i + 1].start_ms - words[i].end_ms >= RELAY_GAP_MS; + let fat_evidence = width > expect * FAT_RATIO && width - expect > FAT_EXCESS_MS; + if !gap_evidence && !fat_evidence { + continue; + } + let follow_syl: f64 = syl[i + 1..].iter().sum(); + let follow_width: f64 = words[i + 1..] + .iter() + .map(|w| (w.end_ms - w.start_ms) as f64) + .sum(); + if follow_syl <= 0.0 || follow_width >= follow_syl * rate * PACK_RATIO { + // Followers are healthy — a real pause, or a real hold. + continue; + } + if fat_evidence && !gap_evidence { + // Loud-echo fat word: the energy gate saw no decay, so cap by + // syllable budget before relaying into the room this frees. + let cap = (expect * FAT_CAP_RATIO).max(FAT_CAP_MIN_MS) as i64; + words[i].end_ms = words[i].end_ms.min(words[i].start_ms + cap); + } + let base = words[i].end_ms as f64; + let room = (line_end as f64 - base).max(1.0); + let mut acc = 0.0f64; + for k in i + 1..words.len() { + let prop_start = base + room * (acc / follow_syl); + acc += syl[k]; + let prop_end = base + room * (acc / follow_syl); + let s = (CTC_BLEND * words[k].start_ms as f64 + (1.0 - CTC_BLEND) * prop_start) as i64; + let e = (CTC_BLEND * words[k].end_ms as f64 + (1.0 - CTC_BLEND) * prop_end) as i64; + words[k].start_ms = s; + words[k].end_ms = e.max(s + ENV_FRAME_MS); + } + break; + } + // 3) Width floor: a word may grow into the gap after it, never into its + // neighbor. + for i in 0..words.len() { + let next_start = words.get(i + 1).map(|w| w.start_ms).unwrap_or(i64::MAX); + let want = words[i].start_ms + MIN_WIDTH_MS; + if words[i].end_ms < want { + words[i].end_ms = want.min(next_start).max(words[i].start_ms + ENV_FRAME_MS); + } + } +} +const FRAME_MS: u32 = 50; +/// Singing-rate prior, ms per syllable-weight unit. v1 stretched words to +/// fill whatever window the next cue left, so pacing tracked the LAYOUT +/// instead of the voice — a chorus crawled and a tight verse sprinted. Words +/// now move at a bounded singing rate and surplus window time is simply not +/// singing time. +const RATE_MIN_MS: f32 = 130.0; +const RATE_MAX_MS: f32 = 420.0; +/// Bridge sub-gaps inside a sung phrase (breaths, melisma dips). +const BRIDGE_MS: i64 = 150; +/// A line's search window never runs past this, no matter how far away the +/// next cue is — the sweep-crawl lesson: unmarked instrumental stretches are +/// not singing time. +const MAX_WINDOW_MS: i64 = 12_000; +const MIN_WINDOW_MS: i64 = 500; +/// A sustained final syllable may cross the next cue by this much. +const OVERLAP_MS: i64 = 350; + +#[derive(serde::Deserialize, Clone)] +pub struct PrepLine { + pub index: u32, + pub time_ms: i64, + pub text: String, +} + +#[derive(serde::Serialize, serde::Deserialize, Clone)] +pub struct WordSpan { + pub text: String, + pub start_ms: i64, + pub end_ms: i64, +} + +#[derive(serde::Serialize, serde::Deserialize, Clone)] +pub struct ArtifactLine { + pub index: u32, + pub confidence: f32, + /// True = these words came from real forced alignment: they are ABSOLUTE + /// track times, independent of the source doc's cues, so display must + /// NOT shift them by doc offsets. False = estimate laid out from the + /// cues; it moves with them. + #[serde(default)] + pub ctc: bool, + pub words: Vec, +} + +#[derive(serde::Serialize, serde::Deserialize, Clone)] +pub struct WordTimingArtifact { + pub version: u32, + pub aligner: String, + pub estimated: bool, + pub lines: Vec, +} + +struct Job { + key: String, + path: PathBuf, + lines: Vec, + duration_ms: Option, + app: tauri::AppHandle, +} + +struct WorkerState { + queue: VecDeque, + inflight: HashSet, + running: bool, +} + +static WORKER: OnceLock> = OnceLock::new(); + +fn worker() -> &'static Mutex { + WORKER.get_or_init(|| { + Mutex::new(WorkerState { + queue: VecDeque::new(), + inflight: HashSet::new(), + running: false, + }) + }) +} + +fn wordtime_dir(app: &tauri::AppHandle) -> Option { + let cfg = crate::config::load(app); + let dir = crate::config::ensure_library_data_dir(app, &cfg) + .ok()? + .join("lyrics") + .join("wordtime"); + std::fs::create_dir_all(&dir).ok()?; + Some(dir) +} + +/// FNV-1a — stable, dependency-free; these are cache keys, not security. +/// Shared with the setlist listener's artifact keys. +pub(crate) fn fnv(bytes: &[u8]) -> u64 { + let mut hash: u64 = 0xcbf29ce484222325; + for b in bytes { + hash ^= *b as u64; + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +/// Artifact identity: the FILE (path + size + mtime — a re-rip re-prepares), +/// the LYRIC DOC (every cue + every word feeds the hash), and the aligner +/// version. Any of the three changing is a different artifact. +fn artifact_key(path: &Path, lines: &[PrepLine], aligner: &str) -> String { + let meta = std::fs::metadata(path).ok(); + let size = meta.as_ref().map(|m| m.len()).unwrap_or(0); + let mtime = meta + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0); + let mut doc = String::new(); + for l in lines { + doc.push_str(&format!("{}|{}|{}\n", l.index, l.time_ms, l.text)); + } + format!( + "{:016x}-{:016x}-{}", + fnv(path.to_string_lossy().to_lowercase().as_bytes()) ^ size ^ (mtime << 20), + fnv(doc.as_bytes()), + aligner, + ) +} + +fn artifact_path(app: &tauri::AppHandle, key: &str) -> Option { + Some(wordtime_dir(app)?.join(format!("{key}.json"))) +} + +fn read_artifact(path: &Path) -> Option { + let bytes = std::fs::read(path).ok()?; + serde_json::from_slice(&bytes).ok() +} + +/// Aligner evidence for the doc-level auto offset: when an EXISTING artifact +/// (never computed here — this is a cache read on the lyrics-get path) shows +/// the sung words consistently starting N ms away from their lines' cues, +/// that median IS the doc's timing error. Only confident forced-aligned +/// lines vote; a scattered spread (IQR) means verse confusion, not a shift, +/// and yields nothing. Returns the shift to ADD to the doc (negative = +/// lyrics should display earlier). +pub(crate) fn cue_shift_evidence( + app: &tauri::AppHandle, + path: &Path, + lines: &[PrepLine], +) -> Option { + if lines.len() < 5 { + return None; + } + // The tier this machine would prepare TODAY — a v5 (stem) machine reads + // v5 evidence; only ctc-flagged lines vote, so the est tier never can. + let key = artifact_key(path, lines, active_aligner(app)); + let artifact = read_artifact(&artifact_path(app, &key)?)?; + let cue_of: std::collections::HashMap = + lines.iter().map(|l| (l.index, l.time_ms)).collect(); + let mut shifts: Vec = artifact + .lines + .iter() + .filter(|l| l.ctc && l.confidence >= 0.6 && !l.words.is_empty()) + .filter_map(|l| Some(l.words[0].start_ms - cue_of.get(&l.index)?)) + .collect(); + if shifts.len() < 5 { + return None; + } + shifts.sort_unstable(); + let median = shifts[shifts.len() / 2]; + let iqr = shifts[(shifts.len() * 3) / 4] - shifts[shifts.len() / 4]; + if median.abs() < 250 || median.abs() > 3_000 || iqr > 500 { + return None; + } + Some(median) +} + +/// Plain syllable count (vowel groups, min 1) for nuclei matching. +fn syllable_count(word: &str) -> usize { + let mut groups = 0usize; + let mut in_group = false; + for c in word.to_lowercase().chars() { + let vowel = matches!(c, 'a' | 'e' | 'i' | 'o' | 'u' | 'y'); + if vowel && !in_group { + groups += 1; + } + in_group = vowel; + } + groups.max(1) +} + +/// Crude syllable weight: vowel groups (min 1) plus a consonant floor, so +/// "extraordinary" outlasts "a" without pretending to be phonetics. +fn syllable_weight(word: &str) -> f32 { + let mut groups = 0u32; + let mut in_group = false; + for c in word.to_lowercase().chars() { + let vowel = matches!(c, 'a' | 'e' | 'i' | 'o' | 'u' | 'y'); + if vowel && !in_group { + groups += 1; + } + in_group = vowel; + } + groups.max(1) as f32 + 0.6 +} + +/// Distribute one line's words inside its window. v2 rules: +/// - a global activity floor (frames are already track-normalized) plus a +/// soft window-relative term, so quiet passages neither light up on noise +/// nor vanish; +/// - the layout is anchored at the CUE unless the voice audibly starts +/// later, and spans the SINGING-RATE estimate — never the whole window +/// (that is how v1 smeared words across instrumental tails); +/// - words walk active audio at a bounded per-syllable rate, bridging +/// breath-sized dips; anything the evidence cannot carry falls back to the +/// rate-prior layout from the cue at floor confidence. +fn lay_out_line( + text: &str, + window_start: i64, + window_end: i64, + frames: &[f32], + voiced_frames: &[bool], + frame_ms: i64, +) -> (Vec, f32) { + let words: Vec<&str> = text.split_whitespace().collect(); + if words.is_empty() || window_end <= window_start { + return (Vec::new(), 0.0); + } + let weights: Vec = words.iter().map(|w| syllable_weight(w)).collect(); + let total: f32 = weights.iter().sum(); + let window_ms = window_end - window_start; + + let first = ((window_start / frame_ms).max(0) as usize).min(frames.len()); + let last = ((window_end / frame_ms).max(0) as usize).min(frames.len()); + let slice = &frames[first..last]; + let vslice: Vec = (first..last) + .map(|i| voiced_frames.get(i).copied().unwrap_or(false)) + .collect(); + let window_peak = slice.iter().cloned().fold(0.0f32, f32::max); + // v3: VOICING carries the verdict — energy only needs to clear a floor, + // so drums and noise stop counting as singing. + let threshold = (0.10f32).max(window_peak * 0.22); + let mut active: Vec = slice + .iter() + .zip(&vslice) + .map(|(&v, &vo)| vo && v >= threshold) + .collect(); + // Bridge breath-sized dips so a sung phrase reads as one span. + let bridge = (BRIDGE_MS / frame_ms).max(1) as usize; + let mut i = 0; + while i < active.len() { + if !active[i] { + let run_start = i; + while i < active.len() && !active[i] { + i += 1; + } + if run_start > 0 && i < active.len() && i - run_start <= bridge { + for a in &mut active[run_start..i] { + *a = true; + } + } + } else { + i += 1; + } + } + let active_ms = active.iter().filter(|&&a| a).count() as i64 * frame_ms; + let first_active = active + .iter() + .position(|&a| a) + .map(|i| window_start + i as i64 * frame_ms); + + // The rate prior: how long this many syllables plausibly takes. + let est_ms = ((total * 260.0) as i64).clamp(700, window_ms); + let rate = if active_ms > 0 { + (active_ms as f32 / total).clamp(RATE_MIN_MS, RATE_MAX_MS) + } else { + (est_ms as f32 / total).clamp(RATE_MIN_MS, RATE_MAX_MS) + }; + + // Start at the cue unless the voice audibly arrives later (capped: a + // wildly late "first active" is evidence noise, not a 2s+ dead cue). + let start = match first_active { + Some(fa) if fa > window_start + 400 => fa.min(window_start + 2_000), + _ => window_start, + }; + + // v3 first choice: SNAP syllables to detected nuclei. A nucleus is an + // energy peak inside a voiced region, ≥150ms from its neighbour — a sung + // syllable's centre. When the detected count roughly agrees with the + // text's syllable count, words land on real syllables instead of + // proportional guesses (the literature's single biggest model-free win). + let mut nuclei: Vec = Vec::new(); + let min_sep = ((150 / frame_ms.max(1)) as usize).max(2); + for i in 1..active.len().saturating_sub(1) { + if active[i] && slice[i] >= slice[i - 1] && slice[i] > slice[i + 1] { + if let Some(&back) = nuclei.last() { + if i - back < min_sep { + if slice[i] > slice[back] { + *nuclei.last_mut().unwrap() = i; + } + continue; + } + } + nuclei.push(i); + } + } + let sylls: Vec = words.iter().map(|w| syllable_count(w)).collect(); + let s_total: usize = sylls.iter().sum::().max(1); + let n = nuclei.len(); + if n >= (s_total / 2).max(2) && n <= s_total * 2 { + let nucleus_at = |syl: usize| -> i64 { + let idx = if s_total <= 1 { + 0 + } else { + (syl.min(s_total - 1) * (n - 1) + (s_total - 1) / 2) / (s_total - 1) + }; + window_start + nuclei[idx.min(n - 1)] as i64 * frame_ms + }; + let mut out = Vec::with_capacity(words.len()); + let mut syl = 0usize; + let mut prev_end = window_start; + for (wi, w) in words.iter().enumerate() { + let start = (nucleus_at(syl) - 60).max(prev_end).max(window_start); + syl += sylls[wi]; + let mut end = if wi == words.len() - 1 { + (nucleus_at(s_total.saturating_sub(1)) + 250).min(window_end) + } else { + nucleus_at(syl) - 60 + }; + end = end.max(start + 80); + out.push(WordSpan { + text: (*w).into(), + start_ms: start, + end_ms: end, + }); + prev_end = end; + } + let ratio = (n as f32 - s_total as f32).abs() / s_total as f32; + let confidence = (0.55 + 0.30 * (1.0 - ratio.min(1.0))).min(0.85); + return (out, confidence); + } + + let mut out = Vec::with_capacity(words.len()); + let evidence_ok = active_ms >= 240; + if evidence_ok { + // Walk active frames from `start`, spending each word's rate-share of + // sung time; inactive frames advance the clock but cost nothing. + let start_idx = (((start - window_start) / frame_ms).max(0) as usize).min(active.len()); + let mut wi = 0usize; + let mut need = weights[0] * rate; + let mut word_start = start; + for (i, &is_active) in active.iter().enumerate().skip(start_idx) { + if wi >= words.len() { + break; + } + if !is_active { + continue; + } + let t = window_start + i as i64 * frame_ms; + if out.len() == wi && word_start < window_start { + word_start = t; + } + need -= frame_ms as f32; + while need <= 0.0 && wi < words.len() { + out.push(WordSpan { + text: words[wi].into(), + start_ms: word_start, + end_ms: t + frame_ms, + }); + wi += 1; + word_start = t + frame_ms; + if wi < words.len() { + need += weights[wi] * rate; + } + } + } + // Words the active audio could not carry continue at the prior rate. + let mut cursor = out.last().map(|w| w.end_ms).unwrap_or(start); + while out.len() < words.len() { + let wi = out.len(); + let len = (weights[wi] * rate) as i64; + let end = (cursor + len).min(window_end); + out.push(WordSpan { + text: words[wi].into(), + start_ms: cursor, + end_ms: end.max(cursor + 80), + }); + cursor = out.last().unwrap().end_ms; + } + let span_end = start + est_ms; + let covered = active + .iter() + .enumerate() + .filter(|(i, &a)| { + let t = window_start + *i as i64 * frame_ms; + a && t >= start && t < span_end + }) + .count() as i64 + * frame_ms; + let quality = (covered as f32 / est_ms as f32).clamp(0.0, 1.0); + (out, (0.4 + quality * 0.45).min(0.85)) + } else { + // No usable evidence: the rate-prior layout from the cue — bounded by + // the singing estimate, NEVER stretched across the window. + let mut cursor = start as f32; + let per = est_ms as f32 / total; + for (w, weight) in words.iter().zip(&weights) { + let len = per * weight; + out.push(WordSpan { + text: (*w).into(), + start_ms: cursor as i64, + end_ms: (cursor + len) as i64, + }); + cursor += len; + } + (out, 0.25) + } +} + +fn run_job(job: &Job) -> Result { + // Tier B: with the downloaded model present, lines go through CTC forced + // alignment against the real audio; each line that the model cannot + // carry falls back PER LINE to the envelope estimate below. Without the + // model this is exactly the est-v3 worker. With the SEPARATOR model also + // present (the v5 tier), the audio the aligner hears is a learned vocal + // stem — and a per-track separation failure quietly falls back to the + // band-passed mixture, because nothing may ever depend on that module. + // The tier comes from the KEY the job was queued under (its last dash + // segment), so a model download landing mid-queue cannot make the + // artifact's stamp and its filename disagree. + let tag = if job.key.ends_with(ALIGNER_CTC_STEM) { + ALIGNER_CTC_STEM + } else if job.key.ends_with(ALIGNER_CTC) { + ALIGNER_CTC + } else { + ALIGNER + }; + let ctc = tag != ALIGNER; + let pcm16k = if ctc { + let stem = if tag == ALIGNER_CTC_STEM { + match crate::vocal_sep::separate_vocals_16k(&job.app, &job.path) { + Ok(pcm) => Some(pcm), + Err(e) => { + log::warn!("vocal_sep fell back to the mixture: {e}"); + None + } + } + } else { + None + }; + stem.or_else(|| player::vocals::vocal_pcm_16k(&job.path).ok()) + } else { + None + }; + // Referee for the span refinement: the same audio the aligner hears. + let env16k: Option> = pcm16k.as_deref().map(rms_envelope); + let feat = player::vocals::vocal_frames(&job.path, FRAME_MS)?; + let frame_ms = feat.frame_ms as i64; + let frames = feat.energy; + let voiced = feat.voiced; + let mut sorted: Vec<&PrepLine> = job.lines.iter().collect(); + sorted.sort_by_key(|l| l.time_ms); + sorted.retain(|l| !l.text.trim().is_empty()); + // Per-line search windows (the est fallback's bounds, and each + // passage's tail). + let window_end: Vec = sorted + .iter() + .enumerate() + .map(|(i, line)| { + let next_start = sorted + .get(i + 1) + .map(|n| n.time_ms) + .or(job.duration_ms) + .unwrap_or(line.time_ms + MAX_WINDOW_MS); + (next_start + OVERLAP_MS) + .min(line.time_ms + MAX_WINDOW_MS) + .max(line.time_ms + MIN_WINDOW_MS) + }) + .collect(); + // Group consecutive lines into passages: inside one, the cues are only + // hints — a word may land up to PASSAGE_LEAD_MS before its own cue, + // which is how words pull free of a slightly-late source doc. + let mut passages: Vec<(usize, usize)> = Vec::new(); // [from, to) + let mut from = 0usize; + for i in 1..=sorted.len() { + let split = i == sorted.len() + || i - from >= PASSAGE_MAX_LINES + || sorted[i].time_ms - sorted[i - 1].time_ms > PASSAGE_GAP_MS + || sorted[i].time_ms - sorted[from].time_ms > PASSAGE_SPAN_MS; + if split { + passages.push((from, i)); + from = i; + } + } + let mut lines = Vec::with_capacity(sorted.len()); + for &(from, to) in &passages { + let group = &sorted[from..to]; + let passage_result = pcm16k.as_ref().and_then(|pcm| { + let lines: Vec<(&str, i64)> = + group.iter().map(|l| (l.text.as_str(), l.time_ms)).collect(); + align_passage_ctc( + &job.app, + pcm, + &lines, + group[0].time_ms - PASSAGE_LEAD_MS, + window_end[to - 1], + ) + }); + match passage_result { + Some(aligned) => { + for (line, (mut words, confidence)) in group.iter().zip(aligned) { + if let Some(env) = env16k.as_deref() { + refine_ctc_spans(&mut words, env); + } + if !words.is_empty() { + lines.push(ArtifactLine { + index: line.index, + confidence, + ctc: true, + words, + }); + } + } + } + None => { + // Ladder: each line alone through CTC, then the estimate. + for (i, line) in group.iter().enumerate() { + let w_end = window_end[from + i]; + let (words, confidence, ctc) = match pcm16k.as_ref().and_then(|pcm| { + align_line_ctc(&job.app, pcm, &line.text, line.time_ms, w_end) + }) { + Some((mut words, confidence)) => { + if let Some(env) = env16k.as_deref() { + refine_ctc_spans(&mut words, env); + } + (words, confidence, true) + } + None => { + let (words, confidence) = lay_out_line( + &line.text, line.time_ms, w_end, &frames, &voiced, frame_ms, + ); + (words, confidence, false) + } + }; + if !words.is_empty() { + lines.push(ArtifactLine { + index: line.index, + confidence, + ctc, + words, + }); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } + } + } + // Politeness between passages: this thread must never contend with + // the audio path for sustained stretches. + std::thread::sleep(std::time::Duration::from_millis(3)); + } + Ok(WordTimingArtifact { + version: 1, + // The tag the job was KEYED under: v5 stem, v4 mixture, else est. + aligner: tag.into(), + // CTC lines are real alignment; a model-less run is all estimate. + // (Per-line fallbacks under CTC keep their floor confidences.) + estimated: !ctc, + lines, + }) +} + +fn pump() { + loop { + let job = { + let mut w = worker().lock().unwrap(); + match w.queue.pop_front() { + Some(j) => j, + None => { + w.running = false; + return; + } + } + }; + let result = run_job(&job); + let ok = result.is_ok(); + if let Ok(artifact) = result { + // Atomic publish: a partial artifact must never look ready. + if let Some(dest) = artifact_path(&job.app, &job.key) { + let tmp = dest.with_extension("json.writing"); + if std::fs::write(&tmp, serde_json::to_vec(&artifact).unwrap_or_default()) + .is_ok() + { + let _ = std::fs::rename(&tmp, &dest); + } + } + } + { + let mut w = worker().lock().unwrap(); + w.inflight.remove(&job.key); + } + if ok { + let _ = job.app.emit("lyrics:wordtime-ready", &job.key); + } + } +} + +#[derive(serde::Deserialize)] +pub struct StreamRef { + pub username: String, + pub path: String, + pub size: u64, +} + +/// Request word timing for a track + its synced lines. The audio is either a +/// local file or a COMPLETE stream-cache file (a partial download has no +/// tail to measure — the caller's poll simply asks again once the transfer +/// finishes). Returns the artifact immediately on a cache hit; otherwise +/// queues the job (single-flight per key) and returns null — +/// `lyrics:wordtime-ready` carries the key when it lands. +#[tauri::command] +pub async fn lyric_prep_request( + app: tauri::AppHandle, + path: Option, + stream: Option, + duration_secs: Option, + lines: Vec, +) -> Result { + let file = match (&path, &stream) { + (Some(p), _) => PathBuf::from(p), + (None, Some(s)) => { + let cache = crate::playback::cache_path(&app, &s.username, &s.path) + .ok_or("no stream cache dir")?; + let complete = s.size > 0 + && matches!(std::fs::metadata(&cache), Ok(md) if md.len() == s.size); + if !complete { + return Err("stream not fully cached yet".into()); + } + cache + } + (None, None) => return Err("no source".into()), + }; + if !file.is_file() { + return Err("not a local file".into()); + } + if lines.is_empty() { + return Err("no lines".into()); + } + let key = artifact_key(&file, &lines, active_aligner(&app)); + let dest = artifact_path(&app, &key).ok_or("no library data dir")?; + // The blocking stat/read rides a blocking task, never the async runtime. + let cached = tauri::async_runtime::spawn_blocking({ + let dest = dest.clone(); + move || read_artifact(&dest) + }) + .await + .map_err(|e| e.to_string())?; + if let Some(artifact) = cached { + return Ok(serde_json::json!({ "key": key, "artifact": artifact })); + } + let mut w = worker().lock().unwrap(); + if !w.inflight.contains(&key) { + w.inflight.insert(key.clone()); + w.queue.push_back(Job { + key: key.clone(), + path: file, + lines, + duration_ms: duration_secs.map(|d| d as i64 * 1000), + app: app.clone(), + }); + if !w.running { + w.running = true; + std::thread::Builder::new() + .name("lyric-prep".into()) + .spawn(pump) + .map_err(|e| e.to_string())?; + } + } + Ok(serde_json::json!({ "key": key, "artifact": null })) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn word(text: &str, start_ms: i64, end_ms: i64) -> WordSpan { + WordSpan { + text: text.into(), + start_ms, + end_ms, + } + } + + /// Envelope builder: (level, ms) segments at the 20 ms frame. + fn env(segments: &[(f32, i64)]) -> Vec { + let mut out = Vec::new(); + for &(level, ms) in segments { + for _ in 0..(ms / ENV_FRAME_MS) { + out.push(level); + } + } + out + } + + /// A reverb tail decays and is trimmed; a genuinely held note keeps its + /// energy and keeps its span — the pass must tell them apart. + #[test] + fn refine_trims_tails_but_keeps_held_notes() { + // "day": sung 0..400ms, then a -26dB tail to 1400ms. + // "hooold": sung strong through 1600..3400ms. + let mut words = vec![word("day", 0, 1400), word("hooold", 1600, 3400)]; + let e = env(&[(1.0, 400), (0.05, 1200), (0.9, 1800), (0.0, 400)]); + refine_ctc_spans(&mut words, &e); + assert!( + words[0].end_ms <= 520, + "tail must be trimmed near the voice's end, got {}", + words[0].end_ms + ); + assert_eq!( + words[1].end_ms, 3400, + "a held note's sustained energy must survive untrimmed" + ); + } + + /// A packed line relays: the fat first word is trimmed by energy, and + /// the followers spread across the freed room instead of staying + /// stacked at the back. + #[test] + fn refine_relays_packed_followers() { + // "Don't" fat to 2600ms but only sung to ~500ms; followers jammed + // into the last 800ms of a 3400ms line. + let mut words = vec![ + word("Don't", 0, 2600), + word("want", 2600, 2800), + word("out", 2800, 3000), + word("bed", 3000, 3400), + ]; + let e = env(&[(1.0, 500), (0.04, 2100), (0.8, 800)]); + refine_ctc_spans(&mut words, &e); + assert!( + words[0].end_ms <= 620, + "fat word must trim to its sung span, got {}", + words[0].end_ms + ); + assert!( + words[1].start_ms < 1800, + "first follower must relay forward into the freed room, got {}", + words[1].start_ms + ); + for pair in words.windows(2) { + assert!( + pair[1].start_ms >= pair[0].start_ms, + "relay must stay monotone" + ); + } + assert_eq!(words[3].end_ms, 3400, "the line's final end is the anchor"); + } + + /// The loud-echo case: a feedback delay repeats the hook word at + /// near-full volume, so energy never decays and the gate cannot trim — + /// the SHAPE evidence (fat non-final word + packed followers) must cap + /// and relay anyway. + #[test] + fn refine_caps_loud_echo_fat_word() { + let mut words = vec![ + word("Don't", 0, 2600), + word("want", 2600, 2800), + word("out", 2800, 3000), + word("bed", 3000, 3400), + ]; + // Energy loud the whole way — no decay for the gate to find. + let e = env(&[(0.9, 3400)]); + refine_ctc_spans(&mut words, &e); + // rate = 3400/4 = 850; cap = 850*1.5 = 1275. + assert!( + words[0].end_ms <= 1275, + "echo-fat word must be capped by syllable budget, got {}", + words[0].end_ms + ); + assert!( + words[1].start_ms < 2200, + "followers must relay into the capped room, got {}", + words[1].start_ms + ); + assert_eq!(words[3].end_ms, 3400); + } + + /// A real held note with HEALTHY followers is never capped: the packed + /// guard, not the energy, is what protects it. + #[test] + fn refine_keeps_held_word_when_followers_are_healthy() { + // "Dread" held 1.9s of a 4s line, but the followers own their fair + // share — no relay, no cap. + let mut words = vec![ + word("Dread", 0, 1900), + word("follows", 2000, 2900), + word("that", 2950, 3300), + word("lead", 3350, 4000), + ]; + let before: Vec<(i64, i64)> = words.iter().map(|w| (w.start_ms, w.end_ms)).collect(); + let e = env(&[(0.8, 4000)]); + refine_ctc_spans(&mut words, &e); + let after: Vec<(i64, i64)> = words.iter().map(|w| (w.start_ms, w.end_ms)).collect(); + assert_eq!(before, after); + } + + /// Healthy lines pass through untouched — no churn on good CTC output. + #[test] + fn refine_leaves_healthy_lines_alone() { + let mut words = vec![ + word("near", 0, 350), + word("as", 380, 520), + word("far", 540, 980), + ]; + let before: Vec<(i64, i64)> = words.iter().map(|w| (w.start_ms, w.end_ms)).collect(); + let e = env(&[(0.8, 1000)]); + refine_ctc_spans(&mut words, &e); + let after: Vec<(i64, i64)> = words.iter().map(|w| (w.start_ms, w.end_ms)).collect(); + assert_eq!(before, after); + } + + /// Sliver words widen into their following gap, never into a neighbor. + #[test] + fn refine_floors_sliver_widths() { + let mut words = vec![word("a", 1000, 1020), word("sender", 1400, 1900)]; + let e = env(&[(0.7, 2000)]); + refine_ctc_spans(&mut words, &e); + assert_eq!( + words[0].end_ms, 1120, + "sliver must widen to the visible floor" + ); + let mut tight = vec![word("a", 1000, 1020), word("sender", 1060, 1900)]; + refine_ctc_spans(&mut tight, &e); + assert_eq!( + tight[0].end_ms, 1060, + "the floor clamps at the neighbor's start" + ); + } +} diff --git a/src-tauri/src/lyrics.rs b/src-tauri/src/lyrics.rs index 35dac59..a51a86f 100644 --- a/src-tauri/src/lyrics.rs +++ b/src-tauri/src/lyrics.rs @@ -1410,6 +1410,7 @@ pub async fn lyrics_get( let payload = payload_from(user, offset, duration_secs); if payload.status != "none" { return Ok(finish_synced( + &app, payload, entry.auto_sync_off, path.as_deref(), @@ -1514,6 +1515,7 @@ pub async fn lyrics_get( let payload = payload_from(doc, offset, duration_secs); if payload.status == "synced" { return Ok(finish_synced( + &app, payload, entry.auto_sync_off, path.as_deref(), @@ -1603,6 +1605,7 @@ pub async fn lyrics_get( // local copy may be the corrected one). if payload.status == "synced" || payload.status == "instrumental" { return Ok(finish_synced( + &app, payload, entry.auto_sync_off, path.as_deref(), @@ -1638,6 +1641,7 @@ pub async fn lyrics_get( /// Stamp the auto-sync fields onto a winning synced payload. async fn finish_synced( + app: &tauri::AppHandle, mut payload: LyricsPayload, auto_sync_off: bool, path: Option<&str>, @@ -1649,6 +1653,35 @@ async fn finish_synced( let first = payload.synced.first().map(|l| l.time_ms); payload.auto_offset_ms = compute_auto_offset(path, duration_secs, source_duration_secs, first).await; + // Second evidence source, consulted only when the corroborated + // duration/silence probe found nothing: the word-timing artifact. + // When forced alignment has already measured this file against this + // doc and the sung words consistently start N ms from their cues, + // that median is the doc's shift. Cache read only — never computed + // here. (Aligned WORD times themselves are absolute and exempt from + // this offset; only line cues move. See ArtifactLine::ctc.) + if payload.auto_offset_ms == 0 { + if let Some(p) = path { + // The exact shape the word-timing request sends: original + // indexes, empties dropped — the artifact key depends on it. + let lines: Vec = payload + .synced + .iter() + .enumerate() + .map(|(i, l)| crate::lyric_prep::PrepLine { + index: i as u32, + time_ms: l.time_ms as i64, + text: l.text.clone(), + }) + .filter(|l| !l.text.trim().is_empty()) + .collect(); + if let Some(shift) = + crate::lyric_prep::cue_shift_evidence(app, Path::new(p), &lines) + { + payload.auto_offset_ms = shift; + } + } + } } payload } diff --git a/src-tauri/src/playlist_formats.rs b/src-tauri/src/playlist_formats.rs index 91a2fce..f343a65 100644 --- a/src-tauri/src/playlist_formats.rs +++ b/src-tauri/src/playlist_formats.rs @@ -775,7 +775,7 @@ pub fn parse_text(bytes: &[u8]) -> Result { } /// "1:23" → 83_000 ms; "1:23:45" → 5_025_000; rejects anything else. -fn parse_timestamp(s: &str) -> Option { +pub(crate) fn parse_timestamp(s: &str) -> Option { let parts: Vec<&str> = s.split(':').collect(); if !(2..=3).contains(&parts.len()) { return None; diff --git a/src-tauri/src/playlists.rs b/src-tauri/src/playlists.rs index 559896f..aa2c360 100644 --- a/src-tauri/src/playlists.rs +++ b/src-tauri/src/playlists.rs @@ -1,4 +1,4 @@ -//! Playlist commands: CRUD over the occurrence model, covers (custom + +//! Playlist commands: CRUD over the occurrence model, covers (custom + //! composed mosaic), file import with staged preview, and export profiles. //! //! Boundaries, per the architecture audit: parsing happens in @@ -246,6 +246,13 @@ pub struct AddItemInput { pub source_line: Option, #[serde(default)] pub note: Option, + /// Catalog track id for a stream row — the identity anchor (exact id, + /// never a name match). + #[serde(default)] + pub track_id: Option, + /// Cover URL for a stream row, which has no local file to read art from. + #[serde(default)] + pub art_url: Option, } #[tauri::command] @@ -269,6 +276,8 @@ pub async fn playlist_add_items( start_ms: t.start_ms, source_line: t.source_line, note: t.note, + track_id: t.track_id, + art_url: t.art_url, ..Default::default() }) .collect(); @@ -318,7 +327,8 @@ pub async fn playlist_set_order( Ok(()) } -/// Repair one row — attach a found local file or corrected metadata. +/// Repair one row — attach a found local file, corrected metadata, or a +/// stream row's catalog anchor (track id + cover URL). #[tauri::command] #[allow(clippy::too_many_arguments)] pub async fn playlist_relink_item( @@ -332,6 +342,8 @@ pub async fn playlist_relink_item( duration_secs: Option, path: Option, art_path: Option, + track_id: Option, + art_url: Option, ) -> Result<(), String> { let l = lib.library.clone(); tauri::async_runtime::spawn_blocking(move || { @@ -344,6 +356,8 @@ pub async fn playlist_relink_item( duration_secs, path.as_deref(), art_path.as_deref(), + track_id.as_deref(), + art_url.as_deref(), now_secs(), ) }) @@ -583,7 +597,9 @@ pub struct DraftEntry { pub duration_secs: Option, pub start_ms: Option, pub raw: String, - /// "local" (matched a library file) | "unresolved". + /// "local" (matched a library file) | "stream" (full identity, no local + /// file — becomes a Soulseek stream on commit) | "unresolved" (identity + /// too thin to play: ID rows, title-only lines). pub resolution: String, /// How the local match was made: "path" | "relinked" | "tags" — evidence /// the preview shows so an exact path match reads differently from a @@ -591,6 +607,18 @@ pub struct DraftEntry { pub matched_by: Option, pub path: Option, pub art_path: Option, + /// Catalog identity, attached by the review's verification pass (the + /// modal matches stream rows against the Deezer-backed catalog — Kyle: + /// verify against "the streaming library part, not Soulseek"). Flows + /// into the committed row so it plays as a catalog track with artwork. + #[serde(default)] + pub track_id: Option, + #[serde(default)] + pub art_url: Option, + /// The heard release's ISRC, when this row came from the listener door + /// — lets the review resolve the catalog by exact recording id. + #[serde(default)] + pub isrc: Option, } #[derive(serde::Serialize)] @@ -604,7 +632,13 @@ pub struct ImportDraft { pub entries: Vec, pub warnings: Vec, pub resolved_count: u32, + /// Rows that will play as Soulseek streams (identified, not owned). + pub stream_count: u32, pub unresolved_count: u32, + /// A suggested cover image URL (the YouTube doors set the video's own + /// thumbnail). Commit installs it as the playlist's custom cover — + /// failure-soft, a cover is never worth failing an import over. + pub cover_url: Option, } /// Open a playlist file and stage it for review. Parsing and resolution only — @@ -769,6 +803,9 @@ pub async fn playlist_import_folder( matched_by: Some(if indexed.is_some() { "path" } else { "file" }.into()), path: Some(path_string), art_path: indexed.and_then(|r| r.art_path.clone()), + track_id: None, + art_url: None, + isrc: None, } }) .collect(); @@ -782,7 +819,9 @@ pub async fn playlist_import_folder( entries, warnings: Vec::new(), resolved_count: resolved, + stream_count: 0, unresolved_count: 0, + cover_url: None, })) } @@ -875,6 +914,38 @@ pub async fn playlist_import_text( resolve_draft(&lib, parsed, "Pasted tracklist".into(), None).await } +/// Stage a YouTube DJ set for review: fetch the video's own tracklist +/// evidence (chapters → description → top comments, the research bundle's +/// declared-data rungs), then run it through the exact pasted-tracklist +/// pipeline — same parser, same resolution, same review modal, same commit. +/// No audio is fetched and nothing is guessed: a set with no posted +/// tracklist says so instead of inventing one. +#[tauri::command] +pub async fn playlist_import_youtube( + lib: State<'_, LibraryState>, + url: String, +) -> Result { + let fetched = crate::youtube_setlist::fetch(&url).await?; + let parsed = playlist_formats::parse_text(fetched.tracklist.as_bytes())?; + let source = if fetched.author.is_empty() { + fetched.title.clone() + } else { + format!("{} — {}", fetched.author, fetched.title) + }; + let mut draft = resolve_draft(&lib, parsed, source, None).await?; + draft.format = "youtube".into(); + draft.name = Some(fetched.title); + // The video's own thumbnail is the playlist's default face. + if let Some(id) = crate::youtube_setlist::video_id_from_url(&url) { + draft.cover_url = Some(format!("https://i.ytimg.com/vi/{id}/maxresdefault.jpg")); + } + draft + .warnings + .insert(0, format!("Tracklist from {}.", fetched.provenance)); + draft.warnings.extend(fetched.warnings); + Ok(draft) +} + /// Commit a reviewed draft as a new playlist, atomically. #[tauri::command] pub async fn playlist_import_commit( @@ -884,6 +955,7 @@ pub async fn playlist_import_commit( description: Option, kind: String, entries: Vec, + cover_url: Option, ) -> Result { let trimmed = name.trim().to_string(); if trimmed.is_empty() { @@ -897,16 +969,25 @@ pub async fn playlist_import_commit( artist: e.artist, album: e.album, duration_secs: e.duration_secs, - kind: if e.resolution == "local" && e.path.is_some() { - "local".into() + // Empty lets the insert classify: a matched path is local, an + // unmatched row with full identity is a stream, the rest stay + // unresolved placeholders. The one kind the command DOES assert + // is "segment" — a listener row that plays a slice of the set's + // own recording; the classifier would read its path as "local". + kind: if e.resolution == "segment" { + "segment".into() } else { - "unresolved".into() + String::new() }, path: e.path.filter(|p| !p.is_empty()), art_path: e.art_path, source_line: (!e.raw.is_empty()).then_some(e.raw), start_ms: e.start_ms, note: None, + // Catalog identity from the review's verification pass rides + // into the row: the playlist plays it as a catalog track. + track_id: e.track_id.map(|id| id.to_string()), + art_url: e.art_url, }) .collect(); let l = lib.library.clone(); @@ -919,9 +1000,74 @@ pub async fn playlist_import_commit( .map_err(|e| e.to_string())? .map_err(|e| e.to_string())?; changed(&app); + if let Some(url) = cover_url { + // Failure-soft: the playlist is already real; a cover is never + // worth failing it over. + match set_cover_from_url(&app, &lib, id, &url).await { + Ok(()) => changed(&app), + Err(e) => log::debug!("import cover fetch failed: {e}"), + } + } Ok(id) } +/// Download an image URL and install it as the playlist's custom cover — +/// the same square master `playlist_cover_pick` writes. YouTube's +/// maxresdefault 404s on some videos, so that URL falls back to hqdefault. +async fn set_cover_from_url( + app: &tauri::AppHandle, + lib: &State<'_, LibraryState>, + id: i64, + url: &str, +) -> Result<(), String> { + let client = reqwest::Client::builder() + .user_agent("SoulStream/0.1") + .timeout(std::time::Duration::from_secs(15)) + .build() + .map_err(|e| e.to_string())?; + let mut candidates = vec![url.to_string()]; + if let Some(rest) = url.strip_suffix("maxresdefault.jpg") { + candidates.push(format!("{rest}hqdefault.jpg")); + } + let mut bytes: Option> = None; + for candidate in candidates { + if let Ok(resp) = client.get(&candidate).send().await { + if resp.status().is_success() { + if let Ok(body) = resp.bytes().await { + // YouTube's CDN can answer a missing size with a tiny + // gray placeholder instead of a 404 — refuse those. + if body.len() > 4096 { + bytes = Some(body.to_vec()); + break; + } + } + } + } + } + let bytes = bytes.ok_or("no usable image answered")?; + let dir = covers_dir(app)?; + let target = dir.join(format!("{id}.jpg")); + let jpeg = tauri::async_runtime::spawn_blocking(move || -> Result, String> { + let img = + image::load_from_memory(&bytes).map_err(|e| format!("not a usable image: {e}"))?; + Ok(square_jpeg(&img, COVER_MASTER_EDGE)) + }) + .await + .map_err(|e| e.to_string())??; + let temp = dir.join(format!("{id}.importing")); + std::fs::write(&temp, &jpeg).map_err(|e| e.to_string())?; + std::fs::rename(&temp, &target).map_err(|e| e.to_string())?; + let l = lib.library.clone(); + let stored = target.to_string_lossy().into_owned(); + tauri::async_runtime::spawn_blocking(move || { + l.set_playlist_cover(id, Some(&stored), now_secs()) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + Ok(()) +} + /// Resolve parsed entries against the local library, in memory. /// /// The ladder, strongest evidence first (identity rules: an exact locator @@ -931,7 +1077,7 @@ pub async fn playlist_import_commit( /// 3. a unique basename match in the library index; /// 4. a unique normalized (artist, title) tag match; /// 5. a unique normalized title match, when the entry names no artist. -async fn resolve_draft( +pub(crate) async fn resolve_draft( lib: &State<'_, LibraryState>, parsed: playlist_formats::ParsedPlaylist, source_name: String, @@ -980,7 +1126,7 @@ async fn resolve_draft( }; let mut entries = Vec::new(); - let (mut resolved, mut unresolved) = (0u32, 0u32); + let (mut resolved, mut streams, mut unresolved) = (0u32, 0u32, 0u32); let mut any_start = false; for entry in &parsed.entries { let ParsedEntry { @@ -1022,6 +1168,9 @@ async fn resolve_draft( matched_by: Some("path".into()), path: Some(abs), art_path: None, + track_id: None, + art_url: None, + isrc: None, }); resolved += 1; continue; @@ -1090,10 +1239,26 @@ async fn resolve_draft( matched_by: Some(how.into()), path: Some(row.path.clone()), art_path: row.art_path.clone(), + track_id: None, + art_url: None, + isrc: None, }); } None => { - unresolved += 1; + // No library file — but a row with FULL identity becomes a + // STREAM on commit (the insert classifier's rule), so the + // preview says so instead of "unresolved": Kyle read that + // label as "not identified" on a fully-recognized set. + // "unresolved" now means what it says: identity too thin + // to play (ID rows, title-only lines). + let full_identity = title.is_some() + && artist.as_deref().map(|a| !a.trim().is_empty()).unwrap_or(false) + && title.as_deref() != Some("ID"); + if full_identity { + streams += 1; + } else { + unresolved += 1; + } entries.push(DraftEntry { ordinal: *ordinal, title: title.clone().unwrap_or_else(|| "Unknown".into()), @@ -1102,10 +1267,13 @@ async fn resolve_draft( duration_secs: *duration_secs, start_ms: *start_ms, raw: raw.clone(), - resolution: "unresolved".into(), + resolution: if full_identity { "stream" } else { "unresolved" }.into(), matched_by: None, path: None, art_path: None, + track_id: None, + art_url: None, + isrc: None, }); } } @@ -1119,7 +1287,9 @@ async fn resolve_draft( entries, warnings: parsed.warnings, resolved_count: resolved, + stream_count: streams, unresolved_count: unresolved, + cover_url: None, }) } @@ -1277,3 +1447,4 @@ pub async fn playlist_export( .map_err(|e| e.to_string())??; Ok(Some(dest)) } + diff --git a/src-tauri/src/setlist_scan.rs b/src-tauri/src/setlist_scan.rs new file mode 100644 index 0000000..2aeed19 --- /dev/null +++ b/src-tauri/src/setlist_scan.rs @@ -0,0 +1,1005 @@ +//! The DJ-set LISTENER: window the set's audio, recognize each window +//! (shazam.rs), build a consensus setlist, and project it into the same +//! ImportDraft the other import doors produce. The research bundle's +//! phases 5–7 in their honest v1 shape: fixed sampling grid, temporal +//! consensus by track key + offset clustering, Unknown gaps preserved as +//! "ID - ID" rows with timestamps, nothing auto-invented. +//! +//! The boundary math is the live-proven offset trick: Shazam reports where +//! a window sits INSIDE the identified track (millisecond-faithful — see +//! shazam.rs's live test), so `window_position − in_track_offset` is the +//! track's start in the mix, and agreeing windows cluster tightly around +//! the true start even on a sparse grid. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use tauri::Emitter; + +use crate::shazam; + +/// Sampling grid: one 12 s window every 45 s. An hour-long set is ~80 +/// windows ≈ ~3–4 minutes of paced analysis. +const STEP_SECS: usize = 45; +/// Pacing between recognition calls — the Bing lesson: unpaced scraping is +/// how providers break. +const PACE_MS: u64 = 1_800; +/// Two implied starts within this many seconds agree. +const CLUSTER_SECS: f64 = 6.0; +/// A gap this long with no recognized audio earns an explicit ID row. +const ID_GAP_SECS: usize = 2 * STEP_SECS; +/// Longest set the scanner will decode (memory bound: ~4 h of 16 kHz mono +/// f32 ≈ 920 MB). +const MAX_SET_SECS: f64 = 4.0 * 3600.0; + +static CANCEL: AtomicBool = AtomicBool::new(false); + +#[tauri::command] +pub fn playlist_scan_cancel() { + CANCEL.store(true, Ordering::Relaxed); +} + +/// One window's verdict, kept for consensus — and persisted as the +/// listen artifact, so a re-scan of the same audio replays hearings +/// instead of re-paying recognition. +#[derive(Clone, serde::Serialize, serde::Deserialize)] +pub(crate) struct Observation { + pos_secs: f64, + rec: Option, +} + +/// A consensus track occurrence in the set. +pub(crate) struct Occurrence { + pub(crate) start_secs: f64, + pub(crate) artist: String, + pub(crate) title: String, + /// The heard release's ISRC (from the same matched release that names + /// the occurrence) — the review's anchor-exact catalog key. + pub(crate) isrc: Option, + pub(crate) votes: usize, +} + +/// One row of the video's own posted tracklist (chapters / description / +/// comment), time-stamped — the human evidence the listener cross-checks +/// its hearings against. +pub(crate) struct DeclaredRow { + pub(crate) start_secs: f64, + pub(crate) artist: Option, + pub(crate) title: String, +} + +fn base_title(title: &str) -> &str { + title.split(['(', '[']).next().unwrap_or(title).trim() +} + +/// The version qualifier of a title — what sits in (…)/[…] after the base, +/// normalized: "Skanka (Kayzo Remix)" → "kayzo remix"; a plain title → "". +fn variant_qualifier(title: &str) -> String { + let base_len = title.split(['(', '[']).next().unwrap_or(title).len(); + title[base_len..] + .to_lowercase() + .chars() + .map(|c| if c.is_alphanumeric() { c } else { ' ' }) + .collect::() + .split_whitespace() + .collect::>() + .join(" ") +} + +fn norm_name(s: &str) -> String { + s.to_lowercase() + .chars() + .filter(|c| c.is_alphanumeric()) + .collect() +} + +/// The DJ-culture unknown-track token — never an identity to match on. +fn is_id_token(title: &str) -> bool { + let n = norm_name(title); + n == "id" || n == "idid" +} + +/// Project tracklist text (a video's posted list, or one pasted alongside a +/// local file) into declared rows. Only timestamped, titled lines can anchor +/// to the mix — the rest are dropped; parse failure is an empty list. +fn declared_rows(text: &str) -> Vec { + crate::playlist_formats::parse_text(text.as_bytes()) + .map(|parsed| { + parsed + .entries + .into_iter() + .filter_map(|e| { + Some(DeclaredRow { + start_secs: e.start_ms? as f64 / 1000.0, + artist: e.artist, + title: e.title?, + }) + }) + .collect() + }) + .unwrap_or_default() +} + +/// Let the video's posted tracklist name the VERSION where the hearing +/// matched a different release of the same track: the fingerprint can only +/// report the release it matched (a remix's unmodified stretches hit the +/// original it samples), while the DJ's own list is authoritative for what +/// was actually played. Only qualifiers move — a declared row must agree on +/// artist and base title, sit within two grid steps of the heard boundary, +/// and CARRY a version qualifier of its own; a plain declared title never +/// strips a heard one. A corrected row loses its heard ISRC — that anchor +/// belongs to the other release — and resolves by its declared name in the +/// review instead. +pub(crate) fn apply_declared_variants( + confirmed: &mut [Occurrence], + declared: &[DeclaredRow], +) -> Vec { + let mut notes = Vec::new(); + for o in confirmed.iter_mut() { + let heard_base = norm_name(base_title(&o.title)); + if heard_base.is_empty() { + continue; + } + let candidate = declared + .iter() + .filter(|d| !is_id_token(&d.title)) + .filter(|d| (d.start_secs - o.start_secs).abs() <= 2.0 * STEP_SECS as f64) + .filter(|d| { + let declared_base = norm_name(base_title(&d.title)); + !declared_base.is_empty() + && (declared_base == heard_base + || declared_base.contains(&heard_base) + || heard_base.contains(&declared_base)) + }) + .filter(|d| match d.artist.as_deref() { + None => true, + Some(a) => { + let declared_artist = norm_name(a); + let heard_artist = norm_name(&o.artist); + declared_artist == heard_artist + || declared_artist.contains(&heard_artist) + || heard_artist.contains(&declared_artist) + } + }) + .min_by(|a, b| { + (a.start_secs - o.start_secs) + .abs() + .total_cmp(&(b.start_secs - o.start_secs).abs()) + }); + let Some(d) = candidate else { continue }; + let declared_q = variant_qualifier(&d.title); + if declared_q.is_empty() || declared_q == variant_qualifier(&o.title) { + continue; + } + notes.push(format!("{} - {} (heard as \"{}\")", o.artist, d.title, o.title)); + o.title = d.title.clone(); + o.isrc = None; + } + notes +} + +/// The identity two windows must share to be the SAME played track: artist +/// plus the title up to its first parenthetical. Live sets match DIFFERENT +/// RELEASES of one track window to window (studio, "Live At…", edits) — +/// exact-title grouping fragments them (the Cappadocia lesson: one track +/// became four "occurrences"). +fn norm_key(rec: &shazam::Recognition) -> (String, String) { + let base = rec + .title + .split(['(', '[']) + .next() + .unwrap_or(&rec.title) + .trim() + .to_lowercase(); + (rec.artist.to_lowercase(), base) +} + +/// Group observations into occurrences: same normalized identity AND +/// window-adjacent (within 2 grid steps of the cluster's last member). +/// Offset-implied starts refine the boundary when they agree; when a live +/// edit scatters them, the first window's position stands (late, never +/// early — honest). A track genuinely played twice is two clusters because +/// adjacency breaks between them. +pub(crate) fn consensus(observations: &[Observation]) -> Vec { + struct Cluster { + key: (String, String), + first_pos: f64, + last_pos: f64, + implieds: Vec, + recs: Vec, + } + let mut clusters: Vec = Vec::new(); + for obs in observations { + let Some(rec) = &obs.rec else { continue }; + let key = norm_key(rec); + let implied = rec.offset_secs.map(|off| obs.pos_secs - off); + match clusters + .iter_mut() + .rev() + .find(|c| c.key == key && obs.pos_secs - c.last_pos <= 2.0 * STEP_SECS as f64 + 1.0) + { + Some(c) => { + c.last_pos = obs.pos_secs; + c.recs.push(rec.clone()); + if let Some(i) = implied { + c.implieds.push(i); + } + } + None => clusters.push(Cluster { + key, + first_pos: obs.pos_secs, + last_pos: obs.pos_secs, + implieds: implied.into_iter().collect(), + recs: vec![rec.clone()], + }), + } + } + let mut out: Vec = clusters + .into_iter() + .map(|mut c| { + // Start: the tightest agreeing pair of implied starts wins; + // scattered offsets (live edits) fall back to the first window. + let mut start = c.first_pos; + let mut implieds = c.implieds.clone(); + implieds.sort_by(f64::total_cmp); + for pair in implieds.windows(2) { + if pair[1] - pair[0] <= 2.0 * CLUSTER_SECS { + start = (pair[0] + pair[1]) / 2.0; + break; + } + } + if c.implieds.len() == 1 && (c.implieds[0] - c.first_pos).abs() <= STEP_SECS as f64 { + start = c.implieds[0]; + } + // The DISPLAYED identity is the cluster's most-heard exact + // variant, not blindly the first window's — the first window + // sits in the transition zone, the most error-prone hearing + // (a remix's intro matches the original it samples). Ties keep + // the earliest; artist, title and ISRC stay coherent, all from + // the SAME matched release. + let votes = c.recs.len(); + let mut best = 0usize; + let mut best_votes = 0usize; + for i in 0..c.recs.len() { + let n = c + .recs + .iter() + .filter(|r| r.title.eq_ignore_ascii_case(&c.recs[i].title)) + .count(); + if n > best_votes { + best_votes = n; + best = i; + } + } + let id = c.recs.swap_remove(best); + Occurrence { + start_secs: start.max(0.0), + isrc: id.isrc, + artist: id.artist, + title: id.title, + votes, + } + }) + .collect(); + out.sort_by(|a, b| a.start_secs.total_cmp(&b.start_secs)); + out +} + +/// Project occurrences + unrecognized gaps into pasted-tracklist LINES — +/// the exact shapes playlist_formats::parse_text accepts, so the listener +/// rides the same parser/resolver/review/commit as every other door. +pub(crate) fn tracklist_lines( + occurrences: &[Occurrence], + observations: &[Observation], + set_len_secs: f64, + declared: &[DeclaredRow], +) -> (String, Vec) { + let fmt = |s: f64| { + let t = s.max(0.0) as i64; + let (h, m, sec) = (t / 3600, (t % 3600) / 60, t % 60); + if h > 0 { + format!("{h}:{m:02}:{sec:02}") + } else { + format!("{m}:{sec:02}") + } + }; + let mut lines: Vec<(f64, String)> = occurrences + .iter() + .map(|o| { + ( + o.start_secs, + format!("{} {} - {}", fmt(o.start_secs), o.artist, o.title), + ) + }) + .collect(); + // Unknown gaps: a RUN of unrecognized windows ≥ ID_GAP_SECS becomes one + // explicit ID row at the run's start — UNLESS the video's own tracklist + // declares rows inside the run, which then stand in for it (named human + // evidence beats an anonymous placeholder; the fingerprint simply + // doesn't know unreleased material). A gap nothing was declared for + // keeps the honest ID — inventing nothing there is still the rule. + let mut filled: Vec = Vec::new(); + let mut fill_gap = + |start: f64, end: f64, lines: &mut Vec<(f64, String)>, filled: &mut Vec| { + let inside: Vec<&DeclaredRow> = declared + .iter() + .filter(|d| !is_id_token(&d.title)) + .filter(|d| d.start_secs >= start - 1.0 && d.start_secs < end) + .collect(); + if inside.is_empty() { + lines.push((start, format!("{} ID - ID", fmt(start)))); + return; + } + for d in inside { + let line = match d.artist.as_deref() { + Some(a) => format!("{} {} - {}", fmt(d.start_secs), a, d.title), + None => format!("{} {}", fmt(d.start_secs), d.title), + }; + filled.push(match d.artist.as_deref() { + Some(a) => format!("{a} - {}", d.title), + None => d.title.clone(), + }); + lines.push((d.start_secs, line)); + } + }; + let mut gap_start: Option = None; + for obs in observations { + match (&obs.rec, gap_start) { + (Some(_), Some(start)) => { + if obs.pos_secs - start >= ID_GAP_SECS as f64 { + fill_gap(start, obs.pos_secs, &mut lines, &mut filled); + } + gap_start = None; + } + (None, None) => gap_start = Some(obs.pos_secs), + _ => {} + } + } + if let Some(start) = gap_start { + if set_len_secs - start >= ID_GAP_SECS as f64 { + fill_gap(start, set_len_secs, &mut lines, &mut filled); + } + } + lines.sort_by(|a, b| a.0.total_cmp(&b.0)); + let text = lines + .into_iter() + .map(|(_, l)| l) + .collect::>() + .join("\n"); + (text, filled) +} + +/// Listen through a decoded set. Emits `playlist:scan-progress` +/// {stage:"listen", window, total, hit?} as it goes; honors the cancel +/// flag between windows; paces every request. Returns the raw +/// observations — consensus is the caller's, so remembered hearings +/// replay through the same math as fresh ones. +pub(crate) async fn scan_pcm16k( + app: &tauri::AppHandle, + mono16k: &[f32], +) -> Result, String> { + CANCEL.store(false, Ordering::Relaxed); + let total_windows = + (mono16k.len() / (STEP_SECS * shazam::SAMPLE_RATE)).max(1); + let mut observations: Vec = Vec::new(); + let mut consecutive_errors = 0usize; + for w in 0..total_windows { + if CANCEL.load(Ordering::Relaxed) { + return Err("canceled".into()); + } + let start = w * STEP_SECS * shazam::SAMPLE_RATE; + let end = (start + shazam::WINDOW_SECS * shazam::SAMPLE_RATE).min(mono16k.len()); + if end <= start + shazam::SAMPLE_RATE { + break; + } + let pos_secs = (start / shazam::SAMPLE_RATE) as f64; + let rec = match shazam::recognize_window(&mono16k[start..end]).await { + Ok(r) => { + consecutive_errors = 0; + r + } + Err(e) if e.contains("rate-limited") => { + // One long breath, one retry; then give the window up. + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + consecutive_errors += 1; + if consecutive_errors > 4 { + return Err("the recognition service is rate-limiting hard — try again later".into()); + } + shazam::recognize_window(&mono16k[start..end]) + .await + .unwrap_or(None) + } + Err(_) => { + consecutive_errors += 1; + if consecutive_errors > 6 { + return Err( + "the recognition service stopped answering — partial scan abandoned".into(), + ); + } + None + } + }; + let _ = app.emit( + "playlist:scan-progress", + serde_json::json!({ + "stage": "listen", + "window": w + 1, + "total": total_windows, + "hit": rec.as_ref().map(|r| format!("{} - {}", r.artist, r.title)), + }), + ); + observations.push(Observation { pos_secs, rec }); + tokio::time::sleep(std::time::Duration::from_millis(PACE_MS)).await; + } + Ok(observations) +} + +/// The whole listener, YouTube edition: yt-dlp the audio (cache-first) → +/// decode → scan → draft through the standard pipeline. +#[tauri::command] +pub async fn playlist_scan_youtube( + app: tauri::AppHandle, + lib: tauri::State<'_, crate::pipeline::LibraryState>, + url: String, +) -> Result { + let video_id = crate::youtube_setlist::video_id_from_url(&url) + .ok_or("That doesn't look like a YouTube video link")?; + CANCEL.store(false, Ordering::Relaxed); + // The FULL fetch first: a posted tracklist is free supplementary + // evidence for the listener — it names the VERSION played where the + // fingerprint can only name the release it matched, and it fills + // unrecognized gaps. Failure-soft: listener sets usually have no + // posted tracklist, and the light title fetch still names the draft + // (the raw-video-id lesson). + let (page_title, declared) = match crate::youtube_setlist::fetch(&url).await { + Ok(fetched) => ( + Some((fetched.title, fetched.author)), + declared_rows(&fetched.tracklist), + ), + Err(_) => (crate::youtube_setlist::fetch_title(&url).await, Vec::new()), + }; + let audio = { + let app2 = app.clone(); + let vid = video_id.clone(); + tauri::async_runtime::spawn_blocking(move || { + crate::ytdlp::fetch_audio_blocking(&app2, &vid, &CANCEL) + }) + .await + .map_err(|e| e.to_string())?? + }; + let mut draft = + scan_file_into_draft(app, lib, audio, page_title, declared, "the video's tracklist") + .await?; + // The video's own thumbnail is the playlist's default face. + draft.cover_url = Some(format!( + "https://i.ytimg.com/vi/{video_id}/maxresdefault.jpg" + )); + Ok(draft) +} + +/// What the local door's picker hands the modal: enough to show the file +/// while the user decides whether to paste a tracklist alongside it. +#[derive(serde::Serialize)] +pub struct PickedSetFile { + path: String, + name: String, + duration_secs: Option, +} + +/// The dialog half of the local door, split from the scan so the modal can +/// collect an optional pasted tracklist between pick and listen. +#[tauri::command] +pub async fn playlist_pick_set_file( + app: tauri::AppHandle, +) -> Result, String> { + use tauri_plugin_dialog::DialogExt; + let (tx, rx) = tokio::sync::oneshot::channel(); + app.dialog() + .file() + .add_filter("Audio", &["mp3", "m4a", "aac", "flac", "wav", "ogg", "mp4"]) + .pick_file(move |path| { + let _ = tx.send(path.map(|p| p.to_string())); + }); + let Some(path) = rx.await.unwrap_or(None) else { + return Ok(None); + }; + let name = std::path::Path::new(&path) + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.clone()); + // Header-only probe for the panel's length chip; a file lofty can't + // read still listens fine. + let duration_secs = { + let p = path.clone(); + tauri::async_runtime::spawn_blocking(move || { + tagger::read_summary(std::path::Path::new(&p)) + .ok() + .and_then(|s| s.duration_secs) + }) + .await + .unwrap_or(None) + }; + Ok(Some(PickedSetFile { + path, + name, + duration_secs, + })) +} + +/// The whole listener, local-file edition: scan a picked set recording, +/// with an optionally pasted tracklist riding along as the same declared +/// evidence the YouTube door fetches for itself. +#[tauri::command] +pub async fn playlist_scan_file( + app: tauri::AppHandle, + lib: tauri::State<'_, crate::pipeline::LibraryState>, + path: String, + tracklist: Option, +) -> Result { + CANCEL.store(false, Ordering::Relaxed); + let declared = tracklist.as_deref().map(declared_rows).unwrap_or_default(); + let name = std::path::Path::new(&path) + .file_stem() + .map(|s| s.to_string_lossy().into_owned()); + scan_file_into_draft( + app, + lib, + std::path::PathBuf::from(path), + name.map(|n| (n, String::new())), + declared, + "the pasted tracklist", + ) + .await +} + +// ---- the listen artifact (observation cache) -------------------------------- + +/// Bump when the recognizer's output changes shape or meaning (v2: ISRC +/// capture) — an old artifact must not answer for a newer listener. +const LISTEN_VERSION: u32 = 2; + +#[derive(serde::Serialize, serde::Deserialize)] +struct ListenArtifact { + version: u32, + /// The sampling grid the observations were taken on; a different grid + /// is a different listen. + step_secs: usize, + set_len_secs: f64, + observations: Vec, +} + +/// The audio file's identity — the lyric_prep idiom: path + size + mtime, +/// so a re-downloaded (re-encoded) file re-listens. +fn listen_key(audio: &std::path::Path) -> String { + let meta = std::fs::metadata(audio).ok(); + let size = meta.as_ref().map(|m| m.len()).unwrap_or(0); + let mtime = meta + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0); + format!( + "{:016x}-v{}", + crate::lyric_prep::fnv(audio.to_string_lossy().to_lowercase().as_bytes()) + ^ size + ^ (mtime << 20), + LISTEN_VERSION, + ) +} + +fn listen_artifact_path(app: &tauri::AppHandle, key: &str) -> Option { + use tauri::Manager as _; + let dir = app.path().app_data_dir().ok()?.join("setlist-listen"); + std::fs::create_dir_all(&dir).ok()?; + Some(dir.join(format!("{key}.json"))) +} + +fn read_listen_artifact(app: &tauri::AppHandle, key: &str) -> Option { + let bytes = std::fs::read(listen_artifact_path(app, key)?).ok()?; + let artifact: ListenArtifact = serde_json::from_slice(&bytes).ok()?; + (artifact.version == LISTEN_VERSION && artifact.step_secs == STEP_SECS).then_some(artifact) +} + +/// Best-effort — a cache write failure must never fail the scan it stores. +fn write_listen_artifact( + app: &tauri::AppHandle, + key: &str, + observations: &[Observation], + set_len_secs: f64, +) { + let Some(path) = listen_artifact_path(app, key) else { + return; + }; + let artifact = ListenArtifact { + version: LISTEN_VERSION, + step_secs: STEP_SECS, + set_len_secs, + observations: observations.to_vec(), + }; + if let Ok(bytes) = serde_json::to_vec(&artifact) { + let _ = std::fs::write(&path, bytes); + } +} + +async fn scan_file_into_draft( + app: tauri::AppHandle, + lib: tauri::State<'_, crate::pipeline::LibraryState>, + audio: std::path::PathBuf, + identity: Option<(String, String)>, + declared: Vec, + // Where the declared rows came from, for the provenance warnings: + // "the video's tracklist" or "the pasted tracklist". + declared_src: &str, +) -> Result { + // A set already listened to on this machine answers from its listen + // artifact: the audio was cached, but recognition still re-paid ~5 + // paced minutes per re-scan. Hearings don't change; consensus + // re-derives at read time, so tuning it never invalidates one. + let key = listen_key(&audio); + let remembered = read_listen_artifact(&app, &key); + let from_cache = remembered.is_some(); + let (observations, set_len) = match remembered { + Some(artifact) => (artifact.observations, artifact.set_len_secs), + None => { + let _ = app.emit( + "playlist:scan-progress", + serde_json::json!({ "stage": "decode" }), + ); + let decode_path = audio.clone(); + let mono16k = tauri::async_runtime::spawn_blocking(move || { + player::vocals::mono_pcm_16k_streaming(&decode_path, MAX_SET_SECS) + }) + .await + .map_err(|e| e.to_string())??; + let set_len = mono16k.len() as f64 / shazam::SAMPLE_RATE as f64; + let observations = scan_pcm16k(&app, &mono16k).await?; + (observations, set_len) + } + }; + + let occurrences = consensus(&observations); + if occurrences.is_empty() { + return Err( + "Nothing in the set was recognized — it may be all unreleased material, or the \ + service is unreachable. The tracklist doors still work." + .into(), + ); + } + // Rows need corroboration: two agreeing windows (≥ ~90 s of presence). + // Single-window hits are transition-misfire territory — they stay + // EVIDENCE (a warning line), never rows (research §7: low-confidence + // candidates are kept as evidence only). + let (mut confirmed, singles): (Vec<_>, Vec<_>) = + occurrences.into_iter().partition(|o| o.votes >= 2); + if confirmed.is_empty() { + return Err( + "Nothing was corroborated across windows — the set may be mostly unreleased \ + material. The tracklist doors still work." + .into(), + ); + } + // Only a scan worth keeping is worth remembering — a deaf run (the + // service flaking window to window) must re-listen next time. + if !from_cache { + write_listen_artifact(&app, &key, &observations, set_len); + } + let variant_notes = apply_declared_variants(&mut confirmed, &declared); + let (text, gap_filled) = tracklist_lines(&confirmed, &observations, set_len, &declared); + let parsed = crate::playlist_formats::parse_text(text.as_bytes())?; + let (title, author) = identity.unwrap_or_else(|| { + ( + audio + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| "Recognized set".into()), + String::new(), + ) + }); + let source = if author.is_empty() { + title.clone() + } else { + format!("{author} — {title}") + }; + let mut draft = crate::playlists::resolve_draft(&lib, parsed, source, None).await?; + draft.format = "listened".into(); + draft.name = Some(title); + // Each row gets its heard release's ISRC back — matched by the same + // second + artist its projected line carried — so the review can + // resolve the catalog by exact recording id before trying any name. + for e in draft.entries.iter_mut() { + let Some(ms) = e.start_ms else { continue }; + if let Some(o) = confirmed + .iter() + .find(|o| o.start_secs.max(0.0) as i64 == ms / 1000 && o.artist == e.artist) + { + e.isrc = o.isrc.clone(); + } + } + // An unidentified stretch still has its AUDIO — the set recording is on + // disk (the YT cache, or the picked file itself). ID rows keep the + // recording's path and a "segment" resolution: they commit as playable + // SLICES (this row's cue to the next row's), so nothing in the set is + // unplayable just because nobody knows its name yet. + let audio_path = audio.to_string_lossy().into_owned(); + let mut sliced = 0usize; + for e in draft.entries.iter_mut() { + if e.resolution == "unresolved" && e.start_ms.is_some() { + e.resolution = "segment".into(); + e.path = Some(audio_path.clone()); + sliced += 1; + } + } + let matched: usize = observations.iter().filter(|o| o.rec.is_some()).count(); + draft.warnings.insert( + 0, + format!( + "Recognized by listening: {matched} of {} windows matched, {} tracks corroborated.{}", + observations.len(), + confirmed.len(), + if from_cache { + " Remembered from an earlier listen of this audio." + } else { + "" + } + ), + ); + if !singles.is_empty() { + let mut names: Vec = singles + .iter() + .map(|o| format!("{} - {}", o.artist, o.title)) + .collect(); + names.sort(); + names.dedup(); + draft.warnings.push(format!( + "Heard once each, left out of the list (transition flickers, usually): {}.", + names.join(" · ") + )); + } + if !variant_notes.is_empty() { + draft.warnings.push(format!( + "Named by {declared_src} — the recording matched a different release: {}.", + variant_notes.join(" · ") + )); + } + if !gap_filled.is_empty() { + draft.warnings.push(format!( + "Unrecognized stretches filled from {declared_src}: {}.", + gap_filled.join(" · ") + )); + } + if sliced > 0 { + draft.warnings.push(format!( + "{sliced} unidentified stretch{} will play as slices of the set's own recording.", + if sliced == 1 { "" } else { "es" } + )); + } + Ok(draft) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn declared_rows_keep_only_timestamped_titled_lines() { + let rows = + declared_rows("00:00 Bicep - Glue\nFour Tet - Baby\n05:30 Overmono - So U Kno"); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].title, "Glue"); + assert_eq!(rows[0].artist.as_deref(), Some("Bicep")); + assert_eq!(rows[1].start_secs, 330.0); + } + + fn obs(pos: f64, hit: Option<(&str, &str, f64)>) -> Observation { + Observation { + pos_secs: pos, + rec: hit.map(|(artist, title, offset)| shazam::Recognition { + artist: artist.into(), + title: title.into(), + key: format!("{artist}-{title}"), + offset_secs: Some(offset), + album: None, + cover_url: None, + isrc: None, + }), + } + } + + /// Three windows inside one track must collapse to ONE occurrence whose + /// start is the offset-implied track start, not the first window hit. + #[test] + fn consensus_clusters_by_implied_start() { + let observations = vec![ + obs(45.0, Some(("Bicep", "Glue", 40.0))), // implies 5.0 + obs(90.0, Some(("Bicep", "Glue", 85.5))), // implies 4.5 + obs(135.0, Some(("Bicep", "Glue", 130.2))), // implies 4.8 + obs(180.0, None), + obs(225.0, Some(("Overmono", "So U Kno", 11.0))), // implies 214.0 + ]; + let out = consensus(&observations); + assert_eq!(out.len(), 2); + assert_eq!(out[0].title, "Glue"); + assert_eq!(out[0].votes, 3); + assert!((out[0].start_secs - 4.65).abs() < 0.2, "{}", out[0].start_secs); + assert_eq!(out[1].votes, 1); + assert!((out[1].start_secs - 214.0).abs() < 0.1); + } + + /// The live-set lesson: one played track matches DIFFERENT releases per + /// window (title variants, junk offsets). Normalized identity + + /// adjacency must fuse them into ONE occurrence, anchored at the first + /// window when the offsets scatter. + #[test] + fn consensus_fuses_release_variants_of_one_played_track() { + let observations = vec![ + obs(450.0, Some(("Monolink", "Father Ocean (Ben Böhmer Remix)", 30.0))), + obs( + 495.0, + Some(( + "Monolink", + "Father Ocean (Ben Böhmer Remix (Live At The Roundhouse, London))", + 290.0, // wildly different release → scattered implied start + )), + ), + obs(540.0, Some(("Monolink", "Father Ocean (Ben Böhmer Remix)", 121.0))), + ]; + let out = consensus(&observations); + assert_eq!(out.len(), 1, "variants must fuse"); + assert_eq!(out[0].votes, 3); + // Implieds are 420 / 205 / 419 — the 420±1 pair agrees, so the + // boundary comes from it, not from the junk one. + assert!((out[0].start_secs - 419.5).abs() < 1.0, "{}", out[0].start_secs); + } + + /// A track played TWICE stays two occurrences — dedup would erase the + /// set's real shape. + #[test] + fn consensus_keeps_repeated_tracks_apart() { + let observations = vec![ + obs(45.0, Some(("Artist", "Anthem", 30.0))), + obs(1800.0, Some(("Artist", "Anthem", 25.0))), + ]; + let out = consensus(&observations); + assert_eq!(out.len(), 2); + } + + /// The ISRC rides with the SAME release whose artist/title label the + /// occurrence — the majority variant's — so identity stays coherent. + #[test] + fn consensus_carries_the_majority_variants_isrc() { + let mut first = obs(45.0, Some(("Bicep", "Glue", 40.0))); + first.rec.as_mut().unwrap().isrc = Some("GBCFB1700573".into()); + let second = obs(90.0, Some(("Bicep", "Glue", 85.0))); + let out = consensus(&[first, second]); + assert_eq!(out.len(), 1); + assert_eq!(out[0].isrc.as_deref(), Some("GBCFB1700573")); + } + + /// The displayed identity is the cluster's most-heard exact variant, + /// not blindly the first window's — the first window sits in the + /// transition zone, where a remix's intro matches the original it + /// samples. + #[test] + fn consensus_prefers_the_majority_variant() { + let observations = vec![ + obs(45.0, Some(("Hamdi", "Skanka", 30.0))), + obs(90.0, Some(("Hamdi", "Skanka (Kayzo Remix)", 75.0))), + obs(135.0, Some(("Hamdi", "Skanka (Kayzo Remix)", 120.0))), + ]; + let out = consensus(&observations); + assert_eq!(out.len(), 1); + assert_eq!(out[0].title, "Skanka (Kayzo Remix)"); + assert_eq!(out[0].votes, 3); + } + + fn occ_at(start: f64, artist: &str, title: &str, isrc: Option<&str>) -> Occurrence { + Occurrence { + start_secs: start, + artist: artist.into(), + title: title.into(), + isrc: isrc.map(str::to_string), + votes: 2, + } + } + + fn dec(start: f64, artist: Option<&str>, title: &str) -> DeclaredRow { + DeclaredRow { + start_secs: start, + artist: artist.map(str::to_string), + title: title.into(), + } + } + + /// The DJ's posted list names the VERSION: a declared qualifier + /// overrides a heard plain title and drops the heard ISRC (that anchor + /// belongs to the other release). A plain declared title never strips + /// a heard qualifier, and a different artist never matches. + #[test] + fn declared_tracklist_names_the_version() { + let mut confirmed = vec![occ_at(855.0, "Hamdi", "Skanka", Some("QZPLS2163210"))]; + let declared = [dec(850.0, Some("Hamdi"), "Skanka (Kayzo Remix)")]; + let notes = apply_declared_variants(&mut confirmed, &declared); + assert_eq!(confirmed[0].title, "Skanka (Kayzo Remix)"); + assert!( + confirmed[0].isrc.is_none(), + "the heard ISRC must not anchor another release" + ); + assert_eq!(notes.len(), 1); + + let mut keep = vec![occ_at(100.0, "Artist", "Song (Club Mix)", None)]; + assert!( + apply_declared_variants(&mut keep, &[dec(100.0, Some("Artist"), "Song")]).is_empty() + ); + assert_eq!(keep[0].title, "Song (Club Mix)"); + + let mut other = vec![occ_at(100.0, "Artist", "Song", None)]; + assert!(apply_declared_variants( + &mut other, + &[dec(100.0, Some("Someone Else"), "Song (X Remix)")] + ) + .is_empty()); + assert_eq!(other[0].title, "Song"); + } + + /// A declared row landing inside an unrecognized stretch stands in for + /// the anonymous ID row; a stretch nothing was declared for keeps the + /// honest ID. + #[test] + fn declared_rows_fill_recognition_gaps() { + let observations = vec![ + obs(0.0, Some(("A", "One", 0.0))), + obs(45.0, None), + obs(90.0, None), + obs(135.0, None), + obs(180.0, Some(("B", "Two", 10.0))), + ]; + let occ = consensus(&observations); + let declared = [dec(70.0, Some("Secret"), "Weapon (VIP)")]; + let (text, filled) = tracklist_lines(&occ, &observations, 200.0, &declared); + assert!(text.contains("1:10 Secret - Weapon (VIP)"), "{text}"); + assert!(!text.contains("ID - ID"), "{text}"); + assert_eq!(filled, ["Secret - Weapon (VIP)"]); + + let (text2, filled2) = tracklist_lines(&occ, &observations, 200.0, &[]); + assert!(text2.contains("ID - ID"), "{text2}"); + assert!(filled2.is_empty()); + } + + /// Long unrecognized stretches earn an ID row; short ones do not. + #[test] + fn gaps_become_id_rows() { + let observations = vec![ + obs(0.0, Some(("A", "One", 0.0))), + obs(45.0, None), + obs(90.0, None), + obs(135.0, None), + obs(180.0, Some(("B", "Two", 10.0))), + ]; + let occ = consensus(&observations); + let (text, _) = tracklist_lines(&occ, &observations, 200.0, &[]); + assert!(text.contains("ID - ID"), "{text}"); + let short_gap = vec![ + obs(0.0, Some(("A", "One", 0.0))), + obs(45.0, None), + obs(90.0, Some(("B", "Two", 10.0))), + ]; + let occ2 = consensus(&short_gap); + let (text2, _) = tracklist_lines(&occ2, &short_gap, 110.0, &[]); + assert!(!text2.contains("ID - ID"), "{text2}"); + } + + /// The projected lines parse back through the real pasted-text parser + /// as a timestamped setlist. + #[test] + fn lines_round_trip_through_parse_text() { + let observations = vec![ + obs(45.0, Some(("Bicep", "Glue", 40.0))), + obs(90.0, Some(("Bicep", "Glue", 85.0))), + ]; + let occ = consensus(&observations); + let (text, _) = tracklist_lines(&occ, &observations, 120.0, &[]); + let parsed = crate::playlist_formats::parse_text(text.as_bytes()).unwrap(); + assert_eq!(parsed.entries.len(), 1); + assert_eq!(parsed.entries[0].artist.as_deref(), Some("Bicep")); + assert!(parsed.entries[0].start_ms.is_some()); + } +} diff --git a/src-tauri/src/shazam.rs b/src-tauri/src/shazam.rs new file mode 100644 index 0000000..5bab66f --- /dev/null +++ b/src-tauri/src/shazam.rs @@ -0,0 +1,478 @@ +//! Shazam-style audio recognition for the DJ-set setlist assistant — the +//! research bundle's "broader recognition adapter" (feasibility §3 rung 4, +//! plan phase 7), Kyle's pick 2026-08-19: identify a mix's tracks by +//! LISTENING, not just by reading its posted tracklist. +//! +//! The signature algorithm and packet format are reimplemented from the +//! publicly documented Shazam fingerprint (Wang 2003; layout and constants +//! cross-checked against the open-source SongRec project's description of +//! the format). Nothing here links foreign code: plain Rust over the same +//! realfft the vocal separator uses. +//! +//! Posture: in-process, no child processes; ONE request per analyzed +//! window against the free discovery endpoint, paced by the caller +//! (research §11 — rate limits, timeout, cache belong to the job layer, +//! built next). Failure is always soft: a window that cannot be +//! recognized is an honest Unknown, never a guess. + +use realfft::num_complex::Complex; +use realfft::RealFftPlanner; + +/// One recognized window: what Shazam heard and where it claims the window +/// sits inside the identified track. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Recognition { + pub artist: String, + pub title: String, + /// Shazam track key — stable id for consensus grouping. + pub key: String, + /// Offset of the analyzed window within the identified track, seconds, + /// when the response carries one (it usually does). + pub offset_secs: Option, + pub album: Option, + pub cover_url: Option, + /// The matched RELEASE's ISRC when the response carries one — an + /// id-grade identity the review can resolve against the catalog + /// exactly, instead of by name. + pub isrc: Option, +} + +// ---- signature generation (pure DSP) ---------------------------------------- + +const FFT_SIZE: usize = 2048; +const BINS: usize = FFT_SIZE / 2 + 1; // 1025 +const HOP: usize = 128; +/// The generator's working window: 12 s at 16 kHz mono, the length the +/// service expects a tag to be. +pub const WINDOW_SECS: usize = 12; +pub const SAMPLE_RATE: usize = 16_000; + +struct Rings { + samples: [i16; FFT_SIZE], + samples_at: usize, + ffts: Vec<[f32; BINS]>, + ffts_at: u8, + spread: Vec<[f32; BINS]>, + spread_at: u8, + passes: u32, + peaks: [Vec; 4], +} + +struct Peak { + pass: u32, + magnitude: u16, + corrected_bin: u16, +} + +/// Symmetric Hann of length 2048 (numpy-style, N−1 denominator) — the +/// window the reference tables were generated from. +fn hann() -> Vec { + (0..FFT_SIZE) + .map(|i| { + 0.5 - 0.5 + * (2.0 * std::f32::consts::PI * i as f32 / (FFT_SIZE - 1) as f32).cos() + }) + .collect() +} + +/// Build the binary signature for ≤ WINDOW_SECS of 16 kHz mono PCM. +pub fn signature(pcm: &[f32]) -> Vec { + let mut samples: Vec = pcm + .iter() + .map(|&x| (x * 32768.0).clamp(-32768.0, 32767.0) as i16) + .collect(); + let want = WINDOW_SECS * SAMPLE_RATE; + if samples.len() < want { + samples.resize(want, 0); + } else { + samples.truncate(want); + } + + let window = hann(); + let mut planner = RealFftPlanner::::new(); + let fft = planner.plan_fft_forward(FFT_SIZE); + let mut rings = Rings { + samples: [0i16; FFT_SIZE], + samples_at: 0, + ffts: vec![[0.0; BINS]; 256], + ffts_at: 0, + spread: vec![[0.0; BINS]; 256], + spread_at: 0, + passes: 0, + peaks: Default::default(), + }; + let mut re_in = vec![0.0f32; FFT_SIZE]; + let mut cx_out = vec![Complex::default(); BINS]; + + for chunk in samples.chunks_exact(HOP) { + // Ring in the new hop, then window the reordered buffer. + for (i, &s) in chunk.iter().enumerate() { + rings.samples[(rings.samples_at + i) & (FFT_SIZE - 1)] = s; + } + rings.samples_at = (rings.samples_at + HOP) & (FFT_SIZE - 1); + for i in 0..FFT_SIZE { + re_in[i] = + rings.samples[(i + rings.samples_at) & (FFT_SIZE - 1)] as f32 * window[i]; + } + let _ = fft.process(&mut re_in, &mut cx_out); + { + let out = &mut rings.ffts[rings.ffts_at as usize]; + for (o, c) in out.iter_mut().zip(cx_out.iter()) { + *o = ((c.re * c.re + c.im * c.im) / (1 << 17) as f32).max(1e-10); + } + } + rings.ffts_at = rings.ffts_at.wrapping_add(1); + + // Frequency- then time-domain peak spreading. + { + let src = rings.ffts[rings.ffts_at.wrapping_sub(1) as usize]; + let dst = &mut rings.spread[rings.spread_at as usize]; + dst.copy_from_slice(&src); + for i in 0..=BINS - 3 { + dst[i] = dst[i].max(dst[i + 1]).max(dst[i + 2]); + } + let snapshot = *dst; + for former in [1u8, 3, 6] { + let f = &mut rings.spread[rings.spread_at.wrapping_sub(former) as usize]; + for i in 0..BINS { + f[i] = f[i].max(snapshot[i]); + } + } + } + rings.spread_at = rings.spread_at.wrapping_add(1); + rings.passes += 1; + + if rings.passes >= 46 { + recognize_peaks(&mut rings); + } + } + + encode(&rings.peaks, want as u32) +} + +fn recognize_peaks(r: &mut Rings) { + let f46 = r.ffts[r.ffts_at.wrapping_sub(46) as usize]; + let s49 = r.spread[r.spread_at.wrapping_sub(49) as usize]; + for bin in 10..=BINS - 11 { + if f46[bin] < 1.0 / 64.0 || f46[bin] < s49[bin - 1] { + continue; + } + let mut neighbor_max = 0.0f32; + for off in [-10i32, -7, -4, -3, 1, 2, 5, 8] { + neighbor_max = neighbor_max.max(s49[(bin as i32 + off) as usize]); + } + if f46[bin] <= neighbor_max { + continue; + } + let mut other_max = neighbor_max; + for off in [ + -53i32, -45, 165, 172, 179, 186, 193, 200, 214, 221, 228, 235, 242, 249, + ] { + let other = &r.spread[((r.spread_at as i32 + off) & 255) as usize]; + other_max = other_max.max(other[bin - 1]); + } + if f46[bin] <= other_max { + continue; + } + let pass = r.passes - 46; + let mag = f46[bin].ln().max(1.0 / 64.0) * 1477.3 + 6144.0; + let before = f46[bin - 1].ln().max(1.0 / 64.0) * 1477.3 + 6144.0; + let after = f46[bin + 1].ln().max(1.0 / 64.0) * 1477.3 + 6144.0; + let variation_1 = mag * 2.0 - before - after; + if variation_1 < 0.0 { + continue; + } + let variation_2 = (after - before) * 32.0 / variation_1; + let corrected = (bin as i32 * 64 + variation_2 as i32) as u16; + let hz = corrected as f32 * (SAMPLE_RATE as f32 / 2.0 / 1024.0 / 64.0); + let band = match hz as i32 { + 250..=519 => 0, + 520..=1449 => 1, + 1450..=3499 => 2, + 3500..=5500 => 3, + _ => continue, + }; + r.peaks[band].push(Peak { + pass, + magnitude: mag as u16, + corrected_bin: corrected, + }); + } +} + +/// The wire format: 48-byte header (magic, CRC-32 of everything after the +/// first 8 bytes, payload size, 16 kHz marker, sample count + 0.24 s), a +/// type-length echo, then per-band peak runs with delta-coded pass numbers. +fn encode(peaks: &[Vec; 4], number_samples: u32) -> Vec { + let mut out: Vec = Vec::with_capacity(4096); + let w32 = |out: &mut Vec, v: u32| out.extend_from_slice(&v.to_le_bytes()); + let w16 = |out: &mut Vec, v: u16| out.extend_from_slice(&v.to_le_bytes()); + + w32(&mut out, 0xcafe2580); // magic1 + w32(&mut out, 0); // crc32 backfilled + w32(&mut out, 0); // size backfilled + w32(&mut out, 0x94119c00); // magic2 + w32(&mut out, 0); + w32(&mut out, 0); + w32(&mut out, 0); + w32(&mut out, 3u32 << 27); // 16 kHz + w32(&mut out, 0); + w32(&mut out, 0); + w32(&mut out, number_samples + (SAMPLE_RATE as f32 * 0.24) as u32); + w32(&mut out, (15 << 19) + 0x40000); + w32(&mut out, 0x40000000); + w32(&mut out, 0); // size echo backfilled + + for (band, list) in peaks.iter().enumerate() { + if list.is_empty() { + continue; + } + let mut body: Vec = Vec::with_capacity(list.len() * 5); + let mut pass = 0u32; + for p in list { + if p.pass - pass >= 255 { + body.push(0xff); + body.extend_from_slice(&p.pass.to_le_bytes()); + pass = p.pass; + } + body.push((p.pass - pass) as u8); + body.extend_from_slice(&p.magnitude.to_le_bytes()); + body.extend_from_slice(&p.corrected_bin.to_le_bytes()); + pass = p.pass; + } + w32(&mut out, 0x60030040 + band as u32); + w32(&mut out, body.len() as u32); + out.extend_from_slice(&body); + while out.len() % 4 != 0 { + out.push(0); + } + } + let _ = w16; // (kept for symmetry; u16 writes happen inside the body) + + let size = (out.len() - 48) as u32; + out[8..12].copy_from_slice(&size.to_le_bytes()); + out[52..56].copy_from_slice(&size.to_le_bytes()); + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&out[8..]); + let crc = hasher.finalize(); + out[4..8].copy_from_slice(&crc.to_le_bytes()); + out +} + +// ---- the discovery request ---------------------------------------------------- + +/// Pseudo-UUID v4 from OS entropy via the stdlib hasher — unique enough for +/// the tag ids the endpoint expects, no uuid/rand dependency. +fn pseudo_uuid() -> String { + use std::collections::hash_map::RandomState; + use std::hash::{BuildHasher, Hasher}; + let mut parts = [0u64; 2]; + for p in parts.iter_mut() { + let mut h = RandomState::new().build_hasher(); + h.write_u64(std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos() as u64) + .unwrap_or(0)); + *p = h.finish(); + } + let b: Vec = parts.iter().flat_map(|p| p.to_le_bytes()).collect(); + format!( + "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-4{:01x}{:02x}-a{:01x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", + b[0], b[1], b[2], b[3], b[4], b[5], b[6] & 0xf, b[7], b[8] & 0xf, b[9], b[10], b[11], b[12], b[13], b[14], b[15] + ) +} + +/// Recognize one ≤12 s window of 16 kHz mono PCM. `Ok(None)` = the service +/// answered and heard nothing it knows — an honest Unknown. `Err` = the +/// request itself failed (network, rate limit) and the caller may retry +/// later. +pub async fn recognize_window(pcm16k: &[f32]) -> Result, String> { + let sig = signature(pcm16k); + use base64::Engine as _; + let uri = format!( + "data:audio/vnd.shazam.sig;base64,{}", + base64::engine::general_purpose::STANDARD.encode(&sig) + ); + let sample_ms = (pcm16k.len().min(WINDOW_SECS * SAMPLE_RATE) * 1000 / SAMPLE_RATE) as u64; + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + let body = serde_json::json!({ + "geolocation": { "altitude": 300, "latitude": 45, "longitude": 2 }, + "signature": { "samplems": sample_ms, "timestamp": ts, "uri": uri }, + "timestamp": ts, + "timezone": "Europe/Paris", + }); + let url = format!( + "https://amp.shazam.com/discovery/v5/en/US/android/-/tag/{}/{}?sync=true&webv3=true&sampling=true&connected=&shazamapiversion=v3&sharehub=true&video=v3", + pseudo_uuid().to_uppercase(), + pseudo_uuid() + ); + let client = reqwest::Client::builder() + .user_agent("Dalvik/2.1.0 (Linux; U; Android 13; Pixel 7 Build/TQ3A.230901.001)") + .timeout(std::time::Duration::from_secs(20)) + .build() + .map_err(|e| e.to_string())?; + let resp = client + .post(&url) + .header("Content-Language", "en_US") + .json(&body) + .send() + .await + .map_err(|e| format!("Shazam did not answer: {e}"))?; + if resp.status().as_u16() == 429 { + return Err("rate-limited by the recognition service".into()); + } + let v: serde_json::Value = resp + .json() + .await + .map_err(|e| format!("unreadable recognition response: {e}"))?; + let Some(track) = v.get("track") else { + return Ok(None); + }; + let title = track + .get("title") + .and_then(serde_json::Value::as_str) + .unwrap_or("") + .to_string(); + let artist = track + .get("subtitle") + .and_then(serde_json::Value::as_str) + .unwrap_or("") + .to_string(); + if title.is_empty() { + return Ok(None); + } + let key = track + .get("key") + .and_then(serde_json::Value::as_str) + .unwrap_or("") + .to_string(); + let offset_secs = v + .get("matches") + .and_then(|m| m.get(0)) + .and_then(|m| m.get("offset")) + .and_then(serde_json::Value::as_f64); + // Album rides in the metadata sections when present. + let mut album = None; + if let Some(sections) = track.get("sections").and_then(serde_json::Value::as_array) { + for s in sections { + if let Some(meta) = s.get("metadata").and_then(serde_json::Value::as_array) { + for m in meta { + if m.get("title").and_then(serde_json::Value::as_str) == Some("Album") { + album = m + .get("text") + .and_then(serde_json::Value::as_str) + .map(str::to_string); + } + } + } + } + } + let cover_url = track + .get("images") + .and_then(|i| i.get("coverart")) + .and_then(serde_json::Value::as_str) + .map(str::to_string); + let isrc = track + .get("isrc") + .and_then(serde_json::Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string); + Ok(Some(Recognition { + artist, + title, + key, + offset_secs, + album, + cover_url, + isrc, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The packet must round out structurally: magics in place, CRC over + /// bytes 8.., size echoed at 8 and 52, bands tagged 0x60030040+n. + #[test] + fn signature_packet_is_well_formed() { + // A tone sweep gives the recognizer real peaks to encode. + let pcm: Vec = (0..WINDOW_SECS * SAMPLE_RATE) + .map(|i| { + let t = i as f32 / SAMPLE_RATE as f32; + (2.0 * std::f32::consts::PI * (440.0 + 80.0 * t) * t).sin() * 0.4 + }) + .collect(); + let sig = signature(&pcm); + assert!(sig.len() > 64, "packet too small: {}", sig.len()); + assert_eq!(&sig[0..4], &0xcafe2580u32.to_le_bytes()); + assert_eq!(&sig[12..16], &0x94119c00u32.to_le_bytes()); + let size = u32::from_le_bytes(sig[8..12].try_into().unwrap()); + assert_eq!(size as usize, sig.len() - 48); + assert_eq!(&sig[52..56], &size.to_le_bytes()); + let mut h = crc32fast::Hasher::new(); + h.update(&sig[8..]); + assert_eq!(u32::from_le_bytes(sig[4..8].try_into().unwrap()), h.finalize()); + let band_tag = u32::from_le_bytes(sig[56..60].try_into().unwrap()); + assert!( + (0x60030040..=0x60030043).contains(&band_tag), + "first band tag {band_tag:#x}" + ); + } + + /// A pure tone at 1 kHz must place its peaks in band 1 (520–1450 Hz). + #[test] + fn peaks_land_in_the_right_band() { + let pcm: Vec = (0..WINDOW_SECS * SAMPLE_RATE) + .map(|i| (2.0 * std::f32::consts::PI * 1000.0 * i as f32 / 16000.0).sin() * 0.4) + .collect(); + let mut samples: Vec = pcm.iter().map(|&x| (x * 32768.0) as i16).collect(); + samples.truncate(WINDOW_SECS * SAMPLE_RATE); + let sig = signature(&pcm); + // Band 1's tag must appear; bands 0/2/3 may be empty or tiny. + let mut found_band1 = false; + let mut at = 56usize; + while at + 8 <= sig.len() { + let tag = u32::from_le_bytes(sig[at..at + 4].try_into().unwrap()); + let len = u32::from_le_bytes(sig[at + 4..at + 8].try_into().unwrap()) as usize; + if tag == 0x60030041 && len > 50 { + found_band1 = true; + } + at += 8 + len + ((4 - len % 4) % 4); + } + assert!(found_band1, "1 kHz peaks must dominate band 1"); + } + + /// LIVE: recognize a 12 s window of a known library track against the + /// real service. Ignored by default — network, unofficial endpoint — + /// run explicitly: cargo test -p app shazam_live -- --ignored --nocapture + #[test] + #[ignore] + fn shazam_live_known_track() { + let path = std::path::Path::new( + "D:\\Music\\SoulSeek Music\\Panda Bear\\2015 - Panda Bear Meets The Grim Reaper\\1_SEQUENTIAL CIRCUITS.flac", + ); + assert!(path.is_file(), "corpus track missing"); + let [l, r] = player::vocals::separation_input_44k1(path).unwrap(); + let mono = player::vocals::stem_pcm_16k(&l, &r); + // A mid-track window, well into the singing. + let start = 60 * SAMPLE_RATE; + let window = &mono[start..start + WINDOW_SECS * SAMPLE_RATE]; + let rt = tokio::runtime::Runtime::new().unwrap(); + let got = rt.block_on(recognize_window(window)).expect("request failed"); + let rec = got.expect("service heard nothing — signature likely malformed"); + println!( + "LIVE RESULT: {} — {} (key {}, offset {:?}, album {:?}, isrc {:?})", + rec.artist, rec.title, rec.key, rec.offset_secs, rec.album, rec.isrc + ); + assert!( + rec.title.to_lowercase().contains("sequential"), + "wrong track: {} — {}", + rec.artist, + rec.title + ); + } +} diff --git a/src-tauri/src/vocal_sep.rs b/src-tauri/src/vocal_sep.rs new file mode 100644 index 0000000..4c3a3af --- /dev/null +++ b/src-tauri/src/vocal_sep.rs @@ -0,0 +1,499 @@ +//! Learned vocal separation for the lyric aligner — the "honest fix" the +//! vocals.rs reducer comment promises: a real separator lifting the voice +//! out of the band before CTC alignment, instead of band-pass pre-emphasis. +//! +//! Model: UVR-MDX-NET Voc_FT (KUIELab MDX-Net / TFC-TDF v2, the Ultimate +//! Vocal Remover project's vocal fine-tune; MIT with credit). Chosen +//! 2026-08-19 from the verified table in +//! research/lyrics-vocal-separation/candidates.md: 63.7 MiB, ONNX opset 13 — +//! under the inbox-WinML ceiling this machine PROVED by running the MMS +//! aligner (opset 14) — static `input`/`output` [1,4,3072,256] f32, vocals +//! SDR 9.64 on the MVSEP multisong benchmark (+1.5 dB over htdemucs at a +//! fifth of the size; the RoFormer/Demucs ONNX exports are opset 17–23 and +//! cannot load here as published). +//! +//! Posture is lyric_align.rs's exactly: WinML (no onnxruntime, no child +//! processes), download only from the user's explicit Settings click, the +//! session cached in a static — and NOTHING depends on this module. When +//! the model is absent or anything fails, the aligner keeps its band-passed +//! mixture input (the ctc-v4 behavior). +//! +//! The inference recipe is UVR's own non-overlap framing, kept bit-faithful +//! so the model sees what it was trained on: stereo 44.1 kHz → per-chunk +//! centered STFT (n_fft 7680, hop 1024, periodic Hann, reflect pad) → crop +//! to 3072 bins → model (channels [L_re, L_im, R_re, R_im]) → vocal +//! spectrogram → zero-pad the cropped bins back → WOLA iSTFT → trim +//! n_fft/2 from each end → butt-join. Each chunk contributes 253,440 fresh +//! samples (~5.75 s), so a four-minute track is ~42 model evaluations. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use realfft::num_complex::Complex; +use realfft::{ComplexToReal, RealFftPlanner, RealToComplex}; +use tauri::Emitter; +use windows::{ + core::{Interface, HSTRING}, + AI::MachineLearning::{ + LearningModel, LearningModelBinding, LearningModelDevice, LearningModelDeviceKind, + LearningModelSession, TensorFloat, + }, + Win32::System::WinRT::{RoInitialize, RO_INIT_MULTITHREADED}, +}; + +const MODEL_URL: &str = + "https://github.com/TRvlvr/model_repo/releases/download/all_public_uvr_models/UVR-MDX-NET-Voc_FT.onnx"; +/// Exact pinned size, shown in Settings and enforced after download — if the +/// asset is ever republished with different bytes, refuse rather than trust. +/// (Verified against the served bytes AND the file's own protobuf tail: +/// opset 13, `input`/`output` [batch,4,3072,256]. Kim_Vocal_2 is 66,759,214 +/// — a first pin mistakenly used that sibling's size, and this guard caught +/// it exactly as designed.) +const MODEL_BYTES: u64 = 66_762_490; + +// UVR MDX family constants (their model_data.json for this checkpoint: +// dim_f 3072, dim_t 2^8, n_fft 7680; hop is the architecture's fixed 1024). +const N_FFT: usize = 7_680; +const HOP: usize = 1_024; +const BINS: usize = N_FFT / 2 + 1; // 3841 +const DIM_F: usize = 3_072; +const DIM_T: usize = 256; +/// Samples one model window covers: 256 centered STFT frames at hop 1024. +const CHUNK: usize = HOP * (DIM_T - 1); // 261,120 +/// STFT edge effects live inside half a window; UVR trims this from each +/// chunk end and butt-joins the rest — no cross-fade, nothing to tune, and +/// the kept region sits in full window-overlap coverage by construction. +const TRIM: usize = N_FFT / 2; // 3840 +/// Fresh samples each chunk contributes (~5.75 s at 44.1 kHz). +const GEN: usize = CHUNK - 2 * TRIM; // 253,440 +/// Flat f32 count of the model's input and output: [1, 4, 3072, 256]. +const TENSOR_LEN: usize = 4 * DIM_F * DIM_T; + +struct Engine { + session: LearningModelSession, + /// Which device kind actually carried the smoke eval — for the log line. + device: &'static str, +} + +static ENGINE: Mutex> = Mutex::new(None); + +fn model_file(app: &tauri::AppHandle) -> Result { + Ok(crate::lyric_align::models_dir(app)?.join("uvr-mdx-voc-ft.onnx")) +} + +pub fn model_present(app: &tauri::AppHandle) -> bool { + matches!(model_file(app), Ok(m) if m.is_file()) +} + +#[tauri::command] +pub fn vocal_sep_status(app: tauri::AppHandle) -> serde_json::Value { + let size = model_file(&app) + .ok() + .and_then(|p| std::fs::metadata(p).ok()) + .map(|m| m.len()) + .unwrap_or(0); + serde_json::json!({ + "present": model_present(&app), + "bytes": size, + "downloadBytes": MODEL_BYTES, + }) +} + +/// Download the separation model. Runs ONLY from the user's explicit +/// Settings click — the consent-before-download charter rule. Streams to a +/// temp file, verifies the exact pinned byte count, renames atomically, +/// emits `lyrics:sep-model-progress` along the way. +#[tauri::command] +pub async fn vocal_sep_download(app: tauri::AppHandle) -> Result<(), String> { + let client = reqwest::Client::builder() + .user_agent("SoulStream/0.1 (vocal separation)") + .build() + .map_err(|e| e.to_string())?; + let dest = model_file(&app)?; + let tmp = dest.with_extension("onnx.downloading"); + let mut response = client + .get(MODEL_URL) + .send() + .await + .map_err(|e| e.to_string())? + .error_for_status() + .map_err(|e| e.to_string())?; + let total = response.content_length().unwrap_or(MODEL_BYTES); + let mut file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?; + let mut received: u64 = 0; + let mut last_emit: u64 = 0; + use std::io::Write; + while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? { + file.write_all(&chunk).map_err(|e| e.to_string())?; + received += chunk.len() as u64; + if received - last_emit > 2 * 1024 * 1024 { + last_emit = received; + let _ = app.emit( + "lyrics:sep-model-progress", + serde_json::json!({ "received": received, "total": total }), + ); + } + } + drop(file); + if received != MODEL_BYTES { + let _ = std::fs::remove_file(&tmp); + return Err(format!( + "download size mismatch: {received} bytes, expected {MODEL_BYTES}" + )); + } + std::fs::rename(&tmp, &dest).map_err(|e| e.to_string())?; + let _ = app.emit( + "lyrics:sep-model-progress", + serde_json::json!({ "received": received, "total": total, "done": true }), + ); + Ok(()) +} + +fn ensure_engine(app: &tauri::AppHandle) -> Result<(), String> { + let mut guard = ENGINE.lock().unwrap(); + if guard.is_some() { + return Ok(()); + } + let model_path = model_file(app)?; + if !model_path.is_file() { + return Err("separation model not downloaded".into()); + } + // Same WinRT posture as the aligner: initialize, tolerate an existing + // apartment, never uninitialize someone else's. + match unsafe { RoInitialize(RO_INIT_MULTITHREADED) } { + Ok(()) => {} + Err(e) if e.code().0 as u32 == 0x8001_0106 => {} + Err(e) => return Err(format!("WinRT init failed: {e}")), + } + let model = + LearningModel::LoadFromFilePath(&HSTRING::from(model_path.to_string_lossy().as_ref())) + .map_err(|e| format!("could not load separation model: {e}"))?; + // DirectX first (DirectML — the discrete GPU when there is one): a track + // is ~42 conv-net evaluations and the GPU turns minutes into seconds. + // The zeros smoke eval proves the device can actually RUN the graph — + // "loads but cannot execute" is a real DML failure mode — then Cpu is + // the quiet fallback, so this works everywhere the aligner does. + for (kind, name) in [ + (LearningModelDeviceKind::DirectX, "DirectX"), + (LearningModelDeviceKind::Cpu, "Cpu"), + ] { + let Ok(device) = LearningModelDevice::Create(kind) else { + continue; + }; + let Ok(session) = LearningModelSession::CreateFromModelOnDevice(&model, &device) else { + continue; + }; + let zeros = vec![0.0f32; TENSOR_LEN]; + if eval_chunk(&session, &zeros).is_ok() { + *guard = Some(Engine { + session, + device: name, + }); + return Ok(()); + } + } + Err("no WinML device could run the separation model".into()) +} + +fn eval_chunk(session: &LearningModelSession, flat: &[f32]) -> Result, String> { + let input = TensorFloat::CreateFromShapeArrayAndDataArray( + &[1, 4, DIM_F as i64, DIM_T as i64], + flat, + ) + .map_err(|e| format!("could not create input tensor: {e}"))?; + let binding = LearningModelBinding::CreateFromSession(session) + .map_err(|e| format!("could not create binding: {e}"))?; + binding + .Bind(&HSTRING::from("input"), &input) + .map_err(|e| format!("could not bind input: {e}"))?; + let result = session + .Evaluate(&binding, &HSTRING::from("vocal-sep")) + .map_err(|e| format!("inference failed: {e}"))?; + if !result.Succeeded().unwrap_or(false) { + return Err("inference did not succeed".into()); + } + let out: TensorFloat = result + .Outputs() + .and_then(|o| o.Lookup(&HSTRING::from("output"))) + .map_err(|e| format!("output missing: {e}"))? + .cast() + .map_err(|e| format!("output wrong type: {e}"))?; + let values: Vec = out + .GetAsVectorView() + .map_err(|e| format!("could not read output: {e}"))? + .into_iter() + .collect(); + if values.len() != TENSOR_LEN { + return Err(format!("unexpected output length {}", values.len())); + } + Ok(values) +} + +// ---- STFT plumbing (pure DSP — no WinML below this line) ------------------- + +struct Stft { + r2c: Arc>, + c2r: Arc>, + /// Periodic Hann — torch.hann_window's default, what the model trained on. + window: Vec, + /// WOLA denominator Σ w² over the padded chunk layout. The 256 frames + /// tile the padded buffer exactly, so it is identical for every chunk. + envelope: Vec, +} + +impl Stft { + fn new() -> Self { + let mut planner = RealFftPlanner::::new(); + let r2c = planner.plan_fft_forward(N_FFT); + let c2r = planner.plan_fft_inverse(N_FFT); + let window: Vec = (0..N_FFT) + .map(|n| 0.5 - 0.5 * (2.0 * std::f32::consts::PI * n as f32 / N_FFT as f32).cos()) + .collect(); + let mut envelope = vec![1e-8f32; CHUNK + 2 * TRIM]; + for t in 0..DIM_T { + for (n, w) in window.iter().enumerate() { + envelope[t * HOP + n] += w * w; + } + } + Stft { + r2c, + c2r, + window, + envelope, + } + } + + /// Centered STFT of one CHUNK-sized segment: reflect-pad n_fft/2 each + /// side (torch.stft center=True), frames at the hop — frame-major + /// [t * BINS + f]. + fn forward(&self, chunk: &[f32]) -> Result>, String> { + debug_assert_eq!(chunk.len(), CHUNK); + let mut padded = vec![0.0f32; CHUNK + 2 * TRIM]; + padded[TRIM..TRIM + CHUNK].copy_from_slice(chunk); + for i in 1..=TRIM { + padded[TRIM - i] = chunk[i]; + padded[TRIM + CHUNK - 1 + i] = chunk[CHUNK - 1 - i]; + } + let mut spec = vec![Complex::default(); DIM_T * BINS]; + let mut frame = vec![0.0f32; N_FFT]; + for t in 0..DIM_T { + for (n, w) in self.window.iter().enumerate() { + frame[n] = padded[t * HOP + n] * w; + } + self.r2c + .process(&mut frame, &mut spec[t * BINS..(t + 1) * BINS]) + .map_err(|e| format!("stft failed: {e}"))?; + } + Ok(spec) + } + + /// WOLA inverse of a frame-major spectrogram back to CHUNK samples with + /// the centered pad trimmed off — torch.istft's window-envelope + /// normalization. Consumes the spectrogram (the FFT scratches it). + fn inverse(&self, spec: &mut [Complex]) -> Result, String> { + let mut acc = vec![0.0f32; CHUNK + 2 * TRIM]; + let mut frame = vec![0.0f32; N_FFT]; + let scale = 1.0 / N_FFT as f32; // realfft's c2r is unnormalized + for t in 0..DIM_T { + let row = &mut spec[t * BINS..(t + 1) * BINS]; + // The model emits free re/im everywhere; a real inverse needs + // pure-real DC and Nyquist. Nyquist sits in the cropped-zero + // region already, DC gets clamped here. + row[0].im = 0.0; + row[BINS - 1].im = 0.0; + self.c2r + .process(row, &mut frame) + .map_err(|e| format!("istft failed: {e}"))?; + for (n, w) in self.window.iter().enumerate() { + acc[t * HOP + n] += frame[n] * scale * w; + } + } + Ok((TRIM..TRIM + CHUNK) + .map(|i| acc[i] / self.envelope[i]) + .collect()) + } +} + +/// Pack two frame-major spectrograms into the model's flat input — channel +/// order [L_re, L_im, R_re, R_im] (UVR's reshape/permute order), frequency +/// cropped to DIM_F. +fn pack_input(spec_l: &[Complex], spec_r: &[Complex]) -> Vec { + let mut flat = vec![0.0f32; TENSOR_LEN]; + for (c, spec) in [spec_l, spec_r].into_iter().enumerate() { + for f in 0..DIM_F { + for t in 0..DIM_T { + let z = spec[t * BINS + f]; + flat[((2 * c) * DIM_F + f) * DIM_T + t] = z.re; + flat[((2 * c + 1) * DIM_F + f) * DIM_T + t] = z.im; + } + } + } + flat +} + +/// Unpack the model's flat output back into two frame-major spectrograms. +/// Bins at and above DIM_F stay zero — the crop restored as silence, which +/// for a vocal stem is honest (nothing sung lives above ~17.6 kHz). +fn unpack_output(flat: &[f32]) -> (Vec>, Vec>) { + let mut left = vec![Complex::default(); DIM_T * BINS]; + let mut right = vec![Complex::default(); DIM_T * BINS]; + for (c, spec) in [&mut left, &mut right].into_iter().enumerate() { + for f in 0..DIM_F { + for t in 0..DIM_T { + spec[t * BINS + f] = Complex::new( + flat[((2 * c) * DIM_F + f) * DIM_T + t], + flat[((2 * c + 1) * DIM_F + f) * DIM_T + t], + ); + } + } + } + (left, right) +} + +/// Chunk a stereo 44.1 kHz track through `run` (the [1,4,3072,256] → +/// same-shape spectrogram transform) and reassemble the returned stem. +/// UVR's non-overlap framing: TRIM zeros lead the track so chunk c's kept +/// middle lands exactly at c·GEN, edge effects fall in the trimmed halves, +/// and segments butt-join. +fn process_chunks( + stft: &Stft, + left: &[f32], + right: &[f32], + mut run: impl FnMut(&[f32]) -> Result, String>, +) -> Result<(Vec, Vec), String> { + let len = left.len().min(right.len()); + if len < HOP { + return Err("audio too short to separate".into()); + } + let chunks = len.div_ceil(GEN); + let padded_len = TRIM + chunks * GEN + TRIM; + let mut mix_l = vec![0.0f32; padded_len]; + let mut mix_r = vec![0.0f32; padded_len]; + mix_l[TRIM..TRIM + len].copy_from_slice(&left[..len]); + mix_r[TRIM..TRIM + len].copy_from_slice(&right[..len]); + let mut stem_l = vec![0.0f32; chunks * GEN]; + let mut stem_r = vec![0.0f32; chunks * GEN]; + for c in 0..chunks { + let s = c * GEN; + let spec_l = stft.forward(&mix_l[s..s + CHUNK])?; + let spec_r = stft.forward(&mix_r[s..s + CHUNK])?; + let out = run(&pack_input(&spec_l, &spec_r))?; + if out.len() != TENSOR_LEN { + return Err(format!("unexpected chunk output length {}", out.len())); + } + let (mut vl, mut vr) = unpack_output(&out); + let wl = stft.inverse(&mut vl)?; + let wr = stft.inverse(&mut vr)?; + stem_l[s..s + GEN].copy_from_slice(&wl[TRIM..TRIM + GEN]); + stem_r[s..s + GEN].copy_from_slice(&wr[TRIM..TRIM + GEN]); + // Politeness between chunks — this thread must never contend with + // the audio path for sustained stretches (the passage loop's rule). + std::thread::sleep(std::time::Duration::from_millis(3)); + } + stem_l.truncate(len); + stem_r.truncate(len); + Ok((stem_l, stem_r)) +} + +/// Whole-file vocal stem as 16 kHz mono for the CTC aligner: decode → +/// chunked WinML separation → mid fold + anti-alias resample (player crate). +/// Fails soft by design — the caller falls back to the band-passed mixture. +pub fn separate_vocals_16k( + app: &tauri::AppHandle, + path: &std::path::Path, +) -> Result, String> { + if !model_present(app) { + return Err("separation model not downloaded".into()); + } + let [left, right] = player::vocals::separation_input_44k1(path)?; + ensure_engine(app)?; + let guard = ENGINE.lock().unwrap(); + let engine = guard.as_ref().ok_or("engine unavailable")?; + let stft = Stft::new(); + let started = Instant::now(); + let (stem_l, stem_r) = process_chunks(&stft, &left, &right, |flat| { + eval_chunk(&engine.session, flat) + })?; + log::info!( + "vocal_sep: {:.0}s of audio separated in {:.1}s on {}", + stem_l.len() as f32 / 44_100.0, + started.elapsed().as_secs_f32(), + engine.device + ); + Ok(player::vocals::stem_pcm_16k(&stem_l, &stem_r)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The WOLA plumbing alone — forward STFT straight back through the + /// inverse, no crop — must be per-sample exact: the numerator and the + /// envelope share the same window algebra, so any real error here is a + /// tiling or normalization bug, not numerics. + #[test] + fn stft_wola_roundtrip_is_exact() { + let chunk: Vec = (0..CHUNK) + .map(|i| (2.0 * std::f32::consts::PI * 440.0 * i as f32 / 44_100.0).sin() * 0.5) + .collect(); + let stft = Stft::new(); + let mut spec = stft.forward(&chunk).unwrap(); + let back = stft.inverse(&mut spec).unwrap(); + let worst = chunk + .iter() + .zip(&back) + .map(|(a, b)| (a - b).abs()) + .fold(0.0f32, f32::max); + assert!(worst < 1e-4, "un-cropped WOLA must be exact, worst {worst}"); + } + + /// An identity model (output spectrogram = input spectrogram) must hand + /// the track back through the whole STFT → crop → pack → unpack → + /// zero-pad → iSTFT → trim → butt-join pipeline: proves the frame + /// tiling, the chunk joins, and the layout round-trip all at once. + /// Tolerance: the reflect pad puts a derivative kink at each chunk + /// edge whose broadband energy reaches above bin 3072, and the model + /// crop discards it — torch.stft + UVR's own framing lose exactly the + /// same (~1% here); the model trained on that world. + #[test] + fn identity_model_reconstructs_across_chunk_joins() { + let len = GEN + GEN / 2; // two chunks, one join, one truncation + let track: Vec = (0..len) + .map(|i| (2.0 * std::f32::consts::PI * 440.0 * i as f32 / 44_100.0).sin() * 0.5) + .collect(); + let stft = Stft::new(); + let (out_l, out_r) = + process_chunks(&stft, &track, &track, |flat| Ok(flat.to_vec())).unwrap(); + assert_eq!(out_l.len(), len); + assert_eq!(out_r.len(), len); + let mut worst = 0.0f32; + for (a, b) in track.iter().zip(&out_l) { + worst = worst.max((a - b).abs()); + } + assert!( + worst < 2e-2, + "identity round-trip must reconstruct the tone, worst err {worst}" + ); + // The join at GEN must not be a seam beyond that same crop error. + let seam = (GEN - 50..GEN + 50) + .map(|i| (track[i] - out_l[i]).abs()) + .fold(0.0f32, f32::max); + assert!(seam < 2e-2, "butt-join must be seamless, err {seam}"); + } + + /// The kept region of every chunk sits in full window-overlap coverage: + /// the WOLA envelope there is flat and far from zero. + #[test] + fn envelope_is_stable_where_samples_are_kept() { + let stft = Stft::new(); + let kept = &stft.envelope[2 * TRIM..2 * TRIM + GEN]; + let min = kept.iter().cloned().fold(f32::INFINITY, f32::min); + let max = kept.iter().cloned().fold(0.0f32, f32::max); + assert!(min > 2.0, "envelope must be well-conditioned, min {min}"); + assert!( + max / min < 1.5, + "envelope must be near-flat in the kept region, {min}..{max}" + ); + } +} diff --git a/src-tauri/src/youtube_setlist.rs b/src-tauri/src/youtube_setlist.rs new file mode 100644 index 0000000..17ec205 --- /dev/null +++ b/src-tauri/src/youtube_setlist.rs @@ -0,0 +1,652 @@ +//! YouTube DJ-set tracklist extraction — the evidence rungs the feasibility +//! research ranks FIRST (creator- and viewer-declared data): the video's +//! chapters, its description, and timestamped tracklists in top comments. +//! (TODO-Feature-Docs/playlist-management-and-setlist-import, research +//! §3/§15: "the high-value, reliable subset is metadata import … broader +//! automatic recognition must remain optional".) No audio is fetched and +//! nothing is recognized — this is metadata import, honest about it, and a +//! set with no posted tracklist simply reports that instead of guessing. +//! +//! Everything runs in-process over reqwest (the no-child-processes posture): +//! one GET of the watch page, plus at most one innertube POST for comments +//! when the page itself carried no tracklist. The extracted lines feed the +//! EXISTING pasted-tracklist pipeline (playlist_formats::parse_text → +//! resolve_draft → review modal → commit), so every downstream behavior — +//! setlist stamps, library matching, stream fallbacks, unresolved rows — +//! is the one already shipped and tested. + +use serde_json::Value; + +/// What one video yielded: identity for the draft header plus the chosen +/// tracklist text and where it came from. +pub struct SetlistFetch { + pub title: String, + pub author: String, + pub tracklist: String, + /// "chapters" | "description" | "a comment by …" — shown in the modal's + /// warnings so the evidence rung is visible (research §3: "show which + /// rung produced a result"). + pub provenance: String, + pub warnings: Vec, +} + +/// Minimum timestamped lines before a block counts as a tracklist — one or +/// two stamps are a lyric quote or a "best drop" comment, not a setlist. +const MIN_TRACKLIST_LINES: usize = 3; +/// Top-level comments scanned for a tracklist before giving up. +const MAX_COMMENTS_SCANNED: usize = 40; + +// ---- URL handling ----------------------------------------------------------- + +/// Accepts watch/short/live/embed/youtu.be URLs (and a bare 11-char id) and +/// returns the canonical video id. +pub fn video_id_from_url(url: &str) -> Option { + let s = url.trim(); + let is_id = |c: &str| { + c.len() == 11 + && c.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + }; + if is_id(s) { + return Some(s.to_string()); + } + let after_scheme = s.split("://").nth(1).unwrap_or(s); + let (host, rest) = after_scheme.split_once('/')?; + let host = host.split('@').next_back().unwrap_or(host).to_lowercase(); + let host = host.split(':').next().unwrap_or(&host); + let path_q = rest.split('#').next().unwrap_or(rest); + let (path, query) = match path_q.split_once('?') { + Some((p, q)) => (p, Some(q)), + None => (path_q, None), + }; + let candidate = if host == "youtu.be" { + path.split('/').next().map(str::to_string) + } else if host.ends_with("youtube.com") || host.ends_with("youtube-nocookie.com") { + let segs: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); + match segs.first().copied() { + Some("watch") => query.and_then(|q| { + q.split('&') + .find_map(|kv| kv.strip_prefix("v=").map(str::to_string)) + }), + Some("shorts") | Some("live") | Some("embed") | Some("v") => { + segs.get(1).map(|s| s.to_string()) + } + _ => None, + } + } else { + None + }; + candidate.filter(|c| is_id(c)) +} + +// ---- watch-page JSON slicing -------------------------------------------------- + +/// Slice the JSON object that begins at the first `{` after `needle`, +/// brace-counting with string/escape awareness — the watch page embeds its +/// data as `var ytInitialData = {...};` inside script tags, megabytes long. +pub(crate) fn slice_json_after<'a>(html: &'a str, needle: &str) -> Option<&'a str> { + let at = html.find(needle)?; + let rest = &html[at + needle.len()..]; + let open = rest.find('{')?; + let bytes = rest.as_bytes(); + let mut depth = 0usize; + let mut in_str = false; + let mut escaped = false; + for (i, &b) in bytes.iter().enumerate().skip(open) { + if in_str { + if escaped { + escaped = false; + } else if b == b'\\' { + escaped = true; + } else if b == b'"' { + in_str = false; + } + continue; + } + match b { + b'"' => in_str = true, + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + return Some(&rest[open..=i]); + } + } + _ => {} + } + } + None +} + +/// Depth-first search for every object under `root` that contains `key`, +/// applying `f` to the value. Youtube's trees are deep, volatile, and +/// renderer-name-happy; searching for leaf keys survives their reshuffles +/// better than hardcoded paths. +fn walk<'a>(root: &'a Value, key: &str, f: &mut dyn FnMut(&'a Value)) { + match root { + Value::Object(map) => { + if let Some(v) = map.get(key) { + f(v); + } + for v in map.values() { + walk(v, key, f); + } + } + Value::Array(items) => { + for v in items { + walk(v, key, f); + } + } + _ => {} + } +} + +/// First string under any of `keys` in reading order — for label-ish leaves +/// ("simpleText", nested runs) where youtube varies the wrapper. +fn first_string(v: &Value, keys: &[&str]) -> Option { + match v { + Value::String(s) => Some(s.clone()), + _ => { + for k in keys { + let mut found = None; + walk(v, k, &mut |x| { + if found.is_none() { + if let Some(s) = x.as_str() { + found = Some(s.to_string()); + } + } + }); + if found.is_some() { + return found; + } + } + None + } + } +} + +// ---- the three evidence rungs ------------------------------------------------- + +/// A line "looks timestamped" when it carries a `m:ss`-or-longer stamp plus +/// some letters — the same acceptance the pasted-tracklist parser applies, +/// reused for candidate SCORING before anything is parsed for real. +pub(crate) fn is_timestamped_line(line: &str) -> bool { + let has_letters = line.chars().filter(|c| c.is_alphabetic()).count() >= 2; + if !has_letters { + return false; + } + line.split(|c: char| c.is_whitespace() || c == '[' || c == ']' || c == '(' || c == ')') + .any(|tok| { + let t = tok.trim_matches(|c: char| !c.is_ascii_digit() && c != ':'); + crate::playlist_formats::parse_timestamp(t).is_some() + }) +} + +/// Chapters out of ytInitialData: every `chapterRenderer` carries a title +/// and `timeRangeStartMillis`. Ordered, creator-authored — the top rung. +pub(crate) fn chapters_tracklist(initial_data: &Value) -> Option { + let mut rows: Vec<(i64, String)> = Vec::new(); + walk(initial_data, "chapterRenderer", &mut |ch| { + let ms = ch + .get("timeRangeStartMillis") + .and_then(Value::as_i64) + .unwrap_or(-1); + let title = ch + .get("title") + .and_then(|t| first_string(t, &["simpleText", "text"])); + if ms >= 0 { + if let Some(t) = title { + rows.push((ms, t)); + } + } + }); + if rows.len() < MIN_TRACKLIST_LINES { + return None; + } + rows.sort_by_key(|(ms, _)| *ms); + rows.dedup(); + Some( + rows.into_iter() + .map(|(ms, t)| format!("{} {}", fmt_ts(ms), t)) + .collect::>() + .join("\n"), + ) +} + +/// The description's best block: the longest run of timestamped lines with +/// at most one untimed line between them (headers like "Tracklist:" break +/// nothing; prose paragraphs do). +pub(crate) fn description_tracklist(description: &str) -> Option { + fn flush<'a>( + current: &mut Vec<&'a str>, + stamped: &mut usize, + best: &mut Option<(usize, Vec<&'a str>)>, + ) { + if *stamped >= MIN_TRACKLIST_LINES + && best.as_ref().map(|(n, _)| *stamped > *n).unwrap_or(true) + { + *best = Some((*stamped, current.clone())); + } + current.clear(); + *stamped = 0; + } + let mut best: Option<(usize, Vec<&str>)> = None; + let mut current: Vec<&str> = Vec::new(); + let mut gap = 0usize; + let mut stamped = 0usize; + for line in description.lines() { + let trimmed = line.trim(); + if is_timestamped_line(trimmed) { + current.push(trimmed); + stamped += 1; + gap = 0; + } else if !current.is_empty() && !trimmed.is_empty() && gap == 0 { + // One stray untimed line rides along (a wrapped title, an "ID"), + // a second breaks the block. + gap = 1; + } else if !current.is_empty() { + flush(&mut current, &mut stamped, &mut best); + gap = 0; + } + } + flush(&mut current, &mut stamped, &mut best); + best.map(|(_, lines)| lines.join("\n")) +} + +/// Comment texts out of an innertube `next` response. Youtube has shipped +/// two shapes in the wild — legacy `contentText` runs and the newer +/// `commentEntityPayload` — so both are walked; each yields (author, text). +pub(crate) fn comment_texts(next_response: &Value) -> Vec<(String, String)> { + let mut out: Vec<(String, String)> = Vec::new(); + walk(next_response, "commentEntityPayload", &mut |p| { + let text = p + .get("properties") + .and_then(|pr| pr.get("content")) + .and_then(|c| c.get("content")) + .and_then(Value::as_str); + let author = p + .get("author") + .and_then(|a| a.get("displayName")) + .and_then(Value::as_str) + .unwrap_or("a commenter"); + if let Some(t) = text { + out.push((author.to_string(), t.to_string())); + } + }); + walk(next_response, "commentRenderer", &mut |c| { + let text = c + .get("contentText") + .map(|ct| collect_runs(ct)) + .filter(|t| !t.is_empty()); + let author = c + .get("authorText") + .and_then(|a| first_string(a, &["simpleText"])) + .unwrap_or_else(|| "a commenter".into()); + if let Some(t) = text { + out.push((author, t)); + } + }); + out.truncate(MAX_COMMENTS_SCANNED); + out +} + +/// Flatten a `runs`/`simpleText` rich-text node into plain text, preserving +/// the run boundaries' own newlines (timestamps arrive as separate runs). +fn collect_runs(v: &Value) -> String { + if let Some(s) = v.get("simpleText").and_then(Value::as_str) { + return s.to_string(); + } + let mut out = String::new(); + walk(v, "text", &mut |t| { + if let Some(s) = t.as_str() { + out.push_str(s); + } + }); + out +} + +/// The comments continuation token for the initial `next` call, discovered +/// generically: the token whose enclosing area names the comment section. +pub(crate) fn comments_token(initial_data: &Value) -> Option { + // The comment section's continuation is the one inside an + // itemSectionRenderer whose identifier mentions comments. + let mut token: Option = None; + walk(initial_data, "itemSectionRenderer", &mut |sec| { + if token.is_some() { + return; + } + let ident = sec + .get("sectionIdentifier") + .and_then(Value::as_str) + .unwrap_or(""); + if !ident.contains("comment") { + return; + } + walk(sec, "continuationCommand", &mut |cc| { + if token.is_none() { + if let Some(t) = cc.get("token").and_then(Value::as_str) { + token = Some(t.to_string()); + } + } + }); + }); + token +} + +fn fmt_ts(ms: i64) -> String { + let total = ms / 1000; + let h = total / 3600; + let m = (total % 3600) / 60; + let s = total % 60; + if h > 0 { + format!("{h}:{m:02}:{s:02}") + } else { + format!("{m}:{s:02}") + } +} + +/// Innertube client version, read from the page so the comments call speaks +/// the same dialect the page was served with; a dated fallback otherwise. +pub(crate) fn client_version(html: &str) -> String { + slice_after_quote(html, "\"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"") + .unwrap_or("2.20260101.00.00") + .to_string() +} + +fn slice_after_quote<'a>(html: &'a str, needle: &str) -> Option<&'a str> { + let at = html.find(needle)?; + let rest = &html[at + needle.len()..]; + rest.split('"').next() +} + +// ---- the fetch ---------------------------------------------------------------- + +/// Just the video's identity (title, author) — for the LISTENER's draft +/// header, whose sets typically have NO posted tracklist (that's why they +/// get listened to): the full `fetch` returns Err for those and took the +/// title down with it (Kyle's first own-set run was named by raw video id). +pub async fn fetch_title(url: &str) -> Option<(String, String)> { + let id = video_id_from_url(url)?; + let client = reqwest::Client::builder() + .user_agent( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \ + Chrome/126.0.0.0 Safari/537.36", + ) + .timeout(std::time::Duration::from_secs(15)) + .build() + .ok()?; + let html = client + .get(format!("https://www.youtube.com/watch?v={id}&hl=en")) + .header("Cookie", "CONSENT=YES+1; SOCS=CAI") + .send() + .await + .ok()? + .text() + .await + .ok()?; + let player: Value = slice_json_after(&html, "ytInitialPlayerResponse") + .and_then(|s| serde_json::from_str(s).ok())?; + let details = player.get("videoDetails")?; + let title = details.get("title").and_then(Value::as_str)?.to_string(); + let author = details + .get("author") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + Some((title, author)) +} + +/// Fetch one video's metadata and pick its best tracklist. At most two +/// requests; every rung failure downgrades to the next with a warning +/// instead of failing the whole import (research §12). +pub async fn fetch(url: &str) -> Result { + let id = video_id_from_url(url).ok_or("That doesn't look like a YouTube video link")?; + let client = reqwest::Client::builder() + .user_agent( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \ + Chrome/126.0.0.0 Safari/537.36", + ) + .timeout(std::time::Duration::from_secs(25)) + .build() + .map_err(|e| e.to_string())?; + let html = client + .get(format!("https://www.youtube.com/watch?v={id}&hl=en")) + .header("Cookie", "CONSENT=YES+1; SOCS=CAI") + .header("Accept-Language", "en") + .send() + .await + .map_err(|e| format!("YouTube did not answer: {e}"))? + .error_for_status() + .map_err(|e| format!("YouTube refused the page: {e}"))? + .text() + .await + .map_err(|e| e.to_string())?; + + let player: Value = slice_json_after(&html, "ytInitialPlayerResponse") + .and_then(|s| serde_json::from_str(s).ok()) + .ok_or("Could not read the video's data — is the link public?")?; + // Unavailable/private/region-locked videos still serve a page — with a + // playability error instead of details. Say that plainly (research §8). + let playability = player + .get("playabilityStatus") + .and_then(|p| p.get("status")) + .and_then(Value::as_str) + .unwrap_or("OK"); + if playability != "OK" && player.get("videoDetails").is_none() { + let reason = player + .get("playabilityStatus") + .and_then(|p| p.get("reason")) + .and_then(Value::as_str) + .unwrap_or("it may be private, removed, or region-locked"); + return Err(format!("YouTube says this video is unavailable — {reason}")); + } + let details = player.get("videoDetails").cloned().unwrap_or(Value::Null); + let title = details + .get("title") + .and_then(Value::as_str) + .unwrap_or("YouTube set") + .to_string(); + let author = details + .get("author") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let description = details + .get("shortDescription") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let initial_data: Value = slice_json_after(&html, "ytInitialData") + .and_then(|s| serde_json::from_str(s).ok()) + .unwrap_or(Value::Null); + + let mut warnings = Vec::new(); + + // Rung 1: chapters. Rung 2: the description's timestamped block. + if let Some(list) = chapters_tracklist(&initial_data) { + return Ok(SetlistFetch { + title, + author, + tracklist: list, + provenance: "the video's chapters".into(), + warnings, + }); + } + if let Some(list) = description_tracklist(&description) { + return Ok(SetlistFetch { + title, + author, + tracklist: list, + provenance: "the video description".into(), + warnings, + }); + } + + // Rung 3: top comments — one innertube call, failure-soft. + match comments_token(&initial_data) { + None => warnings.push("The video page exposed no comments to scan.".into()), + Some(token) => { + let version = client_version(&html); + let body = serde_json::json!({ + "context": { "client": { + "clientName": "WEB", "clientVersion": version, + "hl": "en", "gl": "US", + }}, + "continuation": token, + }); + let resp = client + .post("https://www.youtube.com/youtubei/v1/next?prettyPrint=false") + .json(&body) + .send() + .await; + match resp { + Err(e) => warnings.push(format!("Comments could not be scanned: {e}")), + Ok(r) => match r.json::().await { + Err(e) => warnings.push(format!("Comments could not be read: {e}")), + Ok(next) => { + let mut best: Option<(usize, String, String)> = None; + for (who, text) in comment_texts(&next) { + let stamped = + text.lines().filter(|l| is_timestamped_line(l.trim())).count(); + if stamped >= MIN_TRACKLIST_LINES + && best.as_ref().map(|(n, _, _)| stamped > *n).unwrap_or(true) + { + let block: Vec<&str> = text + .lines() + .map(str::trim) + .filter(|l| is_timestamped_line(l)) + .collect(); + best = Some((stamped, who.clone(), block.join("\n"))); + } + } + if let Some((n, who, list)) = best { + warnings.push(format!( + "Tracklist found in a comment by {who} ({n} timestamped lines) — viewer-supplied, worth a skim." + )); + return Ok(SetlistFetch { + title, + author, + tracklist: list, + provenance: format!("a comment by {who}"), + warnings, + }); + } + warnings.push("No comment carried a timestamped tracklist.".into()); + } + }, + } + } + } + + Err(format!( + "No tracklist found on \"{title}\" — no chapters, no timestamped description, and {} \ + Paste one into the tracklist door if you have it.", + if warnings.iter().any(|w| w.contains("comment")) { + "none of the top comments carry one." + } else { + "the comments could not be scanned." + } + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn video_ids_from_every_url_shape() { + for (url, want) in [ + ("https://www.youtube.com/watch?v=dQw4w9WgXcQ", Some("dQw4w9WgXcQ")), + ("https://youtu.be/dQw4w9WgXcQ?t=42", Some("dQw4w9WgXcQ")), + ("https://www.youtube.com/live/abc-DEF_123?feature=share", Some("abc-DEF_123")), + ("https://music.youtube.com/watch?v=dQw4w9WgXcQ&list=RD", Some("dQw4w9WgXcQ")), + ("www.youtube.com/shorts/abc-DEF_123", Some("abc-DEF_123")), + ("dQw4w9WgXcQ", Some("dQw4w9WgXcQ")), + ("https://example.com/watch?v=dQw4w9WgXcQ", None), + ("https://www.youtube.com/watch?v=tooShort", None), + ] { + assert_eq!( + video_id_from_url(url).as_deref(), + want, + "{url} must yield {want:?}" + ); + } + } + + #[test] + fn json_slicer_handles_braces_in_strings() { + let html = r#""#; + let json = slice_json_after(html, "ytInitialData").unwrap(); + let v: Value = serde_json::from_str(json).unwrap(); + assert_eq!(v["b"]["c"], 1); + } + + #[test] + fn chapters_walk_and_format() { + let data: Value = serde_json::from_str( + r#"{"markersMap":[{"value":{"chapters":[ + {"chapterRenderer":{"title":{"simpleText":"Bicep - Glue"},"timeRangeStartMillis":0}}, + {"chapterRenderer":{"title":{"simpleText":"Overmono - So U Kno"},"timeRangeStartMillis":330000}}, + {"chapterRenderer":{"title":{"simpleText":"Joy O - Sin Palta"},"timeRangeStartMillis":3900000}} + ]}}]}"#, + ) + .unwrap(); + let list = chapters_tracklist(&data).unwrap(); + assert_eq!( + list, + "0:00 Bicep - Glue\n5:30 Overmono - So U Kno\n1:05:00 Joy O - Sin Palta" + ); + } + + #[test] + fn description_block_beats_prose_and_survives_headers() { + let desc = "Recorded live at the warehouse.\nBig thanks to everyone!\n\nTracklist:\n00:00 Artist One - Opener\n04:12 Artist Two - Middle (Remix)\nID - ID\n12:44 Artist Three - Closer\n\nFollow me on socials at 10:00 tonight"; + let block = description_tracklist(desc).unwrap(); + assert_eq!( + block, + "00:00 Artist One - Opener\n04:12 Artist Two - Middle (Remix)\n12:44 Artist Three - Closer" + ); + } + + #[test] + fn too_few_stamps_is_no_tracklist() { + assert!(description_tracklist("00:01 hi\nsome words\n03:30 drop!").is_none()); + assert!(!is_timestamped_line("10000 subscribers!")); + assert!(is_timestamped_line("1:23:45 Artist - Title")); + assert!(is_timestamped_line("Artist - Title [04:20]")); + } + + #[test] + fn comment_payloads_both_shapes() { + let next: Value = serde_json::from_str( + r#"{"frameworkUpdates":{"entityBatchUpdate":{"mutations":[ + {"payload":{"commentEntityPayload":{ + "properties":{"content":{"content":"00:00 A - B\n03:00 C - D\n06:00 E - F"}}, + "author":{"displayName":"@digger"}}}} + ]}}, + "old":[{"commentRenderer":{ + "authorText":{"simpleText":"@legacy"}, + "contentText":{"runs":[{"text":"great set"}]}}}]}"#, + ) + .unwrap(); + let texts = comment_texts(&next); + assert_eq!(texts.len(), 2); + assert_eq!(texts[0].0, "@digger"); + assert!(texts[0].1.contains("03:00 C - D")); + assert_eq!(texts[1], ("@legacy".into(), "great set".into())); + } + + #[test] + fn comments_token_found_by_section_identifier() { + let data: Value = serde_json::from_str( + r#"{"contents":[ + {"itemSectionRenderer":{"sectionIdentifier":"related-items", + "contents":[{"continuationItemRenderer":{"continuationEndpoint":{"continuationCommand":{"token":"WRONG"}}}}]}}, + {"itemSectionRenderer":{"sectionIdentifier":"comment-item-section", + "contents":[{"continuationItemRenderer":{"continuationEndpoint":{"continuationCommand":{"token":"RIGHT"}}}}]}} + ]}"#, + ) + .unwrap(); + assert_eq!(comments_token(&data).as_deref(), Some("RIGHT")); + } +} diff --git a/src-tauri/src/ytdlp.rs b/src-tauri/src/ytdlp.rs new file mode 100644 index 0000000..88d8b21 --- /dev/null +++ b/src-tauri/src/ytdlp.rs @@ -0,0 +1,493 @@ +//! yt-dlp as an OPTIONAL external tool — the acquisition adapter for the +//! DJ-set listener, Kyle's explicit call 2026-08-19 ("okay implement +//! yt-dlp"), relaxing the no-child-processes posture for THIS feature +//! only: anonymous in-process YouTube audio is dead upstream (PO-token +//! wall, verified same day), and the download arms race belongs to the +//! people who fight it full-time. +//! +//! Posture kept from the model downloads: the binary arrives ONLY from an +//! explicit click, from yt-dlp's official GitHub release (a moving target +//! by design — self-updating is why it keeps working, so there is no byte +//! pin; the SOURCE is pinned instead). A PATH-installed yt-dlp is +//! respected before the managed copy. Audio lands in a per-video cache so +//! re-analysis is free (research §11), fetched as the m4a/AAC stream the +//! app's own decoder reads — no ffmpeg, no conversion step. + +use std::path::PathBuf; +use std::process::Stdio; + +use tauri::{Emitter, Manager}; + +/// The NIGHTLY channel, deliberately: as of 2026-08 the stable release +/// 403s on YouTube (extractor drift — burned live TWICE the same evening, +/// the second time when the update button itself overwrote a working +/// nightly with stale stable). Nightly is the yt-dlp project's own +/// recommendation for extractor freshness; against YouTube it is the only +/// channel that works. +const RELEASE_URL: &str = + "https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest/download/yt-dlp.exe"; +/// Deno's official Windows build — the RECOMMENDED EJS runtime (nightly +/// yt-dlp + deno is the one combo proven against YouTube, 2026-08-19). +/// Same posture as yt-dlp: no byte pin, the SOURCE is pinned; consent is +/// the click; the tool's own --version is the sanity check. +const DENO_RELEASE_URL: &str = + "https://github.com/denoland/deno/releases/latest/download/deno-x86_64-pc-windows-msvc.zip"; +/// Windows: no console window for spawned children. +const CREATE_NO_WINDOW: u32 = 0x0800_0000; + +fn tools_dir(app: &tauri::AppHandle) -> Result { + let dir = app + .path() + .app_data_dir() + .map_err(|e| e.to_string())? + .join("tools"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + Ok(dir) +} + +fn managed_exe(app: &tauri::AppHandle) -> Result { + Ok(tools_dir(app)?.join("yt-dlp.exe")) +} + +/// The yt-dlp this machine would use: the managed copy first, then PATH. +pub fn locate(app: &tauri::AppHandle) -> Option { + if let Ok(p) = managed_exe(app) { + if p.is_file() { + return Some(p); + } + } + let path_var = std::env::var_os("PATH")?; + std::env::split_paths(&path_var) + .map(|d| d.join("yt-dlp.exe")) + .find(|c| c.is_file()) +} + +fn tool_version(exe: &PathBuf) -> Option { + use std::os::windows::process::CommandExt; + let out = std::process::Command::new(exe) + .arg("--version") + .creation_flags(CREATE_NO_WINDOW) + .stdin(Stdio::null()) + .output() + .ok()?; + let v = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (!v.is_empty()).then_some(v) +} + +#[tauri::command] +pub fn ytdlp_status(app: tauri::AppHandle) -> serde_json::Value { + // The runtime rides along so the modal can offer the deno download + // exactly when it matters: nothing found, or a PATH node/bun that EJS + // may reject (the home laptop's node v19 lesson). + let runtime = js_runtime(&app); + match locate(&app) { + Some(exe) => serde_json::json!({ + "present": true, + "path": exe.to_string_lossy(), + "managed": managed_exe(&app).map(|m| m == exe).unwrap_or(false), + "version": tool_version(&exe), + "runtime": runtime, + }), + None => serde_json::json!({ "present": false, "runtime": runtime }), + } +} + +/// Download (or update) the managed yt-dlp from the official release. +/// Consent = the click that invoked this. Streams to a temp name, renames +/// atomically, then reports the tool's own --version as the sanity check — +/// there is no byte pin to check against, the release moves by design. +#[tauri::command] +pub async fn ytdlp_download(app: tauri::AppHandle) -> Result { + let client = reqwest::Client::builder() + .user_agent("SoulStream/0.1 (setlist tools)") + .build() + .map_err(|e| e.to_string())?; + let dest = managed_exe(&app)?; + let tmp = dest.with_extension("exe.downloading"); + let mut response = client + .get(RELEASE_URL) + .send() + .await + .map_err(|e| e.to_string())? + .error_for_status() + .map_err(|e| e.to_string())?; + let total = response.content_length().unwrap_or(0); + let mut file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?; + let mut received: u64 = 0; + let mut last_emit: u64 = 0; + use std::io::Write; + while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? { + file.write_all(&chunk).map_err(|e| e.to_string())?; + received += chunk.len() as u64; + if received - last_emit > 512 * 1024 { + last_emit = received; + let _ = app.emit( + "playlist:scan-progress", + serde_json::json!({ "stage": "tool", "received": received, "total": total }), + ); + } + } + drop(file); + if received < 2 * 1024 * 1024 { + let _ = std::fs::remove_file(&tmp); + return Err(format!("suspiciously small download ({received} bytes) — refused")); + } + std::fs::rename(&tmp, &dest).map_err(|e| e.to_string())?; + let version = match tool_version(&dest) { + Some(v) => v, + None => { + let _ = std::fs::remove_file(&dest); + return Err("downloaded tool did not answer --version — removed".into()); + } + }; + Ok(serde_json::json!({ "version": version })) +} + +fn deno_version(exe: &PathBuf) -> Option { + use std::os::windows::process::CommandExt; + let out = std::process::Command::new(exe) + .arg("--version") + .creation_flags(CREATE_NO_WINDOW) + .stdin(Stdio::null()) + .output() + .ok()?; + // First line reads "deno 2.9.5 (stable, release, x86_64-pc-windows-msvc)". + let text = String::from_utf8_lossy(&out.stdout); + let v = text.lines().next()?.split_whitespace().nth(1)?.to_string(); + (!v.is_empty()).then_some(v) +} + +/// Minimal single-entry ZIP extraction, central-directory-driven (the +/// authoritative record even when local headers defer their sizes to data +/// descriptors). Only what deno's release zip needs — stored or deflate, +/// CRC-verified — hand-rolled over the flate2/crc32fast already in the +/// tree rather than growing a zip dependency for one file. +fn extract_zip_entry(bytes: &[u8], want: &str) -> Result, String> { + const EOCD: [u8; 4] = [0x50, 0x4b, 0x05, 0x06]; + const CENTRAL: [u8; 4] = [0x50, 0x4b, 0x01, 0x02]; + const LOCAL: [u8; 4] = [0x50, 0x4b, 0x03, 0x04]; + let u16at = |at: usize| u16::from_le_bytes([bytes[at], bytes[at + 1]]) as usize; + let u32at = |at: usize| u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap()) as usize; + if bytes.len() < 22 { + return Err("archive too small to be a zip".into()); + } + let eocd = (bytes.len().saturating_sub(65_557)..=bytes.len() - 22) + .rev() + .find(|&i| bytes[i..i + 4] == EOCD) + .ok_or("no zip directory found")?; + let mut at = u32at(eocd + 16); + for _ in 0..u16at(eocd + 10) { + if at + 46 > bytes.len() || bytes[at..at + 4] != CENTRAL { + return Err("malformed zip directory".into()); + } + let method = u16at(at + 10); + let crc = u32at(at + 16) as u32; + let comp_size = u32at(at + 20); + let name_len = u16at(at + 28); + let extra_len = u16at(at + 30); + let comment_len = u16at(at + 32); + let local_at = u32at(at + 42); + let name = bytes + .get(at + 46..at + 46 + name_len) + .ok_or("malformed zip directory")?; + if name == want.as_bytes() { + if local_at + 30 > bytes.len() || bytes[local_at..local_at + 4] != LOCAL { + return Err("malformed zip entry".into()); + } + let data_at = local_at + 30 + u16at(local_at + 26) + u16at(local_at + 28); + let data = bytes + .get(data_at..data_at + comp_size) + .ok_or("zip entry data out of range")?; + let out = match method { + 0 => data.to_vec(), + 8 => { + use std::io::Read; + let mut out = Vec::new(); + flate2::read::DeflateDecoder::new(data) + .read_to_end(&mut out) + .map_err(|e| format!("zip inflate failed: {e}"))?; + out + } + m => return Err(format!("unsupported zip method {m}")), + }; + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&out); + if hasher.finalize() != crc { + return Err("zip CRC mismatch — corrupted download, refused".into()); + } + return Ok(out); + } + at += 46 + name_len + extra_len + comment_len; + } + Err(format!("{want} not found in the archive")) +} + +/// Download the managed Deno runtime from denoland's official release and +/// unpack the one file that matters into the tools dir. Progress rides the +/// same event stream as the yt-dlp download, stage "runtime". +#[tauri::command] +pub async fn deno_download(app: tauri::AppHandle) -> Result { + let client = reqwest::Client::builder() + .user_agent("SoulStream/0.1 (setlist tools)") + .build() + .map_err(|e| e.to_string())?; + let mut response = client + .get(DENO_RELEASE_URL) + .send() + .await + .map_err(|e| e.to_string())? + .error_for_status() + .map_err(|e| e.to_string())?; + let total = response.content_length().unwrap_or(0); + let mut zip: Vec = Vec::with_capacity(total as usize); + let mut last_emit: usize = 0; + while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? { + zip.extend_from_slice(&chunk); + if zip.len() - last_emit > 1024 * 1024 { + last_emit = zip.len(); + let _ = app.emit( + "playlist:scan-progress", + serde_json::json!({ "stage": "runtime", "received": zip.len(), "total": total }), + ); + } + } + if zip.len() < 10 * 1024 * 1024 { + return Err(format!( + "suspiciously small download ({} bytes) — refused", + zip.len() + )); + } + let dest = tools_dir(&app)?.join("deno.exe"); + // Inflating ~100 MB is real CPU work — off the async runtime's threads. + let version = tauri::async_runtime::spawn_blocking(move || -> Result { + let exe = extract_zip_entry(&zip, "deno.exe")?; + let tmp = dest.with_extension("exe.downloading"); + std::fs::write(&tmp, &exe).map_err(|e| e.to_string())?; + std::fs::rename(&tmp, &dest).map_err(|e| e.to_string())?; + match deno_version(&dest) { + Some(v) => Ok(v), + None => { + let _ = std::fs::remove_file(&dest); + Err("downloaded deno did not answer --version — removed".into()) + } + } + }) + .await + .map_err(|e| e.to_string())??; + Ok(serde_json::json!({ "version": version })) +} + +/// Where a video's fetched audio lives; keyed by video id so a re-scan of +/// the same set never re-downloads. +fn audio_cache_dir(app: &tauri::AppHandle) -> Result { + let dir = app + .path() + .app_data_dir() + .map_err(|e| e.to_string())? + .join("setlist-audio"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + Ok(dir) +} + +pub fn cached_audio(app: &tauri::AppHandle, video_id: &str) -> Option { + let dir = audio_cache_dir(app).ok()?; + for ext in ["m4a", "mp4", "aac"] { + let p = dir.join(format!("{video_id}.{ext}")); + if p.is_file() { + return Some(p); + } + } + None +} + +/// 2026 yt-dlp needs an external JS runtime for YouTube (the EJS change: +/// "extraction without a JS runtime has been deprecated" — verified live, +/// a runtime-less run 403s). Find one: managed deno first, then PATH +/// deno/node/bun. Returns the `--js-runtimes` value, or None. +fn js_runtime(app: &tauri::AppHandle) -> Option { + if let Ok(dir) = tools_dir(app) { + let managed = dir.join("deno.exe"); + if managed.is_file() { + return Some(format!("deno:{}", managed.to_string_lossy())); + } + } + let path_var = std::env::var_os("PATH")?; + for (exe, name) in [("deno.exe", "deno"), ("node.exe", "node"), ("bun.exe", "bun")] { + if std::env::split_paths(&path_var).any(|d| d.join(exe).is_file()) { + return Some(name.to_string()); + } + } + None +} + +/// Fetch a video's audio-only stream into the cache, emitting download +/// progress parsed from yt-dlp's own lines. Blocking (the caller runs it +/// on a worker thread). Strictly the m4a/AAC ladder — the app decodes that +/// in-process; a set that only offers opus errors honestly rather than +/// handing back a file nothing can read. +pub fn fetch_audio_blocking( + app: &tauri::AppHandle, + video_id: &str, + cancel: &std::sync::atomic::AtomicBool, +) -> Result { + use std::io::BufRead; + use std::os::windows::process::CommandExt; + if let Some(hit) = cached_audio(app, video_id) { + return Ok(hit); + } + let exe = locate(app).ok_or("yt-dlp is not installed yet")?; + let runtime = js_runtime(app).ok_or( + "yt-dlp needs a JavaScript runtime for YouTube now (Deno, Node, or Bun) and none was \ + found on this machine — install one and try again", + )?; + let dir = audio_cache_dir(app)?; + let template = dir.join("%(id)s.%(ext)s"); + let mut child = std::process::Command::new(&exe) + .args([ + "--no-playlist", + "--newline", + "--no-color", + "--js-runtimes", + &runtime, + "-f", + "bestaudio[ext=m4a]/bestaudio[acodec^=mp4a]", + "-o", + &template.to_string_lossy(), + &format!("https://www.youtube.com/watch?v={video_id}"), + ]) + .creation_flags(CREATE_NO_WINDOW) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("could not start yt-dlp: {e}"))?; + let stdout = child.stdout.take().ok_or("no stdout")?; + let reader = std::io::BufReader::new(stdout); + for line in reader.lines().map_while(Result::ok) { + if cancel.load(std::sync::atomic::Ordering::Relaxed) { + let _ = child.kill(); + return Err("canceled".into()); + } + // "[download] 42.3% of 61.53MiB at ..." + if let Some(rest) = line.trim().strip_prefix("[download]") { + if let Some(pct) = rest.trim().split('%').next() { + if let Ok(p) = pct.trim().parse::() { + let _ = app.emit( + "playlist:scan-progress", + serde_json::json!({ "stage": "audio", "pct": p }), + ); + } + } + } + } + let status = child.wait().map_err(|e| e.to_string())?; + if !status.success() { + let mut err = String::new(); + if let Some(mut se) = child.stderr.take() { + use std::io::Read; + let _ = se.read_to_string(&mut err); + } + let tail: String = err.lines().rev().take(3).collect::>().join(" | "); + return Err(format!("yt-dlp failed: {tail}")); + } + cached_audio(app, video_id).ok_or_else(|| { + "yt-dlp finished but no m4a landed — the set may only offer opus audio".into() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Assemble a real zip in memory — local headers, central directory, + /// EOCD — the structure `extract_zip_entry` actually parses. + fn build_zip(entries: &[(&str, &[u8], u16)]) -> Vec { + use std::io::Write as _; + let mut out: Vec = Vec::new(); + let mut central: Vec = Vec::new(); + for (name, data, method) in entries { + let mut hasher = crc32fast::Hasher::new(); + hasher.update(data); + let crc = hasher.finalize(); + let comp: Vec = match method { + 0 => data.to_vec(), + 8 => { + let mut enc = flate2::write::DeflateEncoder::new( + Vec::new(), + flate2::Compression::default(), + ); + enc.write_all(data).unwrap(); + enc.finish().unwrap() + } + _ => unreachable!("test zips are stored or deflated"), + }; + let local_at = out.len() as u32; + out.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); + out.extend_from_slice(&[20, 0, 0, 0]); // version, flags + out.extend_from_slice(&method.to_le_bytes()); + out.extend_from_slice(&[0; 4]); // time, date + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&(comp.len() as u32).to_le_bytes()); + out.extend_from_slice(&(data.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&[0, 0]); // extra_len + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(&comp); + + central.extend_from_slice(&0x0201_4b50u32.to_le_bytes()); + central.extend_from_slice(&[20, 0, 20, 0, 0, 0]); // versions, flags + central.extend_from_slice(&method.to_le_bytes()); + central.extend_from_slice(&[0; 4]); // time, date + central.extend_from_slice(&crc.to_le_bytes()); + central.extend_from_slice(&(comp.len() as u32).to_le_bytes()); + central.extend_from_slice(&(data.len() as u32).to_le_bytes()); + central.extend_from_slice(&(name.len() as u16).to_le_bytes()); + central.extend_from_slice(&[0; 12]); // extra/comment/disk/attrs + central.extend_from_slice(&local_at.to_le_bytes()); + central.extend_from_slice(name.as_bytes()); + } + let cd_at = out.len() as u32; + out.extend_from_slice(¢ral); + out.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); + out.extend_from_slice(&[0; 4]); // disk numbers + out.extend_from_slice(&(entries.len() as u16).to_le_bytes()); + out.extend_from_slice(&(entries.len() as u16).to_le_bytes()); + out.extend_from_slice(&(central.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_at.to_le_bytes()); + out.extend_from_slice(&[0, 0]); // comment_len + out + } + + #[test] + fn extracts_the_named_deflated_entry_among_others() { + let payload: Vec = (0..40_000u32).flat_map(|i| i.to_le_bytes()).collect(); + let zip = build_zip(&[ + ("LICENSE.md", b"decoy text", 8), + ("deno.exe", &payload, 8), + ]); + let got = extract_zip_entry(&zip, "deno.exe").unwrap(); + assert_eq!(got, payload); + } + + #[test] + fn extracts_a_stored_entry() { + let zip = build_zip(&[("deno.exe", b"stored bytes", 0)]); + assert_eq!(extract_zip_entry(&zip, "deno.exe").unwrap(), b"stored bytes"); + } + + #[test] + fn corrupted_data_fails_the_crc_check() { + let mut zip = build_zip(&[("deno.exe", b"soon to be flipped payload", 0)]); + // Flip one payload byte: local header is 30 + name, data follows. + let at = 30 + "deno.exe".len() + 3; + zip[at] ^= 0xff; + let err = extract_zip_entry(&zip, "deno.exe").unwrap_err(); + assert!(err.contains("CRC"), "{err}"); + } + + #[test] + fn a_missing_name_is_an_honest_error() { + let zip = build_zip(&[("other.txt", b"x", 0)]); + assert!(extract_zip_entry(&zip, "deno.exe").is_err()); + } +} diff --git a/src/App.svelte b/src/App.svelte index 54f3b6d..b9dc6e1 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -4,7 +4,6 @@ import PlayerBar from "./components/PlayerBar.svelte"; import NowPlaying from "./components/NowPlaying.svelte"; import ArtworkViewer from "./components/ArtworkViewer.svelte"; - import LyricsOverlay from "./components/LyricsOverlay.svelte"; import LyricsSyncEditor from "./components/LyricsSyncEditor.svelte"; import ImportWindow from "./components/ImportWindow.svelte"; import LoginModal from "./components/LoginModal.svelte"; @@ -23,8 +22,10 @@ import CatalogArtist from "./views/CatalogArtist.svelte"; import CatalogAlbum from "./views/CatalogAlbum.svelte"; import CatalogDiscography from "./views/CatalogDiscography.svelte"; + import Lyrics from "./views/Lyrics.svelte"; import FollowToast from "./components/FollowToast.svelte"; import ImportToast from "./components/ImportToast.svelte"; + import SetlistScanChip from "./components/SetlistScanChip.svelte"; import { ui, initEvents, @@ -48,6 +49,7 @@ catalogArtist: CatalogArtist, catalogAlbum: CatalogAlbum, catalogDiscography: CatalogDiscography, + lyrics: Lyrics, }; const CurrentView = $derived(views[ui.view] ?? Home); @@ -82,6 +84,16 @@ // --bar-h + air rather than --pane-pad. Its ambience wash is the one // exception and deliberately runs to the pane top behind the glass. "home", + // The playlist editor follows the album-page rule: its hero banner + // runs under the frost and the page takes padding-top: --pane-pad. + "playlist", + // The playlist shelf follows the Library exception: its blurred + // ambience pins behind the glass and must begin at the very top of + // the pane, so the page carries its own top inset instead. + "playlists", + // The lyrics room follows the Library exception too: its full-bleed + // duotone backdrop pins behind the glass from the very pane top. + "lyrics", ]); initEvents(); @@ -127,8 +139,8 @@ + - diff --git a/src/components/LyricsOverlay.svelte b/src/components/LyricsOverlay.svelte deleted file mode 100644 index 3d0a706..0000000 --- a/src/components/LyricsOverlay.svelte +++ /dev/null @@ -1,104 +0,0 @@ - - - - -{#if ui.lyricsFull} -
-
-
-

{player.track?.title ?? ""}

- {#if artist}

{artist}

{/if} -
- -
-
- -
-
-{/if} - - diff --git a/src/components/LyricsPane.svelte b/src/components/LyricsPane.svelte index a3a53a2..42e8eb5 100644 --- a/src/components/LyricsPane.svelte +++ b/src/components/LyricsPane.svelte @@ -4,14 +4,15 @@ // picker and a paste door for tracks nothing automatic can cover. Also // rendered by the fullscreen overlay in `big` mode — one component, two // type scales. - import { player, ui, playLocalFile } from "../lib/state.svelte.js"; + import { player, ui, playLocalFile, goToView } from "../lib/state.svelte.js"; import { lyricsStore, lyricIdentity, ensureLyrics, noteTrackChange, activeLineIndex, - activeWordIndex, + wordClockMs, + wordfill, totalOffsetMs, nudgeOffset, setAutoSync, @@ -253,19 +254,26 @@ onclick={() => seekTo(line)} title="Play from here" > - {#if index === active && line.words?.length} - - {@const sungTo = activeWordIndex(line, player.position ?? 0)} - + {#if index === active && lyricsStore.wordFill && line.words?.length} + + {#each line.words as word, wi (wi)} - {word.text} + {word.text} {/each} {:else} @@ -331,7 +339,14 @@ {#snippet tools()} {#if !big} - {/if} @@ -615,20 +630,29 @@ } /* ── karaoke rendering inside the active line ── - Word mode: crisp steps on the source's own word times. */ + Word fill (the wordfill mock's dialed variant A): each word is its own + gradient-clip raster wiping across the word's real sung span — accent + left of the feathered boundary (±8%, mock-dialed), the pane's quieter + unsung tone right of it. --lyr-p per element (inherits: false; + registered @property in Lyrics.svelte). */ .wline { display: block; } - .w { - color: var(--text-secondary); - transition: color 0.12s ease; + .fw { + --lyr-p: -8%; + background-image: linear-gradient( + 90deg, + var(--accent-hover) calc(var(--lyr-p) - 8%), + var(--text-secondary) calc(var(--lyr-p) + 8%) + ); + -webkit-background-clip: text; + background-clip: text; + -webkit-text-fill-color: transparent; + color: transparent; } - .w + .w { + .fw + .fw { margin-left: 0.32em; } - .w.sung { - color: #fff; - } .autochip { flex: none; diff --git a/src/components/NowPlaying.svelte b/src/components/NowPlaying.svelte index d8f6418..3ba50ac 100644 --- a/src/components/NowPlaying.svelte +++ b/src/components/NowPlaying.svelte @@ -15,6 +15,7 @@ openCurrentArtist, openCurrentAlbum, openCurrentSong, + seekPlayer, libraryStore, artBroken, markArtBroken, @@ -943,7 +944,7 @@ const now = performance.now(); if (now - lastWaveSeekAt >= 75) { lastWaveSeekAt = now; - api.playerControl("seek", target); + seekPlayer(target); } } function waveUp(e) { @@ -951,7 +952,7 @@ waveDragging = false; player.scrubbing = false; const target = waveTarget(e); - if (target != null) api.playerControl("seek", target); + if (target != null) seekPlayer(target); try { waveEl.releasePointerCapture(e.pointerId); } catch { @@ -1102,10 +1103,13 @@ album: currentItem.album ?? metaParts[1] ?? null, duration_secs: currentItem.duration ?? null, // A stream item's path names a file on the PEER's - // machine — the row goes in as a placeholder that - // streams on click, not as a fake local file. + // machine — the row goes in as a stream row that + // plays through the resolver, not a fake local + // file. Its catalog id and cover ride along. path: currentItem.stream ? null : currentItem.path, art_path: currentItem.stream ? null : (currentItem.art ?? null), + trackId: currentItem.trackId ?? null, + artUrl: currentItem.stream ? (currentItem.artUrl ?? null) : null, }, ]} /> diff --git a/src/components/PlayerBar.svelte b/src/components/PlayerBar.svelte index 69e3d6c..758a329 100644 --- a/src/components/PlayerBar.svelte +++ b/src/components/PlayerBar.svelte @@ -17,6 +17,8 @@ togglePlayerExpanded, showNowPlaying, saveStream, + goToView, + engineTime, } from "../lib/state.svelte.js"; import * as api from "../lib/api.js"; import { @@ -265,7 +267,11 @@ } function openLyrics() { menuOpen = false; - ui.lyricsFull = true; // the fullscreen overlay (LyricsOverlay.svelte) + // The lyrics PAGE (views/Lyrics.svelte) — the overlay it used to open + // is gone. + goToView("lyrics", { + label: player.track?.title ? `${player.track.title} — Lyrics` : "Lyrics", + }); } // ---- Save to library, from the ⋯ menu --------------------------------- @@ -678,7 +684,9 @@ const now = performance.now(); if (force || now - lastScrubSeekAt >= 75) { lastScrubSeekAt = now; - api.playerControl("seek", target); + // TRACK time in, ENGINE time out — a set-recording slice's + // engine clock is offset by the slice start. + api.playerControl("seek", engineTime(target)); } } @@ -706,7 +714,7 @@ const rate = Math.min(4, Math.max(0.25, Math.abs(e.clientX - scrubLastX) / elapsed * 1.6)); if (activatingScratch || now - lastScrubSeekAt >= 50) { lastScrubSeekAt = now; - api.playerControl("scratch", target, e.clientX < scrubLastX ? -rate : rate); + api.playerControl("scratch", engineTime(target), e.clientX < scrubLastX ? -rate : rate); } } else if (!canScratch) { regularScrub(target); @@ -719,7 +727,7 @@ if (!draggingSeek) return; const target = previewScrub(e, el); if (target != null) { - if (scratchActive) api.playerControl("end-scratch", target); + if (scratchActive) api.playerControl("end-scratch", engineTime(target)); else regularScrub(target, true); // a simple click remains exact seek } draggingSeek = false; diff --git a/src/components/PlaylistImportModal.svelte b/src/components/PlaylistImportModal.svelte index 0c6a016..af4488f 100644 --- a/src/components/PlaylistImportModal.svelte +++ b/src/components/PlaylistImportModal.svelte @@ -3,23 +3,55 @@ // commit. The preview names what matched and what did not; unresolved // rows stay visible placeholders unless the user explicitly leaves them // out. Nothing touches the database until "Create playlist". + // + // Identified-but-unowned rows are VERIFIED against the CATALOG while + // the review is open (Kyle: match them to "the streaming library part, + // not Soulseek") — a strict name+duration match against Deezer-backed + // catalog search. A hit attaches the catalog track id + artwork to the + // row, so the committed playlist plays it as a catalog track. import * as api from "../lib/api.js"; + import { + setlistScan, + startYoutubeListen, + startFileListen, + } from "../lib/setlistScan.svelte.js"; import Icon from "./Icon.svelte"; let { open = $bindable(false), - mode = "file", // "file": a probed draft arrives via `draft`; "paste": textarea first + // "file": a probed draft arrives via `draft`; "paste": textarea + // first; "youtube": a link field first — the set's own tracklist + // evidence (chapters/description/comments) becomes the draft. + mode = "file", draft = null, onimported = () => {}, } = $props(); let pasteText = $state(""); + let youtubeUrl = $state(""); let localDraft = $state(null); + // The local listen door: the picked set recording + its optional + // pasted tracklist (B-flow: pick first, paste after). + let pickedFile = $state(null); // {path, name, duration_secs} + let fileTracklist = $state(""); + let picking = $state(false); let name = $state(""); let includeUnresolved = $state(true); let committing = $state(false); let parsing = $state(false); let error = $state(""); + // The listener: yt-dlp tool state + live scan progress. + let tool = $state(null); + let toolBusy = $state(false); + let denoBusy = $state(false); + // "deno" (managed or PATH) | "node"/"bun" (EJS may reject old ones — + // deno stays on offer) | null (no runtime at all). + const runtimeKind = $derived.by(() => { + const r = tool?.runtime; + if (!r) return null; + return r.startsWith("deno") ? "deno" : r; + }); + let scanning = $state(null); // {stage, pct?, window?, total?, hit?} // The working draft: probed (file mode) or built from the paste box. const working = $derived(localDraft ?? draft); @@ -31,8 +63,108 @@ committing = false; localDraft = null; pasteText = ""; + youtubeUrl = ""; + pickedFile = null; + fileTracklist = ""; includeUnresolved = true; + scanning = null; name = draft?.name ?? ""; + if (mode === "youtube") { + void api.ytdlpStatus().then((s) => (tool = s)).catch(() => {}); + } + if (draft?.entries?.length) startChecks(draft); + }); + $effect(() => { + return api.onEvent("playlist:scan-progress", (p) => { + if (scanning || p?.stage === "tool" || p?.stage === "runtime") + scanning = { ...scanning, ...p }; + }); + }); + + async function downloadTool() { + if (toolBusy) return; + toolBusy = true; + error = ""; + scanning = { stage: "tool" }; + try { + await api.ytdlpDownload(); + tool = await api.ytdlpStatus(); + } catch (e) { + error = String(e); + } finally { + toolBusy = false; + scanning = null; + } + } + + async function downloadDeno() { + if (denoBusy) return; + denoBusy = true; + error = ""; + scanning = { stage: "runtime" }; + try { + await api.denoDownload(); + tool = await api.ytdlpStatus(); + } catch (e) { + error = String(e); + } finally { + denoBusy = false; + scanning = null; + } + } + + // Listens run in the BACKGROUND: starting one closes this modal and + // hands the whole job to the setlistScan store — its chip narrates the + // progress, and the finished draft re-opens the review from there. + function listenYoutube() { + if (!youtubeUrl.trim()) return; + if (!startYoutubeListen(youtubeUrl.trim())) return; + close(); + } + + // The local door picks FIRST: the panel that follows shows the chosen + // file and offers an optional pasted tracklist — the same declared + // evidence the YouTube door fetches for itself — before the listen + // starts. An empty box listens purely by ear. + async function pickLocalFile() { + if (picking || setlistScan.active) return; + picking = true; + error = ""; + try { + const f = await api.playlistPickSetFile(); + if (f) { + pickedFile = f; + fileTracklist = ""; + } + } catch (e) { + error = String(e); + } finally { + picking = false; + } + } + + function listenPicked() { + if (!pickedFile) return; + if (!startFileListen(pickedFile.path, fileTracklist.trim() || null, pickedFile.name)) + return; + close(); + } + + function cancelScan() { + void api.playlistScanCancel(); + } + + const scanLabel = $derived.by(() => { + if (!scanning) return ""; + if (scanning.stage === "tool") return "Downloading yt-dlp…"; + if (scanning.stage === "runtime") + return `Downloading deno…${scanning.total ? ` ${Math.round(((scanning.received ?? 0) / scanning.total) * 100)}%` : ""}`; + if (scanning.stage === "audio") + return `Fetching the set's audio…${scanning.pct != null ? ` ${Math.round(scanning.pct)}%` : ""}`; + if (scanning.stage === "decode") return "Decoding the set…"; + if (scanning.stage === "listen") + return `Listening… window ${scanning.window ?? 0} of ${scanning.total ?? "?"}`; + return "Working…"; }); async function previewPaste() { if (!pasteText.trim() || parsing) return; @@ -45,6 +177,26 @@ } else { localDraft = parsed; if (!name) name = parsed.name ?? ""; + startChecks(parsed); + } + } catch (e) { + error = String(e); + } finally { + parsing = false; + } + } + async function previewYoutube() { + if (!youtubeUrl.trim() || parsing) return; + parsing = true; + error = ""; + try { + const parsed = await api.playlistImportYoutube(youtubeUrl.trim()); + if (!parsed.entries.length) { + error = "The set's tracklist parsed to nothing usable."; + } else { + localDraft = parsed; + name = parsed.name ?? name; + startChecks(parsed); } } catch (e) { error = String(e); @@ -55,11 +207,109 @@ const kept = $derived.by(() => { if (!working) return []; + // Streams are first-class (identified, playable) — the checkbox + // only gates the TRUE unknowns. return includeUnresolved ? working.entries - : working.entries.filter((e) => e.resolution === "local"); + : working.entries.filter((e) => e.resolution !== "unresolved"); }); + // ---- catalog verification of stream rows, live in the review ------- + let checks = $state({}); // ordinal → "checking" | {album, cover} | "none" + let checkGen = 0; + const norm = (s) => + (s ?? "") + .toLowerCase() + .normalize("NFKD") + .replace(/[^\p{L}\p{N}]+/gu, " ") + .trim(); + /// Strict match: at least one exact side (title or artist), both at + /// least partial; duration within ±7 s strengthens; Deezer rank breaks + /// ties so a famous track beats a namesake. + function pickCatalogMatch(tracks, e) { + const wantT = norm(e.title); + const wantA = norm(e.artist); + let best = null; + let bestScore = 0; + for (const t of tracks ?? []) { + const tT = norm(t.title); + const tA = norm(t.artist); + let score = 0; + if (tT === wantT) score += 4; + else if (tT.includes(wantT) || wantT.includes(tT)) score += 2; + if (tA === wantA) score += 4; + else if (tA.includes(wantA) || wantA.includes(tA)) score += 2; + if (score < 6) continue; + if ( + e.duration_secs && + t.duration_secs && + Math.abs(t.duration_secs - e.duration_secs) <= 7 + ) + score += 2; + score += Math.min((t.rank ?? 0) / 1_000_000, 1); + if (score > bestScore) { + bestScore = score; + best = t; + } + } + return best; + } + /// Kick the walk OUTSIDE any effect's tracking zone: verifyStreams + /// reads and writes `checks`, and doing that inside an $effect made + /// every row verdict re-trigger the effect, which bumped the + /// generation and killed its own walk — row one froze at "matching…" + /// forever. Explicit kicks from the draft-arrival sites, via a + /// microtask, keep reactivity out of the loop. + function startChecks(w) { + checks = {}; + const gen = ++checkGen; + queueMicrotask(() => void verifyStreams(w, gen)); + } + async function verifyStreams(w, gen) { + for (const e of w.entries) { + if (e.resolution !== "stream") continue; + if (gen !== checkGen || !open) return; + checks[e.ordinal] = "checking"; + try { + // Anchor-exact first: a row the listener heard carries the + // matched release's ISRC, and one /track/isrc: lookup beats + // any name matcher. A miss falls through to the name walk. + let hit = null; + let anchored = false; + if (e.isrc) { + hit = await api.catalogTrackByIsrc(e.isrc); + if (gen !== checkGen) return; + anchored = !!hit; + } + if (!hit) { + const res = await api.catalogSearch(`${e.artist} ${e.title}`); + if (gen !== checkGen) return; + hit = pickCatalogMatch(res?.tracks, e); + } + if (hit) { + // Attach catalog identity to the row commit will send. + e.track_id = Number(hit.id); + e.art_url = hit.cover ?? null; + if (!e.album) e.album = hit.album ?? null; + if (!e.duration_secs) e.duration_secs = hit.duration_secs ?? null; + checks[e.ordinal] = { + album: hit.album ?? "", + cover: hit.cover ?? null, + anchored, + }; + } else { + checks[e.ordinal] = "none"; + } + } catch { + if (gen === checkGen) checks[e.ordinal] = "none"; + } + await new Promise((r) => setTimeout(r, 250)); + } + } + const checkedFound = $derived( + Object.values(checks).filter((v) => v && v !== "none" && v !== "checking").length, + ); + async function commit() { if (!working || committing) return; const finalName = (name || working.name || "Imported playlist").trim(); @@ -71,6 +321,7 @@ working.description ?? null, working.kind, kept, + working.cover_url ?? null, ); onimported(id, finalName); } catch (e) { @@ -80,6 +331,7 @@ } function close() { + checkGen += 1; // stop any verification walk open = false; } @@ -111,6 +363,8 @@ soulstream: "SoulStream backup", text: "Pasted text", folder: "Folder", + youtube: "YouTube", + listened: "Recognized by ear", }; @@ -132,6 +386,10 @@

{#if working} Review import + {:else if pickedFile} + Listen to a set recording + {:else if mode === "youtube"} + Import a DJ set from YouTube {:else} Paste a tracklist {/if} @@ -142,35 +400,177 @@ {#if !working} -

- One track per line. Timestamps make it a setlist — - 00:00 Artist - Title keeps its place in the mix. -

- - {#if error}

{error}

{/if} -
- - -
+ {#if pickedFile} +
+ + {pickedFile.name} + {#if pickedFile.duration_secs != null} + {fmtTime(pickedFile.duration_secs * 1000)} + {/if} +
+

+ Listening identifies the tracks by sound. If the set came with a + tracklist, paste it too — it names the exact version played + where the ear can only name the release it matched, and fills + stretches nothing recognized. Leave it empty to listen purely by ear. +

+ +
+ Optional. Only timestamped lines count — the stamp anchors each + name to its place in the mix. +
+ {#if error}

{error}

{/if} +
+ + +
+ {:else if mode === "youtube"} + {#if scanning} +
+ +
{scanLabel}
+ {#if scanning.hit} +
{scanning.hit}
+ {/if} +
+
+ +
+ {:else} +

+ Paste a set's link. Fetch tracklist reads the video's own + chapters, description, or a commented tracklist — free and instant. + Listen downloads the audio and identifies the tracks by + sound: a few minutes, and unreleased material stays an honest ID. +

+ e.key === "Enter" && previewYoutube()} + /> +
+ {#if tool?.present} + + yt-dlp {tool.version ?? ""}{tool.managed ? "" : " (from PATH)"} + + + {:else} + Listening needs the downloader tool: + + {/if} + · + {#if runtimeKind === "deno"} + deno + {:else if runtimeKind} + runtime: {runtimeKind} + + {:else} + and a JS runtime: + + {/if} + · + +
+ {#if error}

{error}

{/if} +
+ + + +
+ {/if} + {:else} +

+ One track per line. Timestamps make it a setlist — + 00:00 Artist - Title keeps its place in the mix. +

+ + {#if error}

{error}

{/if} +
+ + +
+ {/if} {:else}
{FORMAT_LABEL[working.format] ?? working.format} {#if working.kind === "setlist"}Setlist{/if} {working.source_name} - {working.resolved_count} matched + {working.resolved_count} in library + {#if working.stream_count > 0} +  · {checkedFound}/{working.stream_count} in catalog + {/if} {#if working.unresolved_count > 0} -  · {working.unresolved_count} unresolved +  · {working.unresolved_count} unknown {/if}
@@ -187,8 +587,8 @@ {#each working.entries as e (e.ordinal)}
{e.ordinal + 1} {#if working.kind === "setlist"} @@ -215,8 +615,41 @@ ? "relinked" : "tags"} + {:else if e.resolution === "stream"} + {#if checks[e.ordinal] === "checking"} + matching… + {:else if checks[e.ordinal] && checks[e.ordinal] !== "none"} + + + catalog + + {:else if checks[e.ordinal] === "none"} + + not in catalog + + {:else} + + stream + + {/if} + {:else if e.resolution === "segment"} + + set audio + {:else} - + unresolved {/if} @@ -242,7 +675,7 @@
{#if error}

{error}

{/if}