dj sets + lyrics: a mix that names its own tracks, and words pulled off the separated voice
This commit is contained in:
1 parent
4efe85b6bf
commit
1dd8573808
41 files changed
+9288
-647
No files matched your search
@@ -76,6 +76,8 @@ pub struct Settings {
|
||||
pub stream_source_prefs: config::SourcePrefs,
|
||||
pub download_source_prefs: config::SourcePrefs,
|
||||
pub volume_normalization: bool,
|
||||
/// Word-by-word lyric fill — experimental, default off.
|
||||
pub word_timing: bool,
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
@@ -114,6 +116,7 @@ pub async fn get_settings(
|
||||
stream_source_prefs: cfg.stream_source_prefs,
|
||||
download_source_prefs: cfg.download_source_prefs,
|
||||
volume_normalization: config::volume_normalization(&cfg),
|
||||
word_timing: config::word_timing(&cfg),
|
||||
username: cfg.username,
|
||||
})
|
||||
}
|
||||
@@ -141,6 +144,7 @@ pub async fn set_settings(
|
||||
stream_source_prefs: Option<config::SourcePrefs>,
|
||||
download_source_prefs: Option<config::SourcePrefs>,
|
||||
volume_normalization: Option<bool>,
|
||||
word_timing: Option<bool>,
|
||||
) -> Result<(), String> {
|
||||
let mut cfg = config::load(&app);
|
||||
let library_folder_moved = download_dir
|
||||
@@ -209,6 +213,9 @@ pub async fn set_settings(
|
||||
// takes effect from the next track.
|
||||
crate::loudness::set_enabled(&app, enabled);
|
||||
}
|
||||
if word_timing.is_some() {
|
||||
cfg.word_timing = word_timing;
|
||||
}
|
||||
config::save(&app, &cfg);
|
||||
|
||||
// A new library folder is a new library. Re-base immediately rather than
|
||||
@@ -1550,6 +1557,15 @@ pub async fn catalog_search(
|
||||
Ok(state.metadata.catalog_search(&query).await)
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn catalog_track_by_isrc(
|
||||
state: State<'_, LibraryState>,
|
||||
isrc: String,
|
||||
) -> Result<Option<metadata::CatalogTrack>, String> {
|
||||
state.touch_catalog_activity();
|
||||
Ok(state.metadata.catalog_track_by_isrc(&isrc).await)
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn catalog_artist(
|
||||
app: tauri::AppHandle,
|
||||
@@ -4051,6 +4067,7 @@ pub fn play_local(
|
||||
app: tauri::AppHandle,
|
||||
path: String,
|
||||
duration: Option<f64>,
|
||||
start_secs: Option<f64>,
|
||||
) -> Result<(), String> {
|
||||
if std::fs::metadata(&path).is_err() {
|
||||
return Err("file not found".into());
|
||||
@@ -4064,7 +4081,11 @@ pub fn play_local(
|
||||
0,
|
||||
None,
|
||||
Box::new(move || {
|
||||
app2.state::<PlayerState>().player.play_local(&path2, duration);
|
||||
// `start_secs` = play a SLICE of the file from there (the
|
||||
// setlist segment rows); plain tracks pass None.
|
||||
app2.state::<PlayerState>()
|
||||
.player
|
||||
.play_local_from(&path2, duration, start_secs);
|
||||
}),
|
||||
);
|
||||
Ok(())
|
||||
@@ -4407,6 +4428,8 @@ pub async fn waveform(
|
||||
players: State<'_, PlayerState>,
|
||||
path: String,
|
||||
buckets: usize,
|
||||
start_secs: Option<f64>,
|
||||
end_secs: Option<f64>,
|
||||
) -> Result<Vec<f32>, String> {
|
||||
const MAX_WAVEFORM_BUCKETS: usize = 4096;
|
||||
if buckets > MAX_WAVEFORM_BUCKETS {
|
||||
@@ -4417,7 +4440,20 @@ pub async fn waveform(
|
||||
if buckets == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let key = format!("{path}#{buckets}");
|
||||
// A range = a setlist SLICE's lane; each slice of one file caches
|
||||
// separately under its own key.
|
||||
let ranged = start_secs.is_some() || end_secs.is_some();
|
||||
let key = if ranged {
|
||||
format!(
|
||||
"{path}#{buckets}@{:.3}-{}",
|
||||
start_secs.unwrap_or(0.0),
|
||||
end_secs
|
||||
.map(|e| format!("{e:.3}"))
|
||||
.unwrap_or_else(|| "end".into()),
|
||||
)
|
||||
} else {
|
||||
format!("{path}#{buckets}")
|
||||
};
|
||||
let job = players.waveform_job(&key);
|
||||
let result = async {
|
||||
let job_guard = job.clone().lock_owned().await;
|
||||
@@ -4426,7 +4462,11 @@ pub async fn waveform(
|
||||
}
|
||||
let p = std::path::PathBuf::from(&path);
|
||||
let (job_guard, peaks) = tauri::async_runtime::spawn_blocking(move || {
|
||||
let peaks = player::compute_waveform(&p, buckets);
|
||||
let peaks = if ranged {
|
||||
player::compute_waveform_range(&p, buckets, start_secs.unwrap_or(0.0), end_secs)
|
||||
} else {
|
||||
player::compute_waveform(&p, buckets)
|
||||
};
|
||||
(job_guard, peaks)
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -139,6 +139,7 @@ pub struct AppConfig {
|
||||
/// every track plays at roughly the same perceived level. None → true.
|
||||
#[serde(default)]
|
||||
pub volume_normalization: Option<bool>,
|
||||
pub word_timing: Option<bool>,
|
||||
}
|
||||
|
||||
/// Default longest edge for embedded covers when the user hasn't chosen.
|
||||
@@ -180,6 +181,14 @@ pub fn volume_normalization(cfg: &AppConfig) -> bool {
|
||||
cfg.volume_normalization.unwrap_or(true)
|
||||
}
|
||||
|
||||
/// Word-by-word lyric timing (the karaoke fill) — EXPERIMENTAL, off by
|
||||
/// default: generated word spans still drift on dense or echo-heavy mixes,
|
||||
/// and the solid line-level sync must be what users live with until the
|
||||
/// feature earns its keep (Kyle, 2026-08-19).
|
||||
pub fn word_timing(cfg: &AppConfig) -> bool {
|
||||
cfg.word_timing.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// One-time migration after the identifier rename: the app-data dir moved
|
||||
/// from `com.trifecta.soulseek-plus` to the new identifier's folder. Rename
|
||||
/// the old dir into place so the library DB, config and stream cache
|
||||
|
||||
@@ -6,15 +6,22 @@ mod hero_crop;
|
||||
mod import;
|
||||
mod hero_detector;
|
||||
mod loudness;
|
||||
mod lyric_align;
|
||||
mod lyric_prep;
|
||||
mod lyrics;
|
||||
mod pipeline;
|
||||
mod playback;
|
||||
mod playlist_formats;
|
||||
mod playlists;
|
||||
mod preview;
|
||||
mod setlist_scan;
|
||||
mod shazam;
|
||||
mod smtc;
|
||||
mod state;
|
||||
mod thumbbar;
|
||||
mod vocal_sep;
|
||||
mod youtube_setlist;
|
||||
mod ytdlp;
|
||||
|
||||
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
||||
pub fn run() {
|
||||
@@ -114,7 +121,15 @@ pub fn run() {
|
||||
playlists::playlist_import_probe,
|
||||
playlists::playlist_import_folder,
|
||||
playlists::playlist_import_text,
|
||||
playlists::playlist_import_youtube,
|
||||
playlists::playlist_import_commit,
|
||||
setlist_scan::playlist_scan_youtube,
|
||||
setlist_scan::playlist_pick_set_file,
|
||||
setlist_scan::playlist_scan_file,
|
||||
setlist_scan::playlist_scan_cancel,
|
||||
ytdlp::ytdlp_status,
|
||||
ytdlp::ytdlp_download,
|
||||
ytdlp::deno_download,
|
||||
playlists::playlist_export,
|
||||
playlists::library_track_search,
|
||||
lyrics::lyrics_get,
|
||||
@@ -128,6 +143,7 @@ pub fn run() {
|
||||
lyrics::lyrics_save_text,
|
||||
lyrics::lyrics_clear,
|
||||
commands::catalog_search,
|
||||
commands::catalog_track_by_isrc,
|
||||
import::import_scan,
|
||||
import::import_portable_root,
|
||||
import::import_resolve_root,
|
||||
@@ -187,6 +203,11 @@ pub fn run() {
|
||||
preview::player_fade_resume,
|
||||
diagnostics::debug_stats,
|
||||
smtc::smtc_metadata,
|
||||
lyric_prep::lyric_prep_request,
|
||||
lyric_align::lyric_model_status,
|
||||
lyric_align::lyric_model_download,
|
||||
vocal_sep::vocal_sep_status,
|
||||
vocal_sep::vocal_sep_download,
|
||||
])
|
||||
.run({
|
||||
#[allow(unused_mut)]
|
||||
|
||||
@@ -0,0 +1,525 @@
|
||||
//! Tier B word alignment: CTC forced alignment with a small acoustic model
|
||||
//! through WinML — the same built-in runtime the hero detector uses, so no
|
||||
//! new dependency and no child processes (the preparation-worker charter).
|
||||
//!
|
||||
//! The model (MMS-300m forced-aligner, ONNX f32 export — Meta's multilingual
|
||||
//! wav2vec2 fine-tuned specifically FOR forced alignment, the checkpoint
|
||||
//! torchaudio's alignment API standardized on) emits per-20ms CHARACTER
|
||||
//! probabilities; a Viterbi pass aligns the KNOWN lyric text against them —
|
||||
//! no transcription, no guessing, just "when does each known word occur".
|
||||
//! Its vocab is lowercase romanized characters with NO word separator and
|
||||
//! blank = `<blank>`; the loader reads all of that from the repo's own
|
||||
//! vocab.json rather than hardcoding (the vocab-never-hardcoded rule).
|
||||
//! It is downloaded ONLY from the user's explicit click in Settings
|
||||
//! (`lyric_model_download`) — the charter's consent rule — verified by
|
||||
//! length, and loaded lazily. When it is absent or fails, the worker's
|
||||
//! envelope estimate stands; nothing ever depends on this module working.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use tauri::{Emitter, Manager};
|
||||
use windows::{
|
||||
core::{Interface, HSTRING},
|
||||
AI::MachineLearning::{
|
||||
LearningModel, LearningModelBinding, LearningModelDevice, LearningModelDeviceKind,
|
||||
LearningModelSession, TensorFloat,
|
||||
},
|
||||
Win32::System::WinRT::{RoInitialize, RO_INIT_MULTITHREADED},
|
||||
};
|
||||
|
||||
const MODEL_URL: &str =
|
||||
"https://huggingface.co/onnx-community/mms-300m-1130-forced-aligner-ONNX/resolve/main/onnx/model.onnx";
|
||||
const VOCAB_URL: &str =
|
||||
"https://huggingface.co/onnx-community/mms-300m-1130-forced-aligner-ONNX/resolve/main/vocab.json";
|
||||
/// Exact f32 model size, shown in Settings and sanity-checked after download.
|
||||
const MODEL_BYTES: u64 = 1_262_529_881;
|
||||
/// wav2vec2-family conv stride: 320 samples at 16 kHz (MMS keeps it).
|
||||
const FRAME_MS: i64 = 20;
|
||||
|
||||
struct Engine {
|
||||
session: LearningModelSession,
|
||||
/// char → token id, from the repo's own vocab.json (never hardcoded).
|
||||
chars: HashMap<char, usize>,
|
||||
vocab_len: usize,
|
||||
blank: usize,
|
||||
/// Word-separator token — wav2vec2-960h had `|`, MMS has none.
|
||||
sep: Option<usize>,
|
||||
}
|
||||
|
||||
static ENGINE: Mutex<Option<Engine>> = Mutex::new(None);
|
||||
|
||||
pub(crate) fn models_dir(app: &tauri::AppHandle) -> Result<PathBuf, String> {
|
||||
let dir = app
|
||||
.path()
|
||||
.app_data_dir()
|
||||
.map_err(|e| e.to_string())?
|
||||
.join("lyric-models");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
fn model_file(app: &tauri::AppHandle) -> Result<PathBuf, String> {
|
||||
Ok(models_dir(app)?.join("mms-300m-fa.onnx"))
|
||||
}
|
||||
|
||||
fn vocab_file(app: &tauri::AppHandle) -> Result<PathBuf, String> {
|
||||
Ok(models_dir(app)?.join("mms-300m-fa.vocab.json"))
|
||||
}
|
||||
|
||||
pub fn model_present(app: &tauri::AppHandle) -> bool {
|
||||
matches!((model_file(app), vocab_file(app)), (Ok(m), Ok(v)) if m.is_file() && v.is_file())
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub fn lyric_model_status(app: tauri::AppHandle) -> serde_json::Value {
|
||||
let size = model_file(&app)
|
||||
.ok()
|
||||
.and_then(|p| std::fs::metadata(p).ok())
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0);
|
||||
serde_json::json!({
|
||||
"present": model_present(&app),
|
||||
"bytes": size,
|
||||
// The tier the machine would actually prepare under right now —
|
||||
// ctc-v5 once the separator model is ALSO on disk.
|
||||
"aligner": crate::lyric_prep::active_aligner(&app),
|
||||
"downloadBytes": MODEL_BYTES,
|
||||
})
|
||||
}
|
||||
|
||||
/// Download the model + vocab. Runs ONLY from the user's explicit Settings
|
||||
/// click — the consent-before-download charter rule. Streams to a temp file,
|
||||
/// renames atomically, emits `lyrics:model-progress` along the way.
|
||||
#[tauri::command]
|
||||
pub async fn lyric_model_download(app: tauri::AppHandle) -> Result<(), String> {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent("SoulStream/0.1 (lyric alignment)")
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
// Vocab first — tiny, and the engine cannot load without it.
|
||||
let vocab_dest = vocab_file(&app)?;
|
||||
let vocab = client
|
||||
.get(VOCAB_URL)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.error_for_status()
|
||||
.map_err(|e| e.to_string())?
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
serde_json::from_slice::<HashMap<String, usize>>(&vocab)
|
||||
.map_err(|_| "vocab.json did not parse".to_string())?;
|
||||
std::fs::write(&vocab_dest, &vocab).map_err(|e| e.to_string())?;
|
||||
|
||||
let dest = model_file(&app)?;
|
||||
let tmp = dest.with_extension("onnx.downloading");
|
||||
let mut response = client
|
||||
.get(MODEL_URL)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.error_for_status()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let total = response.content_length().unwrap_or(0);
|
||||
let mut file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?;
|
||||
let mut received: u64 = 0;
|
||||
let mut last_emit: u64 = 0;
|
||||
use std::io::Write;
|
||||
while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? {
|
||||
file.write_all(&chunk).map_err(|e| e.to_string())?;
|
||||
received += chunk.len() as u64;
|
||||
if received - last_emit > 4 * 1024 * 1024 {
|
||||
last_emit = received;
|
||||
let _ = app.emit(
|
||||
"lyrics:model-progress",
|
||||
serde_json::json!({ "received": received, "total": total }),
|
||||
);
|
||||
}
|
||||
}
|
||||
drop(file);
|
||||
if total > 0 && received != total {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err(format!("download truncated: {received} of {total} bytes"));
|
||||
}
|
||||
std::fs::rename(&tmp, &dest).map_err(|e| e.to_string())?;
|
||||
// The superseded wav2vec2 tier's files are dead weight once this lands.
|
||||
if let Ok(dir) = models_dir(&app) {
|
||||
for legacy in ["wav2vec2-base-960h.onnx", "wav2vec2-base-960h.vocab.json"] {
|
||||
let _ = std::fs::remove_file(dir.join(legacy));
|
||||
}
|
||||
}
|
||||
let _ = app.emit(
|
||||
"lyrics:model-progress",
|
||||
serde_json::json!({ "received": received, "total": total, "done": true }),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_engine(app: &tauri::AppHandle) -> Result<(), String> {
|
||||
let mut guard = ENGINE.lock().unwrap();
|
||||
if guard.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
let model_path = model_file(app)?;
|
||||
if !model_path.is_file() {
|
||||
return Err("model not downloaded".into());
|
||||
}
|
||||
let raw = std::fs::read(vocab_file(app)?).map_err(|e| e.to_string())?;
|
||||
let vocab: HashMap<String, usize> =
|
||||
serde_json::from_slice(&raw).map_err(|_| "vocab.json did not parse".to_string())?;
|
||||
let vocab_len = vocab.values().copied().max().unwrap_or(0) + 1;
|
||||
// CTC blank: MMS names it `<blank>` (its pad_token_id points there);
|
||||
// the wav2vec2-960h family used `<pad>` itself. Prefer the explicit one.
|
||||
let blank = *vocab
|
||||
.get("<blank>")
|
||||
.or_else(|| vocab.get("<pad>"))
|
||||
.ok_or("vocab has no blank token")?;
|
||||
let mut chars = HashMap::new();
|
||||
for (token, id) in &vocab {
|
||||
let mut it = token.chars();
|
||||
if let (Some(c), None) = (it.next(), it.next()) {
|
||||
chars.insert(c, *id);
|
||||
}
|
||||
}
|
||||
let sep = chars.get(&'|').copied();
|
||||
// Match the hero detector's WinRT posture: initialize, tolerate an
|
||||
// existing apartment, never uninitialize someone else's.
|
||||
match unsafe { RoInitialize(RO_INIT_MULTITHREADED) } {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.code().0 as u32 == 0x8001_0106 => {}
|
||||
Err(e) => return Err(format!("WinRT init failed: {e}")),
|
||||
}
|
||||
let model =
|
||||
LearningModel::LoadFromFilePath(&HSTRING::from(model_path.to_string_lossy().as_ref()))
|
||||
.map_err(|e| format!("could not load alignment model: {e}"))?;
|
||||
let device = LearningModelDevice::Create(LearningModelDeviceKind::Cpu)
|
||||
.map_err(|e| format!("could not create WinML device: {e}"))?;
|
||||
let session = LearningModelSession::CreateFromModelOnDevice(&model, &device)
|
||||
.map_err(|e| format!("could not create alignment session: {e}"))?;
|
||||
*guard = Some(Engine {
|
||||
session,
|
||||
chars,
|
||||
vocab_len,
|
||||
blank,
|
||||
sep,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bare-ASCII fold for the Latin letters MMS's romanized vocab expects:
|
||||
/// Western lyrics keep their diacritics, the model's chars do not. Anything
|
||||
/// outside this stays unmapped and the line falls back to the estimate.
|
||||
fn fold_latin(c: char) -> char {
|
||||
match c {
|
||||
'à'..='å' | 'À'..='Å' => 'a',
|
||||
'è'..='ë' | 'È'..='Ë' => 'e',
|
||||
'ì'..='ï' | 'Ì'..='Ï' => 'i',
|
||||
'ò'..='ö' | 'Ò'..='Ö' | 'ø' | 'Ø' => 'o',
|
||||
'ù'..='ü' | 'Ù'..='Ü' => 'u',
|
||||
'ý' | 'ÿ' | 'Ý' => 'y',
|
||||
'ñ' | 'Ñ' => 'n',
|
||||
'ç' | 'Ç' => 'c',
|
||||
// Typographic apostrophes down to the vocab's plain one.
|
||||
'’' | 'ʼ' | '`' => '\'',
|
||||
_ => c,
|
||||
}
|
||||
}
|
||||
|
||||
pub struct AlignedWord {
|
||||
pub start_ms: i64,
|
||||
pub end_ms: i64,
|
||||
}
|
||||
|
||||
pub struct AlignedLine {
|
||||
pub words: Vec<AlignedWord>,
|
||||
pub confidence: f32,
|
||||
}
|
||||
|
||||
/// Force-align a PASSAGE — one or more consecutive lines' words — against a
|
||||
/// 16 kHz mono window. Returns per-line word times RELATIVE to the window
|
||||
/// plus a per-line confidence, or Err when the model cannot carry the
|
||||
/// passage (the caller retries per line, then falls back to the estimate).
|
||||
///
|
||||
/// Aligning lines TOGETHER is what lets word boundaries pull free of a
|
||||
/// slightly-late source cue: inside a passage the line cues are only hints,
|
||||
/// not walls — the research bundle's "whole-song alignment using line
|
||||
/// starts as hints", bounded to passage windows. `floors_ms[l]` is line
|
||||
/// l's earliest permitted emission (window-relative): without that floor,
|
||||
/// a repeating chorus lets the path lock whole lines onto the PREVIOUS
|
||||
/// acoustic instance of the same words (the charter's "competing repeated
|
||||
/// sections") — hint means "at most this much early", never "anywhere".
|
||||
/// `ceils_ms[l]` is the floor's mirror, the latest permitted emission:
|
||||
/// without it a line may smear arbitrarily LATE across its successors'
|
||||
/// audio when the space between lines still sounds vocal-ish (a separated
|
||||
/// stem keeps reverb/delay tails), sustaining one token for seconds at
|
||||
/// HIGH margin confidence — free decoding hears the same tail, so the
|
||||
/// score cannot object. Hint means "at most this much late", too.
|
||||
pub fn align_passage(
|
||||
app: &tauri::AppHandle,
|
||||
pcm: &[f32],
|
||||
lines: &[Vec<String>],
|
||||
floors_ms: &[i64],
|
||||
ceils_ms: &[i64],
|
||||
) -> Result<Vec<AlignedLine>, String> {
|
||||
let word_total: usize = lines.iter().map(|l| l.len()).sum();
|
||||
if pcm.len() < 1600
|
||||
|| word_total == 0
|
||||
|| floors_ms.len() != lines.len()
|
||||
|| ceils_ms.len() != lines.len()
|
||||
{
|
||||
return Err("window too short".into());
|
||||
}
|
||||
ensure_engine(app)?;
|
||||
let guard = ENGINE.lock().unwrap();
|
||||
let engine = guard.as_ref().ok_or("engine unavailable")?;
|
||||
|
||||
// Zero-mean / unit-variance, the model's expected normalization.
|
||||
let mean = pcm.iter().sum::<f32>() / pcm.len() as f32;
|
||||
let var = pcm.iter().map(|s| (s - mean) * (s - mean)).sum::<f32>() / pcm.len() as f32;
|
||||
let scale = 1.0 / (var.sqrt() + 1e-7);
|
||||
let normed: Vec<f32> = pcm.iter().map(|s| (s - mean) * scale).collect();
|
||||
|
||||
let input = TensorFloat::CreateFromShapeArrayAndDataArray(&[1, normed.len() as i64], &normed)
|
||||
.map_err(|e| format!("could not create input tensor: {e}"))?;
|
||||
let binding = LearningModelBinding::CreateFromSession(&engine.session)
|
||||
.map_err(|e| format!("could not create binding: {e}"))?;
|
||||
binding
|
||||
.Bind(&HSTRING::from("input_values"), &input)
|
||||
.map_err(|e| format!("could not bind input: {e}"))?;
|
||||
let result = engine
|
||||
.session
|
||||
.Evaluate(&binding, &HSTRING::from("lyric-align"))
|
||||
.map_err(|e| format!("inference failed: {e}"))?;
|
||||
if !result.Succeeded().unwrap_or(false) {
|
||||
return Err("inference did not succeed".into());
|
||||
}
|
||||
let logits: TensorFloat = result
|
||||
.Outputs()
|
||||
.and_then(|o| o.Lookup(&HSTRING::from("logits")))
|
||||
.map_err(|e| format!("logits missing: {e}"))?
|
||||
.cast()
|
||||
.map_err(|e| format!("logits wrong type: {e}"))?;
|
||||
let values: Vec<f32> = logits
|
||||
.GetAsVectorView()
|
||||
.map_err(|e| format!("could not read logits: {e}"))?
|
||||
.into_iter()
|
||||
.collect();
|
||||
let v = engine.vocab_len;
|
||||
if v == 0 || values.len() % v != 0 {
|
||||
return Err(format!("unexpected logits length {}", values.len()));
|
||||
}
|
||||
let t_frames = values.len() / v;
|
||||
|
||||
// log-softmax per frame.
|
||||
let mut logp = vec![0.0f32; values.len()];
|
||||
for t in 0..t_frames {
|
||||
let row = &values[t * v..(t + 1) * v];
|
||||
let max = row.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
|
||||
let sum: f32 = row.iter().map(|x| (x - max).exp()).sum();
|
||||
let log_z = max + sum.ln();
|
||||
for (i, x) in row.iter().enumerate() {
|
||||
logp[t * v + i] = x - log_z;
|
||||
}
|
||||
}
|
||||
|
||||
// Token sequence: each token knows its WORD (passage-global index), each
|
||||
// word knows its line; a separator joins words only when the vocab has
|
||||
// one (wav2vec2-960h did) — MMS aligns bare chars. Mapping is
|
||||
// vocab-driven and case-agnostic, with a Latin fold so accented lyrics
|
||||
// reach the romanized vocab. A word that maps to nothing (digits,
|
||||
// symbols, other scripts) sinks the whole passage to the per-line
|
||||
// ladder rather than being silently skipped.
|
||||
let map_char = |c: char| -> Option<usize> {
|
||||
engine
|
||||
.chars
|
||||
.get(&c)
|
||||
.or_else(|| c.to_lowercase().next().and_then(|l| engine.chars.get(&l)))
|
||||
.or_else(|| c.to_uppercase().next().and_then(|u| engine.chars.get(&u)))
|
||||
.or_else(|| {
|
||||
let f = fold_latin(c);
|
||||
engine
|
||||
.chars
|
||||
.get(&f)
|
||||
.or_else(|| f.to_uppercase().next().and_then(|u| engine.chars.get(&u)))
|
||||
})
|
||||
.copied()
|
||||
};
|
||||
let mut tokens: Vec<(usize, usize)> = Vec::new();
|
||||
// Per-token emission bounds, frames — the line's cue-derived floor and
|
||||
// its successor-derived ceiling.
|
||||
let mut floor_frame: Vec<usize> = Vec::new();
|
||||
let mut ceil_frame: Vec<usize> = Vec::new();
|
||||
let mut next_wi = 0usize;
|
||||
for (li, line) in lines.iter().enumerate() {
|
||||
let floor = (floors_ms[li].max(0) / FRAME_MS) as usize;
|
||||
let ceil = (ceils_ms[li].max(0) / FRAME_MS) as usize;
|
||||
for word in line {
|
||||
let wi = next_wi;
|
||||
next_wi += 1;
|
||||
if wi > 0 {
|
||||
if let Some(sep) = engine.sep {
|
||||
tokens.push((sep, wi));
|
||||
floor_frame.push(floor);
|
||||
ceil_frame.push(ceil);
|
||||
}
|
||||
}
|
||||
let before = tokens.len();
|
||||
for c in word.chars() {
|
||||
if let Some(id) = map_char(c) {
|
||||
tokens.push((id, wi));
|
||||
floor_frame.push(floor);
|
||||
ceil_frame.push(ceil);
|
||||
}
|
||||
}
|
||||
if tokens.len() == before {
|
||||
return Err("word has no mappable characters".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
let n = tokens.len();
|
||||
if t_frames < n {
|
||||
return Err("window shorter than token sequence".into());
|
||||
}
|
||||
|
||||
// Real CTC trellis: at each frame stay on blank, SUSTAIN the current
|
||||
// token by re-emitting it (the self-loop), or advance by emitting the
|
||||
// next. The self-loop turns single emission frames into token SPANS —
|
||||
// without it every word "ended" one frame after it started and sustained
|
||||
// notes were invisible. Known approximation, kept deliberately: identical
|
||||
// CONSECUTIVE tokens (double letters, "LL") properly require a mandatory
|
||||
// blank between them; collapsing that costs ~one frame (20ms).
|
||||
const NEG: f32 = f32::NEG_INFINITY;
|
||||
let mut trellis = vec![NEG; (t_frames + 1) * (n + 1)];
|
||||
// Backpointers (0 = blank, 1 = self-loop, 2 = advance) make the
|
||||
// backtrack exact instead of re-deriving float comparisons.
|
||||
let mut back = vec![0u8; (t_frames + 1) * (n + 1)];
|
||||
let idx = |t: usize, j: usize| t * (n + 1) + j;
|
||||
trellis[idx(0, 0)] = 0.0;
|
||||
for t in 0..t_frames {
|
||||
let blank_p = logp[t * v + engine.blank];
|
||||
for j in 0..=n.min(t + 1) {
|
||||
let mut best = trellis[idx(t, j)] + blank_p;
|
||||
let mut way = 0u8;
|
||||
// A token may not be emitted before its line's floor (repeated
|
||||
// choruses would lock onto an earlier acoustic instance) nor at
|
||||
// or past its ceiling (stem reverb tails would smear a sustain
|
||||
// across the successors' audio). A line that cannot fit between
|
||||
// its bounds leaves no path — the caller's ladder handles it.
|
||||
if j > 0 && t >= floor_frame[j - 1] && t < ceil_frame[j - 1] {
|
||||
let tok_p = logp[t * v + tokens[j - 1].0];
|
||||
let sustain = trellis[idx(t, j)] + tok_p;
|
||||
if sustain > best {
|
||||
best = sustain;
|
||||
way = 1;
|
||||
}
|
||||
let advance = trellis[idx(t, j - 1)] + tok_p;
|
||||
if advance > best {
|
||||
best = advance;
|
||||
way = 2;
|
||||
}
|
||||
}
|
||||
trellis[idx(t + 1, j)] = best;
|
||||
back[idx(t + 1, j)] = way;
|
||||
}
|
||||
}
|
||||
if trellis[idx(t_frames, n)] == NEG {
|
||||
return Err("no alignment path".into());
|
||||
}
|
||||
// Backtrack the pointers into per-token spans: an advance opens the
|
||||
// token's span, self-loops extend it. Walking backwards, the first
|
||||
// emission met is the token's LAST frame. Every backtrack step consumes
|
||||
// exactly one frame emitting blank or a token — recording that symbol's
|
||||
// logp gives the path's per-frame score, which per-LINE confidence
|
||||
// needs below.
|
||||
let mut first_emit = vec![usize::MAX; n];
|
||||
let mut last_emit = vec![usize::MAX; n];
|
||||
let mut path_logp = vec![0.0f32; t_frames];
|
||||
let mut j = n;
|
||||
let mut t = t_frames;
|
||||
while t > 0 {
|
||||
match back[idx(t, j)] {
|
||||
2 => {
|
||||
first_emit[j - 1] = t - 1;
|
||||
if last_emit[j - 1] == usize::MAX {
|
||||
last_emit[j - 1] = t - 1;
|
||||
}
|
||||
path_logp[t - 1] = logp[(t - 1) * v + tokens[j - 1].0];
|
||||
j -= 1;
|
||||
}
|
||||
1 => {
|
||||
if last_emit[j - 1] == usize::MAX {
|
||||
last_emit[j - 1] = t - 1;
|
||||
}
|
||||
path_logp[t - 1] = logp[(t - 1) * v + tokens[j - 1].0];
|
||||
}
|
||||
_ => {
|
||||
path_logp[t - 1] = logp[(t - 1) * v + engine.blank];
|
||||
}
|
||||
}
|
||||
t -= 1;
|
||||
}
|
||||
if j != 0 {
|
||||
return Err("backtrack failed".into());
|
||||
}
|
||||
|
||||
// Per-word spans, then grouped into lines.
|
||||
let mut word_span: Vec<Option<(usize, usize)>> = vec![None; word_total];
|
||||
for (k, (id, w)) in tokens.iter().enumerate() {
|
||||
if Some(*id) == engine.sep || first_emit[k] == usize::MAX {
|
||||
continue;
|
||||
}
|
||||
word_span[*w] = Some(match word_span[*w] {
|
||||
None => (first_emit[k], last_emit[k]),
|
||||
Some((f, _)) => (f, last_emit[k]),
|
||||
});
|
||||
}
|
||||
|
||||
// Confidence = margin to FREE decoding, per LINE: exp of the mean
|
||||
// per-frame gap between the forced path and the best unconstrained path
|
||||
// over the line's own sung span — "how close is the known lyric to what
|
||||
// the model actually hears", the calibration CTC-segmentation systems
|
||||
// use. Near 1.0 when text matches audio, sinking as it diverges. (v1
|
||||
// sampled exp(logprob) at each token's single emission frame, where CTC
|
||||
// piles mass on blank — every line pinned to the clamp floor.)
|
||||
let free: Vec<f32> = (0..t_frames)
|
||||
.map(|t| logp[t * v..(t + 1) * v].iter().cloned().fold(NEG, f32::max))
|
||||
.collect();
|
||||
let mut out = Vec::with_capacity(lines.len());
|
||||
let mut wi = 0usize;
|
||||
for line in lines {
|
||||
let mut words_out = Vec::with_capacity(line.len());
|
||||
let mut span_first = usize::MAX;
|
||||
let mut span_last = 0usize;
|
||||
for _ in line {
|
||||
match word_span[wi] {
|
||||
Some((f, l)) => {
|
||||
span_first = span_first.min(f);
|
||||
span_last = span_last.max(l);
|
||||
words_out.push(AlignedWord {
|
||||
start_ms: f as i64 * FRAME_MS,
|
||||
end_ms: (l as i64 + 1) * FRAME_MS,
|
||||
});
|
||||
}
|
||||
None => return Err("word lost in backtrack".into()),
|
||||
}
|
||||
wi += 1;
|
||||
}
|
||||
let confidence = if span_first == usize::MAX {
|
||||
0.05
|
||||
} else {
|
||||
let gap: f32 = (span_first..=span_last.min(t_frames - 1))
|
||||
.map(|t| path_logp[t] - free[t])
|
||||
.sum();
|
||||
let frames = (span_last.min(t_frames - 1) - span_first + 1) as f32;
|
||||
(gap / frames).exp().clamp(0.05, 0.99)
|
||||
};
|
||||
out.push(AlignedLine {
|
||||
words: words_out,
|
||||
confidence,
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1410,6 +1410,7 @@ pub async fn lyrics_get(
|
||||
let payload = payload_from(user, offset, duration_secs);
|
||||
if payload.status != "none" {
|
||||
return Ok(finish_synced(
|
||||
&app,
|
||||
payload,
|
||||
entry.auto_sync_off,
|
||||
path.as_deref(),
|
||||
@@ -1514,6 +1515,7 @@ pub async fn lyrics_get(
|
||||
let payload = payload_from(doc, offset, duration_secs);
|
||||
if payload.status == "synced" {
|
||||
return Ok(finish_synced(
|
||||
&app,
|
||||
payload,
|
||||
entry.auto_sync_off,
|
||||
path.as_deref(),
|
||||
@@ -1603,6 +1605,7 @@ pub async fn lyrics_get(
|
||||
// local copy may be the corrected one).
|
||||
if payload.status == "synced" || payload.status == "instrumental" {
|
||||
return Ok(finish_synced(
|
||||
&app,
|
||||
payload,
|
||||
entry.auto_sync_off,
|
||||
path.as_deref(),
|
||||
@@ -1638,6 +1641,7 @@ pub async fn lyrics_get(
|
||||
|
||||
/// Stamp the auto-sync fields onto a winning synced payload.
|
||||
async fn finish_synced(
|
||||
app: &tauri::AppHandle,
|
||||
mut payload: LyricsPayload,
|
||||
auto_sync_off: bool,
|
||||
path: Option<&str>,
|
||||
@@ -1649,6 +1653,35 @@ async fn finish_synced(
|
||||
let first = payload.synced.first().map(|l| l.time_ms);
|
||||
payload.auto_offset_ms =
|
||||
compute_auto_offset(path, duration_secs, source_duration_secs, first).await;
|
||||
// Second evidence source, consulted only when the corroborated
|
||||
// duration/silence probe found nothing: the word-timing artifact.
|
||||
// When forced alignment has already measured this file against this
|
||||
// doc and the sung words consistently start N ms from their cues,
|
||||
// that median is the doc's shift. Cache read only — never computed
|
||||
// here. (Aligned WORD times themselves are absolute and exempt from
|
||||
// this offset; only line cues move. See ArtifactLine::ctc.)
|
||||
if payload.auto_offset_ms == 0 {
|
||||
if let Some(p) = path {
|
||||
// The exact shape the word-timing request sends: original
|
||||
// indexes, empties dropped — the artifact key depends on it.
|
||||
let lines: Vec<crate::lyric_prep::PrepLine> = payload
|
||||
.synced
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, l)| crate::lyric_prep::PrepLine {
|
||||
index: i as u32,
|
||||
time_ms: l.time_ms as i64,
|
||||
text: l.text.clone(),
|
||||
})
|
||||
.filter(|l| !l.text.trim().is_empty())
|
||||
.collect();
|
||||
if let Some(shift) =
|
||||
crate::lyric_prep::cue_shift_evidence(app, Path::new(p), &lines)
|
||||
{
|
||||
payload.auto_offset_ms = shift;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
payload
|
||||
}
|
||||
|
||||
@@ -775,7 +775,7 @@ pub fn parse_text(bytes: &[u8]) -> Result<ParsedPlaylist, String> {
|
||||
}
|
||||
|
||||
/// "1:23" → 83_000 ms; "1:23:45" → 5_025_000; rejects anything else.
|
||||
fn parse_timestamp(s: &str) -> Option<i64> {
|
||||
pub(crate) fn parse_timestamp(s: &str) -> Option<i64> {
|
||||
let parts: Vec<&str> = s.split(':').collect();
|
||||
if !(2..=3).contains(&parts.len()) {
|
||||
return None;
|
||||
|
||||
+181
-10
@@ -1,4 +1,4 @@
|
||||
//! Playlist commands: CRUD over the occurrence model, covers (custom +
|
||||
//! Playlist commands: CRUD over the occurrence model, covers (custom +
|
||||
//! composed mosaic), file import with staged preview, and export profiles.
|
||||
//!
|
||||
//! Boundaries, per the architecture audit: parsing happens in
|
||||
@@ -246,6 +246,13 @@ pub struct AddItemInput {
|
||||
pub source_line: Option<String>,
|
||||
#[serde(default)]
|
||||
pub note: Option<String>,
|
||||
/// Catalog track id for a stream row — the identity anchor (exact id,
|
||||
/// never a name match).
|
||||
#[serde(default)]
|
||||
pub track_id: Option<String>,
|
||||
/// Cover URL for a stream row, which has no local file to read art from.
|
||||
#[serde(default)]
|
||||
pub art_url: Option<String>,
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
@@ -269,6 +276,8 @@ pub async fn playlist_add_items(
|
||||
start_ms: t.start_ms,
|
||||
source_line: t.source_line,
|
||||
note: t.note,
|
||||
track_id: t.track_id,
|
||||
art_url: t.art_url,
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
@@ -318,7 +327,8 @@ pub async fn playlist_set_order(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Repair one row — attach a found local file or corrected metadata.
|
||||
/// Repair one row — attach a found local file, corrected metadata, or a
|
||||
/// stream row's catalog anchor (track id + cover URL).
|
||||
#[tauri::command]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn playlist_relink_item(
|
||||
@@ -332,6 +342,8 @@ pub async fn playlist_relink_item(
|
||||
duration_secs: Option<u32>,
|
||||
path: Option<String>,
|
||||
art_path: Option<String>,
|
||||
track_id: Option<String>,
|
||||
art_url: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let l = lib.library.clone();
|
||||
tauri::async_runtime::spawn_blocking(move || {
|
||||
@@ -344,6 +356,8 @@ pub async fn playlist_relink_item(
|
||||
duration_secs,
|
||||
path.as_deref(),
|
||||
art_path.as_deref(),
|
||||
track_id.as_deref(),
|
||||
art_url.as_deref(),
|
||||
now_secs(),
|
||||
)
|
||||
})
|
||||
@@ -583,7 +597,9 @@ pub struct DraftEntry {
|
||||
pub duration_secs: Option<u32>,
|
||||
pub start_ms: Option<i64>,
|
||||
pub raw: String,
|
||||
/// "local" (matched a library file) | "unresolved".
|
||||
/// "local" (matched a library file) | "stream" (full identity, no local
|
||||
/// file — becomes a Soulseek stream on commit) | "unresolved" (identity
|
||||
/// too thin to play: ID rows, title-only lines).
|
||||
pub resolution: String,
|
||||
/// How the local match was made: "path" | "relinked" | "tags" — evidence
|
||||
/// the preview shows so an exact path match reads differently from a
|
||||
@@ -591,6 +607,18 @@ pub struct DraftEntry {
|
||||
pub matched_by: Option<String>,
|
||||
pub path: Option<String>,
|
||||
pub art_path: Option<String>,
|
||||
/// Catalog identity, attached by the review's verification pass (the
|
||||
/// modal matches stream rows against the Deezer-backed catalog — Kyle:
|
||||
/// verify against "the streaming library part, not Soulseek"). Flows
|
||||
/// into the committed row so it plays as a catalog track with artwork.
|
||||
#[serde(default)]
|
||||
pub track_id: Option<i64>,
|
||||
#[serde(default)]
|
||||
pub art_url: Option<String>,
|
||||
/// The heard release's ISRC, when this row came from the listener door
|
||||
/// — lets the review resolve the catalog by exact recording id.
|
||||
#[serde(default)]
|
||||
pub isrc: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
@@ -604,7 +632,13 @@ pub struct ImportDraft {
|
||||
pub entries: Vec<DraftEntry>,
|
||||
pub warnings: Vec<String>,
|
||||
pub resolved_count: u32,
|
||||
/// Rows that will play as Soulseek streams (identified, not owned).
|
||||
pub stream_count: u32,
|
||||
pub unresolved_count: u32,
|
||||
/// A suggested cover image URL (the YouTube doors set the video's own
|
||||
/// thumbnail). Commit installs it as the playlist's custom cover —
|
||||
/// failure-soft, a cover is never worth failing an import over.
|
||||
pub cover_url: Option<String>,
|
||||
}
|
||||
|
||||
/// Open a playlist file and stage it for review. Parsing and resolution only —
|
||||
@@ -769,6 +803,9 @@ pub async fn playlist_import_folder(
|
||||
matched_by: Some(if indexed.is_some() { "path" } else { "file" }.into()),
|
||||
path: Some(path_string),
|
||||
art_path: indexed.and_then(|r| r.art_path.clone()),
|
||||
track_id: None,
|
||||
art_url: None,
|
||||
isrc: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
@@ -782,7 +819,9 @@ pub async fn playlist_import_folder(
|
||||
entries,
|
||||
warnings: Vec::new(),
|
||||
resolved_count: resolved,
|
||||
stream_count: 0,
|
||||
unresolved_count: 0,
|
||||
cover_url: None,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -875,6 +914,38 @@ pub async fn playlist_import_text(
|
||||
resolve_draft(&lib, parsed, "Pasted tracklist".into(), None).await
|
||||
}
|
||||
|
||||
/// Stage a YouTube DJ set for review: fetch the video's own tracklist
|
||||
/// evidence (chapters → description → top comments, the research bundle's
|
||||
/// declared-data rungs), then run it through the exact pasted-tracklist
|
||||
/// pipeline — same parser, same resolution, same review modal, same commit.
|
||||
/// No audio is fetched and nothing is guessed: a set with no posted
|
||||
/// tracklist says so instead of inventing one.
|
||||
#[tauri::command]
|
||||
pub async fn playlist_import_youtube(
|
||||
lib: State<'_, LibraryState>,
|
||||
url: String,
|
||||
) -> Result<ImportDraft, String> {
|
||||
let fetched = crate::youtube_setlist::fetch(&url).await?;
|
||||
let parsed = playlist_formats::parse_text(fetched.tracklist.as_bytes())?;
|
||||
let source = if fetched.author.is_empty() {
|
||||
fetched.title.clone()
|
||||
} else {
|
||||
format!("{} — {}", fetched.author, fetched.title)
|
||||
};
|
||||
let mut draft = resolve_draft(&lib, parsed, source, None).await?;
|
||||
draft.format = "youtube".into();
|
||||
draft.name = Some(fetched.title);
|
||||
// The video's own thumbnail is the playlist's default face.
|
||||
if let Some(id) = crate::youtube_setlist::video_id_from_url(&url) {
|
||||
draft.cover_url = Some(format!("https://i.ytimg.com/vi/{id}/maxresdefault.jpg"));
|
||||
}
|
||||
draft
|
||||
.warnings
|
||||
.insert(0, format!("Tracklist from {}.", fetched.provenance));
|
||||
draft.warnings.extend(fetched.warnings);
|
||||
Ok(draft)
|
||||
}
|
||||
|
||||
/// Commit a reviewed draft as a new playlist, atomically.
|
||||
#[tauri::command]
|
||||
pub async fn playlist_import_commit(
|
||||
@@ -884,6 +955,7 @@ pub async fn playlist_import_commit(
|
||||
description: Option<String>,
|
||||
kind: String,
|
||||
entries: Vec<DraftEntry>,
|
||||
cover_url: Option<String>,
|
||||
) -> Result<i64, String> {
|
||||
let trimmed = name.trim().to_string();
|
||||
if trimmed.is_empty() {
|
||||
@@ -897,16 +969,25 @@ pub async fn playlist_import_commit(
|
||||
artist: e.artist,
|
||||
album: e.album,
|
||||
duration_secs: e.duration_secs,
|
||||
kind: if e.resolution == "local" && e.path.is_some() {
|
||||
"local".into()
|
||||
// Empty lets the insert classify: a matched path is local, an
|
||||
// unmatched row with full identity is a stream, the rest stay
|
||||
// unresolved placeholders. The one kind the command DOES assert
|
||||
// is "segment" — a listener row that plays a slice of the set's
|
||||
// own recording; the classifier would read its path as "local".
|
||||
kind: if e.resolution == "segment" {
|
||||
"segment".into()
|
||||
} else {
|
||||
"unresolved".into()
|
||||
String::new()
|
||||
},
|
||||
path: e.path.filter(|p| !p.is_empty()),
|
||||
art_path: e.art_path,
|
||||
source_line: (!e.raw.is_empty()).then_some(e.raw),
|
||||
start_ms: e.start_ms,
|
||||
note: None,
|
||||
// Catalog identity from the review's verification pass rides
|
||||
// into the row: the playlist plays it as a catalog track.
|
||||
track_id: e.track_id.map(|id| id.to_string()),
|
||||
art_url: e.art_url,
|
||||
})
|
||||
.collect();
|
||||
let l = lib.library.clone();
|
||||
@@ -919,9 +1000,74 @@ pub async fn playlist_import_commit(
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
changed(&app);
|
||||
if let Some(url) = cover_url {
|
||||
// Failure-soft: the playlist is already real; a cover is never
|
||||
// worth failing it over.
|
||||
match set_cover_from_url(&app, &lib, id, &url).await {
|
||||
Ok(()) => changed(&app),
|
||||
Err(e) => log::debug!("import cover fetch failed: {e}"),
|
||||
}
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Download an image URL and install it as the playlist's custom cover —
|
||||
/// the same square master `playlist_cover_pick` writes. YouTube's
|
||||
/// maxresdefault 404s on some videos, so that URL falls back to hqdefault.
|
||||
async fn set_cover_from_url(
|
||||
app: &tauri::AppHandle,
|
||||
lib: &State<'_, LibraryState>,
|
||||
id: i64,
|
||||
url: &str,
|
||||
) -> Result<(), String> {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent("SoulStream/0.1")
|
||||
.timeout(std::time::Duration::from_secs(15))
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut candidates = vec![url.to_string()];
|
||||
if let Some(rest) = url.strip_suffix("maxresdefault.jpg") {
|
||||
candidates.push(format!("{rest}hqdefault.jpg"));
|
||||
}
|
||||
let mut bytes: Option<Vec<u8>> = None;
|
||||
for candidate in candidates {
|
||||
if let Ok(resp) = client.get(&candidate).send().await {
|
||||
if resp.status().is_success() {
|
||||
if let Ok(body) = resp.bytes().await {
|
||||
// YouTube's CDN can answer a missing size with a tiny
|
||||
// gray placeholder instead of a 404 — refuse those.
|
||||
if body.len() > 4096 {
|
||||
bytes = Some(body.to_vec());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let bytes = bytes.ok_or("no usable image answered")?;
|
||||
let dir = covers_dir(app)?;
|
||||
let target = dir.join(format!("{id}.jpg"));
|
||||
let jpeg = tauri::async_runtime::spawn_blocking(move || -> Result<Vec<u8>, String> {
|
||||
let img =
|
||||
image::load_from_memory(&bytes).map_err(|e| format!("not a usable image: {e}"))?;
|
||||
Ok(square_jpeg(&img, COVER_MASTER_EDGE))
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
let temp = dir.join(format!("{id}.importing"));
|
||||
std::fs::write(&temp, &jpeg).map_err(|e| e.to_string())?;
|
||||
std::fs::rename(&temp, &target).map_err(|e| e.to_string())?;
|
||||
let l = lib.library.clone();
|
||||
let stored = target.to_string_lossy().into_owned();
|
||||
tauri::async_runtime::spawn_blocking(move || {
|
||||
l.set_playlist_cover(id, Some(&stored), now_secs())
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve parsed entries against the local library, in memory.
|
||||
///
|
||||
/// The ladder, strongest evidence first (identity rules: an exact locator
|
||||
@@ -931,7 +1077,7 @@ pub async fn playlist_import_commit(
|
||||
/// 3. a unique basename match in the library index;
|
||||
/// 4. a unique normalized (artist, title) tag match;
|
||||
/// 5. a unique normalized title match, when the entry names no artist.
|
||||
async fn resolve_draft(
|
||||
pub(crate) async fn resolve_draft(
|
||||
lib: &State<'_, LibraryState>,
|
||||
parsed: playlist_formats::ParsedPlaylist,
|
||||
source_name: String,
|
||||
@@ -980,7 +1126,7 @@ async fn resolve_draft(
|
||||
};
|
||||
|
||||
let mut entries = Vec::new();
|
||||
let (mut resolved, mut unresolved) = (0u32, 0u32);
|
||||
let (mut resolved, mut streams, mut unresolved) = (0u32, 0u32, 0u32);
|
||||
let mut any_start = false;
|
||||
for entry in &parsed.entries {
|
||||
let ParsedEntry {
|
||||
@@ -1022,6 +1168,9 @@ async fn resolve_draft(
|
||||
matched_by: Some("path".into()),
|
||||
path: Some(abs),
|
||||
art_path: None,
|
||||
track_id: None,
|
||||
art_url: None,
|
||||
isrc: None,
|
||||
});
|
||||
resolved += 1;
|
||||
continue;
|
||||
@@ -1090,10 +1239,26 @@ async fn resolve_draft(
|
||||
matched_by: Some(how.into()),
|
||||
path: Some(row.path.clone()),
|
||||
art_path: row.art_path.clone(),
|
||||
track_id: None,
|
||||
art_url: None,
|
||||
isrc: None,
|
||||
});
|
||||
}
|
||||
None => {
|
||||
unresolved += 1;
|
||||
// No library file — but a row with FULL identity becomes a
|
||||
// STREAM on commit (the insert classifier's rule), so the
|
||||
// preview says so instead of "unresolved": Kyle read that
|
||||
// label as "not identified" on a fully-recognized set.
|
||||
// "unresolved" now means what it says: identity too thin
|
||||
// to play (ID rows, title-only lines).
|
||||
let full_identity = title.is_some()
|
||||
&& artist.as_deref().map(|a| !a.trim().is_empty()).unwrap_or(false)
|
||||
&& title.as_deref() != Some("ID");
|
||||
if full_identity {
|
||||
streams += 1;
|
||||
} else {
|
||||
unresolved += 1;
|
||||
}
|
||||
entries.push(DraftEntry {
|
||||
ordinal: *ordinal,
|
||||
title: title.clone().unwrap_or_else(|| "Unknown".into()),
|
||||
@@ -1102,10 +1267,13 @@ async fn resolve_draft(
|
||||
duration_secs: *duration_secs,
|
||||
start_ms: *start_ms,
|
||||
raw: raw.clone(),
|
||||
resolution: "unresolved".into(),
|
||||
resolution: if full_identity { "stream" } else { "unresolved" }.into(),
|
||||
matched_by: None,
|
||||
path: None,
|
||||
art_path: None,
|
||||
track_id: None,
|
||||
art_url: None,
|
||||
isrc: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1119,7 +1287,9 @@ async fn resolve_draft(
|
||||
entries,
|
||||
warnings: parsed.warnings,
|
||||
resolved_count: resolved,
|
||||
stream_count: streams,
|
||||
unresolved_count: unresolved,
|
||||
cover_url: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1277,3 +1447,4 @@ pub async fn playlist_export(
|
||||
.map_err(|e| e.to_string())??;
|
||||
Ok(Some(dest))
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,478 @@
|
||||
//! Shazam-style audio recognition for the DJ-set setlist assistant — the
|
||||
//! research bundle's "broader recognition adapter" (feasibility §3 rung 4,
|
||||
//! plan phase 7), Kyle's pick 2026-08-19: identify a mix's tracks by
|
||||
//! LISTENING, not just by reading its posted tracklist.
|
||||
//!
|
||||
//! The signature algorithm and packet format are reimplemented from the
|
||||
//! publicly documented Shazam fingerprint (Wang 2003; layout and constants
|
||||
//! cross-checked against the open-source SongRec project's description of
|
||||
//! the format). Nothing here links foreign code: plain Rust over the same
|
||||
//! realfft the vocal separator uses.
|
||||
//!
|
||||
//! Posture: in-process, no child processes; ONE request per analyzed
|
||||
//! window against the free discovery endpoint, paced by the caller
|
||||
//! (research §11 — rate limits, timeout, cache belong to the job layer,
|
||||
//! built next). Failure is always soft: a window that cannot be
|
||||
//! recognized is an honest Unknown, never a guess.
|
||||
|
||||
use realfft::num_complex::Complex;
|
||||
use realfft::RealFftPlanner;
|
||||
|
||||
/// One recognized window: what Shazam heard and where it claims the window
|
||||
/// sits inside the identified track.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Recognition {
|
||||
pub artist: String,
|
||||
pub title: String,
|
||||
/// Shazam track key — stable id for consensus grouping.
|
||||
pub key: String,
|
||||
/// Offset of the analyzed window within the identified track, seconds,
|
||||
/// when the response carries one (it usually does).
|
||||
pub offset_secs: Option<f64>,
|
||||
pub album: Option<String>,
|
||||
pub cover_url: Option<String>,
|
||||
/// The matched RELEASE's ISRC when the response carries one — an
|
||||
/// id-grade identity the review can resolve against the catalog
|
||||
/// exactly, instead of by name.
|
||||
pub isrc: Option<String>,
|
||||
}
|
||||
|
||||
// ---- signature generation (pure DSP) ----------------------------------------
|
||||
|
||||
const FFT_SIZE: usize = 2048;
|
||||
const BINS: usize = FFT_SIZE / 2 + 1; // 1025
|
||||
const HOP: usize = 128;
|
||||
/// The generator's working window: 12 s at 16 kHz mono, the length the
|
||||
/// service expects a tag to be.
|
||||
pub const WINDOW_SECS: usize = 12;
|
||||
pub const SAMPLE_RATE: usize = 16_000;
|
||||
|
||||
struct Rings {
|
||||
samples: [i16; FFT_SIZE],
|
||||
samples_at: usize,
|
||||
ffts: Vec<[f32; BINS]>,
|
||||
ffts_at: u8,
|
||||
spread: Vec<[f32; BINS]>,
|
||||
spread_at: u8,
|
||||
passes: u32,
|
||||
peaks: [Vec<Peak>; 4],
|
||||
}
|
||||
|
||||
struct Peak {
|
||||
pass: u32,
|
||||
magnitude: u16,
|
||||
corrected_bin: u16,
|
||||
}
|
||||
|
||||
/// Symmetric Hann of length 2048 (numpy-style, N−1 denominator) — the
|
||||
/// window the reference tables were generated from.
|
||||
fn hann() -> Vec<f32> {
|
||||
(0..FFT_SIZE)
|
||||
.map(|i| {
|
||||
0.5 - 0.5
|
||||
* (2.0 * std::f32::consts::PI * i as f32 / (FFT_SIZE - 1) as f32).cos()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Build the binary signature for ≤ WINDOW_SECS of 16 kHz mono PCM.
|
||||
pub fn signature(pcm: &[f32]) -> Vec<u8> {
|
||||
let mut samples: Vec<i16> = pcm
|
||||
.iter()
|
||||
.map(|&x| (x * 32768.0).clamp(-32768.0, 32767.0) as i16)
|
||||
.collect();
|
||||
let want = WINDOW_SECS * SAMPLE_RATE;
|
||||
if samples.len() < want {
|
||||
samples.resize(want, 0);
|
||||
} else {
|
||||
samples.truncate(want);
|
||||
}
|
||||
|
||||
let window = hann();
|
||||
let mut planner = RealFftPlanner::<f32>::new();
|
||||
let fft = planner.plan_fft_forward(FFT_SIZE);
|
||||
let mut rings = Rings {
|
||||
samples: [0i16; FFT_SIZE],
|
||||
samples_at: 0,
|
||||
ffts: vec![[0.0; BINS]; 256],
|
||||
ffts_at: 0,
|
||||
spread: vec![[0.0; BINS]; 256],
|
||||
spread_at: 0,
|
||||
passes: 0,
|
||||
peaks: Default::default(),
|
||||
};
|
||||
let mut re_in = vec![0.0f32; FFT_SIZE];
|
||||
let mut cx_out = vec![Complex::default(); BINS];
|
||||
|
||||
for chunk in samples.chunks_exact(HOP) {
|
||||
// Ring in the new hop, then window the reordered buffer.
|
||||
for (i, &s) in chunk.iter().enumerate() {
|
||||
rings.samples[(rings.samples_at + i) & (FFT_SIZE - 1)] = s;
|
||||
}
|
||||
rings.samples_at = (rings.samples_at + HOP) & (FFT_SIZE - 1);
|
||||
for i in 0..FFT_SIZE {
|
||||
re_in[i] =
|
||||
rings.samples[(i + rings.samples_at) & (FFT_SIZE - 1)] as f32 * window[i];
|
||||
}
|
||||
let _ = fft.process(&mut re_in, &mut cx_out);
|
||||
{
|
||||
let out = &mut rings.ffts[rings.ffts_at as usize];
|
||||
for (o, c) in out.iter_mut().zip(cx_out.iter()) {
|
||||
*o = ((c.re * c.re + c.im * c.im) / (1 << 17) as f32).max(1e-10);
|
||||
}
|
||||
}
|
||||
rings.ffts_at = rings.ffts_at.wrapping_add(1);
|
||||
|
||||
// Frequency- then time-domain peak spreading.
|
||||
{
|
||||
let src = rings.ffts[rings.ffts_at.wrapping_sub(1) as usize];
|
||||
let dst = &mut rings.spread[rings.spread_at as usize];
|
||||
dst.copy_from_slice(&src);
|
||||
for i in 0..=BINS - 3 {
|
||||
dst[i] = dst[i].max(dst[i + 1]).max(dst[i + 2]);
|
||||
}
|
||||
let snapshot = *dst;
|
||||
for former in [1u8, 3, 6] {
|
||||
let f = &mut rings.spread[rings.spread_at.wrapping_sub(former) as usize];
|
||||
for i in 0..BINS {
|
||||
f[i] = f[i].max(snapshot[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
rings.spread_at = rings.spread_at.wrapping_add(1);
|
||||
rings.passes += 1;
|
||||
|
||||
if rings.passes >= 46 {
|
||||
recognize_peaks(&mut rings);
|
||||
}
|
||||
}
|
||||
|
||||
encode(&rings.peaks, want as u32)
|
||||
}
|
||||
|
||||
fn recognize_peaks(r: &mut Rings) {
|
||||
let f46 = r.ffts[r.ffts_at.wrapping_sub(46) as usize];
|
||||
let s49 = r.spread[r.spread_at.wrapping_sub(49) as usize];
|
||||
for bin in 10..=BINS - 11 {
|
||||
if f46[bin] < 1.0 / 64.0 || f46[bin] < s49[bin - 1] {
|
||||
continue;
|
||||
}
|
||||
let mut neighbor_max = 0.0f32;
|
||||
for off in [-10i32, -7, -4, -3, 1, 2, 5, 8] {
|
||||
neighbor_max = neighbor_max.max(s49[(bin as i32 + off) as usize]);
|
||||
}
|
||||
if f46[bin] <= neighbor_max {
|
||||
continue;
|
||||
}
|
||||
let mut other_max = neighbor_max;
|
||||
for off in [
|
||||
-53i32, -45, 165, 172, 179, 186, 193, 200, 214, 221, 228, 235, 242, 249,
|
||||
] {
|
||||
let other = &r.spread[((r.spread_at as i32 + off) & 255) as usize];
|
||||
other_max = other_max.max(other[bin - 1]);
|
||||
}
|
||||
if f46[bin] <= other_max {
|
||||
continue;
|
||||
}
|
||||
let pass = r.passes - 46;
|
||||
let mag = f46[bin].ln().max(1.0 / 64.0) * 1477.3 + 6144.0;
|
||||
let before = f46[bin - 1].ln().max(1.0 / 64.0) * 1477.3 + 6144.0;
|
||||
let after = f46[bin + 1].ln().max(1.0 / 64.0) * 1477.3 + 6144.0;
|
||||
let variation_1 = mag * 2.0 - before - after;
|
||||
if variation_1 < 0.0 {
|
||||
continue;
|
||||
}
|
||||
let variation_2 = (after - before) * 32.0 / variation_1;
|
||||
let corrected = (bin as i32 * 64 + variation_2 as i32) as u16;
|
||||
let hz = corrected as f32 * (SAMPLE_RATE as f32 / 2.0 / 1024.0 / 64.0);
|
||||
let band = match hz as i32 {
|
||||
250..=519 => 0,
|
||||
520..=1449 => 1,
|
||||
1450..=3499 => 2,
|
||||
3500..=5500 => 3,
|
||||
_ => continue,
|
||||
};
|
||||
r.peaks[band].push(Peak {
|
||||
pass,
|
||||
magnitude: mag as u16,
|
||||
corrected_bin: corrected,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// The wire format: 48-byte header (magic, CRC-32 of everything after the
|
||||
/// first 8 bytes, payload size, 16 kHz marker, sample count + 0.24 s), a
|
||||
/// type-length echo, then per-band peak runs with delta-coded pass numbers.
|
||||
fn encode(peaks: &[Vec<Peak>; 4], number_samples: u32) -> Vec<u8> {
|
||||
let mut out: Vec<u8> = Vec::with_capacity(4096);
|
||||
let w32 = |out: &mut Vec<u8>, v: u32| out.extend_from_slice(&v.to_le_bytes());
|
||||
let w16 = |out: &mut Vec<u8>, v: u16| out.extend_from_slice(&v.to_le_bytes());
|
||||
|
||||
w32(&mut out, 0xcafe2580); // magic1
|
||||
w32(&mut out, 0); // crc32 backfilled
|
||||
w32(&mut out, 0); // size backfilled
|
||||
w32(&mut out, 0x94119c00); // magic2
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, 3u32 << 27); // 16 kHz
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, number_samples + (SAMPLE_RATE as f32 * 0.24) as u32);
|
||||
w32(&mut out, (15 << 19) + 0x40000);
|
||||
w32(&mut out, 0x40000000);
|
||||
w32(&mut out, 0); // size echo backfilled
|
||||
|
||||
for (band, list) in peaks.iter().enumerate() {
|
||||
if list.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut body: Vec<u8> = Vec::with_capacity(list.len() * 5);
|
||||
let mut pass = 0u32;
|
||||
for p in list {
|
||||
if p.pass - pass >= 255 {
|
||||
body.push(0xff);
|
||||
body.extend_from_slice(&p.pass.to_le_bytes());
|
||||
pass = p.pass;
|
||||
}
|
||||
body.push((p.pass - pass) as u8);
|
||||
body.extend_from_slice(&p.magnitude.to_le_bytes());
|
||||
body.extend_from_slice(&p.corrected_bin.to_le_bytes());
|
||||
pass = p.pass;
|
||||
}
|
||||
w32(&mut out, 0x60030040 + band as u32);
|
||||
w32(&mut out, body.len() as u32);
|
||||
out.extend_from_slice(&body);
|
||||
while out.len() % 4 != 0 {
|
||||
out.push(0);
|
||||
}
|
||||
}
|
||||
let _ = w16; // (kept for symmetry; u16 writes happen inside the body)
|
||||
|
||||
let size = (out.len() - 48) as u32;
|
||||
out[8..12].copy_from_slice(&size.to_le_bytes());
|
||||
out[52..56].copy_from_slice(&size.to_le_bytes());
|
||||
let mut hasher = crc32fast::Hasher::new();
|
||||
hasher.update(&out[8..]);
|
||||
let crc = hasher.finalize();
|
||||
out[4..8].copy_from_slice(&crc.to_le_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
// ---- the discovery request ----------------------------------------------------
|
||||
|
||||
/// Pseudo-UUID v4 from OS entropy via the stdlib hasher — unique enough for
|
||||
/// the tag ids the endpoint expects, no uuid/rand dependency.
|
||||
fn pseudo_uuid() -> String {
|
||||
use std::collections::hash_map::RandomState;
|
||||
use std::hash::{BuildHasher, Hasher};
|
||||
let mut parts = [0u64; 2];
|
||||
for p in parts.iter_mut() {
|
||||
let mut h = RandomState::new().build_hasher();
|
||||
h.write_u64(std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.subsec_nanos() as u64)
|
||||
.unwrap_or(0));
|
||||
*p = h.finish();
|
||||
}
|
||||
let b: Vec<u8> = parts.iter().flat_map(|p| p.to_le_bytes()).collect();
|
||||
format!(
|
||||
"{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-4{:01x}{:02x}-a{:01x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}",
|
||||
b[0], b[1], b[2], b[3], b[4], b[5], b[6] & 0xf, b[7], b[8] & 0xf, b[9], b[10], b[11], b[12], b[13], b[14], b[15]
|
||||
)
|
||||
}
|
||||
|
||||
/// Recognize one ≤12 s window of 16 kHz mono PCM. `Ok(None)` = the service
|
||||
/// answered and heard nothing it knows — an honest Unknown. `Err` = the
|
||||
/// request itself failed (network, rate limit) and the caller may retry
|
||||
/// later.
|
||||
pub async fn recognize_window(pcm16k: &[f32]) -> Result<Option<Recognition>, String> {
|
||||
let sig = signature(pcm16k);
|
||||
use base64::Engine as _;
|
||||
let uri = format!(
|
||||
"data:audio/vnd.shazam.sig;base64,{}",
|
||||
base64::engine::general_purpose::STANDARD.encode(&sig)
|
||||
);
|
||||
let sample_ms = (pcm16k.len().min(WINDOW_SECS * SAMPLE_RATE) * 1000 / SAMPLE_RATE) as u64;
|
||||
let ts = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
let body = serde_json::json!({
|
||||
"geolocation": { "altitude": 300, "latitude": 45, "longitude": 2 },
|
||||
"signature": { "samplems": sample_ms, "timestamp": ts, "uri": uri },
|
||||
"timestamp": ts,
|
||||
"timezone": "Europe/Paris",
|
||||
});
|
||||
let url = format!(
|
||||
"https://amp.shazam.com/discovery/v5/en/US/android/-/tag/{}/{}?sync=true&webv3=true&sampling=true&connected=&shazamapiversion=v3&sharehub=true&video=v3",
|
||||
pseudo_uuid().to_uppercase(),
|
||||
pseudo_uuid()
|
||||
);
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent("Dalvik/2.1.0 (Linux; U; Android 13; Pixel 7 Build/TQ3A.230901.001)")
|
||||
.timeout(std::time::Duration::from_secs(20))
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let resp = client
|
||||
.post(&url)
|
||||
.header("Content-Language", "en_US")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Shazam did not answer: {e}"))?;
|
||||
if resp.status().as_u16() == 429 {
|
||||
return Err("rate-limited by the recognition service".into());
|
||||
}
|
||||
let v: serde_json::Value = resp
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| format!("unreadable recognition response: {e}"))?;
|
||||
let Some(track) = v.get("track") else {
|
||||
return Ok(None);
|
||||
};
|
||||
let title = track
|
||||
.get("title")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let artist = track
|
||||
.get("subtitle")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
if title.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let key = track
|
||||
.get("key")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let offset_secs = v
|
||||
.get("matches")
|
||||
.and_then(|m| m.get(0))
|
||||
.and_then(|m| m.get("offset"))
|
||||
.and_then(serde_json::Value::as_f64);
|
||||
// Album rides in the metadata sections when present.
|
||||
let mut album = None;
|
||||
if let Some(sections) = track.get("sections").and_then(serde_json::Value::as_array) {
|
||||
for s in sections {
|
||||
if let Some(meta) = s.get("metadata").and_then(serde_json::Value::as_array) {
|
||||
for m in meta {
|
||||
if m.get("title").and_then(serde_json::Value::as_str) == Some("Album") {
|
||||
album = m
|
||||
.get("text")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let cover_url = track
|
||||
.get("images")
|
||||
.and_then(|i| i.get("coverart"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string);
|
||||
let isrc = track
|
||||
.get("isrc")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string);
|
||||
Ok(Some(Recognition {
|
||||
artist,
|
||||
title,
|
||||
key,
|
||||
offset_secs,
|
||||
album,
|
||||
cover_url,
|
||||
isrc,
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The packet must round out structurally: magics in place, CRC over
|
||||
/// bytes 8.., size echoed at 8 and 52, bands tagged 0x60030040+n.
|
||||
#[test]
|
||||
fn signature_packet_is_well_formed() {
|
||||
// A tone sweep gives the recognizer real peaks to encode.
|
||||
let pcm: Vec<f32> = (0..WINDOW_SECS * SAMPLE_RATE)
|
||||
.map(|i| {
|
||||
let t = i as f32 / SAMPLE_RATE as f32;
|
||||
(2.0 * std::f32::consts::PI * (440.0 + 80.0 * t) * t).sin() * 0.4
|
||||
})
|
||||
.collect();
|
||||
let sig = signature(&pcm);
|
||||
assert!(sig.len() > 64, "packet too small: {}", sig.len());
|
||||
assert_eq!(&sig[0..4], &0xcafe2580u32.to_le_bytes());
|
||||
assert_eq!(&sig[12..16], &0x94119c00u32.to_le_bytes());
|
||||
let size = u32::from_le_bytes(sig[8..12].try_into().unwrap());
|
||||
assert_eq!(size as usize, sig.len() - 48);
|
||||
assert_eq!(&sig[52..56], &size.to_le_bytes());
|
||||
let mut h = crc32fast::Hasher::new();
|
||||
h.update(&sig[8..]);
|
||||
assert_eq!(u32::from_le_bytes(sig[4..8].try_into().unwrap()), h.finalize());
|
||||
let band_tag = u32::from_le_bytes(sig[56..60].try_into().unwrap());
|
||||
assert!(
|
||||
(0x60030040..=0x60030043).contains(&band_tag),
|
||||
"first band tag {band_tag:#x}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A pure tone at 1 kHz must place its peaks in band 1 (520–1450 Hz).
|
||||
#[test]
|
||||
fn peaks_land_in_the_right_band() {
|
||||
let pcm: Vec<f32> = (0..WINDOW_SECS * SAMPLE_RATE)
|
||||
.map(|i| (2.0 * std::f32::consts::PI * 1000.0 * i as f32 / 16000.0).sin() * 0.4)
|
||||
.collect();
|
||||
let mut samples: Vec<i16> = pcm.iter().map(|&x| (x * 32768.0) as i16).collect();
|
||||
samples.truncate(WINDOW_SECS * SAMPLE_RATE);
|
||||
let sig = signature(&pcm);
|
||||
// Band 1's tag must appear; bands 0/2/3 may be empty or tiny.
|
||||
let mut found_band1 = false;
|
||||
let mut at = 56usize;
|
||||
while at + 8 <= sig.len() {
|
||||
let tag = u32::from_le_bytes(sig[at..at + 4].try_into().unwrap());
|
||||
let len = u32::from_le_bytes(sig[at + 4..at + 8].try_into().unwrap()) as usize;
|
||||
if tag == 0x60030041 && len > 50 {
|
||||
found_band1 = true;
|
||||
}
|
||||
at += 8 + len + ((4 - len % 4) % 4);
|
||||
}
|
||||
assert!(found_band1, "1 kHz peaks must dominate band 1");
|
||||
}
|
||||
|
||||
/// LIVE: recognize a 12 s window of a known library track against the
|
||||
/// real service. Ignored by default — network, unofficial endpoint —
|
||||
/// run explicitly: cargo test -p app shazam_live -- --ignored --nocapture
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn shazam_live_known_track() {
|
||||
let path = std::path::Path::new(
|
||||
"D:\\Music\\SoulSeek Music\\Panda Bear\\2015 - Panda Bear Meets The Grim Reaper\\1_SEQUENTIAL CIRCUITS.flac",
|
||||
);
|
||||
assert!(path.is_file(), "corpus track missing");
|
||||
let [l, r] = player::vocals::separation_input_44k1(path).unwrap();
|
||||
let mono = player::vocals::stem_pcm_16k(&l, &r);
|
||||
// A mid-track window, well into the singing.
|
||||
let start = 60 * SAMPLE_RATE;
|
||||
let window = &mono[start..start + WINDOW_SECS * SAMPLE_RATE];
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
let got = rt.block_on(recognize_window(window)).expect("request failed");
|
||||
let rec = got.expect("service heard nothing — signature likely malformed");
|
||||
println!(
|
||||
"LIVE RESULT: {} — {} (key {}, offset {:?}, album {:?}, isrc {:?})",
|
||||
rec.artist, rec.title, rec.key, rec.offset_secs, rec.album, rec.isrc
|
||||
);
|
||||
assert!(
|
||||
rec.title.to_lowercase().contains("sequential"),
|
||||
"wrong track: {} — {}",
|
||||
rec.artist,
|
||||
rec.title
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,499 @@
|
||||
//! Learned vocal separation for the lyric aligner — the "honest fix" the
|
||||
//! vocals.rs reducer comment promises: a real separator lifting the voice
|
||||
//! out of the band before CTC alignment, instead of band-pass pre-emphasis.
|
||||
//!
|
||||
//! Model: UVR-MDX-NET Voc_FT (KUIELab MDX-Net / TFC-TDF v2, the Ultimate
|
||||
//! Vocal Remover project's vocal fine-tune; MIT with credit). Chosen
|
||||
//! 2026-08-19 from the verified table in
|
||||
//! research/lyrics-vocal-separation/candidates.md: 63.7 MiB, ONNX opset 13 —
|
||||
//! under the inbox-WinML ceiling this machine PROVED by running the MMS
|
||||
//! aligner (opset 14) — static `input`/`output` [1,4,3072,256] f32, vocals
|
||||
//! SDR 9.64 on the MVSEP multisong benchmark (+1.5 dB over htdemucs at a
|
||||
//! fifth of the size; the RoFormer/Demucs ONNX exports are opset 17–23 and
|
||||
//! cannot load here as published).
|
||||
//!
|
||||
//! Posture is lyric_align.rs's exactly: WinML (no onnxruntime, no child
|
||||
//! processes), download only from the user's explicit Settings click, the
|
||||
//! session cached in a static — and NOTHING depends on this module. When
|
||||
//! the model is absent or anything fails, the aligner keeps its band-passed
|
||||
//! mixture input (the ctc-v4 behavior).
|
||||
//!
|
||||
//! The inference recipe is UVR's own non-overlap framing, kept bit-faithful
|
||||
//! so the model sees what it was trained on: stereo 44.1 kHz → per-chunk
|
||||
//! centered STFT (n_fft 7680, hop 1024, periodic Hann, reflect pad) → crop
|
||||
//! to 3072 bins → model (channels [L_re, L_im, R_re, R_im]) → vocal
|
||||
//! spectrogram → zero-pad the cropped bins back → WOLA iSTFT → trim
|
||||
//! n_fft/2 from each end → butt-join. Each chunk contributes 253,440 fresh
|
||||
//! samples (~5.75 s), so a four-minute track is ~42 model evaluations.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
use realfft::num_complex::Complex;
|
||||
use realfft::{ComplexToReal, RealFftPlanner, RealToComplex};
|
||||
use tauri::Emitter;
|
||||
use windows::{
|
||||
core::{Interface, HSTRING},
|
||||
AI::MachineLearning::{
|
||||
LearningModel, LearningModelBinding, LearningModelDevice, LearningModelDeviceKind,
|
||||
LearningModelSession, TensorFloat,
|
||||
},
|
||||
Win32::System::WinRT::{RoInitialize, RO_INIT_MULTITHREADED},
|
||||
};
|
||||
|
||||
const MODEL_URL: &str =
|
||||
"https://github.com/TRvlvr/model_repo/releases/download/all_public_uvr_models/UVR-MDX-NET-Voc_FT.onnx";
|
||||
/// Exact pinned size, shown in Settings and enforced after download — if the
|
||||
/// asset is ever republished with different bytes, refuse rather than trust.
|
||||
/// (Verified against the served bytes AND the file's own protobuf tail:
|
||||
/// opset 13, `input`/`output` [batch,4,3072,256]. Kim_Vocal_2 is 66,759,214
|
||||
/// — a first pin mistakenly used that sibling's size, and this guard caught
|
||||
/// it exactly as designed.)
|
||||
const MODEL_BYTES: u64 = 66_762_490;
|
||||
|
||||
// UVR MDX family constants (their model_data.json for this checkpoint:
|
||||
// dim_f 3072, dim_t 2^8, n_fft 7680; hop is the architecture's fixed 1024).
|
||||
const N_FFT: usize = 7_680;
|
||||
const HOP: usize = 1_024;
|
||||
const BINS: usize = N_FFT / 2 + 1; // 3841
|
||||
const DIM_F: usize = 3_072;
|
||||
const DIM_T: usize = 256;
|
||||
/// Samples one model window covers: 256 centered STFT frames at hop 1024.
|
||||
const CHUNK: usize = HOP * (DIM_T - 1); // 261,120
|
||||
/// STFT edge effects live inside half a window; UVR trims this from each
|
||||
/// chunk end and butt-joins the rest — no cross-fade, nothing to tune, and
|
||||
/// the kept region sits in full window-overlap coverage by construction.
|
||||
const TRIM: usize = N_FFT / 2; // 3840
|
||||
/// Fresh samples each chunk contributes (~5.75 s at 44.1 kHz).
|
||||
const GEN: usize = CHUNK - 2 * TRIM; // 253,440
|
||||
/// Flat f32 count of the model's input and output: [1, 4, 3072, 256].
|
||||
const TENSOR_LEN: usize = 4 * DIM_F * DIM_T;
|
||||
|
||||
struct Engine {
|
||||
session: LearningModelSession,
|
||||
/// Which device kind actually carried the smoke eval — for the log line.
|
||||
device: &'static str,
|
||||
}
|
||||
|
||||
static ENGINE: Mutex<Option<Engine>> = Mutex::new(None);
|
||||
|
||||
fn model_file(app: &tauri::AppHandle) -> Result<PathBuf, String> {
|
||||
Ok(crate::lyric_align::models_dir(app)?.join("uvr-mdx-voc-ft.onnx"))
|
||||
}
|
||||
|
||||
pub fn model_present(app: &tauri::AppHandle) -> bool {
|
||||
matches!(model_file(app), Ok(m) if m.is_file())
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub fn vocal_sep_status(app: tauri::AppHandle) -> serde_json::Value {
|
||||
let size = model_file(&app)
|
||||
.ok()
|
||||
.and_then(|p| std::fs::metadata(p).ok())
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0);
|
||||
serde_json::json!({
|
||||
"present": model_present(&app),
|
||||
"bytes": size,
|
||||
"downloadBytes": MODEL_BYTES,
|
||||
})
|
||||
}
|
||||
|
||||
/// Download the separation model. Runs ONLY from the user's explicit
|
||||
/// Settings click — the consent-before-download charter rule. Streams to a
|
||||
/// temp file, verifies the exact pinned byte count, renames atomically,
|
||||
/// emits `lyrics:sep-model-progress` along the way.
|
||||
#[tauri::command]
|
||||
pub async fn vocal_sep_download(app: tauri::AppHandle) -> Result<(), String> {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent("SoulStream/0.1 (vocal separation)")
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let dest = model_file(&app)?;
|
||||
let tmp = dest.with_extension("onnx.downloading");
|
||||
let mut response = client
|
||||
.get(MODEL_URL)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.error_for_status()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let total = response.content_length().unwrap_or(MODEL_BYTES);
|
||||
let mut file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?;
|
||||
let mut received: u64 = 0;
|
||||
let mut last_emit: u64 = 0;
|
||||
use std::io::Write;
|
||||
while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? {
|
||||
file.write_all(&chunk).map_err(|e| e.to_string())?;
|
||||
received += chunk.len() as u64;
|
||||
if received - last_emit > 2 * 1024 * 1024 {
|
||||
last_emit = received;
|
||||
let _ = app.emit(
|
||||
"lyrics:sep-model-progress",
|
||||
serde_json::json!({ "received": received, "total": total }),
|
||||
);
|
||||
}
|
||||
}
|
||||
drop(file);
|
||||
if received != MODEL_BYTES {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err(format!(
|
||||
"download size mismatch: {received} bytes, expected {MODEL_BYTES}"
|
||||
));
|
||||
}
|
||||
std::fs::rename(&tmp, &dest).map_err(|e| e.to_string())?;
|
||||
let _ = app.emit(
|
||||
"lyrics:sep-model-progress",
|
||||
serde_json::json!({ "received": received, "total": total, "done": true }),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_engine(app: &tauri::AppHandle) -> Result<(), String> {
|
||||
let mut guard = ENGINE.lock().unwrap();
|
||||
if guard.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
let model_path = model_file(app)?;
|
||||
if !model_path.is_file() {
|
||||
return Err("separation model not downloaded".into());
|
||||
}
|
||||
// Same WinRT posture as the aligner: initialize, tolerate an existing
|
||||
// apartment, never uninitialize someone else's.
|
||||
match unsafe { RoInitialize(RO_INIT_MULTITHREADED) } {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.code().0 as u32 == 0x8001_0106 => {}
|
||||
Err(e) => return Err(format!("WinRT init failed: {e}")),
|
||||
}
|
||||
let model =
|
||||
LearningModel::LoadFromFilePath(&HSTRING::from(model_path.to_string_lossy().as_ref()))
|
||||
.map_err(|e| format!("could not load separation model: {e}"))?;
|
||||
// DirectX first (DirectML — the discrete GPU when there is one): a track
|
||||
// is ~42 conv-net evaluations and the GPU turns minutes into seconds.
|
||||
// The zeros smoke eval proves the device can actually RUN the graph —
|
||||
// "loads but cannot execute" is a real DML failure mode — then Cpu is
|
||||
// the quiet fallback, so this works everywhere the aligner does.
|
||||
for (kind, name) in [
|
||||
(LearningModelDeviceKind::DirectX, "DirectX"),
|
||||
(LearningModelDeviceKind::Cpu, "Cpu"),
|
||||
] {
|
||||
let Ok(device) = LearningModelDevice::Create(kind) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(session) = LearningModelSession::CreateFromModelOnDevice(&model, &device) else {
|
||||
continue;
|
||||
};
|
||||
let zeros = vec![0.0f32; TENSOR_LEN];
|
||||
if eval_chunk(&session, &zeros).is_ok() {
|
||||
*guard = Some(Engine {
|
||||
session,
|
||||
device: name,
|
||||
});
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err("no WinML device could run the separation model".into())
|
||||
}
|
||||
|
||||
fn eval_chunk(session: &LearningModelSession, flat: &[f32]) -> Result<Vec<f32>, String> {
|
||||
let input = TensorFloat::CreateFromShapeArrayAndDataArray(
|
||||
&[1, 4, DIM_F as i64, DIM_T as i64],
|
||||
flat,
|
||||
)
|
||||
.map_err(|e| format!("could not create input tensor: {e}"))?;
|
||||
let binding = LearningModelBinding::CreateFromSession(session)
|
||||
.map_err(|e| format!("could not create binding: {e}"))?;
|
||||
binding
|
||||
.Bind(&HSTRING::from("input"), &input)
|
||||
.map_err(|e| format!("could not bind input: {e}"))?;
|
||||
let result = session
|
||||
.Evaluate(&binding, &HSTRING::from("vocal-sep"))
|
||||
.map_err(|e| format!("inference failed: {e}"))?;
|
||||
if !result.Succeeded().unwrap_or(false) {
|
||||
return Err("inference did not succeed".into());
|
||||
}
|
||||
let out: TensorFloat = result
|
||||
.Outputs()
|
||||
.and_then(|o| o.Lookup(&HSTRING::from("output")))
|
||||
.map_err(|e| format!("output missing: {e}"))?
|
||||
.cast()
|
||||
.map_err(|e| format!("output wrong type: {e}"))?;
|
||||
let values: Vec<f32> = out
|
||||
.GetAsVectorView()
|
||||
.map_err(|e| format!("could not read output: {e}"))?
|
||||
.into_iter()
|
||||
.collect();
|
||||
if values.len() != TENSOR_LEN {
|
||||
return Err(format!("unexpected output length {}", values.len()));
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
// ---- STFT plumbing (pure DSP — no WinML below this line) -------------------
|
||||
|
||||
struct Stft {
|
||||
r2c: Arc<dyn RealToComplex<f32>>,
|
||||
c2r: Arc<dyn ComplexToReal<f32>>,
|
||||
/// Periodic Hann — torch.hann_window's default, what the model trained on.
|
||||
window: Vec<f32>,
|
||||
/// WOLA denominator Σ w² over the padded chunk layout. The 256 frames
|
||||
/// tile the padded buffer exactly, so it is identical for every chunk.
|
||||
envelope: Vec<f32>,
|
||||
}
|
||||
|
||||
impl Stft {
|
||||
fn new() -> Self {
|
||||
let mut planner = RealFftPlanner::<f32>::new();
|
||||
let r2c = planner.plan_fft_forward(N_FFT);
|
||||
let c2r = planner.plan_fft_inverse(N_FFT);
|
||||
let window: Vec<f32> = (0..N_FFT)
|
||||
.map(|n| 0.5 - 0.5 * (2.0 * std::f32::consts::PI * n as f32 / N_FFT as f32).cos())
|
||||
.collect();
|
||||
let mut envelope = vec![1e-8f32; CHUNK + 2 * TRIM];
|
||||
for t in 0..DIM_T {
|
||||
for (n, w) in window.iter().enumerate() {
|
||||
envelope[t * HOP + n] += w * w;
|
||||
}
|
||||
}
|
||||
Stft {
|
||||
r2c,
|
||||
c2r,
|
||||
window,
|
||||
envelope,
|
||||
}
|
||||
}
|
||||
|
||||
/// Centered STFT of one CHUNK-sized segment: reflect-pad n_fft/2 each
|
||||
/// side (torch.stft center=True), frames at the hop — frame-major
|
||||
/// [t * BINS + f].
|
||||
fn forward(&self, chunk: &[f32]) -> Result<Vec<Complex<f32>>, String> {
|
||||
debug_assert_eq!(chunk.len(), CHUNK);
|
||||
let mut padded = vec![0.0f32; CHUNK + 2 * TRIM];
|
||||
padded[TRIM..TRIM + CHUNK].copy_from_slice(chunk);
|
||||
for i in 1..=TRIM {
|
||||
padded[TRIM - i] = chunk[i];
|
||||
padded[TRIM + CHUNK - 1 + i] = chunk[CHUNK - 1 - i];
|
||||
}
|
||||
let mut spec = vec![Complex::default(); DIM_T * BINS];
|
||||
let mut frame = vec![0.0f32; N_FFT];
|
||||
for t in 0..DIM_T {
|
||||
for (n, w) in self.window.iter().enumerate() {
|
||||
frame[n] = padded[t * HOP + n] * w;
|
||||
}
|
||||
self.r2c
|
||||
.process(&mut frame, &mut spec[t * BINS..(t + 1) * BINS])
|
||||
.map_err(|e| format!("stft failed: {e}"))?;
|
||||
}
|
||||
Ok(spec)
|
||||
}
|
||||
|
||||
/// WOLA inverse of a frame-major spectrogram back to CHUNK samples with
|
||||
/// the centered pad trimmed off — torch.istft's window-envelope
|
||||
/// normalization. Consumes the spectrogram (the FFT scratches it).
|
||||
fn inverse(&self, spec: &mut [Complex<f32>]) -> Result<Vec<f32>, String> {
|
||||
let mut acc = vec![0.0f32; CHUNK + 2 * TRIM];
|
||||
let mut frame = vec![0.0f32; N_FFT];
|
||||
let scale = 1.0 / N_FFT as f32; // realfft's c2r is unnormalized
|
||||
for t in 0..DIM_T {
|
||||
let row = &mut spec[t * BINS..(t + 1) * BINS];
|
||||
// The model emits free re/im everywhere; a real inverse needs
|
||||
// pure-real DC and Nyquist. Nyquist sits in the cropped-zero
|
||||
// region already, DC gets clamped here.
|
||||
row[0].im = 0.0;
|
||||
row[BINS - 1].im = 0.0;
|
||||
self.c2r
|
||||
.process(row, &mut frame)
|
||||
.map_err(|e| format!("istft failed: {e}"))?;
|
||||
for (n, w) in self.window.iter().enumerate() {
|
||||
acc[t * HOP + n] += frame[n] * scale * w;
|
||||
}
|
||||
}
|
||||
Ok((TRIM..TRIM + CHUNK)
|
||||
.map(|i| acc[i] / self.envelope[i])
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// Pack two frame-major spectrograms into the model's flat input — channel
|
||||
/// order [L_re, L_im, R_re, R_im] (UVR's reshape/permute order), frequency
|
||||
/// cropped to DIM_F.
|
||||
fn pack_input(spec_l: &[Complex<f32>], spec_r: &[Complex<f32>]) -> Vec<f32> {
|
||||
let mut flat = vec![0.0f32; TENSOR_LEN];
|
||||
for (c, spec) in [spec_l, spec_r].into_iter().enumerate() {
|
||||
for f in 0..DIM_F {
|
||||
for t in 0..DIM_T {
|
||||
let z = spec[t * BINS + f];
|
||||
flat[((2 * c) * DIM_F + f) * DIM_T + t] = z.re;
|
||||
flat[((2 * c + 1) * DIM_F + f) * DIM_T + t] = z.im;
|
||||
}
|
||||
}
|
||||
}
|
||||
flat
|
||||
}
|
||||
|
||||
/// Unpack the model's flat output back into two frame-major spectrograms.
|
||||
/// Bins at and above DIM_F stay zero — the crop restored as silence, which
|
||||
/// for a vocal stem is honest (nothing sung lives above ~17.6 kHz).
|
||||
fn unpack_output(flat: &[f32]) -> (Vec<Complex<f32>>, Vec<Complex<f32>>) {
|
||||
let mut left = vec![Complex::default(); DIM_T * BINS];
|
||||
let mut right = vec![Complex::default(); DIM_T * BINS];
|
||||
for (c, spec) in [&mut left, &mut right].into_iter().enumerate() {
|
||||
for f in 0..DIM_F {
|
||||
for t in 0..DIM_T {
|
||||
spec[t * BINS + f] = Complex::new(
|
||||
flat[((2 * c) * DIM_F + f) * DIM_T + t],
|
||||
flat[((2 * c + 1) * DIM_F + f) * DIM_T + t],
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
(left, right)
|
||||
}
|
||||
|
||||
/// Chunk a stereo 44.1 kHz track through `run` (the [1,4,3072,256] →
|
||||
/// same-shape spectrogram transform) and reassemble the returned stem.
|
||||
/// UVR's non-overlap framing: TRIM zeros lead the track so chunk c's kept
|
||||
/// middle lands exactly at c·GEN, edge effects fall in the trimmed halves,
|
||||
/// and segments butt-join.
|
||||
fn process_chunks(
|
||||
stft: &Stft,
|
||||
left: &[f32],
|
||||
right: &[f32],
|
||||
mut run: impl FnMut(&[f32]) -> Result<Vec<f32>, String>,
|
||||
) -> Result<(Vec<f32>, Vec<f32>), String> {
|
||||
let len = left.len().min(right.len());
|
||||
if len < HOP {
|
||||
return Err("audio too short to separate".into());
|
||||
}
|
||||
let chunks = len.div_ceil(GEN);
|
||||
let padded_len = TRIM + chunks * GEN + TRIM;
|
||||
let mut mix_l = vec![0.0f32; padded_len];
|
||||
let mut mix_r = vec![0.0f32; padded_len];
|
||||
mix_l[TRIM..TRIM + len].copy_from_slice(&left[..len]);
|
||||
mix_r[TRIM..TRIM + len].copy_from_slice(&right[..len]);
|
||||
let mut stem_l = vec![0.0f32; chunks * GEN];
|
||||
let mut stem_r = vec![0.0f32; chunks * GEN];
|
||||
for c in 0..chunks {
|
||||
let s = c * GEN;
|
||||
let spec_l = stft.forward(&mix_l[s..s + CHUNK])?;
|
||||
let spec_r = stft.forward(&mix_r[s..s + CHUNK])?;
|
||||
let out = run(&pack_input(&spec_l, &spec_r))?;
|
||||
if out.len() != TENSOR_LEN {
|
||||
return Err(format!("unexpected chunk output length {}", out.len()));
|
||||
}
|
||||
let (mut vl, mut vr) = unpack_output(&out);
|
||||
let wl = stft.inverse(&mut vl)?;
|
||||
let wr = stft.inverse(&mut vr)?;
|
||||
stem_l[s..s + GEN].copy_from_slice(&wl[TRIM..TRIM + GEN]);
|
||||
stem_r[s..s + GEN].copy_from_slice(&wr[TRIM..TRIM + GEN]);
|
||||
// Politeness between chunks — this thread must never contend with
|
||||
// the audio path for sustained stretches (the passage loop's rule).
|
||||
std::thread::sleep(std::time::Duration::from_millis(3));
|
||||
}
|
||||
stem_l.truncate(len);
|
||||
stem_r.truncate(len);
|
||||
Ok((stem_l, stem_r))
|
||||
}
|
||||
|
||||
/// Whole-file vocal stem as 16 kHz mono for the CTC aligner: decode →
|
||||
/// chunked WinML separation → mid fold + anti-alias resample (player crate).
|
||||
/// Fails soft by design — the caller falls back to the band-passed mixture.
|
||||
pub fn separate_vocals_16k(
|
||||
app: &tauri::AppHandle,
|
||||
path: &std::path::Path,
|
||||
) -> Result<Vec<f32>, String> {
|
||||
if !model_present(app) {
|
||||
return Err("separation model not downloaded".into());
|
||||
}
|
||||
let [left, right] = player::vocals::separation_input_44k1(path)?;
|
||||
ensure_engine(app)?;
|
||||
let guard = ENGINE.lock().unwrap();
|
||||
let engine = guard.as_ref().ok_or("engine unavailable")?;
|
||||
let stft = Stft::new();
|
||||
let started = Instant::now();
|
||||
let (stem_l, stem_r) = process_chunks(&stft, &left, &right, |flat| {
|
||||
eval_chunk(&engine.session, flat)
|
||||
})?;
|
||||
log::info!(
|
||||
"vocal_sep: {:.0}s of audio separated in {:.1}s on {}",
|
||||
stem_l.len() as f32 / 44_100.0,
|
||||
started.elapsed().as_secs_f32(),
|
||||
engine.device
|
||||
);
|
||||
Ok(player::vocals::stem_pcm_16k(&stem_l, &stem_r))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The WOLA plumbing alone — forward STFT straight back through the
|
||||
/// inverse, no crop — must be per-sample exact: the numerator and the
|
||||
/// envelope share the same window algebra, so any real error here is a
|
||||
/// tiling or normalization bug, not numerics.
|
||||
#[test]
|
||||
fn stft_wola_roundtrip_is_exact() {
|
||||
let chunk: Vec<f32> = (0..CHUNK)
|
||||
.map(|i| (2.0 * std::f32::consts::PI * 440.0 * i as f32 / 44_100.0).sin() * 0.5)
|
||||
.collect();
|
||||
let stft = Stft::new();
|
||||
let mut spec = stft.forward(&chunk).unwrap();
|
||||
let back = stft.inverse(&mut spec).unwrap();
|
||||
let worst = chunk
|
||||
.iter()
|
||||
.zip(&back)
|
||||
.map(|(a, b)| (a - b).abs())
|
||||
.fold(0.0f32, f32::max);
|
||||
assert!(worst < 1e-4, "un-cropped WOLA must be exact, worst {worst}");
|
||||
}
|
||||
|
||||
/// An identity model (output spectrogram = input spectrogram) must hand
|
||||
/// the track back through the whole STFT → crop → pack → unpack →
|
||||
/// zero-pad → iSTFT → trim → butt-join pipeline: proves the frame
|
||||
/// tiling, the chunk joins, and the layout round-trip all at once.
|
||||
/// Tolerance: the reflect pad puts a derivative kink at each chunk
|
||||
/// edge whose broadband energy reaches above bin 3072, and the model
|
||||
/// crop discards it — torch.stft + UVR's own framing lose exactly the
|
||||
/// same (~1% here); the model trained on that world.
|
||||
#[test]
|
||||
fn identity_model_reconstructs_across_chunk_joins() {
|
||||
let len = GEN + GEN / 2; // two chunks, one join, one truncation
|
||||
let track: Vec<f32> = (0..len)
|
||||
.map(|i| (2.0 * std::f32::consts::PI * 440.0 * i as f32 / 44_100.0).sin() * 0.5)
|
||||
.collect();
|
||||
let stft = Stft::new();
|
||||
let (out_l, out_r) =
|
||||
process_chunks(&stft, &track, &track, |flat| Ok(flat.to_vec())).unwrap();
|
||||
assert_eq!(out_l.len(), len);
|
||||
assert_eq!(out_r.len(), len);
|
||||
let mut worst = 0.0f32;
|
||||
for (a, b) in track.iter().zip(&out_l) {
|
||||
worst = worst.max((a - b).abs());
|
||||
}
|
||||
assert!(
|
||||
worst < 2e-2,
|
||||
"identity round-trip must reconstruct the tone, worst err {worst}"
|
||||
);
|
||||
// The join at GEN must not be a seam beyond that same crop error.
|
||||
let seam = (GEN - 50..GEN + 50)
|
||||
.map(|i| (track[i] - out_l[i]).abs())
|
||||
.fold(0.0f32, f32::max);
|
||||
assert!(seam < 2e-2, "butt-join must be seamless, err {seam}");
|
||||
}
|
||||
|
||||
/// The kept region of every chunk sits in full window-overlap coverage:
|
||||
/// the WOLA envelope there is flat and far from zero.
|
||||
#[test]
|
||||
fn envelope_is_stable_where_samples_are_kept() {
|
||||
let stft = Stft::new();
|
||||
let kept = &stft.envelope[2 * TRIM..2 * TRIM + GEN];
|
||||
let min = kept.iter().cloned().fold(f32::INFINITY, f32::min);
|
||||
let max = kept.iter().cloned().fold(0.0f32, f32::max);
|
||||
assert!(min > 2.0, "envelope must be well-conditioned, min {min}");
|
||||
assert!(
|
||||
max / min < 1.5,
|
||||
"envelope must be near-flat in the kept region, {min}..{max}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,652 @@
|
||||
//! YouTube DJ-set tracklist extraction — the evidence rungs the feasibility
|
||||
//! research ranks FIRST (creator- and viewer-declared data): the video's
|
||||
//! chapters, its description, and timestamped tracklists in top comments.
|
||||
//! (TODO-Feature-Docs/playlist-management-and-setlist-import, research
|
||||
//! §3/§15: "the high-value, reliable subset is metadata import … broader
|
||||
//! automatic recognition must remain optional".) No audio is fetched and
|
||||
//! nothing is recognized — this is metadata import, honest about it, and a
|
||||
//! set with no posted tracklist simply reports that instead of guessing.
|
||||
//!
|
||||
//! Everything runs in-process over reqwest (the no-child-processes posture):
|
||||
//! one GET of the watch page, plus at most one innertube POST for comments
|
||||
//! when the page itself carried no tracklist. The extracted lines feed the
|
||||
//! EXISTING pasted-tracklist pipeline (playlist_formats::parse_text →
|
||||
//! resolve_draft → review modal → commit), so every downstream behavior —
|
||||
//! setlist stamps, library matching, stream fallbacks, unresolved rows —
|
||||
//! is the one already shipped and tested.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
/// What one video yielded: identity for the draft header plus the chosen
|
||||
/// tracklist text and where it came from.
|
||||
pub struct SetlistFetch {
|
||||
pub title: String,
|
||||
pub author: String,
|
||||
pub tracklist: String,
|
||||
/// "chapters" | "description" | "a comment by …" — shown in the modal's
|
||||
/// warnings so the evidence rung is visible (research §3: "show which
|
||||
/// rung produced a result").
|
||||
pub provenance: String,
|
||||
pub warnings: Vec<String>,
|
||||
}
|
||||
|
||||
/// Minimum timestamped lines before a block counts as a tracklist — one or
|
||||
/// two stamps are a lyric quote or a "best drop" comment, not a setlist.
|
||||
const MIN_TRACKLIST_LINES: usize = 3;
|
||||
/// Top-level comments scanned for a tracklist before giving up.
|
||||
const MAX_COMMENTS_SCANNED: usize = 40;
|
||||
|
||||
// ---- URL handling -----------------------------------------------------------
|
||||
|
||||
/// Accepts watch/short/live/embed/youtu.be URLs (and a bare 11-char id) and
|
||||
/// returns the canonical video id.
|
||||
pub fn video_id_from_url(url: &str) -> Option<String> {
|
||||
let s = url.trim();
|
||||
let is_id = |c: &str| {
|
||||
c.len() == 11
|
||||
&& c.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
|
||||
};
|
||||
if is_id(s) {
|
||||
return Some(s.to_string());
|
||||
}
|
||||
let after_scheme = s.split("://").nth(1).unwrap_or(s);
|
||||
let (host, rest) = after_scheme.split_once('/')?;
|
||||
let host = host.split('@').next_back().unwrap_or(host).to_lowercase();
|
||||
let host = host.split(':').next().unwrap_or(&host);
|
||||
let path_q = rest.split('#').next().unwrap_or(rest);
|
||||
let (path, query) = match path_q.split_once('?') {
|
||||
Some((p, q)) => (p, Some(q)),
|
||||
None => (path_q, None),
|
||||
};
|
||||
let candidate = if host == "youtu.be" {
|
||||
path.split('/').next().map(str::to_string)
|
||||
} else if host.ends_with("youtube.com") || host.ends_with("youtube-nocookie.com") {
|
||||
let segs: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
|
||||
match segs.first().copied() {
|
||||
Some("watch") => query.and_then(|q| {
|
||||
q.split('&')
|
||||
.find_map(|kv| kv.strip_prefix("v=").map(str::to_string))
|
||||
}),
|
||||
Some("shorts") | Some("live") | Some("embed") | Some("v") => {
|
||||
segs.get(1).map(|s| s.to_string())
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
candidate.filter(|c| is_id(c))
|
||||
}
|
||||
|
||||
// ---- watch-page JSON slicing --------------------------------------------------
|
||||
|
||||
/// Slice the JSON object that begins at the first `{` after `needle`,
|
||||
/// brace-counting with string/escape awareness — the watch page embeds its
|
||||
/// data as `var ytInitialData = {...};` inside script tags, megabytes long.
|
||||
pub(crate) fn slice_json_after<'a>(html: &'a str, needle: &str) -> Option<&'a str> {
|
||||
let at = html.find(needle)?;
|
||||
let rest = &html[at + needle.len()..];
|
||||
let open = rest.find('{')?;
|
||||
let bytes = rest.as_bytes();
|
||||
let mut depth = 0usize;
|
||||
let mut in_str = false;
|
||||
let mut escaped = false;
|
||||
for (i, &b) in bytes.iter().enumerate().skip(open) {
|
||||
if in_str {
|
||||
if escaped {
|
||||
escaped = false;
|
||||
} else if b == b'\\' {
|
||||
escaped = true;
|
||||
} else if b == b'"' {
|
||||
in_str = false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match b {
|
||||
b'"' => in_str = true,
|
||||
b'{' => depth += 1,
|
||||
b'}' => {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
return Some(&rest[open..=i]);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Depth-first search for every object under `root` that contains `key`,
|
||||
/// applying `f` to the value. Youtube's trees are deep, volatile, and
|
||||
/// renderer-name-happy; searching for leaf keys survives their reshuffles
|
||||
/// better than hardcoded paths.
|
||||
fn walk<'a>(root: &'a Value, key: &str, f: &mut dyn FnMut(&'a Value)) {
|
||||
match root {
|
||||
Value::Object(map) => {
|
||||
if let Some(v) = map.get(key) {
|
||||
f(v);
|
||||
}
|
||||
for v in map.values() {
|
||||
walk(v, key, f);
|
||||
}
|
||||
}
|
||||
Value::Array(items) => {
|
||||
for v in items {
|
||||
walk(v, key, f);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// First string under any of `keys` in reading order — for label-ish leaves
|
||||
/// ("simpleText", nested runs) where youtube varies the wrapper.
|
||||
fn first_string(v: &Value, keys: &[&str]) -> Option<String> {
|
||||
match v {
|
||||
Value::String(s) => Some(s.clone()),
|
||||
_ => {
|
||||
for k in keys {
|
||||
let mut found = None;
|
||||
walk(v, k, &mut |x| {
|
||||
if found.is_none() {
|
||||
if let Some(s) = x.as_str() {
|
||||
found = Some(s.to_string());
|
||||
}
|
||||
}
|
||||
});
|
||||
if found.is_some() {
|
||||
return found;
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the three evidence rungs -------------------------------------------------
|
||||
|
||||
/// A line "looks timestamped" when it carries a `m:ss`-or-longer stamp plus
|
||||
/// some letters — the same acceptance the pasted-tracklist parser applies,
|
||||
/// reused for candidate SCORING before anything is parsed for real.
|
||||
pub(crate) fn is_timestamped_line(line: &str) -> bool {
|
||||
let has_letters = line.chars().filter(|c| c.is_alphabetic()).count() >= 2;
|
||||
if !has_letters {
|
||||
return false;
|
||||
}
|
||||
line.split(|c: char| c.is_whitespace() || c == '[' || c == ']' || c == '(' || c == ')')
|
||||
.any(|tok| {
|
||||
let t = tok.trim_matches(|c: char| !c.is_ascii_digit() && c != ':');
|
||||
crate::playlist_formats::parse_timestamp(t).is_some()
|
||||
})
|
||||
}
|
||||
|
||||
/// Chapters out of ytInitialData: every `chapterRenderer` carries a title
|
||||
/// and `timeRangeStartMillis`. Ordered, creator-authored — the top rung.
|
||||
pub(crate) fn chapters_tracklist(initial_data: &Value) -> Option<String> {
|
||||
let mut rows: Vec<(i64, String)> = Vec::new();
|
||||
walk(initial_data, "chapterRenderer", &mut |ch| {
|
||||
let ms = ch
|
||||
.get("timeRangeStartMillis")
|
||||
.and_then(Value::as_i64)
|
||||
.unwrap_or(-1);
|
||||
let title = ch
|
||||
.get("title")
|
||||
.and_then(|t| first_string(t, &["simpleText", "text"]));
|
||||
if ms >= 0 {
|
||||
if let Some(t) = title {
|
||||
rows.push((ms, t));
|
||||
}
|
||||
}
|
||||
});
|
||||
if rows.len() < MIN_TRACKLIST_LINES {
|
||||
return None;
|
||||
}
|
||||
rows.sort_by_key(|(ms, _)| *ms);
|
||||
rows.dedup();
|
||||
Some(
|
||||
rows.into_iter()
|
||||
.map(|(ms, t)| format!("{} {}", fmt_ts(ms), t))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n"),
|
||||
)
|
||||
}
|
||||
|
||||
/// The description's best block: the longest run of timestamped lines with
|
||||
/// at most one untimed line between them (headers like "Tracklist:" break
|
||||
/// nothing; prose paragraphs do).
|
||||
pub(crate) fn description_tracklist(description: &str) -> Option<String> {
|
||||
fn flush<'a>(
|
||||
current: &mut Vec<&'a str>,
|
||||
stamped: &mut usize,
|
||||
best: &mut Option<(usize, Vec<&'a str>)>,
|
||||
) {
|
||||
if *stamped >= MIN_TRACKLIST_LINES
|
||||
&& best.as_ref().map(|(n, _)| *stamped > *n).unwrap_or(true)
|
||||
{
|
||||
*best = Some((*stamped, current.clone()));
|
||||
}
|
||||
current.clear();
|
||||
*stamped = 0;
|
||||
}
|
||||
let mut best: Option<(usize, Vec<&str>)> = None;
|
||||
let mut current: Vec<&str> = Vec::new();
|
||||
let mut gap = 0usize;
|
||||
let mut stamped = 0usize;
|
||||
for line in description.lines() {
|
||||
let trimmed = line.trim();
|
||||
if is_timestamped_line(trimmed) {
|
||||
current.push(trimmed);
|
||||
stamped += 1;
|
||||
gap = 0;
|
||||
} else if !current.is_empty() && !trimmed.is_empty() && gap == 0 {
|
||||
// One stray untimed line rides along (a wrapped title, an "ID"),
|
||||
// a second breaks the block.
|
||||
gap = 1;
|
||||
} else if !current.is_empty() {
|
||||
flush(&mut current, &mut stamped, &mut best);
|
||||
gap = 0;
|
||||
}
|
||||
}
|
||||
flush(&mut current, &mut stamped, &mut best);
|
||||
best.map(|(_, lines)| lines.join("\n"))
|
||||
}
|
||||
|
||||
/// Comment texts out of an innertube `next` response. Youtube has shipped
|
||||
/// two shapes in the wild — legacy `contentText` runs and the newer
|
||||
/// `commentEntityPayload` — so both are walked; each yields (author, text).
|
||||
pub(crate) fn comment_texts(next_response: &Value) -> Vec<(String, String)> {
|
||||
let mut out: Vec<(String, String)> = Vec::new();
|
||||
walk(next_response, "commentEntityPayload", &mut |p| {
|
||||
let text = p
|
||||
.get("properties")
|
||||
.and_then(|pr| pr.get("content"))
|
||||
.and_then(|c| c.get("content"))
|
||||
.and_then(Value::as_str);
|
||||
let author = p
|
||||
.get("author")
|
||||
.and_then(|a| a.get("displayName"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("a commenter");
|
||||
if let Some(t) = text {
|
||||
out.push((author.to_string(), t.to_string()));
|
||||
}
|
||||
});
|
||||
walk(next_response, "commentRenderer", &mut |c| {
|
||||
let text = c
|
||||
.get("contentText")
|
||||
.map(|ct| collect_runs(ct))
|
||||
.filter(|t| !t.is_empty());
|
||||
let author = c
|
||||
.get("authorText")
|
||||
.and_then(|a| first_string(a, &["simpleText"]))
|
||||
.unwrap_or_else(|| "a commenter".into());
|
||||
if let Some(t) = text {
|
||||
out.push((author, t));
|
||||
}
|
||||
});
|
||||
out.truncate(MAX_COMMENTS_SCANNED);
|
||||
out
|
||||
}
|
||||
|
||||
/// Flatten a `runs`/`simpleText` rich-text node into plain text, preserving
|
||||
/// the run boundaries' own newlines (timestamps arrive as separate runs).
|
||||
fn collect_runs(v: &Value) -> String {
|
||||
if let Some(s) = v.get("simpleText").and_then(Value::as_str) {
|
||||
return s.to_string();
|
||||
}
|
||||
let mut out = String::new();
|
||||
walk(v, "text", &mut |t| {
|
||||
if let Some(s) = t.as_str() {
|
||||
out.push_str(s);
|
||||
}
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
/// The comments continuation token for the initial `next` call, discovered
|
||||
/// generically: the token whose enclosing area names the comment section.
|
||||
pub(crate) fn comments_token(initial_data: &Value) -> Option<String> {
|
||||
// The comment section's continuation is the one inside an
|
||||
// itemSectionRenderer whose identifier mentions comments.
|
||||
let mut token: Option<String> = None;
|
||||
walk(initial_data, "itemSectionRenderer", &mut |sec| {
|
||||
if token.is_some() {
|
||||
return;
|
||||
}
|
||||
let ident = sec
|
||||
.get("sectionIdentifier")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("");
|
||||
if !ident.contains("comment") {
|
||||
return;
|
||||
}
|
||||
walk(sec, "continuationCommand", &mut |cc| {
|
||||
if token.is_none() {
|
||||
if let Some(t) = cc.get("token").and_then(Value::as_str) {
|
||||
token = Some(t.to_string());
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
token
|
||||
}
|
||||
|
||||
fn fmt_ts(ms: i64) -> String {
|
||||
let total = ms / 1000;
|
||||
let h = total / 3600;
|
||||
let m = (total % 3600) / 60;
|
||||
let s = total % 60;
|
||||
if h > 0 {
|
||||
format!("{h}:{m:02}:{s:02}")
|
||||
} else {
|
||||
format!("{m}:{s:02}")
|
||||
}
|
||||
}
|
||||
|
||||
/// Innertube client version, read from the page so the comments call speaks
|
||||
/// the same dialect the page was served with; a dated fallback otherwise.
|
||||
pub(crate) fn client_version(html: &str) -> String {
|
||||
slice_after_quote(html, "\"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"")
|
||||
.unwrap_or("2.20260101.00.00")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn slice_after_quote<'a>(html: &'a str, needle: &str) -> Option<&'a str> {
|
||||
let at = html.find(needle)?;
|
||||
let rest = &html[at + needle.len()..];
|
||||
rest.split('"').next()
|
||||
}
|
||||
|
||||
// ---- the fetch ----------------------------------------------------------------
|
||||
|
||||
/// Just the video's identity (title, author) — for the LISTENER's draft
|
||||
/// header, whose sets typically have NO posted tracklist (that's why they
|
||||
/// get listened to): the full `fetch` returns Err for those and took the
|
||||
/// title down with it (Kyle's first own-set run was named by raw video id).
|
||||
pub async fn fetch_title(url: &str) -> Option<(String, String)> {
|
||||
let id = video_id_from_url(url)?;
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
|
||||
Chrome/126.0.0.0 Safari/537.36",
|
||||
)
|
||||
.timeout(std::time::Duration::from_secs(15))
|
||||
.build()
|
||||
.ok()?;
|
||||
let html = client
|
||||
.get(format!("https://www.youtube.com/watch?v={id}&hl=en"))
|
||||
.header("Cookie", "CONSENT=YES+1; SOCS=CAI")
|
||||
.send()
|
||||
.await
|
||||
.ok()?
|
||||
.text()
|
||||
.await
|
||||
.ok()?;
|
||||
let player: Value = slice_json_after(&html, "ytInitialPlayerResponse")
|
||||
.and_then(|s| serde_json::from_str(s).ok())?;
|
||||
let details = player.get("videoDetails")?;
|
||||
let title = details.get("title").and_then(Value::as_str)?.to_string();
|
||||
let author = details
|
||||
.get("author")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
Some((title, author))
|
||||
}
|
||||
|
||||
/// Fetch one video's metadata and pick its best tracklist. At most two
|
||||
/// requests; every rung failure downgrades to the next with a warning
|
||||
/// instead of failing the whole import (research §12).
|
||||
pub async fn fetch(url: &str) -> Result<SetlistFetch, String> {
|
||||
let id = video_id_from_url(url).ok_or("That doesn't look like a YouTube video link")?;
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
|
||||
Chrome/126.0.0.0 Safari/537.36",
|
||||
)
|
||||
.timeout(std::time::Duration::from_secs(25))
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let html = client
|
||||
.get(format!("https://www.youtube.com/watch?v={id}&hl=en"))
|
||||
.header("Cookie", "CONSENT=YES+1; SOCS=CAI")
|
||||
.header("Accept-Language", "en")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("YouTube did not answer: {e}"))?
|
||||
.error_for_status()
|
||||
.map_err(|e| format!("YouTube refused the page: {e}"))?
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let player: Value = slice_json_after(&html, "ytInitialPlayerResponse")
|
||||
.and_then(|s| serde_json::from_str(s).ok())
|
||||
.ok_or("Could not read the video's data — is the link public?")?;
|
||||
// Unavailable/private/region-locked videos still serve a page — with a
|
||||
// playability error instead of details. Say that plainly (research §8).
|
||||
let playability = player
|
||||
.get("playabilityStatus")
|
||||
.and_then(|p| p.get("status"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("OK");
|
||||
if playability != "OK" && player.get("videoDetails").is_none() {
|
||||
let reason = player
|
||||
.get("playabilityStatus")
|
||||
.and_then(|p| p.get("reason"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("it may be private, removed, or region-locked");
|
||||
return Err(format!("YouTube says this video is unavailable — {reason}"));
|
||||
}
|
||||
let details = player.get("videoDetails").cloned().unwrap_or(Value::Null);
|
||||
let title = details
|
||||
.get("title")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("YouTube set")
|
||||
.to_string();
|
||||
let author = details
|
||||
.get("author")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let description = details
|
||||
.get("shortDescription")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let initial_data: Value = slice_json_after(&html, "ytInitialData")
|
||||
.and_then(|s| serde_json::from_str(s).ok())
|
||||
.unwrap_or(Value::Null);
|
||||
|
||||
let mut warnings = Vec::new();
|
||||
|
||||
// Rung 1: chapters. Rung 2: the description's timestamped block.
|
||||
if let Some(list) = chapters_tracklist(&initial_data) {
|
||||
return Ok(SetlistFetch {
|
||||
title,
|
||||
author,
|
||||
tracklist: list,
|
||||
provenance: "the video's chapters".into(),
|
||||
warnings,
|
||||
});
|
||||
}
|
||||
if let Some(list) = description_tracklist(&description) {
|
||||
return Ok(SetlistFetch {
|
||||
title,
|
||||
author,
|
||||
tracklist: list,
|
||||
provenance: "the video description".into(),
|
||||
warnings,
|
||||
});
|
||||
}
|
||||
|
||||
// Rung 3: top comments — one innertube call, failure-soft.
|
||||
match comments_token(&initial_data) {
|
||||
None => warnings.push("The video page exposed no comments to scan.".into()),
|
||||
Some(token) => {
|
||||
let version = client_version(&html);
|
||||
let body = serde_json::json!({
|
||||
"context": { "client": {
|
||||
"clientName": "WEB", "clientVersion": version,
|
||||
"hl": "en", "gl": "US",
|
||||
}},
|
||||
"continuation": token,
|
||||
});
|
||||
let resp = client
|
||||
.post("https://www.youtube.com/youtubei/v1/next?prettyPrint=false")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await;
|
||||
match resp {
|
||||
Err(e) => warnings.push(format!("Comments could not be scanned: {e}")),
|
||||
Ok(r) => match r.json::<Value>().await {
|
||||
Err(e) => warnings.push(format!("Comments could not be read: {e}")),
|
||||
Ok(next) => {
|
||||
let mut best: Option<(usize, String, String)> = None;
|
||||
for (who, text) in comment_texts(&next) {
|
||||
let stamped =
|
||||
text.lines().filter(|l| is_timestamped_line(l.trim())).count();
|
||||
if stamped >= MIN_TRACKLIST_LINES
|
||||
&& best.as_ref().map(|(n, _, _)| stamped > *n).unwrap_or(true)
|
||||
{
|
||||
let block: Vec<&str> = text
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| is_timestamped_line(l))
|
||||
.collect();
|
||||
best = Some((stamped, who.clone(), block.join("\n")));
|
||||
}
|
||||
}
|
||||
if let Some((n, who, list)) = best {
|
||||
warnings.push(format!(
|
||||
"Tracklist found in a comment by {who} ({n} timestamped lines) — viewer-supplied, worth a skim."
|
||||
));
|
||||
return Ok(SetlistFetch {
|
||||
title,
|
||||
author,
|
||||
tracklist: list,
|
||||
provenance: format!("a comment by {who}"),
|
||||
warnings,
|
||||
});
|
||||
}
|
||||
warnings.push("No comment carried a timestamped tracklist.".into());
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(format!(
|
||||
"No tracklist found on \"{title}\" — no chapters, no timestamped description, and {} \
|
||||
Paste one into the tracklist door if you have it.",
|
||||
if warnings.iter().any(|w| w.contains("comment")) {
|
||||
"none of the top comments carry one."
|
||||
} else {
|
||||
"the comments could not be scanned."
|
||||
}
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn video_ids_from_every_url_shape() {
|
||||
for (url, want) in [
|
||||
("https://www.youtube.com/watch?v=dQw4w9WgXcQ", Some("dQw4w9WgXcQ")),
|
||||
("https://youtu.be/dQw4w9WgXcQ?t=42", Some("dQw4w9WgXcQ")),
|
||||
("https://www.youtube.com/live/abc-DEF_123?feature=share", Some("abc-DEF_123")),
|
||||
("https://music.youtube.com/watch?v=dQw4w9WgXcQ&list=RD", Some("dQw4w9WgXcQ")),
|
||||
("www.youtube.com/shorts/abc-DEF_123", Some("abc-DEF_123")),
|
||||
("dQw4w9WgXcQ", Some("dQw4w9WgXcQ")),
|
||||
("https://example.com/watch?v=dQw4w9WgXcQ", None),
|
||||
("https://www.youtube.com/watch?v=tooShort", None),
|
||||
] {
|
||||
assert_eq!(
|
||||
video_id_from_url(url).as_deref(),
|
||||
want,
|
||||
"{url} must yield {want:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_slicer_handles_braces_in_strings() {
|
||||
let html = r#"<script>var ytInitialData = {"a":"br{ace \" quote","b":{"c":1}};</script>"#;
|
||||
let json = slice_json_after(html, "ytInitialData").unwrap();
|
||||
let v: Value = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(v["b"]["c"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chapters_walk_and_format() {
|
||||
let data: Value = serde_json::from_str(
|
||||
r#"{"markersMap":[{"value":{"chapters":[
|
||||
{"chapterRenderer":{"title":{"simpleText":"Bicep - Glue"},"timeRangeStartMillis":0}},
|
||||
{"chapterRenderer":{"title":{"simpleText":"Overmono - So U Kno"},"timeRangeStartMillis":330000}},
|
||||
{"chapterRenderer":{"title":{"simpleText":"Joy O - Sin Palta"},"timeRangeStartMillis":3900000}}
|
||||
]}}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let list = chapters_tracklist(&data).unwrap();
|
||||
assert_eq!(
|
||||
list,
|
||||
"0:00 Bicep - Glue\n5:30 Overmono - So U Kno\n1:05:00 Joy O - Sin Palta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn description_block_beats_prose_and_survives_headers() {
|
||||
let desc = "Recorded live at the warehouse.\nBig thanks to everyone!\n\nTracklist:\n00:00 Artist One - Opener\n04:12 Artist Two - Middle (Remix)\nID - ID\n12:44 Artist Three - Closer\n\nFollow me on socials at 10:00 tonight";
|
||||
let block = description_tracklist(desc).unwrap();
|
||||
assert_eq!(
|
||||
block,
|
||||
"00:00 Artist One - Opener\n04:12 Artist Two - Middle (Remix)\n12:44 Artist Three - Closer"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn too_few_stamps_is_no_tracklist() {
|
||||
assert!(description_tracklist("00:01 hi\nsome words\n03:30 drop!").is_none());
|
||||
assert!(!is_timestamped_line("10000 subscribers!"));
|
||||
assert!(is_timestamped_line("1:23:45 Artist - Title"));
|
||||
assert!(is_timestamped_line("Artist - Title [04:20]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_payloads_both_shapes() {
|
||||
let next: Value = serde_json::from_str(
|
||||
r#"{"frameworkUpdates":{"entityBatchUpdate":{"mutations":[
|
||||
{"payload":{"commentEntityPayload":{
|
||||
"properties":{"content":{"content":"00:00 A - B\n03:00 C - D\n06:00 E - F"}},
|
||||
"author":{"displayName":"@digger"}}}}
|
||||
]}},
|
||||
"old":[{"commentRenderer":{
|
||||
"authorText":{"simpleText":"@legacy"},
|
||||
"contentText":{"runs":[{"text":"great set"}]}}}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let texts = comment_texts(&next);
|
||||
assert_eq!(texts.len(), 2);
|
||||
assert_eq!(texts[0].0, "@digger");
|
||||
assert!(texts[0].1.contains("03:00 C - D"));
|
||||
assert_eq!(texts[1], ("@legacy".into(), "great set".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comments_token_found_by_section_identifier() {
|
||||
let data: Value = serde_json::from_str(
|
||||
r#"{"contents":[
|
||||
{"itemSectionRenderer":{"sectionIdentifier":"related-items",
|
||||
"contents":[{"continuationItemRenderer":{"continuationEndpoint":{"continuationCommand":{"token":"WRONG"}}}}]}},
|
||||
{"itemSectionRenderer":{"sectionIdentifier":"comment-item-section",
|
||||
"contents":[{"continuationItemRenderer":{"continuationEndpoint":{"continuationCommand":{"token":"RIGHT"}}}}]}}
|
||||
]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(comments_token(&data).as_deref(), Some("RIGHT"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,493 @@
|
||||
//! yt-dlp as an OPTIONAL external tool — the acquisition adapter for the
|
||||
//! DJ-set listener, Kyle's explicit call 2026-08-19 ("okay implement
|
||||
//! yt-dlp"), relaxing the no-child-processes posture for THIS feature
|
||||
//! only: anonymous in-process YouTube audio is dead upstream (PO-token
|
||||
//! wall, verified same day), and the download arms race belongs to the
|
||||
//! people who fight it full-time.
|
||||
//!
|
||||
//! Posture kept from the model downloads: the binary arrives ONLY from an
|
||||
//! explicit click, from yt-dlp's official GitHub release (a moving target
|
||||
//! by design — self-updating is why it keeps working, so there is no byte
|
||||
//! pin; the SOURCE is pinned instead). A PATH-installed yt-dlp is
|
||||
//! respected before the managed copy. Audio lands in a per-video cache so
|
||||
//! re-analysis is free (research §11), fetched as the m4a/AAC stream the
|
||||
//! app's own decoder reads — no ffmpeg, no conversion step.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::Stdio;
|
||||
|
||||
use tauri::{Emitter, Manager};
|
||||
|
||||
/// The NIGHTLY channel, deliberately: as of 2026-08 the stable release
|
||||
/// 403s on YouTube (extractor drift — burned live TWICE the same evening,
|
||||
/// the second time when the update button itself overwrote a working
|
||||
/// nightly with stale stable). Nightly is the yt-dlp project's own
|
||||
/// recommendation for extractor freshness; against YouTube it is the only
|
||||
/// channel that works.
|
||||
const RELEASE_URL: &str =
|
||||
"https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest/download/yt-dlp.exe";
|
||||
/// Deno's official Windows build — the RECOMMENDED EJS runtime (nightly
|
||||
/// yt-dlp + deno is the one combo proven against YouTube, 2026-08-19).
|
||||
/// Same posture as yt-dlp: no byte pin, the SOURCE is pinned; consent is
|
||||
/// the click; the tool's own --version is the sanity check.
|
||||
const DENO_RELEASE_URL: &str =
|
||||
"https://github.com/denoland/deno/releases/latest/download/deno-x86_64-pc-windows-msvc.zip";
|
||||
/// Windows: no console window for spawned children.
|
||||
const CREATE_NO_WINDOW: u32 = 0x0800_0000;
|
||||
|
||||
fn tools_dir(app: &tauri::AppHandle) -> Result<PathBuf, String> {
|
||||
let dir = app
|
||||
.path()
|
||||
.app_data_dir()
|
||||
.map_err(|e| e.to_string())?
|
||||
.join("tools");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
fn managed_exe(app: &tauri::AppHandle) -> Result<PathBuf, String> {
|
||||
Ok(tools_dir(app)?.join("yt-dlp.exe"))
|
||||
}
|
||||
|
||||
/// The yt-dlp this machine would use: the managed copy first, then PATH.
|
||||
pub fn locate(app: &tauri::AppHandle) -> Option<PathBuf> {
|
||||
if let Ok(p) = managed_exe(app) {
|
||||
if p.is_file() {
|
||||
return Some(p);
|
||||
}
|
||||
}
|
||||
let path_var = std::env::var_os("PATH")?;
|
||||
std::env::split_paths(&path_var)
|
||||
.map(|d| d.join("yt-dlp.exe"))
|
||||
.find(|c| c.is_file())
|
||||
}
|
||||
|
||||
fn tool_version(exe: &PathBuf) -> Option<String> {
|
||||
use std::os::windows::process::CommandExt;
|
||||
let out = std::process::Command::new(exe)
|
||||
.arg("--version")
|
||||
.creation_flags(CREATE_NO_WINDOW)
|
||||
.stdin(Stdio::null())
|
||||
.output()
|
||||
.ok()?;
|
||||
let v = String::from_utf8_lossy(&out.stdout).trim().to_string();
|
||||
(!v.is_empty()).then_some(v)
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub fn ytdlp_status(app: tauri::AppHandle) -> serde_json::Value {
|
||||
// The runtime rides along so the modal can offer the deno download
|
||||
// exactly when it matters: nothing found, or a PATH node/bun that EJS
|
||||
// may reject (the home laptop's node v19 lesson).
|
||||
let runtime = js_runtime(&app);
|
||||
match locate(&app) {
|
||||
Some(exe) => serde_json::json!({
|
||||
"present": true,
|
||||
"path": exe.to_string_lossy(),
|
||||
"managed": managed_exe(&app).map(|m| m == exe).unwrap_or(false),
|
||||
"version": tool_version(&exe),
|
||||
"runtime": runtime,
|
||||
}),
|
||||
None => serde_json::json!({ "present": false, "runtime": runtime }),
|
||||
}
|
||||
}
|
||||
|
||||
/// Download (or update) the managed yt-dlp from the official release.
|
||||
/// Consent = the click that invoked this. Streams to a temp name, renames
|
||||
/// atomically, then reports the tool's own --version as the sanity check —
|
||||
/// there is no byte pin to check against, the release moves by design.
|
||||
#[tauri::command]
|
||||
pub async fn ytdlp_download(app: tauri::AppHandle) -> Result<serde_json::Value, String> {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent("SoulStream/0.1 (setlist tools)")
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let dest = managed_exe(&app)?;
|
||||
let tmp = dest.with_extension("exe.downloading");
|
||||
let mut response = client
|
||||
.get(RELEASE_URL)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.error_for_status()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let total = response.content_length().unwrap_or(0);
|
||||
let mut file = std::fs::File::create(&tmp).map_err(|e| e.to_string())?;
|
||||
let mut received: u64 = 0;
|
||||
let mut last_emit: u64 = 0;
|
||||
use std::io::Write;
|
||||
while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? {
|
||||
file.write_all(&chunk).map_err(|e| e.to_string())?;
|
||||
received += chunk.len() as u64;
|
||||
if received - last_emit > 512 * 1024 {
|
||||
last_emit = received;
|
||||
let _ = app.emit(
|
||||
"playlist:scan-progress",
|
||||
serde_json::json!({ "stage": "tool", "received": received, "total": total }),
|
||||
);
|
||||
}
|
||||
}
|
||||
drop(file);
|
||||
if received < 2 * 1024 * 1024 {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err(format!("suspiciously small download ({received} bytes) — refused"));
|
||||
}
|
||||
std::fs::rename(&tmp, &dest).map_err(|e| e.to_string())?;
|
||||
let version = match tool_version(&dest) {
|
||||
Some(v) => v,
|
||||
None => {
|
||||
let _ = std::fs::remove_file(&dest);
|
||||
return Err("downloaded tool did not answer --version — removed".into());
|
||||
}
|
||||
};
|
||||
Ok(serde_json::json!({ "version": version }))
|
||||
}
|
||||
|
||||
fn deno_version(exe: &PathBuf) -> Option<String> {
|
||||
use std::os::windows::process::CommandExt;
|
||||
let out = std::process::Command::new(exe)
|
||||
.arg("--version")
|
||||
.creation_flags(CREATE_NO_WINDOW)
|
||||
.stdin(Stdio::null())
|
||||
.output()
|
||||
.ok()?;
|
||||
// First line reads "deno 2.9.5 (stable, release, x86_64-pc-windows-msvc)".
|
||||
let text = String::from_utf8_lossy(&out.stdout);
|
||||
let v = text.lines().next()?.split_whitespace().nth(1)?.to_string();
|
||||
(!v.is_empty()).then_some(v)
|
||||
}
|
||||
|
||||
/// Minimal single-entry ZIP extraction, central-directory-driven (the
|
||||
/// authoritative record even when local headers defer their sizes to data
|
||||
/// descriptors). Only what deno's release zip needs — stored or deflate,
|
||||
/// CRC-verified — hand-rolled over the flate2/crc32fast already in the
|
||||
/// tree rather than growing a zip dependency for one file.
|
||||
fn extract_zip_entry(bytes: &[u8], want: &str) -> Result<Vec<u8>, String> {
|
||||
const EOCD: [u8; 4] = [0x50, 0x4b, 0x05, 0x06];
|
||||
const CENTRAL: [u8; 4] = [0x50, 0x4b, 0x01, 0x02];
|
||||
const LOCAL: [u8; 4] = [0x50, 0x4b, 0x03, 0x04];
|
||||
let u16at = |at: usize| u16::from_le_bytes([bytes[at], bytes[at + 1]]) as usize;
|
||||
let u32at = |at: usize| u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap()) as usize;
|
||||
if bytes.len() < 22 {
|
||||
return Err("archive too small to be a zip".into());
|
||||
}
|
||||
let eocd = (bytes.len().saturating_sub(65_557)..=bytes.len() - 22)
|
||||
.rev()
|
||||
.find(|&i| bytes[i..i + 4] == EOCD)
|
||||
.ok_or("no zip directory found")?;
|
||||
let mut at = u32at(eocd + 16);
|
||||
for _ in 0..u16at(eocd + 10) {
|
||||
if at + 46 > bytes.len() || bytes[at..at + 4] != CENTRAL {
|
||||
return Err("malformed zip directory".into());
|
||||
}
|
||||
let method = u16at(at + 10);
|
||||
let crc = u32at(at + 16) as u32;
|
||||
let comp_size = u32at(at + 20);
|
||||
let name_len = u16at(at + 28);
|
||||
let extra_len = u16at(at + 30);
|
||||
let comment_len = u16at(at + 32);
|
||||
let local_at = u32at(at + 42);
|
||||
let name = bytes
|
||||
.get(at + 46..at + 46 + name_len)
|
||||
.ok_or("malformed zip directory")?;
|
||||
if name == want.as_bytes() {
|
||||
if local_at + 30 > bytes.len() || bytes[local_at..local_at + 4] != LOCAL {
|
||||
return Err("malformed zip entry".into());
|
||||
}
|
||||
let data_at = local_at + 30 + u16at(local_at + 26) + u16at(local_at + 28);
|
||||
let data = bytes
|
||||
.get(data_at..data_at + comp_size)
|
||||
.ok_or("zip entry data out of range")?;
|
||||
let out = match method {
|
||||
0 => data.to_vec(),
|
||||
8 => {
|
||||
use std::io::Read;
|
||||
let mut out = Vec::new();
|
||||
flate2::read::DeflateDecoder::new(data)
|
||||
.read_to_end(&mut out)
|
||||
.map_err(|e| format!("zip inflate failed: {e}"))?;
|
||||
out
|
||||
}
|
||||
m => return Err(format!("unsupported zip method {m}")),
|
||||
};
|
||||
let mut hasher = crc32fast::Hasher::new();
|
||||
hasher.update(&out);
|
||||
if hasher.finalize() != crc {
|
||||
return Err("zip CRC mismatch — corrupted download, refused".into());
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
at += 46 + name_len + extra_len + comment_len;
|
||||
}
|
||||
Err(format!("{want} not found in the archive"))
|
||||
}
|
||||
|
||||
/// Download the managed Deno runtime from denoland's official release and
|
||||
/// unpack the one file that matters into the tools dir. Progress rides the
|
||||
/// same event stream as the yt-dlp download, stage "runtime".
|
||||
#[tauri::command]
|
||||
pub async fn deno_download(app: tauri::AppHandle) -> Result<serde_json::Value, String> {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent("SoulStream/0.1 (setlist tools)")
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut response = client
|
||||
.get(DENO_RELEASE_URL)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.error_for_status()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let total = response.content_length().unwrap_or(0);
|
||||
let mut zip: Vec<u8> = Vec::with_capacity(total as usize);
|
||||
let mut last_emit: usize = 0;
|
||||
while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? {
|
||||
zip.extend_from_slice(&chunk);
|
||||
if zip.len() - last_emit > 1024 * 1024 {
|
||||
last_emit = zip.len();
|
||||
let _ = app.emit(
|
||||
"playlist:scan-progress",
|
||||
serde_json::json!({ "stage": "runtime", "received": zip.len(), "total": total }),
|
||||
);
|
||||
}
|
||||
}
|
||||
if zip.len() < 10 * 1024 * 1024 {
|
||||
return Err(format!(
|
||||
"suspiciously small download ({} bytes) — refused",
|
||||
zip.len()
|
||||
));
|
||||
}
|
||||
let dest = tools_dir(&app)?.join("deno.exe");
|
||||
// Inflating ~100 MB is real CPU work — off the async runtime's threads.
|
||||
let version = tauri::async_runtime::spawn_blocking(move || -> Result<String, String> {
|
||||
let exe = extract_zip_entry(&zip, "deno.exe")?;
|
||||
let tmp = dest.with_extension("exe.downloading");
|
||||
std::fs::write(&tmp, &exe).map_err(|e| e.to_string())?;
|
||||
std::fs::rename(&tmp, &dest).map_err(|e| e.to_string())?;
|
||||
match deno_version(&dest) {
|
||||
Some(v) => Ok(v),
|
||||
None => {
|
||||
let _ = std::fs::remove_file(&dest);
|
||||
Err("downloaded deno did not answer --version — removed".into())
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())??;
|
||||
Ok(serde_json::json!({ "version": version }))
|
||||
}
|
||||
|
||||
/// Where a video's fetched audio lives; keyed by video id so a re-scan of
|
||||
/// the same set never re-downloads.
|
||||
fn audio_cache_dir(app: &tauri::AppHandle) -> Result<PathBuf, String> {
|
||||
let dir = app
|
||||
.path()
|
||||
.app_data_dir()
|
||||
.map_err(|e| e.to_string())?
|
||||
.join("setlist-audio");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
pub fn cached_audio(app: &tauri::AppHandle, video_id: &str) -> Option<PathBuf> {
|
||||
let dir = audio_cache_dir(app).ok()?;
|
||||
for ext in ["m4a", "mp4", "aac"] {
|
||||
let p = dir.join(format!("{video_id}.{ext}"));
|
||||
if p.is_file() {
|
||||
return Some(p);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// 2026 yt-dlp needs an external JS runtime for YouTube (the EJS change:
|
||||
/// "extraction without a JS runtime has been deprecated" — verified live,
|
||||
/// a runtime-less run 403s). Find one: managed deno first, then PATH
|
||||
/// deno/node/bun. Returns the `--js-runtimes` value, or None.
|
||||
fn js_runtime(app: &tauri::AppHandle) -> Option<String> {
|
||||
if let Ok(dir) = tools_dir(app) {
|
||||
let managed = dir.join("deno.exe");
|
||||
if managed.is_file() {
|
||||
return Some(format!("deno:{}", managed.to_string_lossy()));
|
||||
}
|
||||
}
|
||||
let path_var = std::env::var_os("PATH")?;
|
||||
for (exe, name) in [("deno.exe", "deno"), ("node.exe", "node"), ("bun.exe", "bun")] {
|
||||
if std::env::split_paths(&path_var).any(|d| d.join(exe).is_file()) {
|
||||
return Some(name.to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Fetch a video's audio-only stream into the cache, emitting download
|
||||
/// progress parsed from yt-dlp's own lines. Blocking (the caller runs it
|
||||
/// on a worker thread). Strictly the m4a/AAC ladder — the app decodes that
|
||||
/// in-process; a set that only offers opus errors honestly rather than
|
||||
/// handing back a file nothing can read.
|
||||
pub fn fetch_audio_blocking(
|
||||
app: &tauri::AppHandle,
|
||||
video_id: &str,
|
||||
cancel: &std::sync::atomic::AtomicBool,
|
||||
) -> Result<PathBuf, String> {
|
||||
use std::io::BufRead;
|
||||
use std::os::windows::process::CommandExt;
|
||||
if let Some(hit) = cached_audio(app, video_id) {
|
||||
return Ok(hit);
|
||||
}
|
||||
let exe = locate(app).ok_or("yt-dlp is not installed yet")?;
|
||||
let runtime = js_runtime(app).ok_or(
|
||||
"yt-dlp needs a JavaScript runtime for YouTube now (Deno, Node, or Bun) and none was \
|
||||
found on this machine — install one and try again",
|
||||
)?;
|
||||
let dir = audio_cache_dir(app)?;
|
||||
let template = dir.join("%(id)s.%(ext)s");
|
||||
let mut child = std::process::Command::new(&exe)
|
||||
.args([
|
||||
"--no-playlist",
|
||||
"--newline",
|
||||
"--no-color",
|
||||
"--js-runtimes",
|
||||
&runtime,
|
||||
"-f",
|
||||
"bestaudio[ext=m4a]/bestaudio[acodec^=mp4a]",
|
||||
"-o",
|
||||
&template.to_string_lossy(),
|
||||
&format!("https://www.youtube.com/watch?v={video_id}"),
|
||||
])
|
||||
.creation_flags(CREATE_NO_WINDOW)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.map_err(|e| format!("could not start yt-dlp: {e}"))?;
|
||||
let stdout = child.stdout.take().ok_or("no stdout")?;
|
||||
let reader = std::io::BufReader::new(stdout);
|
||||
for line in reader.lines().map_while(Result::ok) {
|
||||
if cancel.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
let _ = child.kill();
|
||||
return Err("canceled".into());
|
||||
}
|
||||
// "[download] 42.3% of 61.53MiB at ..."
|
||||
if let Some(rest) = line.trim().strip_prefix("[download]") {
|
||||
if let Some(pct) = rest.trim().split('%').next() {
|
||||
if let Ok(p) = pct.trim().parse::<f32>() {
|
||||
let _ = app.emit(
|
||||
"playlist:scan-progress",
|
||||
serde_json::json!({ "stage": "audio", "pct": p }),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let status = child.wait().map_err(|e| e.to_string())?;
|
||||
if !status.success() {
|
||||
let mut err = String::new();
|
||||
if let Some(mut se) = child.stderr.take() {
|
||||
use std::io::Read;
|
||||
let _ = se.read_to_string(&mut err);
|
||||
}
|
||||
let tail: String = err.lines().rev().take(3).collect::<Vec<_>>().join(" | ");
|
||||
return Err(format!("yt-dlp failed: {tail}"));
|
||||
}
|
||||
cached_audio(app, video_id).ok_or_else(|| {
|
||||
"yt-dlp finished but no m4a landed — the set may only offer opus audio".into()
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Assemble a real zip in memory — local headers, central directory,
|
||||
/// EOCD — the structure `extract_zip_entry` actually parses.
|
||||
fn build_zip(entries: &[(&str, &[u8], u16)]) -> Vec<u8> {
|
||||
use std::io::Write as _;
|
||||
let mut out: Vec<u8> = Vec::new();
|
||||
let mut central: Vec<u8> = Vec::new();
|
||||
for (name, data, method) in entries {
|
||||
let mut hasher = crc32fast::Hasher::new();
|
||||
hasher.update(data);
|
||||
let crc = hasher.finalize();
|
||||
let comp: Vec<u8> = match method {
|
||||
0 => data.to_vec(),
|
||||
8 => {
|
||||
let mut enc = flate2::write::DeflateEncoder::new(
|
||||
Vec::new(),
|
||||
flate2::Compression::default(),
|
||||
);
|
||||
enc.write_all(data).unwrap();
|
||||
enc.finish().unwrap()
|
||||
}
|
||||
_ => unreachable!("test zips are stored or deflated"),
|
||||
};
|
||||
let local_at = out.len() as u32;
|
||||
out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
|
||||
out.extend_from_slice(&[20, 0, 0, 0]); // version, flags
|
||||
out.extend_from_slice(&method.to_le_bytes());
|
||||
out.extend_from_slice(&[0; 4]); // time, date
|
||||
out.extend_from_slice(&crc.to_le_bytes());
|
||||
out.extend_from_slice(&(comp.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&[0, 0]); // extra_len
|
||||
out.extend_from_slice(name.as_bytes());
|
||||
out.extend_from_slice(&comp);
|
||||
|
||||
central.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
|
||||
central.extend_from_slice(&[20, 0, 20, 0, 0, 0]); // versions, flags
|
||||
central.extend_from_slice(&method.to_le_bytes());
|
||||
central.extend_from_slice(&[0; 4]); // time, date
|
||||
central.extend_from_slice(&crc.to_le_bytes());
|
||||
central.extend_from_slice(&(comp.len() as u32).to_le_bytes());
|
||||
central.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
||||
central.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
central.extend_from_slice(&[0; 12]); // extra/comment/disk/attrs
|
||||
central.extend_from_slice(&local_at.to_le_bytes());
|
||||
central.extend_from_slice(name.as_bytes());
|
||||
}
|
||||
let cd_at = out.len() as u32;
|
||||
out.extend_from_slice(¢ral);
|
||||
out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
|
||||
out.extend_from_slice(&[0; 4]); // disk numbers
|
||||
out.extend_from_slice(&(entries.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(entries.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(central.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&cd_at.to_le_bytes());
|
||||
out.extend_from_slice(&[0, 0]); // comment_len
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_the_named_deflated_entry_among_others() {
|
||||
let payload: Vec<u8> = (0..40_000u32).flat_map(|i| i.to_le_bytes()).collect();
|
||||
let zip = build_zip(&[
|
||||
("LICENSE.md", b"decoy text", 8),
|
||||
("deno.exe", &payload, 8),
|
||||
]);
|
||||
let got = extract_zip_entry(&zip, "deno.exe").unwrap();
|
||||
assert_eq!(got, payload);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_a_stored_entry() {
|
||||
let zip = build_zip(&[("deno.exe", b"stored bytes", 0)]);
|
||||
assert_eq!(extract_zip_entry(&zip, "deno.exe").unwrap(), b"stored bytes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupted_data_fails_the_crc_check() {
|
||||
let mut zip = build_zip(&[("deno.exe", b"soon to be flipped payload", 0)]);
|
||||
// Flip one payload byte: local header is 30 + name, data follows.
|
||||
let at = 30 + "deno.exe".len() + 3;
|
||||
zip[at] ^= 0xff;
|
||||
let err = extract_zip_entry(&zip, "deno.exe").unwrap_err();
|
||||
assert!(err.contains("CRC"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_name_is_an_honest_error() {
|
||||
let zip = build_zip(&[("other.txt", b"x", 0)]);
|
||||
assert!(extract_zip_entry(&zip, "deno.exe").is_err());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user