playlists + lyrics: setlists with pages of their own, and the words timed to the voice

This commit is contained in:
KeelrLeek committed 2026-08-17 18:05:31 -05:00
1 parent c4ff6269a3
commit df3d15553a
49 files changed
+14419 -599

No files matched your search

+14
View File
@@ -8,6 +8,7 @@
mod loudness;
mod source;
pub mod vocals;
mod waveform;
pub use loudness::{analyze_loudness, Analysis, AnalysisSource, Incomplete, LoudnessMeter};
@@ -494,6 +495,19 @@ fn start_decoder(
.into(),
)
.spawn(move || {
// Refuse formats there is no decoder for, BEFORE opening them.
//
// Most simply fail to build and report themselves. WavPack does
// not: a `.wv` carrying the original RIFF header in its wrapper is
// probed as raw PCM, so the decoder reports success and streams the
// compressed body as samples — measured at 136,815 NaN/Inf values
// in a single track, straight into the mixer and the normaliser.
// An extension check is the only thing that can stop that, because
// by the time there is a decoder there is no error to notice.
if let Some(reason) = audio_formats::unplayable_path_reason(&track.path) {
let _ = tx.send(Err((generation, reason.to_string())));
return;
}
let (reader, byte_len): (Box<dyn source::MediaRead>, u64) = match track.stream.clone() {
Some(progress) => {
match GrowingFile::open(&track.path, track.total_bytes, progress) {
+512
View File
@@ -0,0 +1,512 @@
//! Vocal-band DSP for the lyrics feature: a leading-silence probe (auto-sync
//! evidence) and an offline vocal-reduction renderer (the classic band-limited
//! center-cancel, NOT a learned separator — honest about what it is).
//!
//! Everything here is offline file work. Nothing touches the live audio
//! engine: the renderer writes an ordinary WAV that plays through the normal
//! local-file path, so a bug in this module can never destabilize playback.
use std::io::Write as _;
use std::path::Path;
use symphonia::core::audio::SampleBuffer;
use symphonia::core::codecs::DecoderOptions;
use symphonia::core::formats::FormatOptions;
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
use symphonia::core::probe::Hint;
/// Longest source the renderer accepts. A WAV render holds the whole track
/// in memory (16-bit stereo ≈ 10.6 MB/min at 44.1k); twenty minutes bounds
/// that at ~212 MB and excludes accidental full-set files.
const MAX_RENDER_SECS: f64 = 20.0 * 60.0;
/// RMS floor treated as "audible": −50 dBFS, sustained. Modern releases sit
/// far above this; dithered lead-in noise sits below it.
const SILENCE_RMS: f32 = 0.003_16;
/// Frame size for the silence scan (about 20 ms at 44.1 kHz).
const SILENCE_FRAME: usize = 1024;
/// How many consecutive audible frames end the silence (~60 ms) — a single
/// vinyl tick or DC blip must not count as the music starting.
const SILENCE_RUN: usize = 3;
/// The silence probe never reads past this point; a track that has not
/// started within thirty seconds is not a case auto-sync should judge.
const SILENCE_SCAN_SECS: f64 = 30.0;
// ---- decoding --------------------------------------------------------------
struct Decoded {
/// Interleaved f32 frames.
samples: Vec<f32>,
channels: usize,
sample_rate: u32,
}
/// Decode `path` fully (or up to `limit_secs`) into interleaved f32.
fn decode(path: &Path, limit_secs: Option<f64>) -> Result<Decoded, String> {
let file = std::fs::File::open(path).map_err(|e| e.to_string())?;
let mss = MediaSourceStream::new(Box::new(file), Default::default());
let mut hint = Hint::new();
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
hint.with_extension(ext);
}
let probed = symphonia::default::get_probe()
.format(
&hint,
mss,
&FormatOptions::default(),
&MetadataOptions::default(),
)
.map_err(|e| e.to_string())?;
let mut format = probed.format;
let track = format
.default_track()
.ok_or("no default audio track")?
.clone();
let mut decoder = symphonia::default::get_codecs()
.make(&track.codec_params, &DecoderOptions::default())
.map_err(|e| e.to_string())?;
let mut samples = Vec::new();
let mut channels = 0usize;
let mut sample_rate = 0u32;
let mut buffer: Option<SampleBuffer<f32>> = None;
loop {
let packet = match format.next_packet() {
Ok(p) => p,
Err(symphonia::core::errors::Error::IoError(_)) => break,
Err(symphonia::core::errors::Error::ResetRequired) => break,
Err(_) => break,
};
if packet.track_id() != track.id {
continue;
}
let decoded = match decoder.decode(&packet) {
Ok(d) => d,
Err(symphonia::core::errors::Error::DecodeError(_)) => continue,
Err(_) => break,
};
let spec = *decoded.spec();
channels = spec.channels.count();
sample_rate = spec.rate;
let out = buffer.get_or_insert_with(|| {
SampleBuffer::<f32>::new(decoded.capacity() as u64, spec)
});
if out.capacity() < decoded.capacity() * channels {
*out = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
}
out.copy_interleaved_ref(decoded);
samples.extend_from_slice(out.samples());
if let Some(limit) = limit_secs {
if sample_rate > 0
&& (samples.len() / channels.max(1)) as f64 / sample_rate as f64 >= limit
{
break;
}
}
}
if samples.is_empty() || channels == 0 || sample_rate == 0 {
return Err("nothing decodable".into());
}
Ok(Decoded {
samples,
channels,
sample_rate,
})
}
// ---- leading silence -------------------------------------------------------
/// Milliseconds of leading silence in `path`, or None when the file cannot be
/// decoded / never becomes audible inside the scan window.
pub fn leading_silence_ms(path: &Path) -> Option<u64> {
let decoded = decode(path, Some(SILENCE_SCAN_SECS)).ok()?;
let frames_rms = fold_rms(&decoded.samples, decoded.channels, SILENCE_FRAME);
let onset_frame = onset_from_rms(&frames_rms)?;
Some(
(onset_frame as u64 * SILENCE_FRAME as u64 * 1000)
/ decoded.sample_rate.max(1) as u64,
)
}
/// Interleaved samples → per-frame mono RMS.
fn fold_rms(samples: &[f32], channels: usize, frame: usize) -> Vec<f32> {
let channels = channels.max(1);
let per_frame = frame * channels;
samples
.chunks(per_frame)
.map(|chunk| {
let sum: f32 = chunk.iter().map(|s| s * s).sum();
(sum / chunk.len().max(1) as f32).sqrt()
})
.collect()
}
/// First frame index beginning [`SILENCE_RUN`] consecutive audible frames.
fn onset_from_rms(rms: &[f32]) -> Option<usize> {
let mut run = 0usize;
for (index, &value) in rms.iter().enumerate() {
if value > SILENCE_RMS {
run += 1;
if run >= SILENCE_RUN {
return Some(index + 1 - run);
}
} else {
run = 0;
}
}
None
}
// ---- vocal activity --------------------------------------------------------
/// A coarse "is someone likely singing here" envelope: the energy of the
/// CENTER (mid) channel band-passed to the vocal range, one value per
/// `frame_ms`, normalized 0..1 against its own loud end. The same physics
/// the vocal reducer exploits — center-mixed vocals dominate the mid band —
/// read instead of removed. A proxy, not a detector: instruments living
/// center-mid (lead guitar, synth hooks) light it up too, which is why the
/// rough-sync generator that consumes it is labeled ESTIMATED and hands the
/// result to the correction editor rather than presenting it as truth.
pub struct VocalActivity {
pub frame_ms: u32,
/// 0..1 per frame.
pub frames: Vec<f32>,
}
pub fn vocal_activity(path: &Path, frame_ms: u32) -> Result<VocalActivity, String> {
let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
let frame_ms = frame_ms.clamp(40, 500);
let frame_len = (decoded.sample_rate as usize * frame_ms as usize) / 1000;
let channels = decoded.channels;
let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0);
let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0);
let mut frames = Vec::new();
let mut acc = 0.0f64;
let mut n = 0usize;
for frame in decoded.samples.chunks(channels) {
let mid = if channels >= 2 {
0.5 * (frame[0] + frame[1])
} else {
frame[0]
};
let banded = high.process(low.process(mid));
acc += (banded * banded) as f64;
n += 1;
if n >= frame_len {
frames.push((acc / n as f64).sqrt() as f32);
acc = 0.0;
n = 0;
}
}
if n > frame_len / 4 {
frames.push((acc / n.max(1) as f64).sqrt() as f32);
}
if frames.is_empty() {
return Err("nothing decodable".into());
}
// Normalize against the loud end (95th percentile), so one hot chorus
// does not flatten the whole song to zero.
let mut sorted: Vec<f32> = frames.clone();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let p95 = sorted[(sorted.len() * 95 / 100).min(sorted.len() - 1)].max(1e-6);
for value in &mut frames {
*value = (*value / p95).min(1.0);
}
Ok(VocalActivity { frame_ms, frames })
}
// ---- vocal reduction -------------------------------------------------------
/// What the renderer produced (for the caller's labels and sanity checks).
pub struct VocalReduction {
pub duration_secs: f64,
pub sample_rate: u32,
}
/// Render a vocal-reduced copy of `src` to `dest_wav` (16-bit stereo WAV).
///
/// The method is the classic out-of-phase cancellation with band limiting:
/// the center (mid) channel is removed in the vocal band while the lows
/// (bass, kick) and the highest air band are kept, and the side content
/// passes through untouched. It works in proportion to how center-mixed the
/// lead vocal is — modern wide/reverbed vocals survive partially. That limit
/// is inherent to the technique; the honest fix is the future learned
/// separator the research bundle scopes.
pub fn render_vocal_reduced(src: &Path, dest_wav: &Path) -> Result<VocalReduction, String> {
// The decode itself stops just past the cap, so an accidental multi-hour
// file cannot balloon memory before the refusal fires.
let decoded = decode(src, Some(MAX_RENDER_SECS + 2.0))?;
if decoded.channels < 2 {
return Err("This recording is mono — there is no center channel to cancel".into());
}
let frames = decoded.samples.len() / decoded.channels;
let duration = frames as f64 / decoded.sample_rate as f64;
if duration > MAX_RENDER_SECS {
return Err("Track is too long for the instrumental renderer (20 min cap)".into());
}
// First two channels carry the stereo image; extra channels are dropped.
let mut stereo: Vec<f32> = Vec::with_capacity(frames * 2);
for frame in decoded.samples.chunks(decoded.channels) {
stereo.push(frame[0]);
stereo.push(frame[1]);
}
process_center_cancel(&mut stereo, decoded.sample_rate);
write_wav_16(dest_wav, &stereo, decoded.sample_rate)?;
Ok(VocalReduction {
duration_secs: duration,
sample_rate: decoded.sample_rate,
})
}
/// In-place band-limited center cancellation over interleaved stereo.
///
/// out_L = side + LP(mid) + HP(mid); out_R = −side + LP(mid) + HP(mid) —
/// removing the mid band where vocals live (~180 Hz–7.5 kHz) and keeping the
/// side image, the lows, and the air.
fn process_center_cancel(stereo: &mut [f32], sample_rate: u32) {
let mut low = Biquad::lowpass(sample_rate as f32, 180.0);
let mut high = Biquad::highpass(sample_rate as f32, 7_500.0);
let keep_air = sample_rate >= 22_050;
for frame in stereo.chunks_mut(2) {
let (l, r) = (frame[0], frame[1]);
let mid = 0.5 * (l + r);
let side = 0.5 * (l - r);
let lows = low.process(mid);
let air = if keep_air { high.process(mid) } else { 0.0 };
let kept = lows + air;
frame[0] = ((side + kept) * 0.95).clamp(-1.0, 1.0);
frame[1] = ((kept - side) * 0.95).clamp(-1.0, 1.0);
}
}
/// RBJ-cookbook biquad, Butterworth Q.
struct Biquad {
b0: f32,
b1: f32,
b2: f32,
a1: f32,
a2: f32,
x1: f32,
x2: f32,
y1: f32,
y2: f32,
}
impl Biquad {
fn new(b0: f32, b1: f32, b2: f32, a0: f32, a1: f32, a2: f32) -> Biquad {
Biquad {
b0: b0 / a0,
b1: b1 / a0,
b2: b2 / a0,
a1: a1 / a0,
a2: a2 / a0,
x1: 0.0,
x2: 0.0,
y1: 0.0,
y2: 0.0,
}
}
fn lowpass(sample_rate: f32, fc: f32) -> Biquad {
let w = 2.0 * std::f32::consts::PI * fc / sample_rate;
// Butterworth: alpha = sin(w)/(2·Q) with Q = 1/√2.
let alpha = w.sin() * std::f32::consts::FRAC_1_SQRT_2;
let c = w.cos();
Biquad::new(
(1.0 - c) / 2.0,
1.0 - c,
(1.0 - c) / 2.0,
1.0 + alpha,
-2.0 * c,
1.0 - alpha,
)
}
fn highpass(sample_rate: f32, fc: f32) -> Biquad {
let w = 2.0 * std::f32::consts::PI * fc / sample_rate;
let alpha = w.sin() * std::f32::consts::FRAC_1_SQRT_2;
let c = w.cos();
Biquad::new(
(1.0 + c) / 2.0,
-(1.0 + c),
(1.0 + c) / 2.0,
1.0 + alpha,
-2.0 * c,
1.0 - alpha,
)
}
fn process(&mut self, x: f32) -> f32 {
let y = self.b0 * x + self.b1 * self.x1 + self.b2 * self.x2
- self.a1 * self.y1
- self.a2 * self.y2;
self.x2 = self.x1;
self.x1 = x;
self.y2 = self.y1;
self.y1 = y;
y
}
}
/// Minimal 16-bit PCM stereo WAV writer (44-byte canonical header).
fn write_wav_16(dest: &Path, interleaved: &[f32], sample_rate: u32) -> Result<(), String> {
let data_len = (interleaved.len() * 2) as u32;
let mut out = std::io::BufWriter::new(
std::fs::File::create(dest).map_err(|e| e.to_string())?,
);
let byte_rate = sample_rate * 2 * 2;
let mut header = Vec::with_capacity(44);
header.extend_from_slice(b"RIFF");
header.extend_from_slice(&(36 + data_len).to_le_bytes());
header.extend_from_slice(b"WAVEfmt ");
header.extend_from_slice(&16u32.to_le_bytes());
header.extend_from_slice(&1u16.to_le_bytes()); // PCM
header.extend_from_slice(&2u16.to_le_bytes()); // stereo
header.extend_from_slice(&sample_rate.to_le_bytes());
header.extend_from_slice(&byte_rate.to_le_bytes());
header.extend_from_slice(&4u16.to_le_bytes()); // block align
header.extend_from_slice(&16u16.to_le_bytes()); // bits
header.extend_from_slice(b"data");
header.extend_from_slice(&data_len.to_le_bytes());
out.write_all(&header).map_err(|e| e.to_string())?;
let mut bytes = Vec::with_capacity(interleaved.len() * 2);
for &sample in interleaved {
bytes.extend_from_slice(&((sample.clamp(-1.0, 1.0) * 32767.0) as i16).to_le_bytes());
}
out.write_all(&bytes).map_err(|e| e.to_string())?;
out.flush().map_err(|e| e.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
fn rms(stereo: &[f32]) -> f32 {
(stereo.iter().map(|s| s * s).sum::<f32>() / stereo.len() as f32).sqrt()
}
fn sine_stereo(freq: f32, secs: f32, rate: u32, pan: &str) -> Vec<f32> {
let frames = (secs * rate as f32) as usize;
let mut out = Vec::with_capacity(frames * 2);
for i in 0..frames {
let s = (2.0 * std::f32::consts::PI * freq * i as f32 / rate as f32).sin() * 0.5;
match pan {
"center" => {
out.push(s);
out.push(s);
}
"left" => {
out.push(s);
out.push(0.0);
}
// Anti-phase: pure side content.
_ => {
out.push(s);
out.push(-s);
}
}
}
out
}
/// The whole point: a center-panned vocal-band tone dies, a center bass
/// tone survives, side content survives.
#[test]
fn center_vocal_band_is_cancelled_lows_and_sides_survive() {
let rate = 44_100;
let mut vocal = sine_stereo(1_000.0, 0.5, rate, "center");
let before = rms(&vocal);
process_center_cancel(&mut vocal, rate);
assert!(
rms(&vocal) < before * 0.06,
"center 1kHz must drop >24dB, got {} → {}",
before,
rms(&vocal)
);
let mut bass = sine_stereo(60.0, 0.5, rate, "center");
let bass_before = rms(&bass);
process_center_cancel(&mut bass, rate);
assert!(
rms(&bass) > bass_before * 0.6,
"center 60Hz must survive the low-band keep, got {} → {}",
bass_before,
rms(&bass)
);
let mut side = sine_stereo(1_000.0, 0.5, rate, "side");
let side_before = rms(&side);
process_center_cancel(&mut side, rate);
assert!(
rms(&side) > side_before * 0.7,
"side content must pass through, got {} → {}",
side_before,
rms(&side)
);
}
#[test]
fn onset_ignores_a_lone_tick() {
// 40 silent frames, one loud blip, more silence, then real audio.
let mut frames = vec![0.0005f32; 40];
frames[10] = 0.2; // vinyl tick — one frame only
frames.extend(vec![0.1f32; 20]);
assert_eq!(onset_from_rms(&frames), Some(40));
assert_eq!(onset_from_rms(&[0.0001f32; 50]), None, "never audible");
}
/// End-to-end through the WAV writer and back through symphonia: the
/// rendered file decodes, keeps the length, and is quieter in the mid.
#[test]
fn renders_a_playable_wav() {
let dir = std::env::temp_dir().join("ssplus-vocals-test");
std::fs::create_dir_all(&dir).unwrap();
let src = dir.join("src.wav");
let dest = dir.join("out.wav");
let rate = 44_100u32;
// A "song": center 1kHz vocal + left-panned 3kHz guitar.
let mut mix = sine_stereo(1_000.0, 1.0, rate, "center");
let guitar = sine_stereo(3_000.0, 1.0, rate, "left");
for (m, g) in mix.iter_mut().zip(guitar.iter()) {
*m = (*m + g).clamp(-1.0, 1.0);
}
write_wav_16(&src, &mix, rate).unwrap();
let result = render_vocal_reduced(&src, &dest).unwrap();
assert_eq!(result.sample_rate, rate);
assert!((result.duration_secs - 1.0).abs() < 0.05);
let back = decode(&dest, None).unwrap();
assert_eq!(back.channels, 2);
let frames = back.samples.len() / 2;
assert!((frames as f64 / rate as f64 - 1.0).abs() < 0.05);
// Mono files are refused, not mangled.
let mono = dir.join("mono.wav");
{
use std::io::Write as _;
let samples: Vec<f32> = sine_stereo(440.0, 0.2, rate, "center")
.chunks(2)
.map(|c| c[0])
.collect();
let data_len = (samples.len() * 2) as u32;
let mut out = std::fs::File::create(&mono).unwrap();
let mut header = Vec::new();
header.extend_from_slice(b"RIFF");
header.extend_from_slice(&(36 + data_len).to_le_bytes());
header.extend_from_slice(b"WAVEfmt ");
header.extend_from_slice(&16u32.to_le_bytes());
header.extend_from_slice(&1u16.to_le_bytes());
header.extend_from_slice(&1u16.to_le_bytes());
header.extend_from_slice(&rate.to_le_bytes());
header.extend_from_slice(&(rate * 2).to_le_bytes());
header.extend_from_slice(&2u16.to_le_bytes());
header.extend_from_slice(&16u16.to_le_bytes());
header.extend_from_slice(b"data");
header.extend_from_slice(&data_len.to_le_bytes());
out.write_all(&header).unwrap();
for s in &samples {
out.write_all(&((s * 32767.0) as i16).to_le_bytes()).unwrap();
}
}
assert!(render_vocal_reduced(&mono, &dest).is_err());
}
}