forked from Keelrleek/SoulStream
919 lines
34 KiB
Rust
919 lines
34 KiB
Rust
//! Vocal-band DSP for the lyrics feature: a leading-silence probe (auto-sync
|
||
//! evidence) and an offline vocal-reduction renderer (the classic band-limited
|
||
//! center-cancel, NOT a learned separator — honest about what it is).
|
||
//!
|
||
//! Everything here is offline file work. Nothing touches the live audio
|
||
//! engine: the renderer writes an ordinary WAV that plays through the normal
|
||
//! local-file path, so a bug in this module can never destabilize playback.
|
||
|
||
use std::io::Write as _;
|
||
use std::path::Path;
|
||
|
||
use symphonia::core::audio::SampleBuffer;
|
||
use symphonia::core::codecs::DecoderOptions;
|
||
use symphonia::core::formats::FormatOptions;
|
||
use symphonia::core::io::MediaSourceStream;
|
||
use symphonia::core::meta::MetadataOptions;
|
||
use symphonia::core::probe::Hint;
|
||
|
||
/// Longest source the renderer accepts. A WAV render holds the whole track
|
||
/// in memory (16-bit stereo ≈ 10.6 MB/min at 44.1k); twenty minutes bounds
|
||
/// that at ~212 MB and excludes accidental full-set files.
|
||
const MAX_RENDER_SECS: f64 = 20.0 * 60.0;
|
||
|
||
/// RMS floor treated as "audible": −50 dBFS, sustained. Modern releases sit
|
||
/// far above this; dithered lead-in noise sits below it.
|
||
const SILENCE_RMS: f32 = 0.003_16;
|
||
/// Frame size for the silence scan (about 20 ms at 44.1 kHz).
|
||
const SILENCE_FRAME: usize = 1024;
|
||
/// How many consecutive audible frames end the silence (~60 ms) — a single
|
||
/// vinyl tick or DC blip must not count as the music starting.
|
||
const SILENCE_RUN: usize = 3;
|
||
/// The silence probe never reads past this point; a track that has not
|
||
/// started within thirty seconds is not a case auto-sync should judge.
|
||
const SILENCE_SCAN_SECS: f64 = 30.0;
|
||
|
||
// ---- decoding --------------------------------------------------------------
|
||
|
||
struct Decoded {
|
||
/// Interleaved f32 frames.
|
||
samples: Vec<f32>,
|
||
channels: usize,
|
||
sample_rate: u32,
|
||
}
|
||
|
||
/// Decode `path` fully (or up to `limit_secs`) into interleaved f32.
|
||
fn decode(path: &Path, limit_secs: Option<f64>) -> Result<Decoded, String> {
|
||
let file = std::fs::File::open(path).map_err(|e| e.to_string())?;
|
||
let mss = MediaSourceStream::new(Box::new(file), Default::default());
|
||
let mut hint = Hint::new();
|
||
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
|
||
hint.with_extension(ext);
|
||
}
|
||
let probed = symphonia::default::get_probe()
|
||
.format(
|
||
&hint,
|
||
mss,
|
||
&FormatOptions::default(),
|
||
&MetadataOptions::default(),
|
||
)
|
||
.map_err(|e| e.to_string())?;
|
||
let mut format = probed.format;
|
||
let track = format
|
||
.default_track()
|
||
.ok_or("no default audio track")?
|
||
.clone();
|
||
let mut decoder = symphonia::default::get_codecs()
|
||
.make(&track.codec_params, &DecoderOptions::default())
|
||
.map_err(|e| e.to_string())?;
|
||
|
||
let mut samples = Vec::new();
|
||
let mut channels = 0usize;
|
||
let mut sample_rate = 0u32;
|
||
let mut buffer: Option<SampleBuffer<f32>> = None;
|
||
loop {
|
||
let packet = match format.next_packet() {
|
||
Ok(p) => p,
|
||
Err(symphonia::core::errors::Error::IoError(_)) => break,
|
||
Err(symphonia::core::errors::Error::ResetRequired) => break,
|
||
Err(_) => break,
|
||
};
|
||
if packet.track_id() != track.id {
|
||
continue;
|
||
}
|
||
let decoded = match decoder.decode(&packet) {
|
||
Ok(d) => d,
|
||
Err(symphonia::core::errors::Error::DecodeError(_)) => continue,
|
||
Err(_) => break,
|
||
};
|
||
let spec = *decoded.spec();
|
||
channels = spec.channels.count();
|
||
sample_rate = spec.rate;
|
||
let out = buffer.get_or_insert_with(|| {
|
||
SampleBuffer::<f32>::new(decoded.capacity() as u64, spec)
|
||
});
|
||
if out.capacity() < decoded.capacity() * channels {
|
||
*out = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
|
||
}
|
||
out.copy_interleaved_ref(decoded);
|
||
samples.extend_from_slice(out.samples());
|
||
if let Some(limit) = limit_secs {
|
||
if sample_rate > 0
|
||
&& (samples.len() / channels.max(1)) as f64 / sample_rate as f64 >= limit
|
||
{
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
if samples.is_empty() || channels == 0 || sample_rate == 0 {
|
||
return Err("nothing decodable".into());
|
||
}
|
||
Ok(Decoded {
|
||
samples,
|
||
channels,
|
||
sample_rate,
|
||
})
|
||
}
|
||
|
||
// ---- streaming long-form decode ---------------------------------------------
|
||
|
||
/// Streaming 16 kHz mono decode for LONG sources (DJ sets, hour-plus mixes):
|
||
/// packets are downmixed and resampled as they arrive, so memory holds only
|
||
/// the 16 kHz mono result (~230 MB per hour as f32) and never the source-rate
|
||
/// PCM. The whole-file `decode` above deliberately caps at 20 minutes; this
|
||
/// is the path past that cap, bounded by `max_secs` of OUTPUT.
|
||
pub fn mono_pcm_16k_streaming(path: &Path, max_secs: f64) -> Result<Vec<f32>, String> {
|
||
let file = std::fs::File::open(path).map_err(|e| e.to_string())?;
|
||
let mss = MediaSourceStream::new(Box::new(file), Default::default());
|
||
let mut hint = Hint::new();
|
||
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
|
||
hint.with_extension(ext);
|
||
}
|
||
let probed = symphonia::default::get_probe()
|
||
.format(
|
||
&hint,
|
||
mss,
|
||
&FormatOptions::default(),
|
||
&MetadataOptions::default(),
|
||
)
|
||
.map_err(|e| e.to_string())?;
|
||
let mut format = probed.format;
|
||
let track = format
|
||
.default_track()
|
||
.ok_or("no default audio track")?
|
||
.clone();
|
||
let mut decoder = symphonia::default::get_codecs()
|
||
.make(&track.codec_params, &DecoderOptions::default())
|
||
.map_err(|e| e.to_string())?;
|
||
|
||
let mut out: Vec<f32> = Vec::new();
|
||
let cap = (max_secs * 16_000.0) as usize;
|
||
let mut resampler: Option<StreamResampler> = None;
|
||
let mut buffer: Option<SampleBuffer<f32>> = None;
|
||
loop {
|
||
if out.len() >= cap {
|
||
break;
|
||
}
|
||
let packet = match format.next_packet() {
|
||
Ok(p) => p,
|
||
Err(_) => break,
|
||
};
|
||
if packet.track_id() != track.id {
|
||
continue;
|
||
}
|
||
let decoded = match decoder.decode(&packet) {
|
||
Ok(d) => d,
|
||
Err(symphonia::core::errors::Error::DecodeError(_)) => continue,
|
||
Err(_) => break,
|
||
};
|
||
let spec = *decoded.spec();
|
||
let channels = spec.channels.count().max(1);
|
||
let buf = buffer.get_or_insert_with(|| {
|
||
SampleBuffer::<f32>::new(decoded.capacity() as u64, spec)
|
||
});
|
||
if buf.capacity() < decoded.capacity() * channels {
|
||
*buf = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
|
||
}
|
||
buf.copy_interleaved_ref(decoded);
|
||
let rs = resampler.get_or_insert_with(|| StreamResampler::new(spec.rate, 16_000));
|
||
rs.push_interleaved(buf.samples(), channels, &mut out);
|
||
}
|
||
if out.is_empty() {
|
||
return Err("nothing decodable".into());
|
||
}
|
||
out.truncate(cap);
|
||
Ok(out)
|
||
}
|
||
|
||
/// Linear resampler with carry across pushes, so packet boundaries do not
|
||
/// click or drift: keeps the previous source sample and a fractional read
|
||
/// position between calls.
|
||
struct StreamResampler {
|
||
ratio: f64,
|
||
/// Fractional position into the CURRENT push, may start negative into
|
||
/// the carried sample.
|
||
pos: f64,
|
||
carry: f32,
|
||
has_carry: bool,
|
||
passthrough: bool,
|
||
}
|
||
|
||
impl StreamResampler {
|
||
fn new(src_rate: u32, dst_rate: u32) -> Self {
|
||
StreamResampler {
|
||
ratio: src_rate as f64 / dst_rate as f64,
|
||
pos: 0.0,
|
||
carry: 0.0,
|
||
has_carry: false,
|
||
passthrough: src_rate == dst_rate,
|
||
}
|
||
}
|
||
|
||
/// Downmix `interleaved` to mono and append the resampled result to
|
||
/// `out`.
|
||
fn push_interleaved(&mut self, interleaved: &[f32], channels: usize, out: &mut Vec<f32>) {
|
||
let frames = interleaved.len() / channels.max(1);
|
||
if frames == 0 {
|
||
return;
|
||
}
|
||
let mono = |i: usize| -> f32 {
|
||
let row = &interleaved[i * channels..(i + 1) * channels];
|
||
row.iter().sum::<f32>() / channels as f32
|
||
};
|
||
if self.passthrough {
|
||
out.extend((0..frames).map(mono));
|
||
return;
|
||
}
|
||
// Source timeline for this push: sample -1 is the carried one.
|
||
loop {
|
||
let i0 = self.pos.floor();
|
||
let frac = (self.pos - i0) as f32;
|
||
let i0 = i0 as isize;
|
||
if i0 + 1 >= frames as isize {
|
||
break;
|
||
}
|
||
let a = if i0 < 0 {
|
||
if self.has_carry { self.carry } else { mono(0) }
|
||
} else {
|
||
mono(i0 as usize)
|
||
};
|
||
let b = mono((i0 + 1) as usize);
|
||
out.push(a + (b - a) * frac);
|
||
self.pos += self.ratio;
|
||
}
|
||
self.carry = mono(frames - 1);
|
||
self.has_carry = true;
|
||
self.pos -= frames as f64;
|
||
}
|
||
}
|
||
|
||
// ---- leading silence -------------------------------------------------------
|
||
|
||
/// Milliseconds of leading silence in `path`, or None when the file cannot be
|
||
/// decoded / never becomes audible inside the scan window.
|
||
pub fn leading_silence_ms(path: &Path) -> Option<u64> {
|
||
let decoded = decode(path, Some(SILENCE_SCAN_SECS)).ok()?;
|
||
let frames_rms = fold_rms(&decoded.samples, decoded.channels, SILENCE_FRAME);
|
||
let onset_frame = onset_from_rms(&frames_rms)?;
|
||
Some(
|
||
(onset_frame as u64 * SILENCE_FRAME as u64 * 1000)
|
||
/ decoded.sample_rate.max(1) as u64,
|
||
)
|
||
}
|
||
|
||
/// Interleaved samples → per-frame mono RMS.
|
||
fn fold_rms(samples: &[f32], channels: usize, frame: usize) -> Vec<f32> {
|
||
let channels = channels.max(1);
|
||
let per_frame = frame * channels;
|
||
samples
|
||
.chunks(per_frame)
|
||
.map(|chunk| {
|
||
let sum: f32 = chunk.iter().map(|s| s * s).sum();
|
||
(sum / chunk.len().max(1) as f32).sqrt()
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
/// First frame index beginning [`SILENCE_RUN`] consecutive audible frames.
|
||
fn onset_from_rms(rms: &[f32]) -> Option<usize> {
|
||
let mut run = 0usize;
|
||
for (index, &value) in rms.iter().enumerate() {
|
||
if value > SILENCE_RMS {
|
||
run += 1;
|
||
if run >= SILENCE_RUN {
|
||
return Some(index + 1 - run);
|
||
}
|
||
} else {
|
||
run = 0;
|
||
}
|
||
}
|
||
None
|
||
}
|
||
|
||
// ---- vocal activity --------------------------------------------------------
|
||
|
||
/// A coarse "is someone likely singing here" envelope: the energy of the
|
||
/// CENTER (mid) channel band-passed to the vocal range, one value per
|
||
/// `frame_ms`, normalized 0..1 against its own loud end. The same physics
|
||
/// the vocal reducer exploits — center-mixed vocals dominate the mid band —
|
||
/// read instead of removed. A proxy, not a detector: instruments living
|
||
/// center-mid (lead guitar, synth hooks) light it up too, which is why the
|
||
/// rough-sync generator that consumes it is labeled ESTIMATED and hands the
|
||
/// result to the correction editor rather than presenting it as truth.
|
||
pub struct VocalActivity {
|
||
pub frame_ms: u32,
|
||
/// 0..1 per frame.
|
||
pub frames: Vec<f32>,
|
||
}
|
||
|
||
pub fn vocal_activity(path: &Path, frame_ms: u32) -> Result<VocalActivity, String> {
|
||
let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
|
||
let frame_ms = frame_ms.clamp(40, 500);
|
||
let frame_len = (decoded.sample_rate as usize * frame_ms as usize) / 1000;
|
||
let channels = decoded.channels;
|
||
let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0);
|
||
let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0);
|
||
let mut frames = Vec::new();
|
||
let mut acc = 0.0f64;
|
||
let mut n = 0usize;
|
||
for frame in decoded.samples.chunks(channels) {
|
||
let mid = if channels >= 2 {
|
||
0.5 * (frame[0] + frame[1])
|
||
} else {
|
||
frame[0]
|
||
};
|
||
let banded = high.process(low.process(mid));
|
||
acc += (banded * banded) as f64;
|
||
n += 1;
|
||
if n >= frame_len {
|
||
frames.push((acc / n as f64).sqrt() as f32);
|
||
acc = 0.0;
|
||
n = 0;
|
||
}
|
||
}
|
||
if n > frame_len / 4 {
|
||
frames.push((acc / n.max(1) as f64).sqrt() as f32);
|
||
}
|
||
if frames.is_empty() {
|
||
return Err("nothing decodable".into());
|
||
}
|
||
// Normalize against the loud end (95th percentile), so one hot chorus
|
||
// does not flatten the whole song to zero.
|
||
let mut sorted: Vec<f32> = frames.clone();
|
||
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||
let p95 = sorted[(sorted.len() * 95 / 100).min(sorted.len() - 1)].max(1e-6);
|
||
for value in &mut frames {
|
||
*value = (*value / p95).min(1.0);
|
||
}
|
||
Ok(VocalActivity { frame_ms, frames })
|
||
}
|
||
|
||
// ---- mono PCM for the CTC aligner ------------------------------------------
|
||
|
||
/// Whole-file VOCAL-FORWARD mono PCM at 16 kHz for the CTC aligner: the mid
|
||
/// (center) channel through the same 180 Hz – 7.5 kHz band the reducer and
|
||
/// the activity envelope use, linearly resampled. Cutting bass/kick
|
||
/// fundamentals and cymbal air before the model's per-window normalization
|
||
/// leaves the voice owning more of the input's dynamic range — cheap
|
||
/// pre-emphasis, NOT separation (center-mixed instruments remain). The
|
||
/// low-pass doubles as a crude anti-alias guard for the 16 kHz resample.
|
||
pub fn vocal_pcm_16k(path: &Path) -> Result<Vec<f32>, String> {
|
||
let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
|
||
let ch = decoded.channels.max(1);
|
||
let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0);
|
||
let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0);
|
||
let mono: Vec<f32> = decoded
|
||
.samples
|
||
.chunks(ch)
|
||
.map(|f| high.process(low.process(f.iter().sum::<f32>() / ch as f32)))
|
||
.collect();
|
||
let src = decoded.sample_rate as f64;
|
||
if (src - 16_000.0).abs() < 1.0 {
|
||
return Ok(mono);
|
||
}
|
||
let ratio = 16_000.0 / src;
|
||
let out_len = (mono.len() as f64 * ratio) as usize;
|
||
let mut out = Vec::with_capacity(out_len);
|
||
for i in 0..out_len {
|
||
let pos = i as f64 / ratio;
|
||
let i0 = pos as usize;
|
||
let frac = (pos - i0 as f64) as f32;
|
||
let a = mono.get(i0).copied().unwrap_or(0.0);
|
||
let b = mono.get(i0 + 1).copied().unwrap_or(a);
|
||
out.push(a + (b - a) * frac);
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
/// Stereo 44.1 kHz planar PCM for the learned vocal separator: any layout
|
||
/// folds to two channels (mono duplicates its one, extra channels beyond the
|
||
/// front pair are ignored), any rate linearly resampled. Linear interpolation
|
||
/// is deliberate — the stem feeds ALIGNMENT, not ears, the same tradeoff
|
||
/// `vocal_pcm_16k` already makes for its own input.
|
||
pub fn separation_input_44k1(path: &Path) -> Result<[Vec<f32>; 2], String> {
|
||
let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
|
||
let ch = decoded.channels.max(1);
|
||
let frames = decoded.samples.len() / ch;
|
||
let mut left = Vec::with_capacity(frames);
|
||
let mut right = Vec::with_capacity(frames);
|
||
for f in decoded.samples.chunks(ch) {
|
||
let l = f[0];
|
||
left.push(l);
|
||
right.push(if f.len() > 1 { f[1] } else { l });
|
||
}
|
||
Ok([
|
||
resample_linear(left, decoded.sample_rate as f64, 44_100.0),
|
||
resample_linear(right, decoded.sample_rate as f64, 44_100.0),
|
||
])
|
||
}
|
||
|
||
/// Aligner input from a separated vocal stem (44.1 kHz planar stereo): mid
|
||
/// fold, a 60 Hz rumble high-pass, the same 7.5 kHz low-pass `vocal_pcm_16k`
|
||
/// uses as its 16 kHz anti-alias guard, then the resample. No 180 Hz floor
|
||
/// here — that floor exists to fight INSTRUMENTS sharing the band, and a
|
||
/// separated stem has (mostly) none left to fight.
|
||
pub fn stem_pcm_16k(left: &[f32], right: &[f32]) -> Vec<f32> {
|
||
let mut rumble = Biquad::highpass(44_100.0, 60.0);
|
||
let mut guard = Biquad::lowpass(44_100.0, 7_500.0);
|
||
let n = left.len().min(right.len());
|
||
let mono: Vec<f32> = (0..n)
|
||
.map(|i| guard.process(rumble.process((left[i] + right[i]) * 0.5)))
|
||
.collect();
|
||
resample_linear(mono, 44_100.0, 16_000.0)
|
||
}
|
||
|
||
/// Linear resampler shared by the separator inputs (see the tradeoff note on
|
||
/// `separation_input_44k1`). Within 1 Hz of the target it hands the buffer
|
||
/// back untouched — whole-track copies are not free at 44.1 kHz.
|
||
fn resample_linear(src_pcm: Vec<f32>, src: f64, dst: f64) -> Vec<f32> {
|
||
if (src - dst).abs() < 1.0 {
|
||
return src_pcm;
|
||
}
|
||
let ratio = dst / src;
|
||
let out_len = (src_pcm.len() as f64 * ratio) as usize;
|
||
let mut out = Vec::with_capacity(out_len);
|
||
for i in 0..out_len {
|
||
let pos = i as f64 / ratio;
|
||
let i0 = pos as usize;
|
||
let frac = (pos - i0 as f64) as f32;
|
||
let a = src_pcm.get(i0).copied().unwrap_or(0.0);
|
||
let b = src_pcm.get(i0 + 1).copied().unwrap_or(a);
|
||
out.push(a + (b - a) * frac);
|
||
}
|
||
out
|
||
}
|
||
|
||
// ---- vocal features (word-timing worker) -----------------------------------
|
||
|
||
/// Per-frame vocal evidence for the lyric word-timing worker: the band
|
||
/// energy `vocal_activity` measures PLUS a voicing verdict — vocals are
|
||
/// PITCHED, so an autocorrelation clarity check on a decimated mid channel
|
||
/// separates sung frames from drums and noise that light the band up.
|
||
pub struct VocalFrames {
|
||
pub frame_ms: u32,
|
||
/// 0..1 per frame, normalized like `vocal_activity`.
|
||
pub energy: Vec<f32>,
|
||
/// Frame carries periodic (pitched) content in the vocal F0 range.
|
||
pub voiced: Vec<bool>,
|
||
}
|
||
|
||
pub fn vocal_frames(path: &Path, frame_ms: u32) -> Result<VocalFrames, String> {
|
||
let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
|
||
let frame_ms = frame_ms.clamp(40, 500);
|
||
let frame_len = (decoded.sample_rate as usize * frame_ms as usize) / 1000;
|
||
let channels = decoded.channels;
|
||
let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0);
|
||
let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0);
|
||
|
||
// Decimate the mid channel to ~8 kHz for the pitch check — full-rate
|
||
// autocorrelation across a song is debug-build poison, and F0 up to
|
||
// 500 Hz needs nothing like 44.1 kHz.
|
||
let decim = (decoded.sample_rate as usize / 8_000).max(1);
|
||
let target_rate = decoded.sample_rate as usize / decim;
|
||
let mut energy = Vec::new();
|
||
let mut voiced = Vec::new();
|
||
let mut acc = 0.0f64;
|
||
let mut n = 0usize;
|
||
let mut pitch_buf: Vec<f32> = Vec::with_capacity(frame_len / decim + 2);
|
||
let mut decim_acc = 0.0f32;
|
||
let mut decim_n = 0usize;
|
||
for frame in decoded.samples.chunks(channels) {
|
||
let mid = if channels >= 2 {
|
||
0.5 * (frame[0] + frame[1])
|
||
} else {
|
||
frame[0]
|
||
};
|
||
let banded = high.process(low.process(mid));
|
||
acc += (banded * banded) as f64;
|
||
n += 1;
|
||
decim_acc += banded;
|
||
decim_n += 1;
|
||
if decim_n >= decim {
|
||
pitch_buf.push(decim_acc / decim as f32);
|
||
decim_acc = 0.0;
|
||
decim_n = 0;
|
||
}
|
||
if n >= frame_len {
|
||
energy.push((acc / n as f64).sqrt() as f32);
|
||
voiced.push(frame_is_voiced(&pitch_buf, target_rate));
|
||
pitch_buf.clear();
|
||
acc = 0.0;
|
||
n = 0;
|
||
}
|
||
}
|
||
if n > frame_len / 4 {
|
||
energy.push((acc / n.max(1) as f64).sqrt() as f32);
|
||
voiced.push(frame_is_voiced(&pitch_buf, target_rate));
|
||
}
|
||
if energy.is_empty() {
|
||
return Err("nothing decodable".into());
|
||
}
|
||
let mut sorted: Vec<f32> = energy.clone();
|
||
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||
let p95 = sorted[(sorted.len() * 95 / 100).min(sorted.len() - 1)].max(1e-6);
|
||
for value in &mut energy {
|
||
*value = (*value / p95).min(1.0);
|
||
}
|
||
Ok(VocalFrames {
|
||
frame_ms,
|
||
energy,
|
||
voiced,
|
||
})
|
||
}
|
||
|
||
/// Normalized-autocorrelation clarity over the sung F0 range (80–500 Hz).
|
||
/// A frame is voiced when some lag in that range correlates strongly —
|
||
/// periodic content — rather than merely being loud.
|
||
fn frame_is_voiced(buf: &[f32], rate: usize) -> bool {
|
||
if buf.len() < 64 || rate == 0 {
|
||
return false;
|
||
}
|
||
let energy: f32 = buf.iter().map(|s| s * s).sum();
|
||
if energy < 1e-7 {
|
||
return false;
|
||
}
|
||
let lag_min = (rate / 500).max(2);
|
||
let lag_max = (rate / 80).min(buf.len() / 2);
|
||
if lag_min >= lag_max {
|
||
return false;
|
||
}
|
||
let mut best = 0.0f32;
|
||
let mut lag = lag_min;
|
||
// Stride lags by 2: clarity peaks are broad; halving the work loses
|
||
// nothing a boolean cares about.
|
||
while lag < lag_max {
|
||
let n = buf.len() - lag;
|
||
let mut num = 0.0f32;
|
||
let mut den_a = 0.0f32;
|
||
let mut den_b = 0.0f32;
|
||
for i in 0..n {
|
||
num += buf[i] * buf[i + lag];
|
||
den_a += buf[i] * buf[i];
|
||
den_b += buf[i + lag] * buf[i + lag];
|
||
}
|
||
let den = (den_a * den_b).sqrt().max(1e-9);
|
||
best = best.max(num / den);
|
||
lag += 2;
|
||
}
|
||
best >= 0.55
|
||
}
|
||
|
||
// ---- vocal reduction -------------------------------------------------------
|
||
|
||
/// What the renderer produced (for the caller's labels and sanity checks).
|
||
pub struct VocalReduction {
|
||
pub duration_secs: f64,
|
||
pub sample_rate: u32,
|
||
}
|
||
|
||
/// Render a vocal-reduced copy of `src` to `dest_wav` (16-bit stereo WAV).
|
||
///
|
||
/// The method is the classic out-of-phase cancellation with band limiting:
|
||
/// the center (mid) channel is removed in the vocal band while the lows
|
||
/// (bass, kick) and the highest air band are kept, and the side content
|
||
/// passes through untouched. It works in proportion to how center-mixed the
|
||
/// lead vocal is — modern wide/reverbed vocals survive partially. That limit
|
||
/// is inherent to the technique; the honest fix is the future learned
|
||
/// separator the research bundle scopes.
|
||
pub fn render_vocal_reduced(src: &Path, dest_wav: &Path) -> Result<VocalReduction, String> {
|
||
// The decode itself stops just past the cap, so an accidental multi-hour
|
||
// file cannot balloon memory before the refusal fires.
|
||
let decoded = decode(src, Some(MAX_RENDER_SECS + 2.0))?;
|
||
if decoded.channels < 2 {
|
||
return Err("This recording is mono — there is no center channel to cancel".into());
|
||
}
|
||
let frames = decoded.samples.len() / decoded.channels;
|
||
let duration = frames as f64 / decoded.sample_rate as f64;
|
||
if duration > MAX_RENDER_SECS {
|
||
return Err("Track is too long for the instrumental renderer (20 min cap)".into());
|
||
}
|
||
// First two channels carry the stereo image; extra channels are dropped.
|
||
let mut stereo: Vec<f32> = Vec::with_capacity(frames * 2);
|
||
for frame in decoded.samples.chunks(decoded.channels) {
|
||
stereo.push(frame[0]);
|
||
stereo.push(frame[1]);
|
||
}
|
||
process_center_cancel(&mut stereo, decoded.sample_rate);
|
||
write_wav_16(dest_wav, &stereo, decoded.sample_rate)?;
|
||
Ok(VocalReduction {
|
||
duration_secs: duration,
|
||
sample_rate: decoded.sample_rate,
|
||
})
|
||
}
|
||
|
||
/// In-place band-limited center cancellation over interleaved stereo.
|
||
///
|
||
/// out_L = side + LP(mid) + HP(mid); out_R = −side + LP(mid) + HP(mid) —
|
||
/// removing the mid band where vocals live (~180 Hz–7.5 kHz) and keeping the
|
||
/// side image, the lows, and the air.
|
||
fn process_center_cancel(stereo: &mut [f32], sample_rate: u32) {
|
||
let mut low = Biquad::lowpass(sample_rate as f32, 180.0);
|
||
let mut high = Biquad::highpass(sample_rate as f32, 7_500.0);
|
||
let keep_air = sample_rate >= 22_050;
|
||
for frame in stereo.chunks_mut(2) {
|
||
let (l, r) = (frame[0], frame[1]);
|
||
let mid = 0.5 * (l + r);
|
||
let side = 0.5 * (l - r);
|
||
let lows = low.process(mid);
|
||
let air = if keep_air { high.process(mid) } else { 0.0 };
|
||
let kept = lows + air;
|
||
frame[0] = ((side + kept) * 0.95).clamp(-1.0, 1.0);
|
||
frame[1] = ((kept - side) * 0.95).clamp(-1.0, 1.0);
|
||
}
|
||
}
|
||
|
||
/// RBJ-cookbook biquad, Butterworth Q.
|
||
struct Biquad {
|
||
b0: f32,
|
||
b1: f32,
|
||
b2: f32,
|
||
a1: f32,
|
||
a2: f32,
|
||
x1: f32,
|
||
x2: f32,
|
||
y1: f32,
|
||
y2: f32,
|
||
}
|
||
|
||
impl Biquad {
|
||
fn new(b0: f32, b1: f32, b2: f32, a0: f32, a1: f32, a2: f32) -> Biquad {
|
||
Biquad {
|
||
b0: b0 / a0,
|
||
b1: b1 / a0,
|
||
b2: b2 / a0,
|
||
a1: a1 / a0,
|
||
a2: a2 / a0,
|
||
x1: 0.0,
|
||
x2: 0.0,
|
||
y1: 0.0,
|
||
y2: 0.0,
|
||
}
|
||
}
|
||
fn lowpass(sample_rate: f32, fc: f32) -> Biquad {
|
||
let w = 2.0 * std::f32::consts::PI * fc / sample_rate;
|
||
// Butterworth: alpha = sin(w)/(2·Q) with Q = 1/√2.
|
||
let alpha = w.sin() * std::f32::consts::FRAC_1_SQRT_2;
|
||
let c = w.cos();
|
||
Biquad::new(
|
||
(1.0 - c) / 2.0,
|
||
1.0 - c,
|
||
(1.0 - c) / 2.0,
|
||
1.0 + alpha,
|
||
-2.0 * c,
|
||
1.0 - alpha,
|
||
)
|
||
}
|
||
fn highpass(sample_rate: f32, fc: f32) -> Biquad {
|
||
let w = 2.0 * std::f32::consts::PI * fc / sample_rate;
|
||
let alpha = w.sin() * std::f32::consts::FRAC_1_SQRT_2;
|
||
let c = w.cos();
|
||
Biquad::new(
|
||
(1.0 + c) / 2.0,
|
||
-(1.0 + c),
|
||
(1.0 + c) / 2.0,
|
||
1.0 + alpha,
|
||
-2.0 * c,
|
||
1.0 - alpha,
|
||
)
|
||
}
|
||
fn process(&mut self, x: f32) -> f32 {
|
||
let y = self.b0 * x + self.b1 * self.x1 + self.b2 * self.x2
|
||
- self.a1 * self.y1
|
||
- self.a2 * self.y2;
|
||
self.x2 = self.x1;
|
||
self.x1 = x;
|
||
self.y2 = self.y1;
|
||
self.y1 = y;
|
||
y
|
||
}
|
||
}
|
||
|
||
/// Minimal 16-bit PCM stereo WAV writer (44-byte canonical header).
|
||
fn write_wav_16(dest: &Path, interleaved: &[f32], sample_rate: u32) -> Result<(), String> {
|
||
let data_len = (interleaved.len() * 2) as u32;
|
||
let mut out = std::io::BufWriter::new(
|
||
std::fs::File::create(dest).map_err(|e| e.to_string())?,
|
||
);
|
||
let byte_rate = sample_rate * 2 * 2;
|
||
let mut header = Vec::with_capacity(44);
|
||
header.extend_from_slice(b"RIFF");
|
||
header.extend_from_slice(&(36 + data_len).to_le_bytes());
|
||
header.extend_from_slice(b"WAVEfmt ");
|
||
header.extend_from_slice(&16u32.to_le_bytes());
|
||
header.extend_from_slice(&1u16.to_le_bytes()); // PCM
|
||
header.extend_from_slice(&2u16.to_le_bytes()); // stereo
|
||
header.extend_from_slice(&sample_rate.to_le_bytes());
|
||
header.extend_from_slice(&byte_rate.to_le_bytes());
|
||
header.extend_from_slice(&4u16.to_le_bytes()); // block align
|
||
header.extend_from_slice(&16u16.to_le_bytes()); // bits
|
||
header.extend_from_slice(b"data");
|
||
header.extend_from_slice(&data_len.to_le_bytes());
|
||
out.write_all(&header).map_err(|e| e.to_string())?;
|
||
let mut bytes = Vec::with_capacity(interleaved.len() * 2);
|
||
for &sample in interleaved {
|
||
bytes.extend_from_slice(&((sample.clamp(-1.0, 1.0) * 32767.0) as i16).to_le_bytes());
|
||
}
|
||
out.write_all(&bytes).map_err(|e| e.to_string())?;
|
||
out.flush().map_err(|e| e.to_string())
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
fn rms(stereo: &[f32]) -> f32 {
|
||
(stereo.iter().map(|s| s * s).sum::<f32>() / stereo.len() as f32).sqrt()
|
||
}
|
||
|
||
fn sine_stereo(freq: f32, secs: f32, rate: u32, pan: &str) -> Vec<f32> {
|
||
let frames = (secs * rate as f32) as usize;
|
||
let mut out = Vec::with_capacity(frames * 2);
|
||
for i in 0..frames {
|
||
let s = (2.0 * std::f32::consts::PI * freq * i as f32 / rate as f32).sin() * 0.5;
|
||
match pan {
|
||
"center" => {
|
||
out.push(s);
|
||
out.push(s);
|
||
}
|
||
"left" => {
|
||
out.push(s);
|
||
out.push(0.0);
|
||
}
|
||
// Anti-phase: pure side content.
|
||
_ => {
|
||
out.push(s);
|
||
out.push(-s);
|
||
}
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
/// The streaming resampler must produce the same stream regardless of
|
||
/// how the source is chopped into pushes — no clicks, no drift at
|
||
/// packet boundaries.
|
||
#[test]
|
||
fn stream_resampler_is_chunking_invariant() {
|
||
let src_rate = 44_100u32;
|
||
let n = 44_100usize;
|
||
let stereo: Vec<f32> = (0..n)
|
||
.flat_map(|i| {
|
||
let s = (2.0 * std::f32::consts::PI * 440.0 * i as f32 / 44_100.0).sin();
|
||
[s, s]
|
||
})
|
||
.collect();
|
||
let mut one_shot = Vec::new();
|
||
let mut a = StreamResampler::new(src_rate, 16_000);
|
||
a.push_interleaved(&stereo, 2, &mut one_shot);
|
||
let mut chunked = Vec::new();
|
||
let mut b = StreamResampler::new(src_rate, 16_000);
|
||
let mut at = 0usize;
|
||
for (k, step) in [431usize, 977, 1153, 64, 2048].iter().cycle().enumerate() {
|
||
if at >= stereo.len() {
|
||
break;
|
||
}
|
||
let end = (at + step * 2).min(stereo.len());
|
||
b.push_interleaved(&stereo[at..end], 2, &mut chunked);
|
||
at = end;
|
||
assert!(k < 200_000, "runaway");
|
||
}
|
||
assert!(
|
||
(one_shot.len() as i64 - chunked.len() as i64).abs() <= 2,
|
||
"length drift: {} vs {}",
|
||
one_shot.len(),
|
||
chunked.len()
|
||
);
|
||
let worst = one_shot
|
||
.iter()
|
||
.zip(&chunked)
|
||
.map(|(x, y)| (x - y).abs())
|
||
.fold(0.0f32, f32::max);
|
||
assert!(worst < 1e-4, "boundary artifacts, worst {worst}");
|
||
}
|
||
|
||
/// The stem→aligner path: a vocal-band tone passes the 60 Hz / 7.5 kHz
|
||
/// hygiene filters and lands at 16 kHz with its energy intact.
|
||
#[test]
|
||
fn stem_pcm_16k_resamples_and_keeps_vocal_band() {
|
||
let frames = 44_100usize;
|
||
let mono: Vec<f32> = (0..frames)
|
||
.map(|i| (2.0 * std::f32::consts::PI * 1_000.0 * i as f32 / 44_100.0).sin() * 0.5)
|
||
.collect();
|
||
let out = stem_pcm_16k(&mono, &mono);
|
||
assert!(
|
||
(15_998..=16_000).contains(&out.len()),
|
||
"one second must land at ~16000 samples, got {}",
|
||
out.len()
|
||
);
|
||
let level = rms(&out);
|
||
assert!(
|
||
level > 0.3 && level < 0.4,
|
||
"1kHz mid tone must pass the stem filters near unity, rms {level}"
|
||
);
|
||
assert!(out.iter().all(|s| s.is_finite()));
|
||
}
|
||
|
||
/// The whole point: a center-panned vocal-band tone dies, a center bass
|
||
/// tone survives, side content survives.
|
||
#[test]
|
||
fn center_vocal_band_is_cancelled_lows_and_sides_survive() {
|
||
let rate = 44_100;
|
||
let mut vocal = sine_stereo(1_000.0, 0.5, rate, "center");
|
||
let before = rms(&vocal);
|
||
process_center_cancel(&mut vocal, rate);
|
||
assert!(
|
||
rms(&vocal) < before * 0.06,
|
||
"center 1kHz must drop >24dB, got {} → {}",
|
||
before,
|
||
rms(&vocal)
|
||
);
|
||
|
||
let mut bass = sine_stereo(60.0, 0.5, rate, "center");
|
||
let bass_before = rms(&bass);
|
||
process_center_cancel(&mut bass, rate);
|
||
assert!(
|
||
rms(&bass) > bass_before * 0.6,
|
||
"center 60Hz must survive the low-band keep, got {} → {}",
|
||
bass_before,
|
||
rms(&bass)
|
||
);
|
||
|
||
let mut side = sine_stereo(1_000.0, 0.5, rate, "side");
|
||
let side_before = rms(&side);
|
||
process_center_cancel(&mut side, rate);
|
||
assert!(
|
||
rms(&side) > side_before * 0.7,
|
||
"side content must pass through, got {} → {}",
|
||
side_before,
|
||
rms(&side)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn onset_ignores_a_lone_tick() {
|
||
// 40 silent frames, one loud blip, more silence, then real audio.
|
||
let mut frames = vec![0.0005f32; 40];
|
||
frames[10] = 0.2; // vinyl tick — one frame only
|
||
frames.extend(vec![0.1f32; 20]);
|
||
assert_eq!(onset_from_rms(&frames), Some(40));
|
||
assert_eq!(onset_from_rms(&[0.0001f32; 50]), None, "never audible");
|
||
}
|
||
|
||
/// End-to-end through the WAV writer and back through symphonia: the
|
||
/// rendered file decodes, keeps the length, and is quieter in the mid.
|
||
#[test]
|
||
fn renders_a_playable_wav() {
|
||
let dir = std::env::temp_dir().join("ssplus-vocals-test");
|
||
std::fs::create_dir_all(&dir).unwrap();
|
||
let src = dir.join("src.wav");
|
||
let dest = dir.join("out.wav");
|
||
let rate = 44_100u32;
|
||
// A "song": center 1kHz vocal + left-panned 3kHz guitar.
|
||
let mut mix = sine_stereo(1_000.0, 1.0, rate, "center");
|
||
let guitar = sine_stereo(3_000.0, 1.0, rate, "left");
|
||
for (m, g) in mix.iter_mut().zip(guitar.iter()) {
|
||
*m = (*m + g).clamp(-1.0, 1.0);
|
||
}
|
||
write_wav_16(&src, &mix, rate).unwrap();
|
||
|
||
let result = render_vocal_reduced(&src, &dest).unwrap();
|
||
assert_eq!(result.sample_rate, rate);
|
||
assert!((result.duration_secs - 1.0).abs() < 0.05);
|
||
|
||
let back = decode(&dest, None).unwrap();
|
||
assert_eq!(back.channels, 2);
|
||
let frames = back.samples.len() / 2;
|
||
assert!((frames as f64 / rate as f64 - 1.0).abs() < 0.05);
|
||
|
||
// Mono files are refused, not mangled.
|
||
let mono = dir.join("mono.wav");
|
||
{
|
||
use std::io::Write as _;
|
||
let samples: Vec<f32> = sine_stereo(440.0, 0.2, rate, "center")
|
||
.chunks(2)
|
||
.map(|c| c[0])
|
||
.collect();
|
||
let data_len = (samples.len() * 2) as u32;
|
||
let mut out = std::fs::File::create(&mono).unwrap();
|
||
let mut header = Vec::new();
|
||
header.extend_from_slice(b"RIFF");
|
||
header.extend_from_slice(&(36 + data_len).to_le_bytes());
|
||
header.extend_from_slice(b"WAVEfmt ");
|
||
header.extend_from_slice(&16u32.to_le_bytes());
|
||
header.extend_from_slice(&1u16.to_le_bytes());
|
||
header.extend_from_slice(&1u16.to_le_bytes());
|
||
header.extend_from_slice(&rate.to_le_bytes());
|
||
header.extend_from_slice(&(rate * 2).to_le_bytes());
|
||
header.extend_from_slice(&2u16.to_le_bytes());
|
||
header.extend_from_slice(&16u16.to_le_bytes());
|
||
header.extend_from_slice(b"data");
|
||
header.extend_from_slice(&data_len.to_le_bytes());
|
||
out.write_all(&header).unwrap();
|
||
for s in &samples {
|
||
out.write_all(&((s * 32767.0) as i16).to_le_bytes()).unwrap();
|
||
}
|
||
}
|
||
assert!(render_vocal_reduced(&mono, &dest).is_err());
|
||
}
|
||
}
|