dj sets + lyrics: a mix that names its own tracks, and words pulled off the separated voice

crates/player/src/lib.rs

@@ -13,7 +13,7 @@ mod waveform;
 
 pub use loudness::{analyze_loudness, Analysis, AnalysisSource, Incomplete, LoudnessMeter};
 pub use source::{GrowingFile, StreamProgress, Terminal};
-pub use waveform::compute_waveform;
+pub use waveform::{compute_waveform, compute_waveform_range};
 
 use std::collections::VecDeque;
 use std::path::{Path, PathBuf};

crates/player/src/vocals.rs

@@ -115,6 +115,138 @@ fn decode(path: &Path, limit_secs: Option<f64>) -> Result<Decoded, String> {
     })
 }
 
+// ---- streaming long-form decode ---------------------------------------------
+
+/// Streaming 16 kHz mono decode for LONG sources (DJ sets, hour-plus mixes):
+/// packets are downmixed and resampled as they arrive, so memory holds only
+/// the 16 kHz mono result (~230 MB per hour as f32) and never the source-rate
+/// PCM. The whole-file `decode` above deliberately caps at 20 minutes; this
+/// is the path past that cap, bounded by `max_secs` of OUTPUT.
+pub fn mono_pcm_16k_streaming(path: &Path, max_secs: f64) -> Result<Vec<f32>, String> {
+    let file = std::fs::File::open(path).map_err(|e| e.to_string())?;
+    let mss = MediaSourceStream::new(Box::new(file), Default::default());
+    let mut hint = Hint::new();
+    if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
+        hint.with_extension(ext);
+    }
+    let probed = symphonia::default::get_probe()
+        .format(
+            &hint,
+            mss,
+            &FormatOptions::default(),
+            &MetadataOptions::default(),
+        )
+        .map_err(|e| e.to_string())?;
+    let mut format = probed.format;
+    let track = format
+        .default_track()
+        .ok_or("no default audio track")?
+        .clone();
+    let mut decoder = symphonia::default::get_codecs()
+        .make(&track.codec_params, &DecoderOptions::default())
+        .map_err(|e| e.to_string())?;
+
+    let mut out: Vec<f32> = Vec::new();
+    let cap = (max_secs * 16_000.0) as usize;
+    let mut resampler: Option<StreamResampler> = None;
+    let mut buffer: Option<SampleBuffer<f32>> = None;
+    loop {
+        if out.len() >= cap {
+            break;
+        }
+        let packet = match format.next_packet() {
+            Ok(p) => p,
+            Err(_) => break,
+        };
+        if packet.track_id() != track.id {
+            continue;
+        }
+        let decoded = match decoder.decode(&packet) {
+            Ok(d) => d,
+            Err(symphonia::core::errors::Error::DecodeError(_)) => continue,
+            Err(_) => break,
+        };
+        let spec = *decoded.spec();
+        let channels = spec.channels.count().max(1);
+        let buf = buffer.get_or_insert_with(|| {
+            SampleBuffer::<f32>::new(decoded.capacity() as u64, spec)
+        });
+        if buf.capacity() < decoded.capacity() * channels {
+            *buf = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
+        }
+        buf.copy_interleaved_ref(decoded);
+        let rs = resampler.get_or_insert_with(|| StreamResampler::new(spec.rate, 16_000));
+        rs.push_interleaved(buf.samples(), channels, &mut out);
+    }
+    if out.is_empty() {
+        return Err("nothing decodable".into());
+    }
+    out.truncate(cap);
+    Ok(out)
+}
+
+/// Linear resampler with carry across pushes, so packet boundaries do not
+/// click or drift: keeps the previous source sample and a fractional read
+/// position between calls.
+struct StreamResampler {
+    ratio: f64,
+    /// Fractional position into the CURRENT push, may start negative into
+    /// the carried sample.
+    pos: f64,
+    carry: f32,
+    has_carry: bool,
+    passthrough: bool,
+}
+
+impl StreamResampler {
+    fn new(src_rate: u32, dst_rate: u32) -> Self {
+        StreamResampler {
+            ratio: src_rate as f64 / dst_rate as f64,
+            pos: 0.0,
+            carry: 0.0,
+            has_carry: false,
+            passthrough: src_rate == dst_rate,
+        }
+    }
+
+    /// Downmix `interleaved` to mono and append the resampled result to
+    /// `out`.
+    fn push_interleaved(&mut self, interleaved: &[f32], channels: usize, out: &mut Vec<f32>) {
+        let frames = interleaved.len() / channels.max(1);
+        if frames == 0 {
+            return;
+        }
+        let mono = |i: usize| -> f32 {
+            let row = &interleaved[i * channels..(i + 1) * channels];
+            row.iter().sum::<f32>() / channels as f32
+        };
+        if self.passthrough {
+            out.extend((0..frames).map(mono));
+            return;
+        }
+        // Source timeline for this push: sample -1 is the carried one.
+        loop {
+            let i0 = self.pos.floor();
+            let frac = (self.pos - i0) as f32;
+            let i0 = i0 as isize;
+            if i0 + 1 >= frames as isize {
+                break;
+            }
+            let a = if i0 < 0 {
+                if self.has_carry { self.carry } else { mono(0) }
+            } else {
+                mono(i0 as usize)
+            };
+            let b = mono((i0 + 1) as usize);
+            out.push(a + (b - a) * frac);
+            self.pos += self.ratio;
+        }
+        self.carry = mono(frames - 1);
+        self.has_carry = true;
+        self.pos -= frames as f64;
+    }
+}
+
 // ---- leading silence -------------------------------------------------------
 
 /// Milliseconds of leading silence in `path`, or None when the file cannot be
@@ -216,6 +348,216 @@ pub fn vocal_activity(path: &Path, frame_ms: u32) -> Result<VocalActivity, Strin
     Ok(VocalActivity { frame_ms, frames })
 }
 
+// ---- mono PCM for the CTC aligner ------------------------------------------
+
+/// Whole-file VOCAL-FORWARD mono PCM at 16 kHz for the CTC aligner: the mid
+/// (center) channel through the same 180 Hz – 7.5 kHz band the reducer and
+/// the activity envelope use, linearly resampled. Cutting bass/kick
+/// fundamentals and cymbal air before the model's per-window normalization
+/// leaves the voice owning more of the input's dynamic range — cheap
+/// pre-emphasis, NOT separation (center-mixed instruments remain). The
+/// low-pass doubles as a crude anti-alias guard for the 16 kHz resample.
+pub fn vocal_pcm_16k(path: &Path) -> Result<Vec<f32>, String> {
+    let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
+    let ch = decoded.channels.max(1);
+    let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0);
+    let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0);
+    let mono: Vec<f32> = decoded
+        .samples
+        .chunks(ch)
+        .map(|f| high.process(low.process(f.iter().sum::<f32>() / ch as f32)))
+        .collect();
+    let src = decoded.sample_rate as f64;
+    if (src - 16_000.0).abs() < 1.0 {
+        return Ok(mono);
+    }
+    let ratio = 16_000.0 / src;
+    let out_len = (mono.len() as f64 * ratio) as usize;
+    let mut out = Vec::with_capacity(out_len);
+    for i in 0..out_len {
+        let pos = i as f64 / ratio;
+        let i0 = pos as usize;
+        let frac = (pos - i0 as f64) as f32;
+        let a = mono.get(i0).copied().unwrap_or(0.0);
+        let b = mono.get(i0 + 1).copied().unwrap_or(a);
+        out.push(a + (b - a) * frac);
+    }
+    Ok(out)
+}
+
+/// Stereo 44.1 kHz planar PCM for the learned vocal separator: any layout
+/// folds to two channels (mono duplicates its one, extra channels beyond the
+/// front pair are ignored), any rate linearly resampled. Linear interpolation
+/// is deliberate — the stem feeds ALIGNMENT, not ears, the same tradeoff
+/// `vocal_pcm_16k` already makes for its own input.
+pub fn separation_input_44k1(path: &Path) -> Result<[Vec<f32>; 2], String> {
+    let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
+    let ch = decoded.channels.max(1);
+    let frames = decoded.samples.len() / ch;
+    let mut left = Vec::with_capacity(frames);
+    let mut right = Vec::with_capacity(frames);
+    for f in decoded.samples.chunks(ch) {
+        let l = f[0];
+        left.push(l);
+        right.push(if f.len() > 1 { f[1] } else { l });
+    }
+    Ok([
+        resample_linear(left, decoded.sample_rate as f64, 44_100.0),
+        resample_linear(right, decoded.sample_rate as f64, 44_100.0),
+    ])
+}
+
+/// Aligner input from a separated vocal stem (44.1 kHz planar stereo): mid
+/// fold, a 60 Hz rumble high-pass, the same 7.5 kHz low-pass `vocal_pcm_16k`
+/// uses as its 16 kHz anti-alias guard, then the resample. No 180 Hz floor
+/// here — that floor exists to fight INSTRUMENTS sharing the band, and a
+/// separated stem has (mostly) none left to fight.
+pub fn stem_pcm_16k(left: &[f32], right: &[f32]) -> Vec<f32> {
+    let mut rumble = Biquad::highpass(44_100.0, 60.0);
+    let mut guard = Biquad::lowpass(44_100.0, 7_500.0);
+    let n = left.len().min(right.len());
+    let mono: Vec<f32> = (0..n)
+        .map(|i| guard.process(rumble.process((left[i] + right[i]) * 0.5)))
+        .collect();
+    resample_linear(mono, 44_100.0, 16_000.0)
+}
+
+/// Linear resampler shared by the separator inputs (see the tradeoff note on
+/// `separation_input_44k1`). Within 1 Hz of the target it hands the buffer
+/// back untouched — whole-track copies are not free at 44.1 kHz.
+fn resample_linear(src_pcm: Vec<f32>, src: f64, dst: f64) -> Vec<f32> {
+    if (src - dst).abs() < 1.0 {
+        return src_pcm;
+    }
+    let ratio = dst / src;
+    let out_len = (src_pcm.len() as f64 * ratio) as usize;
+    let mut out = Vec::with_capacity(out_len);
+    for i in 0..out_len {
+        let pos = i as f64 / ratio;
+        let i0 = pos as usize;
+        let frac = (pos - i0 as f64) as f32;
+        let a = src_pcm.get(i0).copied().unwrap_or(0.0);
+        let b = src_pcm.get(i0 + 1).copied().unwrap_or(a);
+        out.push(a + (b - a) * frac);
+    }
+    out
+}
+
+// ---- vocal features (word-timing worker) -----------------------------------
+
+/// Per-frame vocal evidence for the lyric word-timing worker: the band
+/// energy `vocal_activity` measures PLUS a voicing verdict — vocals are
+/// PITCHED, so an autocorrelation clarity check on a decimated mid channel
+/// separates sung frames from drums and noise that light the band up.
+pub struct VocalFrames {
+    pub frame_ms: u32,
+    /// 0..1 per frame, normalized like `vocal_activity`.
+    pub energy: Vec<f32>,
+    /// Frame carries periodic (pitched) content in the vocal F0 range.
+    pub voiced: Vec<bool>,
+}
+
+pub fn vocal_frames(path: &Path, frame_ms: u32) -> Result<VocalFrames, String> {
+    let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
+    let frame_ms = frame_ms.clamp(40, 500);
+    let frame_len = (decoded.sample_rate as usize * frame_ms as usize) / 1000;
+    let channels = decoded.channels;
+    let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0);
+    let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0);
+
+    // Decimate the mid channel to ~8 kHz for the pitch check — full-rate
+    // autocorrelation across a song is debug-build poison, and F0 up to
+    // 500 Hz needs nothing like 44.1 kHz.
+    let decim = (decoded.sample_rate as usize / 8_000).max(1);
+    let target_rate = decoded.sample_rate as usize / decim;
+    let mut energy = Vec::new();
+    let mut voiced = Vec::new();
+    let mut acc = 0.0f64;
+    let mut n = 0usize;
+    let mut pitch_buf: Vec<f32> = Vec::with_capacity(frame_len / decim + 2);
+    let mut decim_acc = 0.0f32;
+    let mut decim_n = 0usize;
+    for frame in decoded.samples.chunks(channels) {
+        let mid = if channels >= 2 {
+            0.5 * (frame[0] + frame[1])
+        } else {
+            frame[0]
+        };
+        let banded = high.process(low.process(mid));
+        acc += (banded * banded) as f64;
+        n += 1;
+        decim_acc += banded;
+        decim_n += 1;
+        if decim_n >= decim {
+            pitch_buf.push(decim_acc / decim as f32);
+            decim_acc = 0.0;
+            decim_n = 0;
+        }
+        if n >= frame_len {
+            energy.push((acc / n as f64).sqrt() as f32);
+            voiced.push(frame_is_voiced(&pitch_buf, target_rate));
+            pitch_buf.clear();
+            acc = 0.0;
+            n = 0;
+        }
+    }
+    if n > frame_len / 4 {
+        energy.push((acc / n.max(1) as f64).sqrt() as f32);
+        voiced.push(frame_is_voiced(&pitch_buf, target_rate));
+    }
+    if energy.is_empty() {
+        return Err("nothing decodable".into());
+    }
+    let mut sorted: Vec<f32> = energy.clone();
+    sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
+    let p95 = sorted[(sorted.len() * 95 / 100).min(sorted.len() - 1)].max(1e-6);
+    for value in &mut energy {
+        *value = (*value / p95).min(1.0);
+    }
+    Ok(VocalFrames {
+        frame_ms,
+        energy,
+        voiced,
+    })
+}
+
+/// Normalized-autocorrelation clarity over the sung F0 range (80–500 Hz).
+/// A frame is voiced when some lag in that range correlates strongly —
+/// periodic content — rather than merely being loud.
+fn frame_is_voiced(buf: &[f32], rate: usize) -> bool {
+    if buf.len() < 64 || rate == 0 {
+        return false;
+    }
+    let energy: f32 = buf.iter().map(|s| s * s).sum();
+    if energy < 1e-7 {
+        return false;
+    }
+    let lag_min = (rate / 500).max(2);
+    let lag_max = (rate / 80).min(buf.len() / 2);
+    if lag_min >= lag_max {
+        return false;
+    }
+    let mut best = 0.0f32;
+    let mut lag = lag_min;
+    // Stride lags by 2: clarity peaks are broad; halving the work loses
+    // nothing a boolean cares about.
+    while lag < lag_max {
+        let n = buf.len() - lag;
+        let mut num = 0.0f32;
+        let mut den_a = 0.0f32;
+        let mut den_b = 0.0f32;
+        for i in 0..n {
+            num += buf[i] * buf[i + lag];
+            den_a += buf[i] * buf[i];
+            den_b += buf[i + lag] * buf[i + lag];
+        }
+        let den = (den_a * den_b).sqrt().max(1e-9);
+        best = best.max(num / den);
+        lag += 2;
+    }
+    best >= 0.55
+}
+
 // ---- vocal reduction -------------------------------------------------------
 
 /// What the renderer produced (for the caller's labels and sanity checks).
@@ -407,6 +749,70 @@ mod tests {
         out
     }
 
+    /// The streaming resampler must produce the same stream regardless of
+    /// how the source is chopped into pushes — no clicks, no drift at
+    /// packet boundaries.
+    #[test]
+    fn stream_resampler_is_chunking_invariant() {
+        let src_rate = 44_100u32;
+        let n = 44_100usize;
+        let stereo: Vec<f32> = (0..n)
+            .flat_map(|i| {
+                let s = (2.0 * std::f32::consts::PI * 440.0 * i as f32 / 44_100.0).sin();
+                [s, s]
+            })
+            .collect();
+        let mut one_shot = Vec::new();
+        let mut a = StreamResampler::new(src_rate, 16_000);
+        a.push_interleaved(&stereo, 2, &mut one_shot);
+        let mut chunked = Vec::new();
+        let mut b = StreamResampler::new(src_rate, 16_000);
+        let mut at = 0usize;
+        for (k, step) in [431usize, 977, 1153, 64, 2048].iter().cycle().enumerate() {
+            if at >= stereo.len() {
+                break;
+            }
+            let end = (at + step * 2).min(stereo.len());
+            b.push_interleaved(&stereo[at..end], 2, &mut chunked);
+            at = end;
+            assert!(k < 200_000, "runaway");
+        }
+        assert!(
+            (one_shot.len() as i64 - chunked.len() as i64).abs() <= 2,
+            "length drift: {} vs {}",
+            one_shot.len(),
+            chunked.len()
+        );
+        let worst = one_shot
+            .iter()
+            .zip(&chunked)
+            .map(|(x, y)| (x - y).abs())
+            .fold(0.0f32, f32::max);
+        assert!(worst < 1e-4, "boundary artifacts, worst {worst}");
+    }
+
+    /// The stem→aligner path: a vocal-band tone passes the 60 Hz / 7.5 kHz
+    /// hygiene filters and lands at 16 kHz with its energy intact.
+    #[test]
+    fn stem_pcm_16k_resamples_and_keeps_vocal_band() {
+        let frames = 44_100usize;
+        let mono: Vec<f32> = (0..frames)
+            .map(|i| (2.0 * std::f32::consts::PI * 1_000.0 * i as f32 / 44_100.0).sin() * 0.5)
+            .collect();
+        let out = stem_pcm_16k(&mono, &mono);
+        assert!(
+            (15_998..=16_000).contains(&out.len()),
+            "one second must land at ~16000 samples, got {}",
+            out.len()
+        );
+        let level = rms(&out);
+        assert!(
+            level > 0.3 && level < 0.4,
+            "1kHz mid tone must pass the stem filters near unity, rms {level}"
+        );
+        assert!(out.iter().all(|s| s.is_finite()));
+    }
+
     /// The whole point: a center-panned vocal-band tone dies, a center bass
     /// tone survives, side content survives.
     #[test]

crates/player/src/waveform.rs

@@ -10,10 +10,11 @@ use std::path::Path;
 
 use symphonia::core::audio::SampleBuffer;
 use symphonia::core::codecs::DecoderOptions;
-use symphonia::core::formats::FormatOptions;
+use symphonia::core::formats::{FormatOptions, SeekMode, SeekTo};
 use symphonia::core::io::MediaSourceStream;
 use symphonia::core::meta::MetadataOptions;
 use symphonia::core::probe::Hint;
+use symphonia::core::units::Time;
 
 /// Peaks folded over this many frames before resampling to the final bucket
 /// count. Keeps memory bounded regardless of track length.
@@ -31,6 +32,25 @@ pub fn compute_waveform(path: &Path, buckets: usize) -> Vec<f32> {
     }
 }
 
+/// Like [`compute_waveform`], but for one SLICE of the file — the setlist
+/// segment rows' lane. `end_secs: None` runs to the file's end. The decode
+/// seeks coarsely and then trims by packet timestamps, so a 4-minute slice
+/// of a 90-minute set costs a 4-minute decode, not a 90-minute one.
+pub fn compute_waveform_range(
+    path: &Path,
+    buckets: usize,
+    start_secs: f64,
+    end_secs: Option<f64>,
+) -> Vec<f32> {
+    match coarse_rms_range(path, start_secs, end_secs).map(|c| resample(&c, buckets)) {
+        Ok(v) => v,
+        Err(e) => {
+            log::debug!("range waveform for {} failed: {e}", path.display());
+            Vec::new()
+        }
+    }
+}
+
 fn try_compute(path: &Path, buckets: usize) -> Result<Vec<f32>, String> {
     Ok(resample(&coarse_rms(path)?, buckets))
 }
@@ -41,6 +61,14 @@ fn try_compute(path: &Path, buckets: usize) -> Result<Vec<f32>, String> {
 /// against a real decode without re-reading the file per variant — see
 /// `bucketing_strategies_on_a_real_track`.
 fn coarse_rms(path: &Path) -> Result<Vec<f32>, String> {
+    coarse_rms_range(path, 0.0, None)
+}
+
+fn coarse_rms_range(
+    path: &Path,
+    start_secs: f64,
+    end_secs: Option<f64>,
+) -> Result<Vec<f32>, String> {
     let file = std::fs::File::open(path).map_err(|e| e.to_string())?;
     let mss = MediaSourceStream::new(Box::new(file), Default::default());
 
@@ -73,14 +101,34 @@ fn coarse_rms(path: &Path) -> Result<Vec<f32>, String> {
         .default_track()
         .ok_or_else(|| "no default track".to_string())?;
     let track_id = track.id;
+    let time_base = track.codec_params.time_base;
+    let sample_rate = track.codec_params.sample_rate;
     let mut decoder = symphonia::default::get_codecs()
         .make(&track.codec_params, &DecoderOptions::default())
         .map_err(|e| e.to_string())?;
 
+    if start_secs > 0.0 {
+        // A coarse seek lands at a sync point near (usually before) the
+        // target; the timestamp gate in the loop trims the difference. A
+        // format that cannot seek just decodes from the top — slower, same
+        // result.
+        let _ = format.seek(
+            SeekMode::Coarse,
+            SeekTo::Time {
+                time: Time::from(start_secs),
+                track_id: Some(track_id),
+            },
+        );
+        decoder.reset();
+    }
+
     let mut sample_buf: Option<SampleBuffer<f32>> = None;
     let mut coarse: Vec<f32> = Vec::new();
     let mut win_energy = 0f64;
     let mut win_count = 0usize;
+    // When the container carries no timestamps, the slice is bounded by
+    // counted frames from wherever the seek landed instead.
+    let mut frames_seen: u64 = 0;
 
     loop {
         let packet = match format.next_packet() {
@@ -90,6 +138,22 @@ fn coarse_rms(path: &Path) -> Result<Vec<f32>, String> {
         if packet.track_id() != track_id {
             continue;
         }
+        // Where this packet sits in the file, when the format can say.
+        let packet_secs = time_base.map(|tb| {
+            let t = tb.calc_time(packet.ts());
+            t.seconds as f64 + t.frac
+        });
+        if let Some(t) = packet_secs {
+            if t < start_secs {
+                // Before the slice: decode and discard, keeping codec state
+                // warm across the boundary.
+                let _ = decoder.decode(&packet);
+                continue;
+            }
+            if end_secs.is_some_and(|end| t >= end) {
+                break;
+            }
+        }
         let decoded = match decoder.decode(&packet) {
             Ok(d) => d,
             Err(symphonia::core::errors::Error::DecodeError(_)) => continue,
@@ -115,6 +179,14 @@ fn coarse_rms(path: &Path) -> Result<Vec<f32>, String> {
                 win_count = 0;
             }
         }
+        if packet_secs.is_none() {
+            frames_seen += (sb.samples().len() / channels) as u64;
+            if let (Some(end), Some(rate)) = (end_secs, sample_rate) {
+                if frames_seen as f64 / rate as f64 >= end - start_secs {
+                    break;
+                }
+            }
+        }
     }
     if win_count > 0 {
         coarse.push((win_energy / win_count as f64).sqrt() as f32);