Keelrleek/SoulStream
2
1 Fork 1
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

dj sets + lyrics: a mix that names its own tracks, and words pulled off the separated voice

Browse Source
This commit is contained in:
KeelrLeek committed 2026-08-24 10:49:10 -05:00
1 parent 4efe85b6bf
commit 1dd8573808
41 files changed
+9288 -647
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/library/src/lib.rs
+116 -20
View File
@@ -93,11 +93,12 @@ pub struct PlaylistItem {
pub artist: String,
pub album: Option<String>,
pub duration_secs: Option<u32>,
/// The playable local file, when resolution found one. An unresolved
/// import row keeps its metadata and order with no path at all.
/// The playable local file, when resolution found one. A stream or
/// unresolved row keeps its metadata and order with no path at all.
pub path: Option<String>,
pub art_path: Option<String>,
/// "local" (has a playable path) | "unresolved" (import placeholder).
/// "local" (a file on disk) | "stream" (full identity, plays through the
/// resolver) | "unresolved" (import placeholder awaiting repair).
pub kind: String,
/// Sanitized provenance: the raw locator or pasted line this row came
/// from. Repair tools read it; nothing ever executes it.
@@ -106,6 +107,11 @@ pub struct PlaylistItem {
/// begins, milliseconds.
pub start_ms: Option<i64>,
pub note: Option<String>,
/// A stream row's identity anchor: the catalog track id (exact id, never
/// a name match — the same idiom as plays.catalog_album_id).
pub track_id: Option<String>,
/// A stream row's cover, which has no local file to be read from.
pub art_url: Option<String>,
}
/// Insert payload for one playlist row (`add_playlist_items` / `import_playlist`).
@@ -117,11 +123,34 @@ pub struct NewPlaylistItem {
pub duration_secs: Option<u32>,
pub path: Option<String>,
pub art_path: Option<String>,
/// "local" | "unresolved"; empty defaults to "local" when a path exists.
/// "local" | "stream" | "unresolved"; empty lets the insert classify:
/// a path is local, a full identity streams, anything weaker is a
/// placeholder (see `pathless_kind`).
pub kind: String,
pub source_line: Option<String>,
pub start_ms: Option<i64>,
pub note: Option<String>,
pub track_id: Option<String>,
pub art_url: Option<String>,
}
/// Classify a row that names no local file. A full identity — an artist and
/// a real title, never the DJ unknown-token "ID" — plays through the normal
/// resolver exactly like a catalog track, so it is a first-class 'stream'
/// row. Anything weaker stays an 'unresolved' placeholder for the repair
/// tools. (The same gate the playlist page always used to decide whether a
/// click streams or opens repair.)
fn pathless_kind(artist: &str, title: &str) -> &'static str {
let t = title.trim();
if !artist.trim().is_empty()
&& !t.is_empty()
&& !t.eq_ignore_ascii_case("id")
&& !t.eq_ignore_ascii_case("i.d.")
{
"stream"
} else {
"unresolved"
}
}
/// One row of the whole-library matching index (playlist import resolution:
@@ -292,7 +321,9 @@ impl Library {
source_line TEXT,
start_ms INTEGER,
note TEXT,
added_at INTEGER NOT NULL DEFAULT 0
added_at INTEGER NOT NULL DEFAULT 0,
track_id TEXT,
art_url TEXT
);
CREATE INDEX IF NOT EXISTS idx_playlist_items
ON playlist_items(playlist_id, position);
@@ -392,6 +423,22 @@ impl Library {
"ALTER TABLE playlists ADD COLUMN revision INTEGER NOT NULL DEFAULT 0",
[],
);
// Playlist rows split into local vs stream (2026-08-18): a row with a
// full identity and no file is a STREAMING track, not an error state.
// Its anchor is the catalog track id (exact id, never a name match —
// the plays.catalog_album_id idiom) with the cover URL riding along,
// since a stream row has no local file to read artwork from.
let _ = conn.execute("ALTER TABLE playlist_items ADD COLUMN track_id TEXT", []);
let _ = conn.execute("ALTER TABLE playlist_items ADD COLUMN art_url TEXT", []);
// Reclassify rows stored before the split: anything 'unresolved' that
// carries an artist and a real title already streamed on click — the
// kind now says so. Idempotent, so it simply runs on every open.
let _ = conn.execute(
"UPDATE playlist_items SET kind = 'stream'
WHERE kind = 'unresolved' AND TRIM(artist) <> '' AND TRIM(title) <> ''
AND LOWER(TRIM(title)) NOT IN ('id', 'i.d.')",
[],
);
migrate_playlist_rows(&mut conn)?;
Ok(Library {
conn: Mutex::new(conn),
@@ -1126,7 +1173,7 @@ impl Library {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT id, position, title, artist, album, duration_secs, path, art_path,
kind, source_line, start_ms, note
kind, source_line, start_ms, note, track_id, art_url
FROM playlist_items WHERE playlist_id = ?1 ORDER BY position",
)?;
let rows = stmt.query_map([playlist_id], |r| {
@@ -1143,6 +1190,8 @@ impl Library {
source_line: r.get(9)?,
start_ms: r.get(10)?,
note: r.get(11)?,
track_id: r.get(12)?,
art_url: r.get(13)?,
})
})?;
rows.collect::<rusqlite::Result<Vec<_>>>().map_err(Into::into)
@@ -1198,12 +1247,17 @@ impl Library {
let mut ins = tx.prepare(
"INSERT INTO playlist_items
(playlist_id, position, title, artist, album, duration_secs,
path, art_path, kind, source_line, start_ms, note, added_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)",
path, art_path, kind, source_line, start_ms, note, added_at,
track_id, art_url)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
)?;
for (offset, item) in items.iter().enumerate() {
let kind = if item.kind.is_empty() {
if item.path.is_some() { "local" } else { "unresolved" }
if item.path.is_some() {
"local"
} else {
pathless_kind(&item.artist, &item.title)
}
} else {
&item.kind
};
@@ -1221,6 +1275,8 @@ impl Library {
item.start_ms,
item.note,
now,
item.track_id,
item.art_url,
])?;
}
}
@@ -1286,7 +1342,8 @@ impl Library {
}
/// Repair one row: attach a playable local file (or correct its display
/// metadata). Passing `path` Some flips the row to "local".
/// metadata). Passing `path` Some flips the row to "local". `track_id`
/// and `art_url` enrich a stream row's catalog anchor and cover.
#[allow(clippy::too_many_arguments)]
pub fn relink_playlist_item(
&self,
@@ -1298,6 +1355,8 @@ impl Library {
duration_secs: Option<u32>,
path: Option<&str>,
art_path: Option<&str>,
track_id: Option<&str>,
art_url: Option<&str>,
now: i64,
) -> Result<()> {
let mut conn = self.conn.lock().unwrap();
@@ -1310,9 +1369,11 @@ impl Library {
duration_secs = COALESCE(?6, duration_secs),
path = COALESCE(?7, path),
art_path = COALESCE(?8, art_path),
track_id = COALESCE(?9, track_id),
art_url = COALESCE(?10, art_url),
kind = CASE WHEN ?7 IS NOT NULL THEN 'local' ELSE kind END
WHERE playlist_id = ?1 AND id = ?2",
rusqlite::params![playlist_id, item_id, title, artist, album, duration_secs, path, art_path],
rusqlite::params![playlist_id, item_id, title, artist, album, duration_secs, path, art_path, track_id, art_url],
)?;
Self::touch_playlist(&tx, playlist_id, now)?;
tx.commit()?;
@@ -1337,9 +1398,11 @@ impl Library {
tx.execute(
"INSERT INTO playlist_items
(playlist_id, position, title, artist, album, duration_secs,
path, art_path, kind, source_line, start_ms, note, added_at)
path, art_path, kind, source_line, start_ms, note, added_at,
track_id, art_url)
SELECT ?2, position, title, artist, album, duration_secs,
path, art_path, kind, source_line, start_ms, note, ?3
path, art_path, kind, source_line, start_ms, note, ?3,
track_id, art_url
FROM playlist_items WHERE playlist_id = ?1",
rusqlite::params![id, new_id, now],
)?;
@@ -1369,12 +1432,17 @@ impl Library {
let mut ins = tx.prepare(
"INSERT INTO playlist_items
(playlist_id, position, title, artist, album, duration_secs,
path, art_path, kind, source_line, start_ms, note, added_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)",
path, art_path, kind, source_line, start_ms, note, added_at,
track_id, art_url)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)",
)?;
for (position, item) in items.iter().enumerate() {
let kind = if item.kind.is_empty() {
if item.path.is_some() { "local" } else { "unresolved" }
if item.path.is_some() {
"local"
} else {
pathless_kind(&item.artist, &item.title)
}
} else {
&item.kind
};
@@ -1392,6 +1460,8 @@ impl Library {
item.start_ms,
item.note,
now,
item.track_id,
item.art_url,
])?;
}
}
@@ -2261,7 +2331,9 @@ mod tests {
art_path: art.map(String::from),
..Default::default()
};
// Two copies of the same file, on purpose, plus an unresolved import row.
// Two copies of the same file, on purpose, plus a pathless row with a
// full identity (a streaming track, first-class) and a genuinely
// unknown one (the DJ token "ID" stays a placeholder for repair).
lib.add_playlist_items(
id,
&[
@@ -2274,10 +2346,31 @@ mod tests {
110,
)
.unwrap();
lib.add_playlist_items(
id,
&[NewPlaylistItem {
title: "ID".into(),
artist: "Artist".into(),
..Default::default()
}],
None,
111,
)
.unwrap();
let items = lib.playlist_items(id).unwrap();
assert_eq!(items.len(), 5);
assert_eq!(
items[3].kind, "stream",
"a pathless row with full identity streams — never dropped, never an error"
);
assert_eq!(
items[4].kind, "unresolved",
"the DJ unknown-token stays a placeholder for repair"
);
lib.remove_playlist_items(id, &[items[4].id], 112).unwrap();
let items = lib.playlist_items(id).unwrap();
assert_eq!(items.len(), 4);
assert_eq!(items[3].kind, "unresolved", "no path means placeholder, never dropped");
// Reorder: move the SECOND "One" first; the two copies stay distinct.
let ids: Vec<i64> = items.iter().map(|i| i.id).collect();
@@ -2308,9 +2401,10 @@ mod tests {
assert_eq!(items.len(), 4);
// Aggregates: unresolved count, duration sum, distinct art seeds.
// "Ghost" is a stream row (full identity), so nothing is unresolved.
let meta = lib.playlist_meta(id).unwrap().unwrap();
assert_eq!(meta.track_count, 4);
assert_eq!(meta.unresolved_count, 1);
assert_eq!(meta.unresolved_count, 0);
assert_eq!(meta.duration_secs, 400, "all four rows carry 100s");
assert_eq!(meta.art_seeds, vec!["C:\\m\\a.jpg".to_string(), "C:\\m\\b.jpg".into()]);
assert!(meta.revision > 0);
@@ -2349,7 +2443,7 @@ mod tests {
.unwrap();
let rows = lib.playlist_items(imported).unwrap();
assert_eq!(rows.len(), 2);
assert_eq!(rows[0].kind, "unresolved");
assert_eq!(rows[0].kind, "stream", "full identity, no file: streams");
assert_eq!(rows[1].kind, "local");
assert_eq!(rows[1].start_ms, Some(3_600_000));
let meta = lib.playlist_meta(imported).unwrap().unwrap();
@@ -2365,6 +2459,8 @@ mod tests {
Some(240),
Some("C:\\m\\opener.mp3"),
None,
None,
None,
170,
)
.unwrap();
crates/metadata/src/lib.rs
+21
View File
@@ -4500,6 +4500,27 @@ impl MetadataClient {
}
}
/// One catalog track by ISRC — the anchor-exact lookup for rows whose
/// identity came from AUDIO RECOGNITION (the recognizer reports the
/// matched release's ISRC). An id-grade identity, never a name guess;
/// `None` is an honest "that release isn't in the catalog".
pub async fn catalog_track_by_isrc(&self, isrc: &str) -> Option<CatalogTrack> {
let clean: String = isrc
.chars()
.filter(|c| c.is_ascii_alphanumeric())
.map(|c| c.to_ascii_uppercase())
.collect();
// ISRCs are exactly 12 characters (CC-XXX-YY-NNNNN without the
// dashes); anything else would only ever be a malformed URL.
if clean.len() != 12 {
return None;
}
let body = self
.deezer(&format!("https://api.deezer.com/track/isrc:{clean}"))
.await?;
parse_track(&body)
}
/// The two things the search hero can say about an artist that Deezer
/// cannot: what they sound like, and how big they are.
///
crates/player/src/lib.rs
+1 -1
View File
@@ -13,7 +13,7 @@ mod waveform;
pub use loudness::{analyze_loudness, Analysis, AnalysisSource, Incomplete, LoudnessMeter};
pub use source::{GrowingFile, StreamProgress, Terminal};
pub use waveform::compute_waveform;
pub use waveform::{compute_waveform, compute_waveform_range};
use std::collections::VecDeque;
use std::path::{Path, PathBuf};
crates/player/src/vocals.rs
+406
View File
@@ -115,6 +115,138 @@ fn decode(path: &Path, limit_secs: Option<f64>) -> Result<Decoded, String> {
})
}
// ---- streaming long-form decode ---------------------------------------------
/// Streaming 16 kHz mono decode for LONG sources (DJ sets, hour-plus mixes):
/// packets are downmixed and resampled as they arrive, so memory holds only
/// the 16 kHz mono result (~230 MB per hour as f32) and never the source-rate
/// PCM. The whole-file `decode` above deliberately caps at 20 minutes; this
/// is the path past that cap, bounded by `max_secs` of OUTPUT.
pub fn mono_pcm_16k_streaming(path: &Path, max_secs: f64) -> Result<Vec<f32>, String> {
let file = std::fs::File::open(path).map_err(|e| e.to_string())?;
let mss = MediaSourceStream::new(Box::new(file), Default::default());
let mut hint = Hint::new();
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
hint.with_extension(ext);
}
let probed = symphonia::default::get_probe()
.format(
&hint,
mss,
&FormatOptions::default(),
&MetadataOptions::default(),
)
.map_err(|e| e.to_string())?;
let mut format = probed.format;
let track = format
.default_track()
.ok_or("no default audio track")?
.clone();
let mut decoder = symphonia::default::get_codecs()
.make(&track.codec_params, &DecoderOptions::default())
.map_err(|e| e.to_string())?;
let mut out: Vec<f32> = Vec::new();
let cap = (max_secs * 16_000.0) as usize;
let mut resampler: Option<StreamResampler> = None;
let mut buffer: Option<SampleBuffer<f32>> = None;
loop {
if out.len() >= cap {
break;
}
let packet = match format.next_packet() {
Ok(p) => p,
Err(_) => break,
};
if packet.track_id() != track.id {
continue;
}
let decoded = match decoder.decode(&packet) {
Ok(d) => d,
Err(symphonia::core::errors::Error::DecodeError(_)) => continue,
Err(_) => break,
};
let spec = *decoded.spec();
let channels = spec.channels.count().max(1);
let buf = buffer.get_or_insert_with(|| {
SampleBuffer::<f32>::new(decoded.capacity() as u64, spec)
});
if buf.capacity() < decoded.capacity() * channels {
*buf = SampleBuffer::<f32>::new(decoded.capacity() as u64, spec);
}
buf.copy_interleaved_ref(decoded);
let rs = resampler.get_or_insert_with(|| StreamResampler::new(spec.rate, 16_000));
rs.push_interleaved(buf.samples(), channels, &mut out);
}
if out.is_empty() {
return Err("nothing decodable".into());
}
out.truncate(cap);
Ok(out)
}
/// Linear resampler with carry across pushes, so packet boundaries do not
/// click or drift: keeps the previous source sample and a fractional read
/// position between calls.
struct StreamResampler {
ratio: f64,
/// Fractional position into the CURRENT push, may start negative into
/// the carried sample.
pos: f64,
carry: f32,
has_carry: bool,
passthrough: bool,
}
impl StreamResampler {
fn new(src_rate: u32, dst_rate: u32) -> Self {
StreamResampler {
ratio: src_rate as f64 / dst_rate as f64,
pos: 0.0,
carry: 0.0,
has_carry: false,
passthrough: src_rate == dst_rate,
}
}
/// Downmix `interleaved` to mono and append the resampled result to
/// `out`.
fn push_interleaved(&mut self, interleaved: &[f32], channels: usize, out: &mut Vec<f32>) {
let frames = interleaved.len() / channels.max(1);
if frames == 0 {
return;
}
let mono = |i: usize| -> f32 {
let row = &interleaved[i * channels..(i + 1) * channels];
row.iter().sum::<f32>() / channels as f32
};
if self.passthrough {
out.extend((0..frames).map(mono));
return;
}
// Source timeline for this push: sample -1 is the carried one.
loop {
let i0 = self.pos.floor();
let frac = (self.pos - i0) as f32;
let i0 = i0 as isize;
if i0 + 1 >= frames as isize {
break;
}
let a = if i0 < 0 {
if self.has_carry { self.carry } else { mono(0) }
} else {
mono(i0 as usize)
};
let b = mono((i0 + 1) as usize);
out.push(a + (b - a) * frac);
self.pos += self.ratio;
}
self.carry = mono(frames - 1);
self.has_carry = true;
self.pos -= frames as f64;
}
}
// ---- leading silence -------------------------------------------------------
/// Milliseconds of leading silence in `path`, or None when the file cannot be
@@ -216,6 +348,216 @@ pub fn vocal_activity(path: &Path, frame_ms: u32) -> Result<VocalActivity, Strin
Ok(VocalActivity { frame_ms, frames })
}
// ---- mono PCM for the CTC aligner ------------------------------------------
/// Whole-file VOCAL-FORWARD mono PCM at 16 kHz for the CTC aligner: the mid
/// (center) channel through the same 180 Hz – 7.5 kHz band the reducer and
/// the activity envelope use, linearly resampled. Cutting bass/kick
/// fundamentals and cymbal air before the model's per-window normalization
/// leaves the voice owning more of the input's dynamic range — cheap
/// pre-emphasis, NOT separation (center-mixed instruments remain). The
/// low-pass doubles as a crude anti-alias guard for the 16 kHz resample.
pub fn vocal_pcm_16k(path: &Path) -> Result<Vec<f32>, String> {
let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
let ch = decoded.channels.max(1);
let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0);
let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0);
let mono: Vec<f32> = decoded
.samples
.chunks(ch)
.map(|f| high.process(low.process(f.iter().sum::<f32>() / ch as f32)))
.collect();
let src = decoded.sample_rate as f64;
if (src - 16_000.0).abs() < 1.0 {
return Ok(mono);
}
let ratio = 16_000.0 / src;
let out_len = (mono.len() as f64 * ratio) as usize;
let mut out = Vec::with_capacity(out_len);
for i in 0..out_len {
let pos = i as f64 / ratio;
let i0 = pos as usize;
let frac = (pos - i0 as f64) as f32;
let a = mono.get(i0).copied().unwrap_or(0.0);
let b = mono.get(i0 + 1).copied().unwrap_or(a);
out.push(a + (b - a) * frac);
}
Ok(out)
}
/// Stereo 44.1 kHz planar PCM for the learned vocal separator: any layout
/// folds to two channels (mono duplicates its one, extra channels beyond the
/// front pair are ignored), any rate linearly resampled. Linear interpolation
/// is deliberate — the stem feeds ALIGNMENT, not ears, the same tradeoff
/// `vocal_pcm_16k` already makes for its own input.
pub fn separation_input_44k1(path: &Path) -> Result<[Vec<f32>; 2], String> {
let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
let ch = decoded.channels.max(1);
let frames = decoded.samples.len() / ch;
let mut left = Vec::with_capacity(frames);
let mut right = Vec::with_capacity(frames);
for f in decoded.samples.chunks(ch) {
let l = f[0];
left.push(l);
right.push(if f.len() > 1 { f[1] } else { l });
}
Ok([
resample_linear(left, decoded.sample_rate as f64, 44_100.0),
resample_linear(right, decoded.sample_rate as f64, 44_100.0),
])
}
/// Aligner input from a separated vocal stem (44.1 kHz planar stereo): mid
/// fold, a 60 Hz rumble high-pass, the same 7.5 kHz low-pass `vocal_pcm_16k`
/// uses as its 16 kHz anti-alias guard, then the resample. No 180 Hz floor
/// here — that floor exists to fight INSTRUMENTS sharing the band, and a
/// separated stem has (mostly) none left to fight.
pub fn stem_pcm_16k(left: &[f32], right: &[f32]) -> Vec<f32> {
let mut rumble = Biquad::highpass(44_100.0, 60.0);
let mut guard = Biquad::lowpass(44_100.0, 7_500.0);
let n = left.len().min(right.len());
let mono: Vec<f32> = (0..n)
.map(|i| guard.process(rumble.process((left[i] + right[i]) * 0.5)))
.collect();
resample_linear(mono, 44_100.0, 16_000.0)
}
/// Linear resampler shared by the separator inputs (see the tradeoff note on
/// `separation_input_44k1`). Within 1 Hz of the target it hands the buffer
/// back untouched — whole-track copies are not free at 44.1 kHz.
fn resample_linear(src_pcm: Vec<f32>, src: f64, dst: f64) -> Vec<f32> {
if (src - dst).abs() < 1.0 {
return src_pcm;
}
let ratio = dst / src;
let out_len = (src_pcm.len() as f64 * ratio) as usize;
let mut out = Vec::with_capacity(out_len);
for i in 0..out_len {
let pos = i as f64 / ratio;
let i0 = pos as usize;
let frac = (pos - i0 as f64) as f32;
let a = src_pcm.get(i0).copied().unwrap_or(0.0);
let b = src_pcm.get(i0 + 1).copied().unwrap_or(a);
out.push(a + (b - a) * frac);
}
out
}
// ---- vocal features (word-timing worker) -----------------------------------
/// Per-frame vocal evidence for the lyric word-timing worker: the band
/// energy `vocal_activity` measures PLUS a voicing verdict — vocals are
/// PITCHED, so an autocorrelation clarity check on a decimated mid channel
/// separates sung frames from drums and noise that light the band up.
pub struct VocalFrames {
pub frame_ms: u32,
/// 0..1 per frame, normalized like `vocal_activity`.
pub energy: Vec<f32>,
/// Frame carries periodic (pitched) content in the vocal F0 range.
pub voiced: Vec<bool>,
}
pub fn vocal_frames(path: &Path, frame_ms: u32) -> Result<VocalFrames, String> {
let decoded = decode(path, Some(MAX_RENDER_SECS + 2.0))?;
let frame_ms = frame_ms.clamp(40, 500);
let frame_len = (decoded.sample_rate as usize * frame_ms as usize) / 1000;
let channels = decoded.channels;
let mut low = Biquad::highpass(decoded.sample_rate as f32, 180.0);
let mut high = Biquad::lowpass(decoded.sample_rate as f32, 7_500.0);
// Decimate the mid channel to ~8 kHz for the pitch check — full-rate
// autocorrelation across a song is debug-build poison, and F0 up to
// 500 Hz needs nothing like 44.1 kHz.
let decim = (decoded.sample_rate as usize / 8_000).max(1);
let target_rate = decoded.sample_rate as usize / decim;
let mut energy = Vec::new();
let mut voiced = Vec::new();
let mut acc = 0.0f64;
let mut n = 0usize;
let mut pitch_buf: Vec<f32> = Vec::with_capacity(frame_len / decim + 2);
let mut decim_acc = 0.0f32;
let mut decim_n = 0usize;
for frame in decoded.samples.chunks(channels) {
let mid = if channels >= 2 {
0.5 * (frame[0] + frame[1])
} else {
frame[0]
};
let banded = high.process(low.process(mid));
acc += (banded * banded) as f64;
n += 1;
decim_acc += banded;
decim_n += 1;
if decim_n >= decim {
pitch_buf.push(decim_acc / decim as f32);
decim_acc = 0.0;
decim_n = 0;
}
if n >= frame_len {
energy.push((acc / n as f64).sqrt() as f32);
voiced.push(frame_is_voiced(&pitch_buf, target_rate));
pitch_buf.clear();
acc = 0.0;
n = 0;
}
}
if n > frame_len / 4 {
energy.push((acc / n.max(1) as f64).sqrt() as f32);
voiced.push(frame_is_voiced(&pitch_buf, target_rate));
}
if energy.is_empty() {
return Err("nothing decodable".into());
}
let mut sorted: Vec<f32> = energy.clone();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let p95 = sorted[(sorted.len() * 95 / 100).min(sorted.len() - 1)].max(1e-6);
for value in &mut energy {
*value = (*value / p95).min(1.0);
}
Ok(VocalFrames {
frame_ms,
energy,
voiced,
})
}
/// Normalized-autocorrelation clarity over the sung F0 range (80–500 Hz).
/// A frame is voiced when some lag in that range correlates strongly —
/// periodic content — rather than merely being loud.
fn frame_is_voiced(buf: &[f32], rate: usize) -> bool {
if buf.len() < 64 || rate == 0 {
return false;
}
let energy: f32 = buf.iter().map(|s| s * s).sum();
if energy < 1e-7 {
return false;
}
let lag_min = (rate / 500).max(2);
let lag_max = (rate / 80).min(buf.len() / 2);
if lag_min >= lag_max {
return false;
}
let mut best = 0.0f32;
let mut lag = lag_min;
// Stride lags by 2: clarity peaks are broad; halving the work loses
// nothing a boolean cares about.
while lag < lag_max {
let n = buf.len() - lag;
let mut num = 0.0f32;
let mut den_a = 0.0f32;
let mut den_b = 0.0f32;
for i in 0..n {
num += buf[i] * buf[i + lag];
den_a += buf[i] * buf[i];
den_b += buf[i + lag] * buf[i + lag];
}
let den = (den_a * den_b).sqrt().max(1e-9);
best = best.max(num / den);
lag += 2;
}
best >= 0.55
}
// ---- vocal reduction -------------------------------------------------------
/// What the renderer produced (for the caller's labels and sanity checks).
@@ -407,6 +749,70 @@ mod tests {
out
}
/// The streaming resampler must produce the same stream regardless of
/// how the source is chopped into pushes — no clicks, no drift at
/// packet boundaries.
#[test]
fn stream_resampler_is_chunking_invariant() {
let src_rate = 44_100u32;
let n = 44_100usize;
let stereo: Vec<f32> = (0..n)
.flat_map(|i| {
let s = (2.0 * std::f32::consts::PI * 440.0 * i as f32 / 44_100.0).sin();
[s, s]
})
.collect();
let mut one_shot = Vec::new();
let mut a = StreamResampler::new(src_rate, 16_000);
a.push_interleaved(&stereo, 2, &mut one_shot);
let mut chunked = Vec::new();
let mut b = StreamResampler::new(src_rate, 16_000);
let mut at = 0usize;
for (k, step) in [431usize, 977, 1153, 64, 2048].iter().cycle().enumerate() {
if at >= stereo.len() {
break;
}
let end = (at + step * 2).min(stereo.len());
b.push_interleaved(&stereo[at..end], 2, &mut chunked);
at = end;
assert!(k < 200_000, "runaway");
}
assert!(
(one_shot.len() as i64 - chunked.len() as i64).abs() <= 2,
"length drift: {} vs {}",
one_shot.len(),
chunked.len()
);
let worst = one_shot
.iter()
.zip(&chunked)
.map(|(x, y)| (x - y).abs())
.fold(0.0f32, f32::max);
assert!(worst < 1e-4, "boundary artifacts, worst {worst}");
}
/// The stem→aligner path: a vocal-band tone passes the 60 Hz / 7.5 kHz
/// hygiene filters and lands at 16 kHz with its energy intact.
#[test]
fn stem_pcm_16k_resamples_and_keeps_vocal_band() {
let frames = 44_100usize;
let mono: Vec<f32> = (0..frames)
.map(|i| (2.0 * std::f32::consts::PI * 1_000.0 * i as f32 / 44_100.0).sin() * 0.5)
.collect();
let out = stem_pcm_16k(&mono, &mono);
assert!(
(15_998..=16_000).contains(&out.len()),
"one second must land at ~16000 samples, got {}",
out.len()
);
let level = rms(&out);
assert!(
level > 0.3 && level < 0.4,
"1kHz mid tone must pass the stem filters near unity, rms {level}"
);
assert!(out.iter().all(|s| s.is_finite()));
}
/// The whole point: a center-panned vocal-band tone dies, a center bass
/// tone survives, side content survives.
#[test]
crates/player/src/waveform.rs
+73 -1
View File
@@ -10,10 +10,11 @@ use std::path::Path;
use symphonia::core::audio::SampleBuffer;
use symphonia::core::codecs::DecoderOptions;
use symphonia::core::formats::FormatOptions;
use symphonia::core::formats::{FormatOptions, SeekMode, SeekTo};
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
use symphonia::core::probe::Hint;
use symphonia::core::units::Time;
/// Peaks folded over this many frames before resampling to the final bucket
/// count. Keeps memory bounded regardless of track length.
@@ -31,6 +32,25 @@ pub fn compute_waveform(path: &Path, buckets: usize) -> Vec<f32> {
}
}
/// Like [`compute_waveform`], but for one SLICE of the file — the setlist
/// segment rows' lane. `end_secs: None` runs to the file's end. The decode
/// seeks coarsely and then trims by packet timestamps, so a 4-minute slice
/// of a 90-minute set costs a 4-minute decode, not a 90-minute one.
pub fn compute_waveform_range(
path: &Path,
buckets: usize,
start_secs: f64,
end_secs: Option<f64>,
) -> Vec<f32> {
match coarse_rms_range(path, start_secs, end_secs).map(|c| resample(&c, buckets)) {
Ok(v) => v,
Err(e) => {
log::debug!("range waveform for {} failed: {e}", path.display());
Vec::new()
}
}
}
fn try_compute(path: &Path, buckets: usize) -> Result<Vec<f32>, String> {
Ok(resample(&coarse_rms(path)?, buckets))
}
@@ -41,6 +61,14 @@ fn try_compute(path: &Path, buckets: usize) -> Result<Vec<f32>, String> {
/// against a real decode without re-reading the file per variant — see
/// `bucketing_strategies_on_a_real_track`.
fn coarse_rms(path: &Path) -> Result<Vec<f32>, String> {
coarse_rms_range(path, 0.0, None)
}
fn coarse_rms_range(
path: &Path,
start_secs: f64,
end_secs: Option<f64>,
) -> Result<Vec<f32>, String> {
let file = std::fs::File::open(path).map_err(|e| e.to_string())?;
let mss = MediaSourceStream::new(Box::new(file), Default::default());
@@ -73,14 +101,34 @@ fn coarse_rms(path: &Path) -> Result<Vec<f32>, String> {
.default_track()
.ok_or_else(|| "no default track".to_string())?;
let track_id = track.id;
let time_base = track.codec_params.time_base;
let sample_rate = track.codec_params.sample_rate;
let mut decoder = symphonia::default::get_codecs()
.make(&track.codec_params, &DecoderOptions::default())
.map_err(|e| e.to_string())?;
if start_secs > 0.0 {
// A coarse seek lands at a sync point near (usually before) the
// target; the timestamp gate in the loop trims the difference. A
// format that cannot seek just decodes from the top — slower, same
// result.
let _ = format.seek(
SeekMode::Coarse,
SeekTo::Time {
time: Time::from(start_secs),
track_id: Some(track_id),
},
);
decoder.reset();
}
let mut sample_buf: Option<SampleBuffer<f32>> = None;
let mut coarse: Vec<f32> = Vec::new();
let mut win_energy = 0f64;
let mut win_count = 0usize;
// When the container carries no timestamps, the slice is bounded by
// counted frames from wherever the seek landed instead.
let mut frames_seen: u64 = 0;
loop {
let packet = match format.next_packet() {
@@ -90,6 +138,22 @@ fn coarse_rms(path: &Path) -> Result<Vec<f32>, String> {
if packet.track_id() != track_id {
continue;
}
// Where this packet sits in the file, when the format can say.
let packet_secs = time_base.map(|tb| {
let t = tb.calc_time(packet.ts());
t.seconds as f64 + t.frac
});
if let Some(t) = packet_secs {
if t < start_secs {
// Before the slice: decode and discard, keeping codec state
// warm across the boundary.
let _ = decoder.decode(&packet);
continue;
}
if end_secs.is_some_and(|end| t >= end) {
break;
}
}
let decoded = match decoder.decode(&packet) {
Ok(d) => d,
Err(symphonia::core::errors::Error::DecodeError(_)) => continue,
@@ -115,6 +179,14 @@ fn coarse_rms(path: &Path) -> Result<Vec<f32>, String> {
win_count = 0;
}
}
if packet_secs.is_none() {
frames_seen += (sb.samples().len() / channels) as u64;
if let (Some(end), Some(rate)) = (end_secs, sample_rate) {
if frames_seen as f64 / rate as f64 >= end - start_secs {
break;
}
}
}
}
if win_count > 0 {
coarse.push((win_energy / win_count as f64).sqrt() as f32);