forked from Keelrleek/SoulStream
dj sets + lyrics: a mix that names its own tracks, and words pulled off the separated voice
This commit is contained in:
KeelrLeek committed
1 parent
4efe85b6bf
commit
1dd8573808
41 files changed
+9288
-647
No files matched your search
@@ -0,0 +1,478 @@
|
||||
//! Shazam-style audio recognition for the DJ-set setlist assistant — the
|
||||
//! research bundle's "broader recognition adapter" (feasibility §3 rung 4,
|
||||
//! plan phase 7), Kyle's pick 2026-08-19: identify a mix's tracks by
|
||||
//! LISTENING, not just by reading its posted tracklist.
|
||||
//!
|
||||
//! The signature algorithm and packet format are reimplemented from the
|
||||
//! publicly documented Shazam fingerprint (Wang 2003; layout and constants
|
||||
//! cross-checked against the open-source SongRec project's description of
|
||||
//! the format). Nothing here links foreign code: plain Rust over the same
|
||||
//! realfft the vocal separator uses.
|
||||
//!
|
||||
//! Posture: in-process, no child processes; ONE request per analyzed
|
||||
//! window against the free discovery endpoint, paced by the caller
|
||||
//! (research §11 — rate limits, timeout, cache belong to the job layer,
|
||||
//! built next). Failure is always soft: a window that cannot be
|
||||
//! recognized is an honest Unknown, never a guess.
|
||||
|
||||
use realfft::num_complex::Complex;
|
||||
use realfft::RealFftPlanner;
|
||||
|
||||
/// One recognized window: what Shazam heard and where it claims the window
|
||||
/// sits inside the identified track.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Recognition {
|
||||
pub artist: String,
|
||||
pub title: String,
|
||||
/// Shazam track key — stable id for consensus grouping.
|
||||
pub key: String,
|
||||
/// Offset of the analyzed window within the identified track, seconds,
|
||||
/// when the response carries one (it usually does).
|
||||
pub offset_secs: Option<f64>,
|
||||
pub album: Option<String>,
|
||||
pub cover_url: Option<String>,
|
||||
/// The matched RELEASE's ISRC when the response carries one — an
|
||||
/// id-grade identity the review can resolve against the catalog
|
||||
/// exactly, instead of by name.
|
||||
pub isrc: Option<String>,
|
||||
}
|
||||
|
||||
// ---- signature generation (pure DSP) ----------------------------------------
|
||||
|
||||
const FFT_SIZE: usize = 2048;
|
||||
const BINS: usize = FFT_SIZE / 2 + 1; // 1025
|
||||
const HOP: usize = 128;
|
||||
/// The generator's working window: 12 s at 16 kHz mono, the length the
|
||||
/// service expects a tag to be.
|
||||
pub const WINDOW_SECS: usize = 12;
|
||||
pub const SAMPLE_RATE: usize = 16_000;
|
||||
|
||||
struct Rings {
|
||||
samples: [i16; FFT_SIZE],
|
||||
samples_at: usize,
|
||||
ffts: Vec<[f32; BINS]>,
|
||||
ffts_at: u8,
|
||||
spread: Vec<[f32; BINS]>,
|
||||
spread_at: u8,
|
||||
passes: u32,
|
||||
peaks: [Vec<Peak>; 4],
|
||||
}
|
||||
|
||||
struct Peak {
|
||||
pass: u32,
|
||||
magnitude: u16,
|
||||
corrected_bin: u16,
|
||||
}
|
||||
|
||||
/// Symmetric Hann of length 2048 (numpy-style, N−1 denominator) — the
|
||||
/// window the reference tables were generated from.
|
||||
fn hann() -> Vec<f32> {
|
||||
(0..FFT_SIZE)
|
||||
.map(|i| {
|
||||
0.5 - 0.5
|
||||
* (2.0 * std::f32::consts::PI * i as f32 / (FFT_SIZE - 1) as f32).cos()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Build the binary signature for ≤ WINDOW_SECS of 16 kHz mono PCM.
|
||||
pub fn signature(pcm: &[f32]) -> Vec<u8> {
|
||||
let mut samples: Vec<i16> = pcm
|
||||
.iter()
|
||||
.map(|&x| (x * 32768.0).clamp(-32768.0, 32767.0) as i16)
|
||||
.collect();
|
||||
let want = WINDOW_SECS * SAMPLE_RATE;
|
||||
if samples.len() < want {
|
||||
samples.resize(want, 0);
|
||||
} else {
|
||||
samples.truncate(want);
|
||||
}
|
||||
|
||||
let window = hann();
|
||||
let mut planner = RealFftPlanner::<f32>::new();
|
||||
let fft = planner.plan_fft_forward(FFT_SIZE);
|
||||
let mut rings = Rings {
|
||||
samples: [0i16; FFT_SIZE],
|
||||
samples_at: 0,
|
||||
ffts: vec![[0.0; BINS]; 256],
|
||||
ffts_at: 0,
|
||||
spread: vec![[0.0; BINS]; 256],
|
||||
spread_at: 0,
|
||||
passes: 0,
|
||||
peaks: Default::default(),
|
||||
};
|
||||
let mut re_in = vec![0.0f32; FFT_SIZE];
|
||||
let mut cx_out = vec![Complex::default(); BINS];
|
||||
|
||||
for chunk in samples.chunks_exact(HOP) {
|
||||
// Ring in the new hop, then window the reordered buffer.
|
||||
for (i, &s) in chunk.iter().enumerate() {
|
||||
rings.samples[(rings.samples_at + i) & (FFT_SIZE - 1)] = s;
|
||||
}
|
||||
rings.samples_at = (rings.samples_at + HOP) & (FFT_SIZE - 1);
|
||||
for i in 0..FFT_SIZE {
|
||||
re_in[i] =
|
||||
rings.samples[(i + rings.samples_at) & (FFT_SIZE - 1)] as f32 * window[i];
|
||||
}
|
||||
let _ = fft.process(&mut re_in, &mut cx_out);
|
||||
{
|
||||
let out = &mut rings.ffts[rings.ffts_at as usize];
|
||||
for (o, c) in out.iter_mut().zip(cx_out.iter()) {
|
||||
*o = ((c.re * c.re + c.im * c.im) / (1 << 17) as f32).max(1e-10);
|
||||
}
|
||||
}
|
||||
rings.ffts_at = rings.ffts_at.wrapping_add(1);
|
||||
|
||||
// Frequency- then time-domain peak spreading.
|
||||
{
|
||||
let src = rings.ffts[rings.ffts_at.wrapping_sub(1) as usize];
|
||||
let dst = &mut rings.spread[rings.spread_at as usize];
|
||||
dst.copy_from_slice(&src);
|
||||
for i in 0..=BINS - 3 {
|
||||
dst[i] = dst[i].max(dst[i + 1]).max(dst[i + 2]);
|
||||
}
|
||||
let snapshot = *dst;
|
||||
for former in [1u8, 3, 6] {
|
||||
let f = &mut rings.spread[rings.spread_at.wrapping_sub(former) as usize];
|
||||
for i in 0..BINS {
|
||||
f[i] = f[i].max(snapshot[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
rings.spread_at = rings.spread_at.wrapping_add(1);
|
||||
rings.passes += 1;
|
||||
|
||||
if rings.passes >= 46 {
|
||||
recognize_peaks(&mut rings);
|
||||
}
|
||||
}
|
||||
|
||||
encode(&rings.peaks, want as u32)
|
||||
}
|
||||
|
||||
fn recognize_peaks(r: &mut Rings) {
|
||||
let f46 = r.ffts[r.ffts_at.wrapping_sub(46) as usize];
|
||||
let s49 = r.spread[r.spread_at.wrapping_sub(49) as usize];
|
||||
for bin in 10..=BINS - 11 {
|
||||
if f46[bin] < 1.0 / 64.0 || f46[bin] < s49[bin - 1] {
|
||||
continue;
|
||||
}
|
||||
let mut neighbor_max = 0.0f32;
|
||||
for off in [-10i32, -7, -4, -3, 1, 2, 5, 8] {
|
||||
neighbor_max = neighbor_max.max(s49[(bin as i32 + off) as usize]);
|
||||
}
|
||||
if f46[bin] <= neighbor_max {
|
||||
continue;
|
||||
}
|
||||
let mut other_max = neighbor_max;
|
||||
for off in [
|
||||
-53i32, -45, 165, 172, 179, 186, 193, 200, 214, 221, 228, 235, 242, 249,
|
||||
] {
|
||||
let other = &r.spread[((r.spread_at as i32 + off) & 255) as usize];
|
||||
other_max = other_max.max(other[bin - 1]);
|
||||
}
|
||||
if f46[bin] <= other_max {
|
||||
continue;
|
||||
}
|
||||
let pass = r.passes - 46;
|
||||
let mag = f46[bin].ln().max(1.0 / 64.0) * 1477.3 + 6144.0;
|
||||
let before = f46[bin - 1].ln().max(1.0 / 64.0) * 1477.3 + 6144.0;
|
||||
let after = f46[bin + 1].ln().max(1.0 / 64.0) * 1477.3 + 6144.0;
|
||||
let variation_1 = mag * 2.0 - before - after;
|
||||
if variation_1 < 0.0 {
|
||||
continue;
|
||||
}
|
||||
let variation_2 = (after - before) * 32.0 / variation_1;
|
||||
let corrected = (bin as i32 * 64 + variation_2 as i32) as u16;
|
||||
let hz = corrected as f32 * (SAMPLE_RATE as f32 / 2.0 / 1024.0 / 64.0);
|
||||
let band = match hz as i32 {
|
||||
250..=519 => 0,
|
||||
520..=1449 => 1,
|
||||
1450..=3499 => 2,
|
||||
3500..=5500 => 3,
|
||||
_ => continue,
|
||||
};
|
||||
r.peaks[band].push(Peak {
|
||||
pass,
|
||||
magnitude: mag as u16,
|
||||
corrected_bin: corrected,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// The wire format: 48-byte header (magic, CRC-32 of everything after the
|
||||
/// first 8 bytes, payload size, 16 kHz marker, sample count + 0.24 s), a
|
||||
/// type-length echo, then per-band peak runs with delta-coded pass numbers.
|
||||
fn encode(peaks: &[Vec<Peak>; 4], number_samples: u32) -> Vec<u8> {
|
||||
let mut out: Vec<u8> = Vec::with_capacity(4096);
|
||||
let w32 = |out: &mut Vec<u8>, v: u32| out.extend_from_slice(&v.to_le_bytes());
|
||||
let w16 = |out: &mut Vec<u8>, v: u16| out.extend_from_slice(&v.to_le_bytes());
|
||||
|
||||
w32(&mut out, 0xcafe2580); // magic1
|
||||
w32(&mut out, 0); // crc32 backfilled
|
||||
w32(&mut out, 0); // size backfilled
|
||||
w32(&mut out, 0x94119c00); // magic2
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, 3u32 << 27); // 16 kHz
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, 0);
|
||||
w32(&mut out, number_samples + (SAMPLE_RATE as f32 * 0.24) as u32);
|
||||
w32(&mut out, (15 << 19) + 0x40000);
|
||||
w32(&mut out, 0x40000000);
|
||||
w32(&mut out, 0); // size echo backfilled
|
||||
|
||||
for (band, list) in peaks.iter().enumerate() {
|
||||
if list.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut body: Vec<u8> = Vec::with_capacity(list.len() * 5);
|
||||
let mut pass = 0u32;
|
||||
for p in list {
|
||||
if p.pass - pass >= 255 {
|
||||
body.push(0xff);
|
||||
body.extend_from_slice(&p.pass.to_le_bytes());
|
||||
pass = p.pass;
|
||||
}
|
||||
body.push((p.pass - pass) as u8);
|
||||
body.extend_from_slice(&p.magnitude.to_le_bytes());
|
||||
body.extend_from_slice(&p.corrected_bin.to_le_bytes());
|
||||
pass = p.pass;
|
||||
}
|
||||
w32(&mut out, 0x60030040 + band as u32);
|
||||
w32(&mut out, body.len() as u32);
|
||||
out.extend_from_slice(&body);
|
||||
while out.len() % 4 != 0 {
|
||||
out.push(0);
|
||||
}
|
||||
}
|
||||
let _ = w16; // (kept for symmetry; u16 writes happen inside the body)
|
||||
|
||||
let size = (out.len() - 48) as u32;
|
||||
out[8..12].copy_from_slice(&size.to_le_bytes());
|
||||
out[52..56].copy_from_slice(&size.to_le_bytes());
|
||||
let mut hasher = crc32fast::Hasher::new();
|
||||
hasher.update(&out[8..]);
|
||||
let crc = hasher.finalize();
|
||||
out[4..8].copy_from_slice(&crc.to_le_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
// ---- the discovery request ----------------------------------------------------
|
||||
|
||||
/// Pseudo-UUID v4 from OS entropy via the stdlib hasher — unique enough for
|
||||
/// the tag ids the endpoint expects, no uuid/rand dependency.
|
||||
fn pseudo_uuid() -> String {
|
||||
use std::collections::hash_map::RandomState;
|
||||
use std::hash::{BuildHasher, Hasher};
|
||||
let mut parts = [0u64; 2];
|
||||
for p in parts.iter_mut() {
|
||||
let mut h = RandomState::new().build_hasher();
|
||||
h.write_u64(std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.subsec_nanos() as u64)
|
||||
.unwrap_or(0));
|
||||
*p = h.finish();
|
||||
}
|
||||
let b: Vec<u8> = parts.iter().flat_map(|p| p.to_le_bytes()).collect();
|
||||
format!(
|
||||
"{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-4{:01x}{:02x}-a{:01x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}",
|
||||
b[0], b[1], b[2], b[3], b[4], b[5], b[6] & 0xf, b[7], b[8] & 0xf, b[9], b[10], b[11], b[12], b[13], b[14], b[15]
|
||||
)
|
||||
}
|
||||
|
||||
/// Recognize one ≤12 s window of 16 kHz mono PCM. `Ok(None)` = the service
|
||||
/// answered and heard nothing it knows — an honest Unknown. `Err` = the
|
||||
/// request itself failed (network, rate limit) and the caller may retry
|
||||
/// later.
|
||||
pub async fn recognize_window(pcm16k: &[f32]) -> Result<Option<Recognition>, String> {
|
||||
let sig = signature(pcm16k);
|
||||
use base64::Engine as _;
|
||||
let uri = format!(
|
||||
"data:audio/vnd.shazam.sig;base64,{}",
|
||||
base64::engine::general_purpose::STANDARD.encode(&sig)
|
||||
);
|
||||
let sample_ms = (pcm16k.len().min(WINDOW_SECS * SAMPLE_RATE) * 1000 / SAMPLE_RATE) as u64;
|
||||
let ts = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
let body = serde_json::json!({
|
||||
"geolocation": { "altitude": 300, "latitude": 45, "longitude": 2 },
|
||||
"signature": { "samplems": sample_ms, "timestamp": ts, "uri": uri },
|
||||
"timestamp": ts,
|
||||
"timezone": "Europe/Paris",
|
||||
});
|
||||
let url = format!(
|
||||
"https://amp.shazam.com/discovery/v5/en/US/android/-/tag/{}/{}?sync=true&webv3=true&sampling=true&connected=&shazamapiversion=v3&sharehub=true&video=v3",
|
||||
pseudo_uuid().to_uppercase(),
|
||||
pseudo_uuid()
|
||||
);
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent("Dalvik/2.1.0 (Linux; U; Android 13; Pixel 7 Build/TQ3A.230901.001)")
|
||||
.timeout(std::time::Duration::from_secs(20))
|
||||
.build()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let resp = client
|
||||
.post(&url)
|
||||
.header("Content-Language", "en_US")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Shazam did not answer: {e}"))?;
|
||||
if resp.status().as_u16() == 429 {
|
||||
return Err("rate-limited by the recognition service".into());
|
||||
}
|
||||
let v: serde_json::Value = resp
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| format!("unreadable recognition response: {e}"))?;
|
||||
let Some(track) = v.get("track") else {
|
||||
return Ok(None);
|
||||
};
|
||||
let title = track
|
||||
.get("title")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let artist = track
|
||||
.get("subtitle")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
if title.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let key = track
|
||||
.get("key")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let offset_secs = v
|
||||
.get("matches")
|
||||
.and_then(|m| m.get(0))
|
||||
.and_then(|m| m.get("offset"))
|
||||
.and_then(serde_json::Value::as_f64);
|
||||
// Album rides in the metadata sections when present.
|
||||
let mut album = None;
|
||||
if let Some(sections) = track.get("sections").and_then(serde_json::Value::as_array) {
|
||||
for s in sections {
|
||||
if let Some(meta) = s.get("metadata").and_then(serde_json::Value::as_array) {
|
||||
for m in meta {
|
||||
if m.get("title").and_then(serde_json::Value::as_str) == Some("Album") {
|
||||
album = m
|
||||
.get("text")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let cover_url = track
|
||||
.get("images")
|
||||
.and_then(|i| i.get("coverart"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string);
|
||||
let isrc = track
|
||||
.get("isrc")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string);
|
||||
Ok(Some(Recognition {
|
||||
artist,
|
||||
title,
|
||||
key,
|
||||
offset_secs,
|
||||
album,
|
||||
cover_url,
|
||||
isrc,
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The packet must round out structurally: magics in place, CRC over
|
||||
/// bytes 8.., size echoed at 8 and 52, bands tagged 0x60030040+n.
|
||||
#[test]
|
||||
fn signature_packet_is_well_formed() {
|
||||
// A tone sweep gives the recognizer real peaks to encode.
|
||||
let pcm: Vec<f32> = (0..WINDOW_SECS * SAMPLE_RATE)
|
||||
.map(|i| {
|
||||
let t = i as f32 / SAMPLE_RATE as f32;
|
||||
(2.0 * std::f32::consts::PI * (440.0 + 80.0 * t) * t).sin() * 0.4
|
||||
})
|
||||
.collect();
|
||||
let sig = signature(&pcm);
|
||||
assert!(sig.len() > 64, "packet too small: {}", sig.len());
|
||||
assert_eq!(&sig[0..4], &0xcafe2580u32.to_le_bytes());
|
||||
assert_eq!(&sig[12..16], &0x94119c00u32.to_le_bytes());
|
||||
let size = u32::from_le_bytes(sig[8..12].try_into().unwrap());
|
||||
assert_eq!(size as usize, sig.len() - 48);
|
||||
assert_eq!(&sig[52..56], &size.to_le_bytes());
|
||||
let mut h = crc32fast::Hasher::new();
|
||||
h.update(&sig[8..]);
|
||||
assert_eq!(u32::from_le_bytes(sig[4..8].try_into().unwrap()), h.finalize());
|
||||
let band_tag = u32::from_le_bytes(sig[56..60].try_into().unwrap());
|
||||
assert!(
|
||||
(0x60030040..=0x60030043).contains(&band_tag),
|
||||
"first band tag {band_tag:#x}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A pure tone at 1 kHz must place its peaks in band 1 (520–1450 Hz).
|
||||
#[test]
|
||||
fn peaks_land_in_the_right_band() {
|
||||
let pcm: Vec<f32> = (0..WINDOW_SECS * SAMPLE_RATE)
|
||||
.map(|i| (2.0 * std::f32::consts::PI * 1000.0 * i as f32 / 16000.0).sin() * 0.4)
|
||||
.collect();
|
||||
let mut samples: Vec<i16> = pcm.iter().map(|&x| (x * 32768.0) as i16).collect();
|
||||
samples.truncate(WINDOW_SECS * SAMPLE_RATE);
|
||||
let sig = signature(&pcm);
|
||||
// Band 1's tag must appear; bands 0/2/3 may be empty or tiny.
|
||||
let mut found_band1 = false;
|
||||
let mut at = 56usize;
|
||||
while at + 8 <= sig.len() {
|
||||
let tag = u32::from_le_bytes(sig[at..at + 4].try_into().unwrap());
|
||||
let len = u32::from_le_bytes(sig[at + 4..at + 8].try_into().unwrap()) as usize;
|
||||
if tag == 0x60030041 && len > 50 {
|
||||
found_band1 = true;
|
||||
}
|
||||
at += 8 + len + ((4 - len % 4) % 4);
|
||||
}
|
||||
assert!(found_band1, "1 kHz peaks must dominate band 1");
|
||||
}
|
||||
|
||||
/// LIVE: recognize a 12 s window of a known library track against the
|
||||
/// real service. Ignored by default — network, unofficial endpoint —
|
||||
/// run explicitly: cargo test -p app shazam_live -- --ignored --nocapture
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn shazam_live_known_track() {
|
||||
let path = std::path::Path::new(
|
||||
"D:\\Music\\SoulSeek Music\\Panda Bear\\2015 - Panda Bear Meets The Grim Reaper\\1_SEQUENTIAL CIRCUITS.flac",
|
||||
);
|
||||
assert!(path.is_file(), "corpus track missing");
|
||||
let [l, r] = player::vocals::separation_input_44k1(path).unwrap();
|
||||
let mono = player::vocals::stem_pcm_16k(&l, &r);
|
||||
// A mid-track window, well into the singing.
|
||||
let start = 60 * SAMPLE_RATE;
|
||||
let window = &mono[start..start + WINDOW_SECS * SAMPLE_RATE];
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
let got = rt.block_on(recognize_window(window)).expect("request failed");
|
||||
let rec = got.expect("service heard nothing — signature likely malformed");
|
||||
println!(
|
||||
"LIVE RESULT: {} — {} (key {}, offset {:?}, album {:?}, isrc {:?})",
|
||||
rec.artist, rec.title, rec.key, rec.offset_secs, rec.album, rec.isrc
|
||||
);
|
||||
assert!(
|
||||
rec.title.to_lowercase().contains("sequential"),
|
||||
"wrong track: {} — {}",
|
||||
rec.artist,
|
||||
rec.title
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user