//! Album metadata + cover art: MusicBrainz (canonical names, release MBID) //! and Cover Art Archive, with the iTunes Search API as art fallback. //! MusicBrainz is rate-limited to 1 request/second per their policy. //! Deezer's public API backs the catalog browser (search, artist pages, //! tracklists) — free, keyless, and it exposes record types and previews. use futures::{ future::BoxFuture, stream::{self, FuturesUnordered, StreamExt}, FutureExt, }; use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::{Mutex, OnceCell}; const USER_AGENT: &str = "SoulSeekPlus/0.1 (dev build)"; /// Per-request failure tracing for latency/reliability audits. Enabled with /// `SOULSTREAM_NET_TRACE=1`; zero-cost when off. Failures are otherwise /// swallowed into `None` by design, which makes a provider outage or rate /// limit indistinguishable from "resource does not exist" when auditing. fn net_trace(url: &str, msg: &str) { static ON: std::sync::OnceLock = std::sync::OnceLock::new(); if *ON.get_or_init(|| std::env::var("SOULSTREAM_NET_TRACE").is_ok_and(|v| v == "1")) { eprintln!("NET_TRACE\t{msg}\t{url}"); } } /// One disk-cache entry: `{at, url, body}`. The stored URL guards against a /// (vanishingly unlikely) 64-bit filename hash collision serving the wrong /// resource. fn read_disk_cache(path: &std::path::Path, url: &str) -> Option { let raw = std::fs::read(path).ok()?; let entry: serde_json::Value = serde_json::from_slice(&raw).ok()?; if entry.get("url").and_then(|v| v.as_str()) != Some(url) { return None; } let at = entry.get("at").and_then(|v| v.as_u64())?; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .ok()? .as_secs(); if now.saturating_sub(at) > DISK_CACHE_TTL.as_secs() { return None; } entry.get("body").cloned() } /// Remove cache files stale past twice the TTL (they can never be read /// again — reads reject anything past one TTL). Runs once at client /// construction; a directory scan of a few thousand small files is /// milliseconds and keeps the cache from growing without bound. fn prune_disk_cache(dir: &std::path::Path) { let Some(cutoff) = std::time::SystemTime::now().checked_sub(DISK_CACHE_TTL * 2) else { return; }; let Ok(entries) = std::fs::read_dir(dir) else { return; }; for entry in entries.flatten() { let path = entry.path(); if path.extension().and_then(|ext| ext.to_str()) != Some("json") { continue; } let stale = entry .metadata() .ok() .and_then(|meta| meta.modified().ok()) .is_some_and(|modified| modified < cutoff); if stale { let _ = std::fs::remove_file(&path); } } } fn write_disk_cache(path: &std::path::Path, url: &str, body: &serde_json::Value) { let Ok(now) = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) else { return; }; let entry = serde_json::json!({ "at": now.as_secs(), "url": url, "body": body }); if let Ok(raw) = serde_json::to_vec(&entry) { // Best-effort: a failed write only costs a refetch tomorrow. let _ = std::fs::write(path, raw); } } /// Identity-anchored facts from an artist's MusicBrainz page: the links that /// identify them elsewhere, plus the descriptive fields that can stand in for /// a biography when no prose source has one. #[derive(Default)] struct ArtistIdentityFacts { wikidata_id: Option, wikipedia_title: Option, discogs_id: Option, disambiguation: Option, kind: Option, area: Option, tags: Vec, } impl ArtistIdentityFacts { /// Last-resort biography built only from MusicBrainz's own fields. It /// never asserts anything the entry does not say, which is why it is safe /// where a fuzzy text match is not. fn compose_bio(&self, artist: &str) -> Option { let mut sentences = Vec::new(); match (self.disambiguation.as_deref(), self.kind.as_deref()) { // MusicBrainz's disambiguation is already a description // ("David Ameal, techno / breakbeat producer"). (Some(note), _) => sentences.push(format!("{artist}: {note}.")), (None, Some(kind)) => { let descriptor = if kind.eq_ignore_ascii_case("person") { "a solo artist".to_string() } else { format!("a {}", kind.to_lowercase()) }; match self.area.as_deref() { Some(area) => sentences.push(format!("{artist} is {descriptor} from {area}.")), None => sentences.push(format!("{artist} is {descriptor}.")), } } (None, None) => { if let Some(area) = self.area.as_deref() { sentences.push(format!("{artist} is from {area}.")); } } } if !self.tags.is_empty() { sentences.push(format!("Associated genres: {}.", self.tags.join(", "))); } (!sentences.is_empty()).then(|| sentences.join(" ")) } } /// One canonical MusicBrainz artist URL, shared by the bio tiers and the /// socials lookup so both ride a single cached response. fn artist_detail_url(mbid: &str) -> String { format!("https://musicbrainz.org/ws/2/artist/{mbid}?inc=url-rels+tags&fmt=json") } /// The artist's most-voted genre tags, strongest first. fn top_musicbrainz_tags(body: &serde_json::Value) -> Vec { let Some(tags) = body.get("tags").and_then(|value| value.as_array()) else { return Vec::new(); }; let mut ranked: Vec<(i64, String)> = tags .iter() .filter_map(|tag| { let name = tag.get("name").and_then(|value| value.as_str())?; let count = tag .get("count") .and_then(|value| value.as_i64()) .unwrap_or(0); (count >= 0).then(|| (count, name.to_string())) }) .collect(); ranked.sort_by(|left, right| right.0.cmp(&left.0).then(left.1.cmp(&right.1))); ranked.truncate(4); ranked.into_iter().map(|(_, name)| name).collect() } /// Discogs profiles carry the site's own markup — `[a=Artist]`, `[l=Label]`, /// `[b]`, and `[url=…]text[/url]`. Render it as the plain prose the card wants. fn strip_discogs_markup(profile: &str) -> String { let mut out = String::with_capacity(profile.len()); let mut rest = profile; while let Some(open) = rest.find('[') { out.push_str(&rest[..open]); let Some(close) = rest[open..].find(']') else { rest = &rest[open..]; break; }; let tag = &rest[open + 1..open + close]; // `[a=Name]` / `[l=Name]` / `[m=Name]` keep the name they wrap. if let Some((prefix, value)) = tag.split_once('=') { if matches!(prefix, "a" | "l" | "m" | "r") { out.push_str(value); } } rest = &rest[open + close + 1..]; } out.push_str(rest); out.split_whitespace().collect::>().join(" ") } /// Pull the fields the news panel needs out of a Google News RSS body. /// /// A real XML parser is not warranted for one provider whose format is four /// flat tags inside `` blocks — the same judgment the Bing scrape made. /// Google encodes item titles as plain text with entities (no CDATA observed), /// appends " - Publication" to every title, and stamps RFC 2822 pubDates. fn parse_news_rss(body: &str) -> Vec { fn tag(block: &str, name: &str) -> Option { let open = format!("<{name}"); let start = block.find(&open)?; let start = block[start..].find('>')? + start + 1; let end = block[start..].find(&format!(""))? + start; Some(block[start..end].trim().to_string()) } fn decode(s: &str) -> String { // The entities Google actually emits, plus numeric forms. let mut out = s .replace("&", "&") .replace("<", "<") .replace(">", ">") .replace(""", "\"") .replace("'", "'"); while let Some(start) = out.find("&#") { let Some(len) = out[start..].find(';') else { break }; let code = &out[start + 2..start + len]; let ch = code .strip_prefix('x') .map_or_else(|| code.parse::().ok(), |h| u32::from_str_radix(h, 16).ok()) .and_then(char::from_u32); match ch { Some(c) => out.replace_range(start..start + len + 1, &c.to_string()), None => break, } } out } /// "Mon, 28 Jul 2026 07:00:00 GMT" → unix seconds. Hand-rolled because /// nothing else in the workspace needs a date crate. fn rfc2822_secs(s: &str) -> i64 { let parts: Vec<&str> = s.split_whitespace().collect(); // [Mon,] 28 Jul 2026 07:00:00 GMT let off = usize::from(parts.first().is_some_and(|p| p.ends_with(','))); let (Some(day), Some(mon), Some(year), Some(time)) = ( parts.get(off).and_then(|v| v.parse::().ok()), parts.get(off + 1), parts.get(off + 2).and_then(|v| v.parse::().ok()), parts.get(off + 3), ) else { return 0; }; const MONTHS: [&str; 12] = [ "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", ]; let Some(month) = MONTHS.iter().position(|m| m.eq_ignore_ascii_case(mon)) else { return 0; }; let mut t = time.split(':').filter_map(|v| v.parse::().ok()); let (h, mi, sec) = ( t.next().unwrap_or(0), t.next().unwrap_or(0), t.next().unwrap_or(0), ); // Civil date → days since epoch (Howard Hinnant's algorithm). let month = month as i64 + 1; let y = if month <= 2 { year - 1 } else { year }; let era = y.div_euclid(400); let yoe = y - era * 400; let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; let days = era * 146097 + doe - 719468; days * 86400 + h * 3600 + mi * 60 + sec } /// The `url` attribute of an opening tag: ``. /// `tag()` above returns element TEXT and has no attribute support. fn attr(block: &str, name: &str, key: &str) -> Option { let open = format!("<{name}"); let start = block.find(&open)?; let head_end = block[start..].find('>')? + start; let head = &block[start..head_end]; let at = head.find(&format!("{key}=\""))? + key.len() + 2; let end = head[at..].find('"')? + at; Some(decode(head[at..end].trim())) } /// "https://www.wwd.com/x" → "wwd.com". Bare host, no scheme, no `www.`. fn host_of(url: &str) -> String { let rest = url.split_once("://").map_or(url, |(_, r)| r); let host = rest.split(['/', '?', '#']).next().unwrap_or(""); host.trim_start_matches("www.").to_ascii_lowercase() } let mut out = Vec::new(); for block in body.split("").skip(1) { let block = block.split("").next().unwrap_or(""); let Some(raw_title) = tag(block, "title") else { continue }; let Some(link) = tag(block, "link") else { continue }; let source = tag(block, "source").map(|s| decode(&s)).unwrap_or_default(); let host = attr(block, "source", "url").map(|u| host_of(&u)).unwrap_or_default(); let mut title = decode(&raw_title); // "Headline - Publication" → "Headline" when the tail names the source. if !source.is_empty() { if let Some(stripped) = title.strip_suffix(&format!(" - {source}")) { title = stripped.to_string(); } } let date = tag(block, "pubDate").map(|d| rfc2822_secs(&d)).unwrap_or(0); out.push(ArtistNewsItem { title, url: decode(&link), source, date, host, others: 0 }); } // NO CAP AND NO DATE SORT HERE. // // This used to `break` at 12 and then sort by date. The feed returns 100 // items in Google's RELEVANCE order, so that kept the twelve Google liked // best and discarded ranks 13-100 — and for an ambiguously named artist // that is exactly backwards: measured on the live feed, every genuine // "toe" item sits at rank 15-88 and every genuine "Pond" item starts at // rank 15. Zero of the top twelve were about either band. No amount of // filtering downstream could recover what was thrown away up here. // // Relevance order is also the better tiebreaker to preserve, so the caller // filters and dedups against the full feed and caps at the end. out } /* =========================== news relevance =============================== * * Deciding whether a headline is about THIS band. Researched and adversarially * verified 2026-08-05 against seven live feeds; the plan and the measurements * live in research/news/PLAN.md. * * The one idea the whole thing rests on: the signal is the name's SYNTACTIC * ROLE, not the headline's topic. The query already contains the word "music", * so every junk item is saturated with music vocabulary — "Every song on the * 'Tip Toe' soundtrack" has a music-press host, a quoted work title and a * music noun, and is still not about the band toe. What separates a real item * is evidence AT THE NAME: a verb it governs, a role noun beside it, its * position. Hence the two-part decision at the bottom — a total, and a * separate floor on the occurrence-level subscore that publisher reputation * can never satisfy on its own. */ /// How much evidence a name needs before it is believed. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum NewsTier { /// Multi-token distinctive name ("King Gizzard & The Lizard Wizard"). Distinctive, /// Any other single-token name ("Weezer", "Wand", "Interpol"). Single, /// A name that is also an ordinary English word ("toe", "Pond", "Girls"). VeryCommon, } impl NewsTier { const fn threshold(self) -> i32 { match self { Self::Distinctive => 0, Self::Single => 3, Self::VeryCommon => 6, } } /// The floor on occurrence-level evidence. `Distinctive` opts out. const fn min_occurrence(self) -> i32 { match self { Self::Distinctive => i32::MIN, Self::Single => 1, Self::VeryCommon => 3, } } } /// Names that are ordinary English words. Deliberately a short explicit list, /// NOT a frequency table: bucketing by corpus rank was measured to send Wand /// from 16 kept to 45 (the feed floods with "Wistoria: Wand and Sword"), and /// toe precision from 1.00 to 0.70. const VERY_COMMON_NAMES: &[&str] = &[ "air", "bush", "cake", "cream", "curve", "eels", "elbow", "everything", "fear", "field", "fire", "fuel", "garbage", "girls", "gorilla", "grass", "heart", "hole", "horse", "james", "journey", "kansas", "keane", "low", "lush", "moon", "mountain", "oasis", "pond", "portal", "prince", "pulp", "queen", "rain", "ride", "river", "rush", "sky", "smoke", "snow", "spirit", "star", "steel", "stone", "sun", "texas", "toe", "train", "war", "water", "wire", "yes", ]; fn is_very_common(token: &str) -> bool { VERY_COMMON_NAMES.contains(&token.to_ascii_lowercase().as_str()) } /// The tier for an artist name. pub fn news_tier(artist: &str) -> NewsTier { let tokens: Vec<&str> = artist.split_whitespace().filter(|t| !t.is_empty()).collect(); match tokens.len() { 0 | 1 => { let t = tokens.first().copied().unwrap_or(""); if is_very_common(t) { NewsTier::VeryCommon } else { NewsTier::Single } } _ => { if tokens.iter().all(|t| is_very_common(t)) { NewsTier::VeryCommon } else { NewsTier::Distinctive } } } } /// Publisher hosts whose presence is evidence AGAINST a music story. Negative /// classes are far more enumerable than a music-press allowlist would be — of /// fourteen genuine `toe` items only one sits on any obvious list — so hosts /// are only ever a nudge, never a gate. const OFF_TOPIC_HOSTS: &[&str] = &[ "legacy.com", "dignitymemorial.com", "echovita.com", "tributearchive.com", "wwd.com", "footwearnews.com", "hypebeast.com", "pagesix.com", "vogue.com", "gamesradar.com", "ign.com", "polygon.com", "kotaku.com", "pcgamer.com", "eurogamer.net", "chowhound.com", "allrecipes.com", "tastingtable.com", "thedailymeal.com", "espn.com", "realtor.com", "zillow.com", ]; const MUSIC_HOSTS: &[&str] = &[ "pitchfork.com", "stereogum.com", "brooklynvegan.com", "consequence.net", "nme.com", "rollingstone.com", "spin.com", "billboard.com", "thelineofbestfit.com", "clashmusic.com", "diymag.com", "loudersound.com", "alternativepress.com", "paste.com", "exclaim.ca", "thequietus.com", "residentadvisor.net", "ra.co", "faroutmagazine.co.uk", "mojo4music.com", ]; /// Verbs a band GOVERNS. Kept deliberately narrow: adding "brings"/"opens" /// admitted "bike ride brings history, music and family activities" and /// "Frog Pond opens 16th season". const CORE_VERBS: &[&str] = &[ "announce", "announces", "announced", "share", "shares", "shared", "release", "releases", "released", "drop", "drops", "dropped", "unveil", "unveils", "unveiled", "reveal", "reveals", "tease", "teases", "teased", "detail", "details", "confirm", "confirms", "confirmed", "debut", "debuts", "premiere", "premieres", "preview", "previews", "stream", "streams", "perform", "performs", "headline", "headlines", "tour", "tours", "remix", "remixes", "reissue", "reissues", "sign", "signs", "cancel", "cancels", "postpone", "postpones", "reunite", "reunites", "reunited", "add", "adds", "plot", "plots", "map", ]; const SOFT_VERBS: &[&str] = &[ "return", "returns", "launch", "launches", "join", "joins", "take", "takes", "bring", "brings", "make", "makes", "open", "opens", "set", "sets", "play", "plays", "go", "goes", ]; const ROLE_NOUNS: &[&str] = &[ "band", "bands", "group", "duo", "trio", "quartet", "singer", "rapper", "outfit", "rockers", "frontman", "frontwoman", "drummer", "guitarist", "bassist", "producer", "act", "collective", ]; const WEAK_ROLES: &[&str] = &["star", "stars", "legend", "legends", "icon", "icons"]; const MUSIC_NOUNS: &[&str] = &[ "album", "albums", "ep", "eps", "lp", "single", "singles", "song", "songs", "track", "tracks", "tour", "shows", "show", "gig", "gigs", "review", "interview", "video", "festival", "vinyl", "reissue", "record", "records", "setlist", "concert", "residency", ]; /// Heads that turn a name into a PLACE or an EVENT rather than a band. const ENTITY_HEADS: &[&str] = &[ "park", "lake", "colony", "county", "hall", "studio", "project", "fest", "festival", "day", "days", "river", "jam", "lily", "market", "club", "school", "church", "arts", "trail", "backlot", "road", "street", "lane", "farm", "inn", "resort", "camp", "beach", "bay", "creek", "valley", "ridge", "grove", "village", "township", "estates", "gardens", ]; const OBIT_MARKERS: &[&str] = &[ "obitu", "passed away", "funeral", "found dead", "in loving memory", "celebration of life", ]; fn words(s: &str) -> Vec { s.split(|c: char| !c.is_alphanumeric() && c != '\'') .filter(|w| !w.is_empty()) .map(|w| w.to_ascii_lowercase()) .collect() } /// The name forms a headline may legitimately use. fn name_variants(artist: &str) -> Vec { let mut out = vec![artist.to_string()]; if artist.contains('&') { out.push(artist.replace('&', "and")); } if artist.to_ascii_lowercase().contains(" and ") { out.push(artist.replacen(" and ", " & ", 1)); } let toks: Vec<&str> = artist.split_whitespace().collect(); if toks.len() >= 4 { // "King Gizzard & The Lizard Wizard" → "King Gizzard". Without this the // hard gate silently drops genuine stories that use the short form. out.push(toks[..2].join(" ")); } out.sort(); out.dedup(); out } /// Byte offsets of every whole-token occurrence of `needle` in `hay`, matched /// case-insensitively on token boundaries. A trailing `'s` still counts — /// without that, "King Gizzard's Stu Mackenzie" fails the hard gate. fn token_occurrences(hay: &str, needle: &str) -> Vec<(usize, usize)> { let h = hay.to_ascii_lowercase(); let n = needle.to_ascii_lowercase(); if n.is_empty() { return Vec::new(); } let mut out = Vec::new(); let mut from = 0; while let Some(rel) = h[from..].find(&n) { let start = from + rel; let end = start + n.len(); let before_ok = start == 0 || !h[..start].chars().next_back().is_some_and(|c| c.is_alphanumeric()); let after = h[end..].chars().next(); let after_ok = match after { None => true, Some(c) if !c.is_alphanumeric() => true, // allow the possessive "'s" which the char scan above already let // through as an apostrophe; a bare alphanumeric is a different word _ => false, }; if before_ok && after_ok { out.push((start, end)); } from = end.max(start + 1); } out } /// Is the character immediately abutting this span an ASCII hyphen? That is /// what makes "peep-toe", "toe-tapping", "head-to-toe" and "Chorn-Pond" not /// the band. Spaced en-dashes ("toe – 'NOW I SEE THE LIGHT'") are a different /// character and are correctly ignored. fn hyphen_abuts(hay: &str, start: usize, end: usize) -> bool { let before = hay[..start].chars().next_back() == Some('-'); let after = hay[end..].chars().next() == Some('-'); before || after } /// Title Case headlines capitalise everything, so an all-lowercase occurrence /// carries no information there. fn is_title_case(title: &str) -> bool { let long: Vec<&str> = title .split_whitespace() .filter(|w| w.chars().filter(|c| c.is_alphabetic()).count() > 3) .collect(); if long.len() < 3 { return true; // too short to tell; do not use the case rule } let capped = long .iter() .filter(|w| w.chars().next().is_some_and(char::is_uppercase)) .count(); capped * 10 > long.len() * 6 } /// Score a headline for being about `artist`. `None` means rejected outright. /// /// See research/news/PLAN.md §2 for the derivation of every weight and for the /// measurements behind the two-part decision at the end. pub fn news_relevance(item: &ArtistNewsItem, artist: &str, tier: NewsTier) -> Option { let title = item.title.as_str(); let lower = title.to_ascii_lowercase(); let variants = name_variants(artist); // ---- hard gate 1: the name must appear as a whole token run ---------- let mut hits: Vec<(usize, usize)> = Vec::new(); for v in &variants { hits.extend(token_occurrences(title, v)); } if hits.is_empty() { return None; } // ---- hard gate 2: every occurrence is inside a hyphenated compound ---- if hits.iter().all(|&(s, e)| hyphen_abuts(title, s, e)) { return None; } hits.retain(|&(s, e)| !hyphen_abuts(title, s, e)); // ---- hard gate 3: canonical capitalisation vs an all-lowercase hit ---- let name_is_capitalised = artist.chars().next().is_some_and(char::is_uppercase); if name_is_capitalised && !is_title_case(title) { let any_cased = hits .iter() .any(|&(s, e)| title[s..e].chars().next().is_some_and(char::is_uppercase)); if !any_cased { return None; } } let has = |list: &[&str], w: &str| list.contains(&w); // ---- document-level signals (counted once) --------------------------- let name_tokens = artist.split_whitespace().count(); let mut doc = match name_tokens { 0 | 1 => 0, 2 => 2, _ => 4, }; if !item.host.is_empty() { if OFF_TOPIC_HOSTS.iter().any(|h| item.host == *h || item.host.ends_with(&format!(".{h}"))) { doc -= 6; } else if MUSIC_HOSTS.iter().any(|h| item.host == *h) { doc += 2; } } if OBIT_MARKERS.iter().any(|m| lower.contains(m)) { doc -= 6; } if title.contains('"') || title.contains('\u{201c}') || title.contains('\u{2018}') { doc += 1; } // ---- occurrence-level signals — keep the MAX over occurrences -------- // A headline may name the artist several ways; only one of them has to // carry the evidence. let mut best_occ = i32::MIN; for &(s, e) in &hits { let mut occ = 0i32; let before_toks = words(&title[..s]); let after_toks = words(&title[e..]); let prev = before_toks.last().map(String::as_str); let next = after_toks.first().map(String::as_str); let next2 = after_toks.get(1).map(String::as_str); // "within 2 tokens" is a DISTANCE, so it reaches index 2. Checking only // the first two positions lost "toe Announces October Shows in Osaka", // where the music noun is the third word after the name. let near: Vec<&str> = after_toks.iter().take(3).map(String::as_str).collect(); if next.is_some_and(|w| has(CORE_VERBS, w)) { occ += 3; } else if next.is_some_and(|w| has(SOFT_VERBS, w)) { occ += 1; } // possessive followed by a music noun within two tokens if title[e..].starts_with('\u{2019}') || title[e..].starts_with('\'') { if next.is_some_and(|w| has(MUSIC_NOUNS, w)) || next2.is_some_and(|w| has(MUSIC_NOUNS, w)) { occ += 3; } } if prev.is_some_and(|w| has(ROLE_NOUNS, w)) || next.is_some_and(|w| has(ROLE_NOUNS, w)) { occ += 3; } else if prev.is_some_and(|w| has(WEAK_ROLES, w)) || next.is_some_and(|w| has(WEAK_ROLES, w)) { occ += 1; } if near.iter().any(|w| has(MUSIC_NOUNS, w)) { occ += 2; } // Head of the headline — or head of what follows a masthead marker, // which is the same claim: "Review: toe, 'NOW I SEE THE LIGHT'". // char_indices, not `rfind(..) + 1`: an en/em dash is three bytes, so // adding one lands mid-character and slicing panics. In an async Tauri // command that panic surfaces as a promise that never resolves, which // is why only artists whose name sits mid-headline hung. let after_marker = title[..s] .char_indices() .rev() .find(|(_, ch)| matches!(ch, ':' | '\u{2013}' | '\u{2014}')) .is_some_and(|(c, ch)| title[c + ch.len_utf8()..s].trim().is_empty()); if before_toks.is_empty() || after_marker { occ += 2; } // " — " / ": " masthead convention let tail = title[e..].trim_start(); if tail.starts_with(':') || tail.starts_with('\u{2013}') || tail.starts_with('\u{2014}') { occ += 2; } // "Review:" / "Listen:" prefix markers if let Some(colon) = title.find(':') { if colon < 40 { let head = title[..colon].to_ascii_lowercase(); if ["review", "interview", "premiere", "listen", "watch", "feature", "video"] .iter() .any(|m| head.contains(m)) { occ += 2; } } } // ---- negative: the name is really part of a longer proper name ---- // Bare-whitespace adjacency ONLY. Treating any adjacent capitalised // token as an extension wrongly kills "Review: toe, …" and // "Pond 'Terrestrials' Review". if s > 0 { let gap = &title[..s]; if gap.ends_with(' ') { let prev_raw = gap.trim_end().rsplit(' ').next().unwrap_or(""); let prev_cap = prev_raw.chars().next().is_some_and(char::is_uppercase); let sentence_initial = gap.trim_end().is_empty(); let known = prev.is_some_and(|w| { has(CORE_VERBS, w) || has(SOFT_VERBS, w) || has(ROLE_NOUNS, w) || has(WEAK_ROLES, w) || has(MUSIC_NOUNS, w) }); if prev_cap && !known && !prev_raw.contains('-') { occ -= if sentence_initial { 2 } else { 4 }; } // "Amanda M. Pond" if prev_raw.len() == 2 && prev_raw.ends_with('.') { occ -= 5; } } } if next.is_some_and(|w| has(ENTITY_HEADS, w)) { occ -= 4; } else if next2.is_some_and(|w| has(ENTITY_HEADS, w)) { occ -= 3; } // "across the Pond", "over the pond" if before_toks.len() >= 2 { let a = before_toks[before_toks.len() - 2].as_str(); let b = before_toks[before_toks.len() - 1].as_str(); if b == "the" && ["across", "over", "from", "in", "into", "beyond"].contains(&a) { occ -= 6; } } if prev.is_some_and(|w| ["a", "an", "the", "to"].contains(&w)) && !artist.starts_with("The ") { occ -= 3; } best_occ = best_occ.max(occ); } if best_occ == i32::MIN { return None; } let total = doc + best_occ; // The occurrence floor is the whole filter. At a single combined threshold // every false positive scored at threshold from document-level signals // alone — "Every song on the 'Tip Toe' soundtrack" is a music-press host // plus a quoted title plus a music noun and zero evidence at the name. // Publisher reputation must never admit an item on its own. if total >= tier.threshold() && best_occ >= tier.min_occurrence() { Some(total) } else { None } } /* ============================ news dedup ================================== * * Collapsing several outlets' reports of ONE story. Runs after the filter and * before the cap. See research/news/PLAN.md §3. * * Two counter-intuitive choices, both measured: * - CONTAINMENT, not Jaccard. Headline lengths are wildly asymmetric — * Stereogum's six-token 'King Gizzard & The Lizard Wizard – "Alien Metal"' * against a thirty-token version of the same story — and Jaccard's union * denominator punishes exactly that (F1 0.61 vs 0.44). * - KEEP the artist-name tokens. Stripping them starves a six-token headline * to two and was measured to lose real merges and invent false ones. The * name is a uniform offset across every pair in a per-artist feed, so it * moves the operating point rather than the ranking — which the raised * 0.6 threshold and the non-name-overlap clause compensate for. */ const NEWS_STOP: &[&str] = &[ "a", "an", "the", "of", "to", "in", "on", "at", "for", "with", "and", "or", "from", "is", "are", "was", "were", "be", "been", "by", "as", "it", "its", "this", "that", "new", "now", "has", "have", "had", "will", "their", "his", "her", "they", "he", "she", "you", "your", "our", "says", "say", "said", "more", "than", "then", "but", "not", "out", "up", "off", "over", "into", "about", "after", "before", "some", "all", "how", "why", "what", "when", "who", "his", "hers", "via", "amp", ]; /// Cap applied AFTER filtering and dedup. The frontend's band asks for up to /// `bandCols * 3` items, so the old 12 starved a wide window. const NEWS_KEEP: usize = 40; /// Score, reject, collapse, cap — the whole pipeline over one parsed feed. /// /// If the filter empties the list, that is the answer: an empty row is /// strictly better than the unfiltered feed, which for an ambiguously named /// artist is 100% junk. There is deliberately no fall back to unfiltered. fn filter_news(raw: Vec, artist: &str, started: Instant) -> Vec { let fetched = raw.len(); let tier = news_tier(artist); let mut scored: Vec<(ArtistNewsItem, i32)> = raw .into_iter() .filter_map(|item| news_relevance(&item, artist, tier).map(|s| (item, s))) .collect(); let kept = scored.len(); // Google's relevance order is the tiebreaker worth preserving, so only // sort by date once the set is final. let mut out = dedup_news(std::mem::take(&mut scored), artist); let deduped = out.len(); // Sort BEFORE the quota so the two survivors of each story are its newest, // and quota BEFORE the cap so the forty slots are forty stories. out.sort_by_key(|item| -item.date); let mut out = news_story_quota(out, artist); let storied = out.len(); out.truncate(NEWS_KEEP); // "fetched 100, kept 0" and "fetched 0" must never read the same — the // trap already recorded for MusicBrainz probing. stage_trace( "artist-news", started, &format!( "{artist} tier={tier:?} fetched={fetched} kept={kept} deduped={deduped} storied={storied}" ), ); out } /// Comparable token set for one headline. The " - Publisher" suffix is already /// stripped by `parse_news_rss`; do not re-implement it here. fn news_signature(title: &str) -> std::collections::BTreeSet { let folded = title .replace('\u{2019}', "'") .replace(['\u{201c}', '\u{201d}'], "\"") .replace(['\u{2013}', '\u{2014}'], "-") .replace('&', " and "); folded .split(|c: char| !c.is_alphanumeric()) .filter(|w| w.len() > 2) .map(|w| w.to_ascii_lowercase()) .filter(|w| !NEWS_STOP.contains(&w.as_str())) .collect() } /// Collapse near-duplicate reports, keeping the best-scoring member of each /// cluster and recording how many others it stands for. /// /// Leader clustering, NOT union-find: union-find at a comparable threshold /// chained 68 of 93 King Gizzard items — a Nonagon Infinity review and a /// Phantom Island review included — into a single "story". fn dedup_news(items: Vec<(ArtistNewsItem, i32)>, artist: &str) -> Vec { let name_toks: std::collections::BTreeSet = news_signature(artist); let sigs: Vec<_> = items.iter().map(|(i, _)| news_signature(&i.title)).collect(); // leader index -> members let mut clusters: Vec<(usize, Vec)> = Vec::new(); 'item: for i in 0..items.len() { for (leader, members) in clusters.iter_mut() { let (a, b) = (&sigs[*leader], &sigs[i]); let inter: Vec<&String> = a.intersection(b).collect(); let (da, db) = (items[*leader].0.date, items[i].0.date); // Google truncates syndicated copies with an ellipsis, which is // exactly the GamesRadar+/inkl.com pair. let ta = items[*leader].0.title.trim_end_matches(['\u{2026}', '.', ' ']); let tb = items[i].0.title.trim_end_matches(['\u{2026}', '.', ' ']); let prefix = ta.len() >= 24 && tb.len() >= 24 && (ta.starts_with(tb) || tb.starts_with(ta)); // `rfc2822_secs` yields 0 on a parse failure, so two unparseable // items would otherwise show |delta| = 0 and always pass the window. let dated = da != 0 && db != 0 && (da - db).abs() <= 2 * 86_400; let denom = a.len().min(b.len()).max(5) as f32; let containment = inter.len() as f32 / denom; let non_name = inter.iter().filter(|t| !name_toks.contains(**t)).count(); let same = prefix || (dated && containment >= 0.6 && inter.len() >= 3 && non_name >= 2); if same { members.push(i); continue 'item; } } clusters.push((i, vec![i])); } let mut out = Vec::new(); for (_, members) in clusters { // Survivor = best score; tie-break to the earliest report. let best = *members .iter() .min_by(|&&x, &&y| { items[y].1 .cmp(&items[x].1) .then_with(|| items[x].0.date.cmp(&items[y].0.date)) }) .expect("cluster is never empty"); let mut item = items[best].0.clone(); item.others = (members.len() - 1) as u32; out.push(item); } out } /* ======================= per-story display quota ========================== * * What dedup cannot reach. Leader clustering collapses REWORDINGS of one * report, but it leaves stragglers: twelve outlets covering the "Alien Metal" * announcement produced one 11-member cluster plus six loners that are similar * to each other and not to the leader that already absorbed the rest. Measured * on the live feed, seven of the visible twelve slots were that one record. * * The remedy is NOT a lower dedup threshold. Lexical similarity merges * unrelated stories long before it catches rewrites — Cleveland's "Beloved band * pulls its music from major streaming service" and InsideHook's "Scammers * Share Fake King Gizzard Songs on Spotify" are the same story and share * literally zero content words. Closing that gap needs sentence embeddings. * See research/news/PLAN.md §3. * * So bound the damage instead: at most two items may name the same WORK. The * key is the album or song title the headline is about, which syndicated * coverage repeats verbatim even when it shares no other vocabulary. Headlines * that name no work are never dropped — an artist's feed can shrink but can * never be emptied by this. */ /// Nouns after which a title is a work title. "toe Announces October Shows" /// carries no work; "Detail New Album Alien Metal" does. const WORK_CUES: &[&str] = &[ "album", "albums", "lp", "ep", "single", "singles", "track", "song", "mixtape", "record", ]; /// Words that are never a work title on their own. Without these, "…Album /// Review" keys as `review` and "…New Song Next Week" as `next week`, and two /// unrelated headlines share a story. const GENERIC_TITLE_WORDS: &[&str] = &[ "review", "reviews", "interview", "interviews", "stream", "listen", "watch", "premiere", "premieres", "video", "videos", "news", "tour", "tours", "dates", "list", "setlist", "week", "weekend", "day", "today", "tomorrow", "release", "releases", "single", "singles", "album", "albums", "ep", "lp", "song", "songs", "track", "tracks", "next", "first", "new", "out", "now", "more", ]; /// A capitalised function word does not continue a title: "New Song With Cindy /// Lee" is a credit, not a work called "With Cindy Lee". const RUN_STOP_WORDS: &[&str] = &["with", "from", "by", "featuring", "feat", "and", "for", "on", "in", "at", "to", "of", "via", "ft"]; /// Every quoted span in a headline, as (contents, byte range covering the /// quotes themselves). /// /// Two rules earn their keep. A straight quote only OPENS at a word boundary — /// otherwise a possessive ("Ware's") opens a span that closes on the next real /// quote and misaligns every span after it. And a curly opener may close on its /// straight twin, because real feeds mix them inside one headline /// (`Produce ‘The Gold Album'`). fn quoted_spans(title: &str) -> Vec<(String, usize, usize)> { let chars: Vec<(usize, char)> = title.char_indices().collect(); let mut out = Vec::new(); let mut i = 0; while i < chars.len() { let (idx, c) = chars[i]; let closers: &[char] = match c { '\u{2018}' => &['\u{2019}', '\''], '\u{201c}' => &['\u{201d}', '"'], '"' => &['"'], '\'' => &['\''], _ => { i += 1; continue; } }; let curly = matches!(c, '\u{2018}' | '\u{201c}'); let opens = curly || i == 0 || !chars[i - 1].1.is_alphanumeric(); let mut advance = 1; if opens { let closed = (i + 1..chars.len()).find(|&j| { closers.contains(&chars[j].1) && (curly || j + 1 >= chars.len() || !chars[j + 1].1.is_alphanumeric()) }); if let Some(j) = closed { let (close_idx, close_ch) = chars[j]; let inner = title[idx + c.len_utf8()..close_idx].to_string(); out.push((inner, idx, close_idx + close_ch.len_utf8())); advance = j - i + 1; } } i += advance; } out } /// Normalise a candidate work title into a comparison key, or reject it. /// /// Artist-name tokens and a leading article come off, which is what unifies /// "Weezer: The Gathering" with "The Gathering" — five headlines about one /// tour that would otherwise split into two keys and pass the quota twice. fn work_key(raw: &str, name_toks: &HashSet) -> Option { let mut toks: Vec = raw .split(|c: char| !c.is_alphanumeric()) .filter(|w| !w.is_empty()) .map(|w| w.to_ascii_lowercase()) .collect(); // A span this long is speech, not a title: "It's 5 days since they released // new music – was starting to think we'd never hear from them again". if toks.len() > 8 { return None; } toks.retain(|w| !name_toks.contains(w)); while matches!(toks.first().map(String::as_str), Some("the" | "a" | "an")) { toks.remove(0); } if toks.is_empty() || toks.len() > 6 { return None; } if toks .iter() .all(|w| GENERIC_TITLE_WORDS.contains(&w.as_str()) || NEWS_STOP.contains(&w.as_str())) { return None; } let key = toks.join(" "); (key.len() >= 3).then_some(key) } fn bare_word(w: &str) -> String { w.chars().filter(|c| c.is_alphanumeric()).collect::().to_ascii_lowercase() } /// Whitespace-separated words with their byte offsets. `split_whitespace` /// alone loses the offsets, and the offsets are what tell a work cue inside a /// quoted span from one outside it. fn offset_words(s: &str) -> Vec<(usize, &str)> { let mut out = Vec::new(); let mut start: Option = None; for (i, c) in s.char_indices() { if c.is_whitespace() { if let Some(at) = start.take() { out.push((at, &s[at..i])); } } else if start.is_none() { start = Some(i); } } if let Some(at) = start { out.push((at, &s[at..])); } out } /// The last whole word before a byte offset. fn word_before(title: &str, at: usize) -> String { title[..at] .split(|c: char| !c.is_alphanumeric()) .rfind(|w| !w.is_empty()) .unwrap_or("") .to_ascii_lowercase() } /// The work a headline is about — an album or song title, normalised — or /// `None` when it names none. `None` is common and is not a failure: a tour /// announcement or an interview is its own story and is never quota'd. pub fn news_story_key(title: &str, artist: &str) -> Option { let name_toks: HashSet = artist .split(|c: char| !c.is_alphanumeric()) .filter(|w| !w.is_empty()) .map(|w| w.to_ascii_lowercase()) .collect(); let raw_spans = quoted_spans(title); let keyed: Vec<(String, usize)> = raw_spans .iter() .filter_map(|(raw, at, _)| work_key(raw, &name_toks).map(|k| (k, *at))) .collect(); // 1. a quoted span right after a work cue IS the work — this outranks a // bare quote, which is as often speech ("'We are truly doomed'"). if let Some((key, _)) = keyed .iter() .find(|(_, at)| WORK_CUES.contains(&word_before(title, *at).as_str())) { return Some(key.clone()); } // 2. an unquoted Title Case run after a work cue, closed by end of headline // or by `:` `,` or a dash. The closing requirement is what keeps // "Fueling Album Speculation [Watch]" from keying as a record. let words = offset_words(title); for i in 0..words.len() { let (at, word) = words[i]; if !WORK_CUES.contains(&bare_word(word).as_str()) { continue; } // A cue INSIDE a quote is part of a title, not an introduction to one: // `A “Sweater Song” Homage` was keying as the work "homage". if raw_spans.iter().any(|&(_, s, e)| at >= s && at < e) { continue; } let mut run: Vec<&str> = Vec::new(); let mut j = i + 1; let mut closed = false; while j < words.len() { let w = words[j].1; if !w.chars().next().is_some_and(|c| c.is_uppercase() || c.is_numeric()) { break; } if RUN_STOP_WORDS.contains(&bare_word(w).as_str()) { break; } run.push(w); j += 1; if w.ends_with([':', ',', '\u{2013}', '\u{2014}']) { closed = true; break; } } if run.is_empty() { continue; } if !closed { closed = j >= words.len() || words[j].1.starts_with([':', ',', '\u{2013}', '\u{2014}']); } if !closed { continue; } if let Some(key) = work_key(&run.join(" "), &name_toks) { return Some(key); } } // 3. any other quoted span keyed.into_iter().next().map(|(key, _)| key) } /// At most this many headlines about one work. Two rather than one: a record /// that is genuinely in the news deserves more than a single line, and the two /// kept are usually the announcement and the release. const NEWS_STORY_MAX: usize = 2; /// Keep at most `NEWS_STORY_MAX` items per work, in the order given — so /// calling this on a date-sorted list keeps the newest of each story. pub fn news_story_quota(items: Vec, artist: &str) -> Vec { let mut seen: HashMap = HashMap::new(); items .into_iter() .filter(|item| match news_story_key(&item.title, artist) { None => true, Some(key) => { let n = seen.entry(key).or_insert(0); *n += 1; *n <= NEWS_STORY_MAX } }) .collect() } /// Wikimedia hosts share one UA policy and one rate limiter, so they share /// this client's compliant UA, pacer, 429 handling and disk cache. Wikidata is /// included: it is the identity anchor that turns a MusicBrainz artist into /// the RIGHT Wikipedia article. fn is_wikimedia_url(url: &str) -> bool { url.starts_with("https://en.wikipedia.org/") || url.starts_with("https://www.wikidata.org/") } /// Per-stage latency tracing for the artist-page pipeline. Enabled with /// `SOULSTREAM_STAGE_TRACE=1`; zero-cost when off. This is how load-time /// audits attribute wall-clock to a stage instead of guessing. fn stage_trace(stage: &str, started: Instant, detail: &str) { static ON: std::sync::OnceLock = std::sync::OnceLock::new(); if *ON.get_or_init(|| std::env::var("SOULSTREAM_STAGE_TRACE").is_ok_and(|v| v == "1")) { eprintln!( "STAGE\t{stage}\t{}ms\t{detail}", started.elapsed().as_millis() ); } } /// How long a provider image host keeps its last liveness verdict. /// /// Long enough that a dead CDN costs one probe rather than one per artist, /// short enough that the hero comes back on its own within minutes of the host /// recovering — no restart, no cache clearing. const IMAGE_HOST_PROBE_TTL: Duration = Duration::from_secs(5 * 60); /// A liveness probe races the page, so it must never become the page's latency. const IMAGE_HOST_PROBE_TIMEOUT: Duration = Duration::from_secs(3); /// The confirming probe runs after the page has already moved on, so it can /// afford to wait out congestion the fast one could not. const IMAGE_HOST_CONFIRM_TIMEOUT: Duration = Duration::from_secs(12); /// Last liveness verdict per image host, shared by the fast probe and by the /// background confirmation that is allowed to overturn it. static IMAGE_HOST_HEALTH: std::sync::OnceLock>> = std::sync::OnceLock::new(); /// Cached catalog responses expire after this long. const CATALOG_TTL: Duration = Duration::from_secs(15 * 60); /// Deezer publishes a budget of 50 requests per 5 seconds per IP and answers an /// exhausted one with **HTTP 200** plus `{"error":{"code":4,...}}` — a body that /// parses perfectly and contains no `data`. One large artist page issues more /// than that on its own (profile + album page + top + up to 50 album details), /// so the burst is paced here rather than left to fail. The budget keeps a small /// margin under the published limit for requests already in flight. const DEEZER_RATE_WINDOW: Duration = Duration::from_secs(5); const DEEZER_RATE_BUDGET: usize = 45; /// A quota answer is retried inside the fetch, so a rejected request never /// reaches a caller as "this resource does not exist". const DEEZER_QUOTA_ATTEMPTS: usize = 3; const DEEZER_QUOTA_BACKOFF: Duration = Duration::from_millis(1600); /// UA for providers that publish a policy about who is calling them. /// /// Wikimedia's requires a client that identifies itself with a contact point; /// the shared default UA reads as an anonymous bot and gets a savage limiter /// (measured 2026-07-22: ~13 requests then HTTP 429 with `Retry-After: 51`, /// which silently zeroed every album description on a cold artist page). A /// compliant UA sustained 45+ back-to-back requests with no rejection. Last.fm /// serves an anonymous UA today but asks API clients to identify themselves, /// and the Popular ordering now depends on it, so it opts in too. /// /// Applied to those two only — every other provider (notably the Bing HTML /// scrape) must keep seeing the default UA. const CONTACT_USER_AGENT: &str = "SoulStream/0.1 (personal desktop music app; contact: https://github.com/KeelrLeek)"; /// Client-side pacing for Wikipedia stays at the measured compliant-UA /// budget (45 back-to-back requests accepted; a cold 14-album page needs /// ~35-40) so one artist page clears in a single window while a runaway /// burst still cannot provoke the limiter. const WIKI_RATE_WINDOW: Duration = Duration::from_secs(10); const WIKI_RATE_BUDGET: usize = 40; /// A 429 is retried after honoring `Retry-After`; anything longer than the /// fail-fast bound means the IP is in a penalty box and waiting inside one /// album-info call would hold the expander open for nothing. const WIKI_429_ATTEMPTS: usize = 3; const WIKI_COOLDOWN_FAIL_FAST: Duration = Duration::from_secs(8); const WIKI_COOLDOWN_DEFAULT: Duration = Duration::from_secs(5); /// Slow-moving editorial providers (MusicBrainz, Wikipedia) are also cached /// on DISK, so an app restart does not re-pay the MusicBrainz identity chain /// (4-6 requests behind the mandatory 1 req/s gate — measured 3.7-8.5 s per /// artist, 65-87% of a cold page's time-to-fully-sorted) or the description /// harvest. Deezer/TheAudioDB stay memory-only: fan counts and fresh /// releases should not be a day stale. const DISK_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60); /// Discogs allows 25 unauthenticated requests per minute per IP. Profiles are /// slow-moving prose, so they are disk cached and this window is only a guard /// against a burst (a library warm-up sweep) tripping the limiter. const DISCOGS_RATE_WINDOW: Duration = Duration::from_secs(60); const DISCOGS_RATE_BUDGET: usize = 18; const IMAGE_CACHE_TTL: Duration = Duration::from_secs(15 * 60); const IMAGE_CACHE_MAX_BYTES: usize = 64 * 1024 * 1024; /// Provider/search images are untrusted remote responses. Never allocate an /// unbounded body merely because a search result claims to be a photograph. const IMAGE_DOWNLOAD_MAX_BYTES: usize = 32 * 1024 * 1024; const HERO_MAX_DECODE_DIMENSION: u32 = 12_000; const HERO_MAX_DECODE_ALLOC_BYTES: u64 = 128 * 1024 * 1024; const COVER_MAX_DECODE_DIMENSION: u32 = 8_000; const COVER_MAX_DECODE_ALLOC_BYTES: u64 = 192 * 1024 * 1024; const FUSED_PORTRAIT_FILE: &str = "fused-portraits-v1.json"; const APPLE_EXACT_ARTIST_CANDIDATE_LIMIT: usize = 8; /// The identity pass may look at far more same-named Apple performers than the /// page-load gate dares to, because it does not have to GUESS which one is /// ours: MusicBrainz has already named the artist behind this Deezer id, so the /// winner is chosen by overlap with that identity instead of by overlap with a /// Deezer list that may itself be fused. Wand has TEN exact-name Apple artists /// and the page-load cap therefore returned an empty, unusable catalogue — /// which is precisely why 46 co-tenant releases reached the page. const APPLE_IDENTITY_ARTIST_CANDIDATE_LIMIT: usize = 20; /// Minimum MusicBrainz release-group titles an Apple candidate must reproduce /// before it is accepted as the same act. One shared title is a coincidence /// between namesakes ("Greatest Hits"); two is a catalogue. const APPLE_IDENTITY_MIN_TITLE_HITS: usize = 2; // Cover verification is background polish, not a reason for four malicious or // unusually large remote rasters to reserve up to 768 MiB at once. Network // fetches still overlap four-wide in the UI; only CPU decode/hash work is // globally capped at two. const COVER_DECODE_CONCURRENCY: usize = 2; type CatalogFetch = Arc>>; type ArtistArtFetch = Arc>; type ArtistHeroSeedFetch = Arc>>; type ImageFetch = Arc>>>>; type HtmlFetch = Arc>>; type HeroScoreFetch = Arc>>; type AppleCandidatesFetch = Arc>>>; /// Shared Fanart.tv *project* API key. Not a secret — it identifies the app and /// is meant to be embedded and shared across all users (same as TheAudioDB's /// public `123` key used below). Rotate at fanart.tv if it is ever abused. /// Unlocks 1920×1080 artist backgrounds, a 2.25× pixel step up from /// TheAudioDB's 1280×720 fanart. const FANART_KEY: &str = "1a2fb0b9fd7fa6fcf68388f78a9cdca6"; /// Shared Last.fm *project* API key, same class as [`FANART_KEY`] and /// TheAudioDB's public `123` — app-wide, disposable, meant to be embedded. /// Rotate at last.fm/api/accounts if it is ever abused. /// Supplies the scrobble counts that order the Popular section: Deezer's own /// `/top` is recency-weighted, so a week-old single outranks the songs the /// artist is actually known for (Taylor Swift's Deezer top three are all from /// the current release; Last.fm's are Blank Space, Cruel Summer, Shake It Off). const LASTFM_KEY: &str = "69f3b3201381b56903eeee31477644a2"; /// Last.fm publishes no per-key quota, only a ~5 req/s per-IP guideline. One /// artist page spends exactly one request and the answer is disk cached for a /// day, so this window is purely a guard against a burst (a library warm-up /// sweep) tripping a limiter we cannot see. Deliberately not the unpaced /// treatment Bing got — that is the one provider that broke this way. const LASTFM_RATE_WINDOW: Duration = Duration::from_secs(1); const LASTFM_RATE_BUDGET: usize = 4; /// Google News RSS has no published budget and no API contract at all — it is /// scraping-adjacent, exactly the situation that broke the Bing hero tier when /// it ran unpaced. Deliberately the most conservative pacer in this file: the /// home page's news walk is a background task with no user waiting on any /// single request. const NEWS_RATE_WINDOW: Duration = Duration::from_secs(10); const NEWS_RATE_BUDGET: usize = 4; /// Headlines move slower than the feed refresh: within one TTL a new story is /// at most this stale, which the home page's 45-day display window absorbs. const NEWS_TTL: Duration = Duration::from_secs(6 * 60 * 60); /// How deep to pull the scrobble table. /// /// Raised from 100 once this table also had to serve per-track play counts on /// ALBUM pages, not just reorder the popular ten. 100 covered the Deezer pool /// fine but is nowhere near an album's deep cuts — measured on Weezer, it hit /// only 2 of Van Weezer's 10 tracks and 5 of the White Album's. /// /// 1000 is one request, the same request, and it covered **56 of 57** album /// tracks measured across Johnny Cash, the Grateful Dead, Miles Davis, Bob /// Dylan and Queen — catalogues where Last.fm holds 150k-390k track titles. /// That works because the long tail is bootlegs and mislabelled scrobbles: a /// real album track is, by definition, one of the popular ones. The cost is /// body size, 77 KB -> ~775 KB per artist, and it is disk cached for a day. const LASTFM_TRACK_LIMIT: usize = 1000; /// How many Deezer tracks are fetched to reorder. Deezer's `/top` is ranked by /// its own recency-weighted popularity, so the canonical songs sit deeper than /// the ten we used to show: measured across 18 artists, Last.fm's canonical top /// ten is only 43% present in Deezer's top 10, but 77% by its top 50. Past 50 /// the curve flattens (79% at 100) and every extra track is one more row that /// can never be displayed. const POPULAR_POOL_LIMIT: usize = 50; /// How many ranked tracks reach the frontend. The UI shows five and expands to /// the rest, so this is the size of the expanded list. const POPULAR_KEEP: usize = 20; /// Release-group paging for the first-release-date pass. 100 is MusicBrainz's /// maximum page; three pages covers a very prolific catalogue (Queen has ~100 /// album groups alone) without ever walking a discography forever behind the /// mandatory 1 req/s gate. const MB_RELEASE_GROUP_PAGE: usize = 100; const MB_RELEASE_GROUP_PAGES: usize = 3; /// How many times the FIRST release-group page is retried before the supplement /// declares MusicBrainz unavailable. /// /// The failures this exists for are transient by nature — a 503 out of the /// 1 req/s limiter, or this machine having throttled itself with probes — and /// the cost of giving up is the whole date + attribution pass for that artist. /// How long the overview holds first paint for the supplements chain (linked /// MBID → Last.fm ranks + original release dates). Warm the chain answers in /// single-digit milliseconds and the budget never bites; cold it is 2.2s+ of /// MusicBrainz behind the 1 req/s gate, and the overview paints without it /// rather than becoming the slow page it exists to preempt. Wide enough for /// a disk-cache read on a busy runtime, an order of magnitude under a cold /// network chain. const OVERVIEW_SUPPLEMENTS_BUDGET: Duration = Duration::from_millis(300); /// Two retries behind a widening backoff is enough to ride out a blip without /// turning a genuine outage into a slow page: the request rides the same /// 1 req/s gate as everything else, and the identity pass is deferred, so /// nothing on screen is waiting on it. const MB_RETRY_ATTEMPTS: u32 = 2; const MB_RETRY_BACKOFF: Duration = Duration::from_millis(1200); /// Release paging for [`MetadataClient::musicbrainz_recordings`]. 100 is /// MusicBrainz's maximum page and each one carries every tracklist on it, so /// two pages reach 200 pressings — comfortably the whole discography for the /// artists this exists for, and bounded because each page costs a second behind /// the 1 req/s gate and roughly 200 KB of body. const MB_RECORDING_PAGE: usize = 100; const MB_RECORDING_PAGES: usize = 2; /// How deep Last.fm's rank order is kept for building Popular. Only /// [`POPULAR_KEEP`] rows are shown, but a track can fail to resolve to anything /// playable, so the list needs headroom to fill from underneath. const LASTFM_RANKED_KEEP: usize = 40; /// How many distinct records one page will look up previews for. Popular is /// twenty rows, so a gap spanning more than a few albums means the artist is /// barely on Apple either and the next lookup is unlikely to pay. Bounded /// because iTunes Search allows roughly 20 calls a minute. const APPLE_PREVIEW_ALBUMS: usize = 4; /// A resolved artist banner rarely changes and costs two external calls, so it /// is cached far longer than ordinary catalog pages. const BANNER_TTL: Duration = Duration::from_secs(24 * 60 * 60); /// A transient provider outage should not immediately relaunch the complete /// TADB → MusicBrainz → Fanart → Bing chain on every repeat navigation. Empty /// art gets only a short reprieve; successful art keeps the full-day TTL. const EMPTY_ART_TTL: Duration = Duration::from_secs(2 * 60); /// Give purpose-built art databases a short head start before the provisional /// stage spends a Bing request. The exhaustive fallback still uses the exact /// provider priority and scoring rules below. const HERO_SEED_BING_DELAY: Duration = Duration::from_millis(1500); #[derive(Debug, Clone)] pub struct AlbumMeta { pub artist: String, pub album: String, /// MusicBrainz release date, usually YYYY-MM-DD but sometimes year-only. pub release_date: Option, pub year: Option, pub release_mbid: Option, pub release_group_mbid: Option, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct CatalogArtist { pub id: u64, pub name: String, pub picture: Option, pub album_count: Option, pub fans: Option, } #[derive(Debug, Clone, serde::Serialize)] pub struct CatalogAlbum { pub id: u64, pub title: String, pub artist: String, pub cover: Option, pub release_date: Option, /// "album" | "ep" | "single" | "compile" (absent in some search results) pub record_type: Option, pub track_count: Option, /// Deezer has no release-group id. When an edition shares a strong ordered /// core-track fingerprint with an original album, both receive the /// original's id here so the UI can fold the musical work reliably instead /// of guessing from display titles alone. pub edition_group: Option, /// Deezer does not expose a soundtrack record type. Album details do, /// however, identify most film/game releases with its Films/Games genre. pub is_soundtrack: bool, /// Deezer has no live record type either, and live albums frequently omit /// any "Live" hint from their title (Animal Collective's "Ballet Slippers", /// "FEELSLive 04/05"). The tracklist is the reliable tell: when a clear /// majority of a release's tracks are marked as live recordings, it is one. /// Only set once album details (hence the tracklist) are known. #[serde(default)] pub is_live: bool, /// True when MusicBrainz types the release group `Mixtape/Street`. /// /// Deezer has no mixtape record type at all, and for several artists the /// mixtape IS the canonical work — Death Grips' "Exmilitary" is their /// debut, and Chance the Rapper's "Acid Rap" and "Coloring Book" carry /// more scrobbles than anything he has released as an album. They cannot /// simply join the Albums grid, though: MusicBrainz files 118 of them for /// Gucci Mane and 51 for Lil Wayne, which would bury a 15-record /// discography. The flag exists so the frontend can give them their own /// tab, where volume costs nothing and nothing has to be hidden. #[serde(default)] pub is_mixtape: bool, /// True when Deezer's album details credit a DIFFERENT artist as the /// release's primary artist — i.e. this artist is a contributor on someone /// else's record. `/artist/{id}/albums` returns those alongside the /// artist's own releases, which is how four Rooby Jeantal lo-fi records /// arrived in Mid-Air Thief's discography. /// /// A flag rather than a filter, deliberately: Deezer marks EVERY /// contributor `role: "Main"`, so a guest spot and a genuine joint album /// (Kanye West on "Watch the Throne", primary artist JAY-Z) are /// indistinguishable here. Only corroboration can separate them, and that /// happens where the MusicBrainz identity is merged. /// /// Only set once album details are known; false means "own release, or not /// yet checked". #[serde(default)] pub appears_on: bool, } #[derive(Debug, Clone, serde::Serialize)] pub struct CatalogTrack { pub id: u64, pub title: String, pub artist: String, /// The performing artist's Deezer id, when the provider gave one alongside /// the name. Search results span many artists, and a NAME is not an /// identity here — see the namesake rules on `lastfm_listener_ranks`. This /// is what lets the search page ask for a row's listener count without ever /// querying a provider by artist name. #[serde(default)] pub artist_id: Option, /// Deezer's own popularity score, 0–1,000,000. Present on search results, /// and the only popularity signal available at FIRST PAINT — Last.fm /// listeners arrive a beat later. The search page ranks its top-result /// candidates on this, so a famous song is not outranked by an obscure /// one-track single that happens to share its name. #[serde(default)] pub rank: Option, pub album: String, pub album_id: Option, pub cover: Option, pub duration_secs: Option, /// 30-second MP3 preview URL. pub preview: Option, /// Position within its disc (Deezer `track_position`). pub track_no: Option, /// Disc number for multi-disc releases (Deezer `disk_number`); `None` or 1 /// for single-disc albums. #[serde(default)] pub disk_number: Option, /// True only for a track on an as-yet-unreleased (future-dated) album that /// is not itself already out as a single. The UI blocks streaming/saving /// these; pre-released singles from an upcoming album stay `false` and /// remain playable. Always `false` for normal, released albums. #[serde(default)] pub unreleased: bool, /// All-time distinct Last.fm listeners, when this track could be anchored /// to one (see [`Self::listeners`] producers in `lastfm_listener_ranks`). /// `None` means Last.fm has no count for it — a track too new or too /// obscure to have been scrobbled, or an artist with no usable anchor at /// all. The UI shows a dash rather than a zero: unknown is not none. #[serde(default)] pub listeners: Option, /// Total Last.fm scrobbles, from the same table as [`Self::listeners`]. /// This is the album tracklist's "Plays" column. `None` means Last.fm has /// no entry for the track — shown as a dash, never as a zero. #[serde(default)] pub plays: Option, /// The Deezer `/top` track id this row was re-attributed away from, when /// [`apply_original_album_attribution`] moved it onto the song's original /// album. `None` means the row is Deezer's own attribution, untouched. /// /// The UI still matches now-playing against this id, so a track started /// from search, the suggest dropdown or an album page — all of which use /// the id Deezer LICENSED — keeps its highlight on this row. #[serde(default)] pub origin_track_id: Option, } /// One entry of Last.fm's top-tracks table, in rank order. #[derive(Debug, Clone)] struct LastfmTrack { /// [`track_title_identity`] of `title`. key: String, /// Last.fm's own spelling, used when no Deezer row supplies one. title: String, stats: TrackStats, } /// Last.fm's top-tracks table, both ways round: by identity for the lookups /// that annotate an existing row, and in rank order for the ones that BUILD the /// Popular list. #[derive(Debug, Clone, Default)] struct LastfmRanks { by_key: HashMap, ranked: Vec, } /// One recording of one song, and the record it is best attributed to. /// Built by [`MetadataClient::musicbrainz_recordings`]. #[derive(Debug, Clone)] struct MbRecording { release_group_id: String, album_title: String, /// MusicBrainz's spelling of the track, used when Deezer has no row to take /// a display title from. title: String, release_date: Option, length_ms: Option, /// Primary type Album with NO secondary type — i.e. a studio record rather /// than a compilation, live album, soundtrack or remix set. plain_album: bool, } impl MbRecording { /// Is this a better home for the song than `other`? /// /// A plain album always beats a typed one, however early the typed one is /// dated — the point of the test is that compilations are dated by their /// own release and would otherwise win on date alone. Between two of the /// same kind, the earlier record wins; a missing date loses to any date. fn outranks(&self, other: &Self) -> bool { match (self.plain_album, other.plain_album) { (true, false) => true, (false, true) => false, _ => match (self.release_date.as_deref(), other.release_date.as_deref()) { (Some(mine), Some(theirs)) => mine < theirs, (Some(_), None) => true, _ => false, }, } } } /// What one MusicBrainz release GROUP says about a work: when it first came /// out, and what kind of release it is. /// /// The two travel together because they only mean anything read together. A /// compilation is dated EARLIER than the works it compiles, so a date carried /// without its own secondary types actively *selects* hits packages — see /// [`apply_original_album_attribution`], which measured 79/80 with the type /// exclusion and 63/80 without it. #[derive(Debug, Clone, Default, PartialEq)] struct ReleaseFacts { /// MusicBrainz `first-release-date` for the group. first: String, /// MusicBrainz `secondary-types`: `Compilation`, `Live`, `Soundtrack`, /// `Remix`, `DJ-mix`, … Empty for a plain studio album. secondary: Vec, /// Several MusicBrainz groups collapsed onto this key carrying DIFFERENT /// first-release dates, so the key cannot say which record a given Deezer /// release is. Nothing is ever dated or excluded from an ambiguous key. /// /// Weezer is the measured case: MusicBrainz files seven self-titled /// albums, all titled plain "Weezer", separated only by `disambiguation` /// ("Blue Album" … "Gold Album"). Taking the earliest dated the 2026 Gold /// Album 1994-05-10 and filed an unreleased record next to the debut. ambiguous: bool, } /// One track's Last.fm scrobble figures. Both arrive in the same /// `artist.gettoptracks` body, so carrying plays alongside listeners costs /// nothing beyond the field. #[derive(Debug, Clone, Copy, Default, PartialEq)] struct TrackStats { /// Total scrobbles — the "Plays" column on an album. plays: u64, /// Distinct listeners — what the Popular list ranks by. listeners: u64, } /// What the MBID-anchored providers contribute to an artist page. Empty on /// every failure path, which degrades the page to Deezer's own data. #[derive(Debug, Default)] struct ArtistSupplements { /// Last.fm scrobble figures, keyed by [`track_title_identity`]. listeners: HashMap, /// MusicBrainz first-release facts, keyed by [`release_date_key`]. first_released: HashMap, /// Last.fm's top tracks IN RANK ORDER, which is what Popular is built from. ranked: Vec, /// Which record each recording belongs to, for Last.fm entries Deezer has /// no pressing of. Empty on the overview stage, which does not pay for it. recordings: HashMap, /// MusicBrainz never answered, as distinct from having nothing to say. /// /// The two used to be the same value — a failed page `break`s and leaves an /// empty map, which reads exactly like "this artist has no release groups". /// One transient blip therefore cost a page its true album dates AND its /// original-album attribution, silently, with nothing in the UI or the /// traces to say so. It also produces false bug reports: My Bloody /// Valentine was recorded as "the supplement is failing to inject Loveless" /// when the supplement was fine and MusicBrainz had simply not replied that /// once. /// /// Callers must not treat this as evidence of anything. See /// [`MetadataClient::musicbrainz_first_release_dates`]. musicbrainz_unavailable: bool, } /// What the search hero shows about an artist beyond Deezer's own fields. /// Every field is independently optional — see [`MetadataClient::artist_profile`]. #[derive(Debug, Clone, Default, serde::Serialize)] pub struct CatalogArtistProfile { /// All-time distinct Last.fm listeners for the artist. pub listeners: Option, /// All-time Last.fm scrobbles for the artist. pub plays: Option, /// MusicBrainz's most-voted genre tags, strongest first. pub tags: Vec, } #[derive(Debug, Clone, serde::Serialize)] pub struct CatalogSearch { pub artists: Vec, pub albums: Vec, pub tracks: Vec, } #[derive(Debug, Clone, serde::Serialize)] pub struct CatalogArtistPage { pub artist: CatalogArtist, /// Artist hero from Fanart.tv, TheAudioDB, or the hardened Bing fallback. pub banner: Option, /// Every qualifying hero from the highest-priority provider that returned /// artwork. The frontend starts at a random entry and rotates through this /// list; lower-quality provider tiers are never mixed into a better tier. pub banners: Vec, /// Set by the Tauri layer when a user-imported per-artist hero overrides /// provider artwork. Always false in provider-only metadata responses. pub banner_custom: bool, /// Fanart.tv and TheAudioDB logo candidates (transparent PNGs). /// The UI scores them for "clean monochrome wordmark", tints the best to the /// accent, else shows the most-liked in full colour, else plain text. pub logos: Vec, /// Deezer's curated top tracks, shown first on the artist profile. pub popular_tracks: Vec, pub albums: Vec, /// Majority genre across the album details already fetched for the /// edition/type analysis — zero extra API calls. `None` when no detail /// carried a usable genre. pub top_genre: Option, /// Official links (homepage + socials) from MusicBrainz url-rels, anchored /// to the Deezer-linked MBID so a namesake never shows another artist's /// accounts. Ordered for display; at most one link per kind. pub socials: Vec, } #[derive(Debug, Clone, serde::Serialize)] pub struct ArtistLink { /// One of: website, instagram, x, facebook, tiktok, youtube, bandcamp, /// soundcloud — the frontend maps these to icons and brand tints. pub kind: String, pub url: String, } /// One headline about a followed artist, for the home page's news panel. /// `date` is unix seconds; freshness policy (the 45-day cap, the review /// filter) is the frontend's display decision, not baked in here. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct ArtistNewsItem { pub title: String, pub url: String, pub source: String, pub date: i64, /// Publisher host from `` — "wwd.com", "pitchfork.com". /// The element's TEXT is a display name ("GamesRadar+") and is not a host; /// the attribute is, and it is present on every item. /// /// `serde(default)` is load-bearing: cached items are read back with /// `from_value::>(..).ok()`, and `.ok()` swallows a /// schema mismatch — a required new field would silently invalidate every /// cached entry and fire a cold refetch for every follow at once, against /// a rate-limited endpoint. #[serde(default)] pub host: String, /// How many near-duplicate reports of this same story were collapsed into /// it, so the UI can say "+N more outlets" instead of dropping them mutely. #[serde(default)] pub others: u32, } /// Lightweight artist identity + artwork response used to paint the profile /// while the deeper discography/edition analysis continues in parallel. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct CatalogArtistArt { pub artist: CatalogArtist, pub banner: Option, pub banners: Vec, pub banner_custom: bool, pub logos: Vec, } /// First catalog-content stage for an artist page. One Deezer round trip yields /// the identity, popular tracks and first 50 curated releases. Provider art, /// exhaustive pagination, per-album classification and MusicBrainz enrichment /// deliberately stay out of this response so the page can become useful while /// the complete catalog continues in the background. #[derive(Debug, Clone, serde::Serialize)] pub struct CatalogArtistOverview { pub artist: CatalogArtist, pub popular_tracks: Vec, pub albums: Vec, } #[derive(Debug, Clone, serde::Serialize)] pub struct CatalogAlbumPage { pub album: CatalogAlbum, pub tracks: Vec, /// Total Last.fm scrobbles for the record, and the distinct listeners /// behind them. `None` when the artist could not be anchored to Last.fm. #[serde(default)] pub album_plays: Option, #[serde(default)] pub album_listeners: Option, /// The totals were summed from this tracklist because Last.fm has no album /// entry of its own — routine for anniversary and deluxe titles. Surfaced /// so the UI can label an approximation rather than pass it off as /// Last.fm's own figure. #[serde(default)] pub totals_summed: bool, } /// Editorial context for one release — a prose description plus genre/label /// facts — sourced from TheAudioDB's album records first (real writing, plus /// label/genre columns) and Wikipedia's page summary as the fallback. All /// fields are best-effort; an empty struct means neither source knew the /// record. #[derive(Debug, Clone, Default, serde::Serialize)] pub struct AlbumInfo { pub description: Option, pub genre: Option, pub label: Option, /// "theaudiodb" or "wikipedia" — which source supplied `description`. pub source: Option, /// Release-type evidence read from the description's LEAD sentence /// ("…is a compilation album by The Killers…"): one of "compilation", /// "live", "remix", "soundtrack", "ep", "album", "song". Prose that opens /// by declaring the record's type is direct editorial evidence the title /// heuristics cannot see — the page filing consumes it. /// /// "song" is the one hint that is not a release type: it reports that the /// only editorial article for this title describes a TRACK. That makes it /// evidence ABOUT the description rather than about the release — the /// filing must not read the album named in such a lead as this release's /// own kind. pub type_hint: Option, } /// Resolved hero artwork for an artist — the wide background banner and the /// band's HD ClearLOGO — resolved together (one Fanart.tv call) and cached as /// a unit. #[derive(Debug, Clone, Default)] struct ArtistArt { banners: Vec, logos: Vec, } /// One Bing result card with its own metadata kept attached all the way through /// scoring. This is intentionally not just a URL: losing the title/source page /// made it possible for an unrelated image to borrow identity text from an /// adjacent result card. #[derive(Debug, Clone)] struct BingImageCandidate { url: String, title: String, page_url: String, identity: BingIdentityConfidence, /// False only for a product/release card retained as identity evidence. /// It may corroborate an artist-named host for sibling press photography, /// but is never downloaded, scored, or returned as a hero itself. display_eligible: bool, query_index: usize, rank_in_query: usize, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum BingIdentityConfidence { /// The image URL names the artist, both the result title and page URL name /// the artist, or a trusted Ticketmaster/Live Nation artist profile owns it. Strong, /// Only one Bing metadata field names the artist. Retained solely as a /// last-resort single-image fallback when no strongly anchored result exists. Weak, } /// Identity scope for a Bing fallback. A globally unambiguous display name can /// use the existing per-card identity rules. So can a name whose requested /// Deezer id demonstrably *owns* it: a web image search ranks by open-web /// prominence, so when one exact-name artist dwarfs its namesakes the results /// are about that artist and the per-card rules are the right filter. /// /// Only a genuinely contested name falls back to per-card catalogue anchoring, /// where the shared name is not evidence and each surviving card must /// independently mention a distinctive release from the Apple catalogue /// selected by the requested Deezer id's weighted release evidence. That tier /// is deliberately narrow: press photography rarely cites a release title, so /// applying it to an uncontested name silently discards the best heroes. #[derive(Debug, Clone, PartialEq, Eq)] enum BingArtistIdentityPolicy { Unambiguous, NameOwner, CatalogAnchored { release_titles: HashSet }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum DeezerExactNameState { Missing, Unambiguous, /// Exact-name namesakes exist, but the requested id decisively out-follows /// all of them, so it is the subject of a name search. Dominant, Ambiguous, } #[derive(Debug, Clone, PartialEq, Eq)] struct ParsedBingImage { url: String, title: String, page_url: String, } /// Pixel-derived quality evidence kept alongside a Bing result through final /// ranking. The difference hash lets the carousel discard the same press photo /// when Bing exposes several crops, sizes, or CDN copies of it. #[derive(Debug, Clone)] struct HeroVisualScore { score: f64, fingerprint: HeroFingerprint, } /// Crop-aware perceptual evidence. Search engines frequently expose one press /// photo through several publishers, aspect crops, and color treatments. #[derive(Debug, Clone)] struct HeroFingerprint { /// Full-frame plus 18 overlapping crop hashes (85% and 70% at a 3x3 grid). dhashes: Vec, /// Coarse 8x8x8 RGB histogram over a fixed 64x64 sample. color_histogram: [u16; 512], } const HERO_EXACT_HASH_MAX_DISTANCE: u32 = 8; const HERO_CROP_HASH_MAX_DISTANCE: u32 = 13; const HERO_COLOR_INTERSECTION_MIN: u32 = 2_250; // about 55% of 64x64 pixels const HERO_SCORE_CONCURRENCY: usize = 4; /// Minimum width:height for a hero candidate. Album sleeves are exactly 1.0; /// every press photo across the probe fixtures measures 1.33 or wider. const HERO_MIN_ASPECT: f64 = 1.15; /// Portrait sources are admitted below this ratio, well clear of the square /// band where release artwork lives. The app crops tall sources to 16:9 around /// the detected faces (`hero_crop::maximal_hero_rect` keeps full width and /// slides the window vertically), so a portrait press photo is usable hero /// material while a 1:1 sleeve is not. const HERO_PORTRAIT_MAX_ASPECT: f64 = 0.92; /// A portrait source is cropped to full-width x (width*9/16), so its width IS /// the resulting hero width. Keep enough pixels that the crop is still a hero. const HERO_PORTRAIT_MIN_WIDTH: u32 = 1000; /// How many of the artist's releases to perceptually hash when deciding whether /// a square candidate is one of their sleeves. const HERO_COVER_HASH_LIMIT: usize = 25; const HERO_SEARCH_WAVE_CONCURRENCY: usize = 3; /// A Bing image page with at most this many cards is a degraded answer, not a /// small result set. Bing's genuine pages run 12-100 cards; the degraded shape /// is one card of something unrelated. const HERO_DEGRADED_PAGE_CARDS: usize = 2; /// Bing's own default adult-content preference, sent as a cookie. The /// `&safesearch=moderate` query parameter measures identical to sending no /// preference at all; this cookie actually selects the bucket. const BING_MODERATE_COOKIE: &str = "SRCHHPGUSR=ADLT=DEMOTE"; const HERO_SCORE_CACHE_MAX_ENTRIES: usize = 128; const HERO_SCORE_RECIPE: &str = "bing-identity-visual-v1"; /// A MusicBrainz studio album injected when Deezer's regional catalog omits the /// release entirely. The synthetic numeric id keeps the existing frontend API /// intact; this map routes an opened card back to its release group. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] struct MbCatalogAlbum { release_group_id: String, /// The ARTIST's MusicBrainz id, not the release group's. /// /// Kept because the Last.fm scrobble table is anchored on exactly this, and /// a supplemented album used to have no way back to it: the play-count pass /// resolved the mbid from a DEEZER artist id, which a record Deezer never /// carried does not have. So the records the supplement exists to rescue — /// Loveless, Isn't Anything, Exmilitary — opened with no Plays column at /// all. The id was in hand when the group was registered; it just was not /// kept. artist_mbid: String, artist: String, title: String, release_date: Option, cover: Option, } /// The deferred MusicBrainz pass — see `catalog_artist_identity`. #[derive(Debug, Clone, Default, serde::Serialize)] pub struct CatalogArtistIdentity { /// Echoes the requested Deezer identity so a late MusicBrainz response can /// never be merged into a different artist after fast navigation. pub artist_id: u64, pub albums: Vec, pub socials: Vec, pub type_hints: Vec, /// Deezer album ids this artist's own identity corroborates. A row flagged /// `appears_on` that appears here is a real collaboration, not a guest /// spot — see [`ReleaseOwnership`]. pub owned_album_ids: Vec, /// Deezer album ids belonging to a different act that shares this display /// name. Non-empty only where co-tenancy was proven. pub foreign_album_ids: Vec, /// True when this Deezer artist record was proven to hold more than one /// act. The Deezer artist PORTRAIT is then untrustworthy by the same /// argument as the releases — it is whichever co-tenant uploaded last. pub fused_record: bool, } fn empty_catalog_artist_identity(artist_id: u64) -> CatalogArtistIdentity { CatalogArtistIdentity { artist_id, ..CatalogArtistIdentity::default() } } /// One MusicBrainz release-group classification hint. The frontend matches it /// to a Deezer/Apple release by normalized title and files that release under /// the tab MusicBrainz's authoritative primary/secondary types imply — the fix /// for compilations, soundtracks and remix albums the providers hand back /// tagged as a plain "album" (e.g. Radiohead's "KID A MNESIA", which MB knows /// is a Compilation). Raw MB type strings are sent through so the release /// taxonomy stays owned in one place, the frontend. #[derive(Debug, Clone, serde::Serialize)] pub struct ReleaseTypeHint { pub title: String, pub primary_type: Option, pub secondary_types: Vec, } /// One release from the iTunes artist lookup. That single unthrottled request /// carries the whole discography — title, date, track count and type — so it /// serves two jobs: the cover resolver's artwork thumbnails, and the release /// supplement that used to require rate-limited MusicBrainz calls. #[derive(Debug, Clone)] struct AppleRelease { title: String, title_norm: String, artwork: String, release_date: Option, track_count: Option, /// iTunes `collectionType`, e.g. "Album" / "Compilation". collection_type: Option, /// iTunes `collectionId` — the key for a follow-up song lookup that yields /// the album's tracklist. `None` when the lookup entry omitted it. collection_id: Option, /// iTunes `copyright`, e.g. "℗ 2014 GOD?". The label is the only field that /// separates a co-tenant from the artist on a fused Deezer record when /// neither provider carries the title (see [`ReleaseOwnership`]). copyright: Option, } /// One identity-checked Apple artist catalogue. Apple can return several /// *exactly* same-named performers for one search (Crazy P currently has a UK /// electronic act, a Latin act and a rapper). `releases` belongs to the single /// candidate whose titles best corroborate the requested Deezer artist's own /// release endpoint. `foreign_titles` contains titles exclusive to the losing /// Apple candidates; those are strong, cheap evidence that a Deezer release was /// fused into the wrong same-named artist id. #[derive(Debug, Clone, Default)] struct AppleArtistCatalog { releases: Vec, foreign_titles: HashSet, /// Deezer genres corroborated only by losing Apple candidates. This removes /// the rest of a fused namesake cluster even when Apple does not carry every /// individual leaked title. Genres also matched by the winner are excluded, /// so collaborations and broad shared styles remain untouched. foreign_genres: HashSet, /// Winning Apple performer and corroboration weight. The latter prevents a /// first-50 overview race from overwriting a stronger full-discography /// decision in the Deezer-id cache. apple_artist_id: Option, evidence_weight: u64, evidence_count: usize, /// Multiple exact performers existed but at least one lookup was missing /// or intentionally capped, so even filtering Deezer's fused payload would /// be guesswork. Overview/full return no stage and retry later. identity_unavailable: bool, /// False when several exact-name Apple candidates tied without any release /// corroboration. In that case Apple must not supplement the page: choosing /// the provider's first search hit would recreate the namesake bug. identity_confirmed: bool, } #[derive(Debug, Clone)] struct AppleArtistCandidate { artist_id: u64, releases: Vec, /// Titles this artist's lookup returns under a DIFFERENT Apple artist id /// whose display credit still names them as a principal — "¥$, Kanye West /// & Ty Dolla $ign", "JAŸ-Z & Kanye West". `releases` cannot hold these: /// it is filtered to the looked-up id so the page supplement never injects /// somebody else's record, which is also why VULTURES 1 and KIDS SEE /// GHOSTS had no corroboration anywhere. /// /// Deliberately NOT merged into `releases`: a guest feature arrives in the /// same bucket ("Estelle" — "American Boy (feat. Kanye West)") and is /// excluded here only because the credit does not name the artist. credited_titles: HashSet, lookup_complete: bool, cacheable: bool, } /// A title from the requested Deezer id's own release endpoint. Deezer can /// fuse namesakes under one numeric id, so the title alone is not enough: /// listener/fan weight makes the established catalogue dominate stray uploads /// while the base point still lets several low-traffic exact matches agree. #[derive(Debug, Clone)] struct DeezerReleaseEvidence { title_norm: String, weight: u64, genre_id: Option, } /// A supplemented Apple album, keyed by its synthetic catalog id so an opened /// card can fetch its tracklist from iTunes. The Apple analogue of /// [`MbCatalogAlbum`]: Apple releases carry large synthetic ids that Deezer's /// `/album/{id}` endpoint cannot resolve, so `catalog_album` routes them here. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] struct AppleCatalogAlbum { collection_id: u64, /// The DEEZER artist id this supplement was fetched for. /// /// Same reason [`MbCatalogAlbum::artist_mbid`] exists: the play-count pass /// needs a way back to the artist, and an Apple-supplemented row has no /// Deezer ALBUM to read one from. Weezer's "Hurley (Deluxe Edition)" is an /// Apple row, and it opened with no Plays column while the plain Deezer /// edition beside it had one. artist_deezer_id: u64, artist: String, title: String, cover: Option, release_date: Option, } /// Up to `n` image URLs from a Fanart.tv array, most-liked first. fn top_liked_urls(body: &serde_json::Value, key: &str, n: usize) -> Vec { let Some(arr) = body.get(key).and_then(|v| v.as_array()) else { return Vec::new(); }; let mut items: Vec<(i64, &str)> = arr .iter() .filter_map(|b| { let url = b .get("url") .and_then(|u| u.as_str()) .filter(|u| u.starts_with("http"))?; let likes = b .get("likes") .and_then(|l| l.as_str()) .and_then(|s| s.parse().ok()) .unwrap_or(0); Some((likes, url)) }) .collect(); items.sort_by(|a, b| b.0.cmp(&a.0)); items .into_iter() .take(n) .map(|(_, u)| u.to_string()) .collect() } fn audiodb_banner_urls(artist: &serde_json::Value) -> Vec { [ "strArtistFanart", "strArtistFanart2", "strArtistFanart3", "strArtistFanart4", ] .iter() .filter_map(|key| { artist .get(*key) .and_then(|value| value.as_str()) .filter(|url| url.starts_with("https://")) .map(String::from) }) .fold(Vec::new(), |mut urls, url| { if !urls.contains(&url) { urls.push(url); } urls }) } pub struct MetadataClient { http: reqwest::Client, /// Emit detailed Bing candidate/score lines for the live probe. Production /// clients leave this false, so normal app logs remain quiet. hero_diagnostics: bool, /// Last MusicBrainz request time, for the 1 rps limit. mb_gate: Mutex, /// Start times of the Deezer requests inside the current rate window, for /// the 50-per-5-seconds budget. deezer_gate: Mutex>, /// URL → (fetched-at, body). Catalog browsing re-requests the same /// pages constantly; a short TTL cache keeps it snappy and polite. catalog_cache: Mutex>, /// Concurrent callers for the same cold URL share one network request. /// The fast artwork response and full page intentionally overlap, so this /// prevents their common profile/provider lookups from being duplicated. catalog_inflight: Mutex>, /// URLs whose most recent failure was a 4xx — an answer about the /// RESOURCE (it does not exist), not about the request. `cached_json` /// deliberately caches no failures, so its `None` conflates "MusicBrainz /// has no such entity" with "MusicBrainz hiccuped"; this marker is what /// lets a caller retry the hiccup without punishing every legitimately /// absent resource with the same ladder. Entries clear on later success. definitive_miss: Mutex>, /// Artist name → (fetched-at, resolved art). Art resolution hits two /// providers (TheAudioDB + Fanart.tv); this keeps repeat artist visits /// instant and keeps us polite to both services. art_cache: Mutex>, art_inflight: Mutex>, /// Purpose-built Fanart/TADB result shared by the fast logo-status command /// and the final art command. Without this monotonic provider stage, two /// independent timeout windows could report [] first and a curated logo /// later, briefly exposing then replacing a generated cover logo. provider_art_cache: Mutex>, provider_art_inflight: Mutex>, /// First trustworthy remote hero candidate. This is deliberately separate /// from the final provider-tier result: the UI may paint its preview while /// the complete quality-ranked set continues resolving. hero_seed_cache: Mutex)>>, hero_seed_inflight: Mutex>, /// Bing result HTML is shared by the fast-seed and exhaustive passes. Raw /// candidate images already use `image_cache`; without this companion cache /// the two stages would still download the same search page twice. html_cache: Mutex>, html_inflight: Mutex>, /// Candidate pixel scores are shared by the provisional and exhaustive Bing /// passes. The key includes every metadata field used by visual rejection, /// so a later stronger result card can never borrow a stale decision. hero_score_cache: Mutex)>>, hero_score_inflight: Mutex>, /// Short-lived raw image cache shared by the HD preview, 4K finisher and /// logo proxy. In particular, preview + 4K no longer download the same hero /// twice at the same moment. image_cache: Mutex>)>>, image_inflight: Mutex>, /// Synthetic catalog id → MusicBrainz release group for studio albums that /// are unavailable in Deezer's current regional response. mb_catalog: Mutex>, /// Deezer identity (or the explicit name-only fallback used by standalone /// cover resolution) → identity-checked Apple catalogue. Artist-page keys /// are always `dz:{id}`; a shared display name can never reuse another /// Deezer artist's chosen Apple performer. apple_albums: Mutex>>, /// Normalized name → every exact-name Apple candidate. Candidate discovery /// is identity-neutral and therefore safe to share; the winning catalogue /// is selected and cached separately under the Deezer id above. apple_candidates: Mutex>>>, /// Concurrent page/cover requests share the bounded Apple search + lookup /// wave instead of stampeding the provider. apple_candidates_inflight: Mutex>, /// The same discovery under [`APPLE_IDENTITY_ARTIST_CANDIDATE_LIMIT`], kept /// apart because the two differ in the one way that matters: above the /// page-load cap the map above holds a deliberate "unresolved" sentinel, /// and the identity pass must not inherit it. apple_identity_candidates: Mutex>>>, apple_identity_candidates_inflight: Mutex>, /// Synthetic catalog id → Apple album for supplemented releases missing from /// Deezer's regional response, so an opened card can fetch its iTunes /// tracklist. Mirrors [`Self::mb_catalog`]. apple_catalog: Mutex>, /// Sliding-window pacer for wikipedia.org, mirroring [`Self::deezer_gate`]. wiki_gate: Mutex>, /// Server-imposed cooldown: set from a 429's `Retry-After`, so concurrent /// description lookups stop piling onto an already-penalized IP. wiki_cooldown: Mutex>, /// Sliding-window pacer for api.discogs.com (see [`DISCOGS_RATE_BUDGET`]). discogs_gate: Mutex>, /// Sliding-window pacer for ws.audioscrobbler.com (see [`LASTFM_RATE_BUDGET`]). lastfm_gate: Mutex>, /// Sliding-window pacer for news.google.com (see [`NEWS_RATE_BUDGET`]). news_gate: Mutex>, /// Query URL → parsed items, so a session refresh costs nothing. Disk /// persistence rides the generic JSON cache beside it. news_cache: Mutex)>>, /// On-disk JSON cache for slow-moving providers (see [`DISK_CACHE_TTL`]). /// `None` (the default, and every test/example) keeps behavior purely /// in-memory. disk_cache_dir: Option, /// Deezer artist id → the portrait to show instead of the one Deezer /// serves, for records proven to hold more than one act. /// /// Deliberately NOT inside [`Self::disk_cache_dir`]: `prune_disk_cache` /// deletes every `.json` there older than twice the provider TTL, and a /// verdict is not a cached response — losing it silently would put another /// artist's face back in the search bar. identity_store_dir: Option, fused_portraits: std::sync::Mutex>>, } impl Default for MetadataClient { fn default() -> Self { Self::new() } } impl MetadataClient { pub fn new() -> Self { MetadataClient { http: reqwest::Client::builder() .user_agent(USER_AGENT) .timeout(Duration::from_secs(15)) .build() .expect("build http client"), hero_diagnostics: false, mb_gate: Mutex::new(Instant::now() - Duration::from_secs(2)), deezer_gate: Mutex::new(VecDeque::new()), catalog_cache: Mutex::new(HashMap::new()), catalog_inflight: Mutex::new(HashMap::new()), definitive_miss: Mutex::new(HashSet::new()), art_cache: Mutex::new(HashMap::new()), art_inflight: Mutex::new(HashMap::new()), provider_art_cache: Mutex::new(HashMap::new()), provider_art_inflight: Mutex::new(HashMap::new()), hero_seed_cache: Mutex::new(HashMap::new()), hero_seed_inflight: Mutex::new(HashMap::new()), html_cache: Mutex::new(HashMap::new()), html_inflight: Mutex::new(HashMap::new()), hero_score_cache: Mutex::new(HashMap::new()), hero_score_inflight: Mutex::new(HashMap::new()), image_cache: Mutex::new(HashMap::new()), image_inflight: Mutex::new(HashMap::new()), mb_catalog: Mutex::new(HashMap::new()), apple_albums: Mutex::new(HashMap::new()), apple_candidates: Mutex::new(HashMap::new()), apple_candidates_inflight: Mutex::new(HashMap::new()), apple_identity_candidates: Mutex::new(HashMap::new()), apple_identity_candidates_inflight: Mutex::new(HashMap::new()), apple_catalog: Mutex::new(HashMap::new()), wiki_gate: Mutex::new(VecDeque::new()), wiki_cooldown: Mutex::new(None), discogs_gate: Mutex::new(VecDeque::new()), lastfm_gate: Mutex::new(VecDeque::new()), news_gate: Mutex::new(VecDeque::new()), news_cache: Mutex::new(HashMap::new()), disk_cache_dir: None, identity_store_dir: None, fused_portraits: std::sync::Mutex::new(HashMap::new()), } } /// Enable the on-disk cache for slow-moving providers (MusicBrainz, /// Wikipedia). The directory is created if missing; failures degrade to /// memory-only behavior. Long-stale entries are pruned at startup — /// expired files are otherwise only ever touched if their exact URL is /// refetched, so without this sweep a browsing-heavy user accumulates /// orphaned per-album files indefinitely. pub fn with_disk_cache(mut self, dir: std::path::PathBuf) -> Self { if std::fs::create_dir_all(&dir).is_ok() { prune_disk_cache(&dir); self.disk_cache_dir = Some(dir); } self } /// Enable the durable identity store — verdicts, not cached responses. /// /// A fused record is discovered by the deferred ownership pass, which costs /// rate-limited MusicBrainz and Apple lookups. The search bar cannot pay /// that on every keystroke, so what it consults is the REMEMBERED answer. /// Once an artist has been opened, every later surface agrees for free. pub fn with_identity_store(mut self, dir: std::path::PathBuf) -> Self { if std::fs::create_dir_all(&dir).is_ok() { if let Ok(raw) = std::fs::read(dir.join(FUSED_PORTRAIT_FILE)) { if let Ok(map) = serde_json::from_slice::>>(&raw) { let mut portraits = self.fused_portraits.lock().unwrap_or_else(|e| e.into_inner()); for (id, portrait) in map { if let Ok(id) = id.parse::() { portraits.insert(id, portrait); } } } } self.identity_store_dir = Some(dir); } self } /// Record that this Deezer artist record holds more than one act, together /// with the portrait to show in its place (`None` when nothing better is /// known — a blank slot is honest, a confident wrong face is not). pub fn remember_fused_portrait(&self, artist_id: u64, portrait: Option) { let snapshot = { let mut portraits = self .fused_portraits .lock() .unwrap_or_else(|e| e.into_inner()); portraits.insert(artist_id, portrait); portraits .iter() .map(|(id, url)| (id.to_string(), url.clone())) .collect::>>() }; let Some(dir) = self.identity_store_dir.as_ref() else { return; }; if let Ok(raw) = serde_json::to_vec(&snapshot) { let _ = std::fs::write(dir.join(FUSED_PORTRAIT_FILE), raw); } } /// `Some(portrait)` when this artist is a known fused record. The inner /// option is the replacement, which may itself be `None`. fn known_fused_portrait(&self, artist_id: u64) -> Option> { self.fused_portraits .lock() .unwrap_or_else(|e| e.into_inner()) .get(&artist_id) .cloned() } /// Swap in the remembered portrait wherever a Deezer artist row surfaces. /// Deezer's `picture` on a fused record is whichever co-tenant uploaded /// last, and it is a real photograph at real dimensions — nothing about the /// image itself gives it away, so only the remembered verdict can. fn apply_known_portrait(&self, artist: &mut CatalogArtist) { if let Some(portrait) = self.known_fused_portrait(artist.id) { artist.picture = portrait; } } /// Disk-cache file for a URL, for qualifying (slow-moving) providers only. fn disk_cache_path(&self, url: &str) -> Option { let dir = self.disk_cache_dir.as_ref()?; let qualifies = url.starts_with("https://musicbrainz.org/") || is_wikimedia_url(url) || url.starts_with("https://api.discogs.com/") // Scrobble totals are cumulative all-time counts in the millions; // a day of drift cannot reorder a top-ten. Being on disk is the // whole point of using them -- an unpaid ordering must not cost a // network round on every revisit, which is exactly what TheAudioDB // suffers for NOT being here. || url.starts_with("https://ws.audioscrobbler.com/") // `artist_news` has always written and read a disk entry here, but // news URLs never qualified, so `disk_cache_path` returned None and // the whole branch was dead: every restart refetched every follow // against a rate-limited endpoint. The 24h TTL this reader enforces // is the outer bound; news carries its own 6h stamp inside the body. || url.starts_with("https://news.google.com/rss/"); if !qualifies { return None; } use std::hash::{Hash, Hasher}; let mut hasher = std::collections::hash_map::DefaultHasher::new(); url.hash(&mut hasher); Some(dir.join(format!("{:016x}.json", hasher.finish()))) } /// Where a supplemented album's identity is kept between runs. /// /// A release Deezer's regional catalog omits is injected with a SYNTHETIC /// id, and the only thing that can turn that id back into a release group /// (or an Apple collection) is [`Self::mb_catalog`] / [`Self::apple_catalog`] /// — two in-memory maps written when an artist page is built. Anything that /// reaches such an album without having built that page in THIS process /// falls through to `https://api.deezer.com/album/{synthetic}`, which /// cannot resolve, and the user is told "album lookup failed". /// /// Measured: King Gizzard's "The Silver Cord" (id 6660585962991362) fails /// cold in 237ms while the real Deezer single beside it (489399625) returns /// 14 tracks. Reaching it from the sidebar's recent places, from a replayed /// artist page, or after the registries hit their 1000-entry `clear()` all /// produce exactly that. Persisting the mapping makes the id resolvable /// again without another artist-page build. fn supplement_cache_path(&self, album_id: u64) -> Option { let dir = self.disk_cache_dir.as_ref()?; Some(dir.join(format!("supplement-{album_id:016x}.json"))) } /// Stable key for the entry above, so a filename collision cannot serve one /// album's identity for another (same guard `read_disk_cache` already applies /// to URLs). fn supplement_cache_key(album_id: u64) -> String { format!("soulstream://supplement/{album_id}") } /// Remember a supplemented album on disk as well as in memory. fn persist_supplement(&self, album_id: u64, entry: &T) { let Some(path) = self.supplement_cache_path(album_id) else { return; }; let Ok(body) = serde_json::to_value(entry) else { return; }; write_disk_cache(&path, &Self::supplement_cache_key(album_id), &body); } /// Read one back. `None` when it was never registered, or is older than /// [`DISK_CACHE_TTL`] — in which case opening the artist page registers it /// again, exactly as before. fn load_supplement(&self, album_id: u64) -> Option { let path = self.supplement_cache_path(album_id)?; let body = read_disk_cache(&path, &Self::supplement_cache_key(album_id))?; serde_json::from_value(body).ok() } /// Enable verbose hero-candidate diagnostics. Intended for /// `hero_source_probe`; it does not alter selection behavior. pub fn with_hero_diagnostics(mut self) -> Self { self.hero_diagnostics = true; self } /// Diagnostic entry point used by the live probe to exercise Bing even /// when Fanart.tv or TheAudioDB would normally satisfy the artist first. #[doc(hidden)] pub async fn debug_bing_hero_banners(&self, artist: &str) -> Vec { self.bing_hero_banners(artist, None, &BingArtistIdentityPolicy::Unambiguous) .await } /// GET a Deezer endpoint with a short-TTL cache. async fn deezer(&self, url: &str) -> Option { self.cached_json(url, false).await } /// Fetch one Deezer track FRESH, bypassing the catalog cache entirely. /// /// This exists for preview-lease minting and must not be used for page /// data: every `/track/{id}` response mints a new signed preview URL that /// expires in ~13–15 minutes (Phase 0), so a cached body is a stale lease /// by construction. Pacing and the compliant UA still apply; two attempts, /// then the caller classifies the failure. The generic cache is also /// UPDATED with the fresh body so later page reads can't resurrect an /// older, already-expired preview URL for this track. pub async fn deezer_track_uncached(&self, id: u64) -> Option { let url = format!("https://api.deezer.com/track/{id}"); for attempt in 0..2u32 { self.deezer_throttle().await; let resp = self.http.get(&url).send().await; match resp { Ok(r) if r.status().is_success() => { let Ok(body) = r.json::().await else { return None; }; // Deezer signals quota inside a 200 body; honor the same // retry the cached path uses before giving up. let quota = body .pointer("/error/code") .and_then(|c| c.as_u64()) .is_some_and(|c| c == 4); if quota && attempt == 0 { continue; } let mut cache = self.catalog_cache.lock().await; cache.insert(url.clone(), (Instant::now(), body.clone())); return Some(body); } _ => { if attempt == 0 { continue; } return None; } } } None } /// GET a MusicBrainz endpoint through the same cache/single-flight layer. /// The one-request-per-second gate belongs inside the cache-miss /// initializer: putting it before the cache lookup made two completely warm /// artist-page reads sleep for roughly two seconds for no network traffic. async fn musicbrainz_json(&self, url: &str) -> Option { self.cached_json(url, true).await } /// The identity gate, plus a rescue for MusicBrainz DUPLICATES. /// /// MusicBrainz carries more than one entry for the same artist more often /// than the strict-equality gate assumes, and TheAudioDB may reference the /// other one. Roosevelt has two — 84825fb6 and d7eecee2, both "Roosevelt", /// Person, DE, "German producer & DJ from Cologne" — and because TheAudioDB /// pointed at the entry Deezer does not link, his logo was thrown away as a /// namesake's. /// /// Names stay untrusted, which is the whole point of the gate: the rescue /// asks whether TheAudioDB's OWN MusicBrainz entry links back to the Deezer /// artist we are resolving. Two MB entries pointing at one Deezer id are the /// same act whatever their ids say. Only reached when the cheap checks fail, /// and it rides the shared cached artist lookup. async fn audiodb_identity_ok( &self, record: &serde_json::Value, deezer_id: u64, linked_mbid: Option<&str>, ) -> bool { if audiodb_artist_matches_identity(record, deezer_id, linked_mbid) { return true; } let Some(record_mbid) = audiodb_record_mbid(record) else { return false; }; let Some(body) = self .musicbrainz_json(&artist_detail_url(record_mbid)) .await else { return false; }; let needle = format!("deezer.com/artist/{deezer_id}"); body.get("relations") .and_then(|value| value.as_array()) .is_some_and(|relations| { relations.iter().any(|relation| { relation .pointer("/url/resource") .and_then(|value| value.as_str()) .is_some_and(|url| { url.trim_end_matches('/').to_lowercase().ends_with(&needle) }) }) }) } async fn cached_json( &self, url: &str, throttle_musicbrainz: bool, ) -> Option { { let cache = self.catalog_cache.lock().await; if let Some((at, body)) = cache.get(url) { if at.elapsed() < CATALOG_TTL { return Some(body.clone()); } } } let fetch = { let mut inflight = self.catalog_inflight.lock().await; inflight .entry(url.to_string()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone() }; let body = fetch .get_or_init(|| async { // Slow-moving providers read through the disk cache first, so // a restart does not re-pay the MusicBrainz 1 req/s chain. let disk = self.disk_cache_path(url); if let Some(path) = disk.as_deref() { if let Some(body) = read_disk_cache(path, url) { return Some(body); } } let deezer = url.starts_with("https://api.deezer.com/"); let wiki = is_wikimedia_url(url); let lastfm = url.starts_with("https://ws.audioscrobbler.com/"); let attempts = if wiki { WIKI_429_ATTEMPTS } else { DEEZER_QUOTA_ATTEMPTS }; for attempt in 0..attempts { if throttle_musicbrainz { self.mb_throttle().await; } if deezer { self.deezer_throttle().await; } if wiki && !self.wiki_throttle().await { net_trace(url, "wiki-cooldown-fail-fast"); return None; } if lastfm { self.lastfm_throttle().await; } // Ok(Some(body)) = usable answer; Ok(None) = final failure; // Err(()) = a retryable Wikipedia 429 (cooldown recorded). let body: Result, ()> = async { let request = if wiki || lastfm { // Identify the app, with a contact — see // [`CONTACT_USER_AGENT`]. The generic default UA is // treated as an anonymous bot and rate limited into // silence. self.http .get(url) .header(reqwest::header::USER_AGENT, CONTACT_USER_AGENT) } else { self.http.get(url) }; let response = match request.send().await { Ok(response) => response, Err(err) => { net_trace(url, &format!("send-error {err}")); return Ok(None); } }; let status = response.status(); if !status.is_success() { if wiki && status == reqwest::StatusCode::TOO_MANY_REQUESTS { let retry_after = response .headers() .get(reqwest::header::RETRY_AFTER) .and_then(|value| value.to_str().ok()) .and_then(|value| value.trim().parse::().ok()) // Clamp before building the Instant: a // hostile/broken header (u64::MAX) would // overflow `Instant + Duration`, which // PANICS. Anything above the fail-fast // bound already means "give up", so an // hour is more than enough. .map(|secs| Duration::from_secs(secs.min(3600))) .unwrap_or(WIKI_COOLDOWN_DEFAULT); net_trace(url, &format!("wiki-429 retry-after={retry_after:?}")); *self.wiki_cooldown.lock().await = Some(Instant::now() + retry_after); return Err(()); } // A 4xx is definitive — the resource is absent, // not the server unwell. Mark it so retry ladders // upstream can tell the two apart (see // `definitive_miss`). if status.is_client_error() { self.definitive_miss.lock().await.insert(url.to_string()); } net_trace(url, &format!("http-{status}")); return Ok(None); } match response.json::().await { Ok(body) => Ok(Some(body)), Err(err) => { net_trace(url, &format!("body-error {err}")); Ok(None) } } } .await; let body = match body { Ok(body) => body, // Retryable 429: the next loop pass waits out the // recorded cooldown (or fails fast on a long ban). Err(()) => continue, }; // A provider error is an answer about the *request*, never // about the resource. Returning it as a body made a rejected // read look like an empty one — and caching it made that // verdict stick for the whole catalog TTL. let Some(body) = body else { return None; }; self.definitive_miss.lock().await.remove(url); if body.get("error").is_none() { if let Some(path) = disk.as_deref() { write_disk_cache(path, url, &body); } return Some(body); } if !deezer_quota_error(&body) || attempt + 1 == attempts { net_trace(url, "provider-error-body"); return None; } net_trace(url, "deezer-quota-retry"); tokio::time::sleep(DEEZER_QUOTA_BACKOFF).await; } None }) .await .clone(); if let Some(body) = body.as_ref() { let mut cache = self.catalog_cache.lock().await; if cache.len() >= 300 { cache.retain(|_, (at, _)| at.elapsed() < CATALOG_TTL); if cache.len() >= 300 && !cache.contains_key(url) { // A large artist can contribute ~70 detail entries. The old // all-or-nothing clear created a cache cliff on roughly the // fifth artist; evict just the oldest response instead. let oldest = cache .iter() .min_by_key(|(_, (at, _))| *at) .map(|(key, _)| key.clone()); if let Some(oldest) = oldest { cache.remove(&oldest); } } } cache.insert(url.to_string(), (Instant::now(), body.clone())); } let mut inflight = self.catalog_inflight.lock().await; if inflight .get(url) .is_some_and(|current| Arc::ptr_eq(current, &fetch)) { inflight.remove(url); } body } /// Hero artwork for the artist — wide background banner + band ClearLOGO — /// best quality first: /// /// 1. **Fanart.tv** `artist4kbackground` (3840×2160) then `artistbackground` /// (1920×1080), plus `hdmusiclogo` (HD transparent PNG), keyed by /// MusicBrainz id — one call yields both. /// 2. **TheAudioDB** 1280×720 artist fanart, restored as the broad-coverage /// fallback and upscaled by the app's dedicated hero proxy. /// 3. **Bing image search**, with Extra-Large + Wide filters, exact artist /// identity gating, commerce/product rejection, and grid-rank decay. /// /// TheAudioDB is queried for its fanart/logo fields, but the MusicBrainz id /// comes from the **Deezer-linked identity first** (`deezer_linked_mbid`); /// TADB's name-matched id is only a fallback. TADB has no record at all for /// many smaller artists — which used to lock Fanart.tv out entirely — and /// a name match can be a namesake, which the linked id cannot. Everything is best-effort with /// tight timeouts so a slow provider never holds up the artist page; if all /// fail, the UI falls back to Deezer's square portrait (banner) and plain /// text (logo). Results are cached (see [`BANNER_TTL`]). async fn artist_hero_seed(&self, artist: &str, deezer_id: u64) -> Option { // A completed final result is also the best possible seed. let cache_key = format!("dz:{deezer_id}"); if let Some((at, cached)) = self.art_cache.lock().await.get(&cache_key) { let ttl = if cached.banners.is_empty() && cached.logos.is_empty() { EMPTY_ART_TTL } else { BANNER_TTL }; if at.elapsed() < ttl { return cached.banners.first().cloned(); } } if let Some((at, cached)) = self.hero_seed_cache.lock().await.get(&cache_key) { let ttl = if cached.is_some() { BANNER_TTL } else { EMPTY_ART_TTL }; if at.elapsed() < ttl { return cached.clone(); } } let fetch = { let mut inflight = self.hero_seed_inflight.lock().await; inflight .entry(cache_key.clone()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone() }; let resolved = fetch .get_or_init(|| self.resolve_hero_seed(artist, deezer_id)) .await .clone(); self.hero_seed_cache .lock() .await .insert(cache_key.clone(), (Instant::now(), resolved.clone())); let mut inflight = self.hero_seed_inflight.lock().await; if inflight .get(&cache_key) .is_some_and(|current| Arc::ptr_eq(current, &fetch)) { inflight.remove(&cache_key); } resolved } /// Return the first identity-safe source that becomes usable. This is only /// a provisional display race: `resolve_art` still chooses the final tier /// with Fanart → TheAudioDB → hardened Bing priority. async fn resolve_hero_seed(&self, artist: &str, deezer_id: u64) -> Option { let mut sources: FuturesUnordered>> = FuturesUnordered::new(); sources.push( async move { let record = self.audiodb_artist(artist).await?; // An exact display name is not an identity: TheAudioDB also // has one record per namesake. Accept its fast banner/MBID only // when the record itself links back to this Deezer id or to the // MusicBrainz id that links back to it. // When Deezer has an authoritative MusicBrainz relationship, // it outranks every name-searched TADB field (including a stale // strDeezerID). The lookup is shared with the Fanart branch, so // this closes a namesake hole without another hot-path request. let linked_mbid = self.deezer_linked_mbid(deezer_id).await; if !self .audiodb_identity_ok(&record, deezer_id, linked_mbid.as_deref()) .await { return None; } if let Some(banner) = audiodb_banner_urls(&record).into_iter().next() { return Some(banner); } // TADB occasionally has the MBID but no fanart of its own. let mbid = record .get("strMusicBrainzID") .and_then(|value| value.as_str()) .filter(|value| !value.is_empty())?; self.fanart_art(mbid).await.0.into_iter().next() } .boxed(), ); sources.push( async move { let mbid = self.deezer_linked_mbid(deezer_id).await?; self.fanart_art(&mbid).await.0.into_iter().next() } .boxed(), ); sources.push( async move { tokio::time::sleep(HERO_SEED_BING_DELAY).await; let policy = self.bing_fallback_policy(artist, deezer_id).await?; self.bing_hero_seed(artist, deezer_id, &policy).await } .boxed(), ); while let Some(source) = sources.next().await { if source.is_some() { return source; } } None } async fn artist_art(&self, artist: &str, deezer_id: u64) -> ArtistArt { // Keyed by the Deezer id, not the display name: two artists can share a // name, and the resolver below is anchored to this identity. let cache_key = format!("dz:{deezer_id}"); if let Some((at, cached)) = self.art_cache.lock().await.get(&cache_key) { let ttl = if cached.banners.is_empty() && cached.logos.is_empty() { EMPTY_ART_TTL } else { BANNER_TTL }; if at.elapsed() < ttl { return cached.clone(); } } let fetch = { let mut inflight = self.art_inflight.lock().await; inflight .entry(cache_key.clone()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone() }; let resolved = fetch .get_or_init(|| self.resolve_art(artist, deezer_id)) .await .clone(); // Successful art is stable for a day. Empty results receive only the // short TTL above: enough to prevent an immediate expensive retry loop, // but not enough to hide a transient provider recovery for long. self.art_cache .lock() .await .insert(cache_key.clone(), (Instant::now(), resolved.clone())); let mut inflight = self.art_inflight.lock().await; if inflight .get(&cache_key) .is_some_and(|current| Arc::ptr_eq(current, &fetch)) { inflight.remove(&cache_key); } resolved } /// Definitive purpose-built provider stage shared by logo status and final /// art. Positive results are stable for a day; an empty timeout result gets /// only the short negative TTL, but every caller in one page generation /// observes the exact same answer. async fn provider_art(&self, artist: &str, deezer_id: u64) -> ArtistArt { let cache_key = format!("dz:{deezer_id}"); if let Some((at, cached)) = self.provider_art_cache.lock().await.get(&cache_key) { let ttl = if cached.banners.is_empty() && cached.logos.is_empty() { EMPTY_ART_TTL } else { BANNER_TTL }; if at.elapsed() < ttl { return cached.clone(); } } let fetch = { let mut inflight = self.provider_art_inflight.lock().await; inflight .entry(cache_key.clone()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone() }; let resolved = fetch .get_or_init(|| self.resolve_provider_art(artist, deezer_id)) .await .clone(); let mut cache = self.provider_art_cache.lock().await; if cache.len() >= 512 && !cache.contains_key(&cache_key) { cache.clear(); } cache.insert(cache_key.clone(), (Instant::now(), resolved.clone())); drop(cache); let mut inflight = self.provider_art_inflight.lock().await; if inflight .get(&cache_key) .is_some_and(|current| Arc::ptr_eq(current, &fetch)) { inflight.remove(&cache_key); } resolved } /// Purpose-built provider artwork only (Fanart.tv + TheAudioDB). This is /// also the definitive, fast answer to "does this artist have a provider /// logo?"; the slow Bing hero fallback is deliberately not part of it. async fn resolve_provider_art(&self, artist: &str, deezer_id: u64) -> ArtistArt { // Begin Fanart as soon as the identity-anchored Deezer→MusicBrainz link // resolves instead of waiting for the unrelated four-second TADB // timeout. TADB still supplies the fallback MBID only when no linked // identity exists, exactly preserving the old safety/priority rule. let linked_fanart = async { match self.deezer_linked_mbid(deezer_id).await { Some(mbid) => { let art = self.fanart_art(&mbid).await; (Some(mbid), art) } None => (None, (Vec::new(), Vec::new())), } }; let (mut tadb, (linked_mbid, (mut banners, mut logos))) = tokio::join!(self.audiodb_artist(artist), linked_fanart); // Fanart.tv's API host and its asset host are different machines, and // a live API is no promise of a live CDN. Verify before these URLs win, // or a dead CDN satisfies every `is_empty()` guard below and silently // locks out both TheAudioDB and Bing. See `image_host_alive`. self.drop_if_host_is_down(&mut banners).await; self.drop_if_host_is_down(&mut logos).await; // A same-name TheAudioDB record may be a completely different act. // Keep it only when one of its provider ids independently closes the // loop back to the requested Deezer identity. In particular, never use // an unlinked TADB logo as the provisional logo before Fanart's curated // result lands. let tadb_identified = match tadb.as_ref() { Some(record) => { self.audiodb_identity_ok(record, deezer_id, linked_mbid.as_deref()) .await } None => false, }; if !tadb_identified { tadb = None; } // TheAudioDB's purpose-made fanart is normally 1280×720. It does not // have Fanart.tv's resolution, but it is vastly better than a missing // hero and usually reads as a deliberate band/artist photo rather than // an encyclopedia performance shot. The frontend hero proxy raises it // to a 1920px-wide render using high-quality local resampling. if banners.is_empty() { banners = tadb.as_ref().map(audiodb_banner_urls).unwrap_or_default(); } // TheAudioDB also carries transparent artist logos. It was already // queried for the MBID, so include this second provider at no extra // network cost and let the UI's cleanliness/width rules rank it. if let Some(logo) = tadb .as_ref() .and_then(|a| a.get("strArtistLogo")) .and_then(|v| v.as_str()) .filter(|url| url.starts_with("https://")) { if !logos.iter().any(|candidate| candidate == logo) { logos.push(logo.to_string()); } } // Same check for whatever TheAudioDB just supplied. Cached per host, so // when Fanart was the dead one this costs nothing, and when TheAudioDB // is the dead one it leaves `banners` empty for the Bing fallback in // `resolve_art` rather than handing the page more unloadable URLs. self.drop_if_host_is_down(&mut banners).await; self.drop_if_host_is_down(&mut logos).await; ArtistArt { banners, logos } } async fn resolve_art(&self, artist: &str, deezer_id: u64) -> ArtistArt { // Give purpose-built providers the same grace period as the progressive // seed path, then overlap Bing discovery when they are still pending. // Provider art always wins, so this changes latency rather than tier // semantics; a fast Fanart/TADB success still launches no Bing traffic. let provider = self.provider_art(artist, deezer_id); tokio::pin!(provider); let grace = tokio::time::sleep(HERO_SEED_BING_DELAY); tokio::pin!(grace); tokio::select! { mut art = &mut provider => { if art.banners.is_empty() { if let Some(policy) = self.bing_fallback_policy(artist, deezer_id).await { art.banners = self .bing_hero_banners(artist, Some(deezer_id), &policy) .await; } } art } _ = &mut grace => { let bing = async { let Some(policy) = self.bing_fallback_policy(artist, deezer_id).await else { return Vec::new(); }; self.bing_hero_banners(artist, Some(deezer_id), &policy) .await }; tokio::pin!(bing); tokio::select! { mut art = &mut provider => { if art.banners.is_empty() { art.banners = bing.await; } art } banners = &mut bing => { let mut art = provider.await; if art.banners.is_empty() { art.banners = banners; } art } } } } } /// Resolve how Bing cards may prove the requested artist identity. /// /// This is a question about the *artist*, not about each photograph: a Bing /// image search returns whatever the open web means by the name, so the /// only thing that has to be established is whether the requested Deezer id /// is that artist. A unique exact name and a decisively dominant one both /// answer yes, and then the established per-card rules (exact tokens, image /// leaf names, co-bill/namesake blocks, visual gates) are the whole filter — /// exactly the behaviour that produced the reviewed hero sets. /// /// Only a genuinely contested name needs more. There, weighted Deezer /// release evidence selects one Apple catalogue and only Strong cards /// mentioning a distinctive winner-only release are eligible. That per-card /// tier must not leak upward: ordinary press photography cites no release /// title, so it discards the best heroes for any artist it touches. /// /// Provider transport failures, an uncorroborated catalogue, and an empty /// anchor set all fail closed, as do incomplete Apple lookups once the /// catalogue actually has to decide the identity. Every network read uses /// the existing cache/singleflight layers shared with the artist-page stages. async fn bing_fallback_policy( &self, artist: &str, deezer_id: u64, ) -> Option { let url = format!( "https://api.deezer.com/search/artist?q={}&limit=25", urlencode(artist) ); let (search, candidates) = tokio::join!(self.deezer(&url), self.apple_artist_candidates(artist),); let deezer_state = search .as_ref() .map(|body| deezer_exact_name_state(body, artist, deezer_id)) .unwrap_or(DeezerExactNameState::Missing); match deezer_state { // No usable catalogue answer at all: the identity is unknown, not // merely contested. DeezerExactNameState::Missing => { log::debug!( "suppressing Bing hero for unavailable exact-name identity {deezer_id} ({artist})" ); return None; } // Sole bearer of the name. Apple namesakes are a discography // concern and are filtered there; they cannot change who a name // search on the open web is about. DeezerExactNameState::Unambiguous => { return Some(BingArtistIdentityPolicy::Unambiguous); } DeezerExactNameState::Dominant => { return Some(BingArtistIdentityPolicy::NameOwner); } DeezerExactNameState::Ambiguous => {} } if candidates .iter() .any(|candidate| !candidate.lookup_complete) { log::debug!( "suppressing Bing hero for incomplete namesake catalogues {deezer_id} ({artist})" ); return None; } let cache_key = format!("dz:{deezer_id}"); let cached_catalog = self .apple_albums .lock() .await .get(&cache_key) .filter(|catalog| catalog.identity_confirmed) .cloned(); let catalog = if let Some(catalog) = cached_catalog { catalog } else { let albums_url = format!("https://api.deezer.com/artist/{deezer_id}/albums?limit=50&index=0"); let album_page = self.deezer(&albums_url).await?; let evidence: Vec = album_page .get("data")? .as_array()? .iter() .filter_map(deezer_release_evidence) .collect(); self.apple_catalog_for_deezer_candidates( deezer_id, artist, candidates.clone(), &evidence, ) .await }; if !catalog.identity_confirmed || catalog.identity_unavailable { log::debug!( "suppressing uncorroborated ambiguous-name Bing hero {deezer_id} ({artist})" ); return None; } let release_titles = bing_catalog_release_anchors(&candidates, &catalog, artist); if release_titles.is_empty() { log::debug!( "suppressing ambiguous-name Bing hero without distinctive catalogue anchors {deezer_id} ({artist})" ); return None; } Some(BingArtistIdentityPolicy::CatalogAnchored { release_titles }) } /// Whether description lookups may match by NAME for this artist. Broader /// than requiring a strictly unambiguous name: a decisively dominant name /// (NameOwner) qualifies too, because the description path carries its own content /// corroboration — the fetched article must read like a release page AND /// name the artist — which a blind editorial match lacks. Restricting this /// to strictly unambiguous names was the regression that silently dropped /// descriptions for every artist with a dead exact-name Deezer dupe row /// (The Killers, King Gizzard). A genuinely contested name still fails /// closed: two real artists sharing one name can also share an album /// title, and no text check can split them. async fn editorial_fallback_allowed(&self, artist: &str, deezer_id: u64) -> bool { matches!( self.bing_fallback_policy(artist, deezer_id).await, Some(BingArtistIdentityPolicy::Unambiguous | BingArtistIdentityPolicy::NameOwner) ) } /// Linked member evidence is a fallback for cards that do not repeat an /// album title (interviews and live photos usually name a frontperson /// instead). It is fetched only for catalogue-anchored namesakes and is /// overlapped with Bing discovery; normal unambiguous artists pay nothing. async fn bing_linked_people( &self, artist: &str, deezer_id: Option, policy: &BingArtistIdentityPolicy, ) -> HashSet { if !matches!(policy, BingArtistIdentityPolicy::CatalogAnchored { .. }) { return HashSet::new(); } let Some(deezer_id) = deezer_id else { return HashSet::new(); }; tokio::time::timeout(Duration::from_secs(3), async { let Some(mbid) = self.deezer_linked_mbid(deezer_id).await else { return HashSet::new(); }; self.artist_identity_people(&mbid, artist).await }) .await .unwrap_or_default() } /// Bing-only long-tail hero tier. The first query mirrors the proven manual /// Extra-Large + Wide `artist band photo` search; stricter portrait/live /// intents add diversity when that recipe is sparse. Merchandise is /// rejected before download and every remaining raster is visually scored. /// Only runs after Fanart.tv and TheAudioDB return no hero. async fn bing_hero_banners( &self, artist: &str, deezer_id: Option, policy: &BingArtistIdentityPolicy, ) -> Vec { let started = Instant::now(); let discovery_started = Instant::now(); // Hashed alongside discovery, not after it: the covers are 250px and // mostly cache hits, so this costs no wall clock, and without them a // square press photo cannot be told from a sleeve and is rejected. let (discovery, linked_people, cover_dhashes) = tokio::join!( tokio::time::timeout( Duration::from_secs(20), self.bing_image_candidates(artist, policy), ), self.bing_linked_people(artist, deezer_id, policy), async { match deezer_id { Some(id) => self.artist_cover_dhashes(id).await, None => Vec::new(), } }, ); let Ok(candidates) = discovery else { if self.hero_diagnostics { eprintln!( "HERO_PERF\t{}\tdiscovery_timeout_ms={}", artist, started.elapsed().as_millis() ); } return Vec::new(); }; let discovery_ms = discovery_started.elapsed().as_millis(); let candidates = filter_bing_candidates_for_policy(candidates, policy, artist, &linked_people); let candidate_count = candidates.len(); let scoring_started = Instant::now(); let scored = self .score_bing_hero_candidates(candidates, artist, &cover_dhashes) .await; let scoring_ms = scoring_started.elapsed().as_millis(); let scored_count = scored.len(); let (selected, duplicates) = select_scored_hero_urls(scored); if self.hero_diagnostics { for (kept, removed) in duplicates { eprintln!("HERO_DEDUPE\t{}\t{}\t{}", artist, kept, removed); } eprintln!( "HERO_PERF\t{}\tdiscovery_ms={}\tscoring_ms={}\ttotal_ms={}\tcandidates={}\tscored={}\tselected={}", artist, discovery_ms, scoring_ms, started.elapsed().as_millis(), candidate_count, scored_count, selected.len(), ); } selected } /// One-wave Bing provisional. Only a fully scored Strong-identity result is /// allowed to paint early; Weak candidates must wait for the exhaustive /// pass because a later Strong result changes the final eligibility set. async fn bing_hero_seed( &self, artist: &str, deezer_id: u64, policy: &BingArtistIdentityPolicy, ) -> Option { tokio::time::timeout(Duration::from_secs(6), async { let (candidates, linked_people) = tokio::join!( self.bing_image_candidates_limited(artist, policy, 1, 1, 3, 3), self.bing_linked_people(artist, Some(deezer_id), policy), ); let candidates = filter_bing_candidates_for_policy(candidates, policy, artist, &linked_people); // No cover hashes on the seed path: it exists to paint something // safe within six seconds, and a square candidate simply waits for // the exhaustive pass to adjudicate it against the catalogue. self.first_scored_strong_hero_seed(candidates, artist, &[]) .await }) .await .ok() .flatten() } /// A provisional hero needs one safe image, not a complete ranking. Race /// only Strong-identity candidates through the unchanged pixel scorer and /// return the first one that passes every quality gate. The exhaustive /// search still scores/ranks the entire set for the final carousel, while /// its shared image/score caches reuse any work completed here. async fn first_scored_strong_hero_seed( &self, candidates: Vec, artist: &str, cover_dhashes: &[u64], ) -> Option { let mut scoring = stream::iter( candidates .into_iter() .filter(|candidate| candidate.identity == BingIdentityConfidence::Strong) .map(|candidate| async move { self.hero_candidate_score(&candidate, artist, cover_dhashes) .await .map(|_| candidate.url) }), ) .buffer_unordered(HERO_SCORE_CONCURRENCY); while let Some(scored) = scoring.next().await { if let Some(url) = scored { if self.hero_diagnostics { eprintln!("HERO_SEED\t{}\t{}", artist, url); } return Some(url); } } None } async fn score_bing_hero_candidates( &self, candidates: Vec, artist: &str, cover_dhashes: &[u64], ) -> Vec<(String, f64, HeroFingerprint, BingIdentityConfidence)> { stream::iter(candidates.into_iter().map(|candidate| async move { let visual = self .hero_candidate_score(&candidate, artist, cover_dhashes) .await?; let base = visual.score; // Relevance order is meaningful within a query, but the old global // rank unfairly buried the first result from later portrait/DJ // intents. Live-performance searches remain a lower-confidence // visual source than a deliberate press/profile photograph. let query_weight = match candidate.query_index { 0 => 1.05, 1 | 2 => 1.0, 3 => 0.96, _ => 0.70, }; let rank_weight = 0.88_f64.powi(candidate.rank_in_query as i32); let identity_weight = match candidate.identity { BingIdentityConfidence::Strong => 1.0, BingIdentityConfidence::Weak => 0.55, }; let profile_weight = if is_ticketing_artist_profile_for_artist( &candidate.page_url, &candidate.url, artist, ) { 1.15 } else { 1.0 }; let score = base * query_weight * rank_weight * identity_weight * profile_weight; if self.hero_diagnostics { eprintln!( "HERO_RANK\t{}\t{:?}\tq{}\tr{}\tbase={:.2}\tfinal={:.2}\t{}\t{}\t{}", artist, candidate.identity, candidate.query_index, candidate.rank_in_query, base, score, diagnostic_field(&candidate.title), candidate.page_url, candidate.url, ); } Some((candidate.url, score, visual.fingerprint, candidate.identity)) })) .buffer_unordered(HERO_SCORE_CONCURRENCY) .filter_map(async move |scored| scored) .collect() .await } /// Bing image-search results as the broadest hero fallback (user /// decision: a personal app, broader sourcing beats a blank hero; /// DuckDuckGo proxies this same index but TLS-fingerprints clients, so /// its endpoint is dead to plain HTTP). Bing's regular image-results page /// serves HTML with entity-encoded result JSON. Guardrails: a /// result only qualifies when its title or source page URL names the artist /// as **exact word tokens** — so "Billy Cobb" can never match a "Billy /// Cobham" photo. Obvious store/product results are rejected before their /// oversized mockups can enter the quality race. Visual penalties catch /// generic white-canvas product renders and likely text-overlay cards whose /// page URL carries no useful hint. Unfetchable or junk URLs self-filter /// afterwards: the scorer must download and decode every candidate before /// it can win. async fn bing_image_candidates( &self, artist: &str, policy: &BingArtistIdentityPolicy, ) -> Vec { self.bing_image_candidates_limited(artist, policy, 5, 3, 12, 3) .await } /// One Bing image read, with the degraded-answer retry. /// /// Bing answers the SAME url with wildly different pages: measured back to /// back, one card on one read and 25-80 on the next. A near-empty page is a /// degraded answer, not an empty result set, so drop it and ask once more /// rather than caching it as this artist's verdict for the next quarter /// hour. The test used to be `is_empty()`, which let the commonest degraded /// shape straight through: a ONE-card page carrying something unrelated /// (15 of 30 waves for MSPAINT and Billy Cobb in one audit). Keep whichever /// read is richer so a thin retry can never lose a good first answer. async fn bing_wave_cards(&self, url: &str, cookie: Option<&str>) -> Vec { let mut parsed = match self.browser_html_cookie(url, cookie).await { Some(page) => parse_bing_image_results(&page), None => Vec::new(), }; if parsed.len() <= HERO_DEGRADED_PAGE_CARDS { self.forget_html(url).await; if let Some(page) = self.browser_html_cookie(url, cookie).await { let retried = parse_bing_image_results(&page); if retried.len() > parsed.len() { parsed = retried; } } } parsed } async fn bing_image_candidates_limited( &self, artist: &str, policy: &BingArtistIdentityPolicy, query_limit: usize, wave_limit: usize, total_limit: usize, per_query_limit: usize, ) -> Vec { // The lead wave deliberately does NOT add Bing's Photo-type classifier: // on the verified Camping in Alaska search that classifier removed a // genuine 2560x1920 Songbyrd press photo. Product/result guards below // are more precise. Later generic intents keep Photo-only filtering. const MANUAL_WAVES: [&str; 3] = [ "+filterui:imagesize-wallpaper+filterui:aspect-wide", "+filterui:imagesize-large+filterui:aspect-wide", "+filterui:imagesize-large", ]; const PHOTO_WAVES: [&str; 3] = [ "+filterui:imagesize-wallpaper+filterui:aspect-wide+filterui:photo-photo", "+filterui:imagesize-large+filterui:photo-photo", "+filterui:photo-photo", ]; // Keep the first query byte-for-byte equivalent to the successful // manual search (not quoted, no negative terms). Result identity and // product gates make it safe, while per-query caps prevent "band" from // crowding out better solo/DJ portraits for non-band artists. let queries = [ format!("{artist} band photo"), // The BARE name, no decoration at all. Our decorated queries can // collapse a rare artist onto a popular entity that shares a word: // every "Hana Vu ... photo" phrasing returns the Japanese girl group // HANA (525 of 525 cards, and quoting does not help), while the bare // "Hana Vu" returns her real press photos -- 16 of 17 pass the // identity gate as Strong (NME, Pitchfork, LA Times, Nylon, Dork). // Safe to widen the funnel here because the identity gate is the // thing that filters: it rejected ~3,000 cards in the audit that // found this, with no observed false rejects. artist.to_string(), format!( "\"{artist}\" musician portrait photo -album -cover -vinyl -poster -flyer -tickets" ), format!( "\"{artist}\" DJ producer profile photo -album -cover -poster -flyer -tickets -event" ), format!( "\"{artist}\" musician interview photo -album -cover -vinyl -poster -flyer" ), format!( "\"{artist}\" live performance photo -album -cover -poster -flyer -tickets" ), ]; let mut out: Vec = Vec::new(); // Why results died, per gate. Every gate below is a silent `continue`, // so a gate that starts rejecting EVERYTHING looks exactly like Bing // returning nothing — which is how this tier has now broken four times // without anything saying so. HERO_WAVE proves the scrape parsed, this // proves what threw the results away. let mut rejected_scheme = 0usize; let mut rejected_identity = 0usize; let mut rejected_surname = 0usize; let mut rejected_multi_artist = 0usize; let mut rejected_unsuitable = 0usize; let mut rejected_uncorroborated = 0usize; let mut parsed_total = 0usize; 'queries: for (query_index, query) in queries.into_iter().take(query_limit).enumerate() { let waves = if query_index == 0 { &MANUAL_WAVES } else { &PHOTO_WAVES }; let query_start = out.len(); let q = urlencode(&query); // Search waves are independent network reads. Start the bounded set // together, but consume it in the original wave order so candidate // membership, metadata upgrades, ranks and early-stop behavior stay // identical to the sequential implementation. // READ BOTH ADULT-CONTENT BUCKETS AND POOL THEM. Bing's safesearch // preference selects which cached result set answers a query, and // any single bucket can be serving a DIFFERENT query's results -- // measured in both directions on the same machine minutes apart: // Fishmans strict 32/35 relevant | unsignalled 0/35 (porn, an // Indonesian chord page, a printable 2026 calendar) // Nikki Nair strict 0/35 | unsignalled 35/35 (every card // a real press photo -- RA, Rinse, Boiler Room, DJ Mag) // So there is no correct global setting: strict alone silently loses // artists Bing can see perfectly well, and no signal at all loses // different ones. Ask both and let the identity gate choose; it // rejected ~3,000 cards in the audit behind this with no observed // false rejects, and it is what keeps the unfiltered bucket's // explicit results out of the pool. let wave_urls = waves .iter() .take(wave_limit) .map(|wave| { let base = format!( "https://www.bing.com/images/search?q={q}&qft={}&form=HDRSC3&first=1&cc=us&setlang=en-us&tsc=ImageHoverTitle", urlencode(wave) ); // Distinct URLs so the html cache keeps the two buckets // apart; the moderate read carries Bing's own default // preference as a cookie instead of the query parameter, // because `&safesearch=moderate` measures identical to // sending nothing while the cookie actually takes effect. (format!("{base}&safesearch=strict"), base) }) .collect::>(); let mut wave_pages = stream::iter(wave_urls) .map(|(strict_url, moderate_url)| async move { let (strict, moderate) = tokio::join!( self.bing_wave_cards(&strict_url, None), self.bing_wave_cards(&moderate_url, Some(BING_MODERATE_COOKIE)), ); // Pool, preferring the strict bucket's ordering. Bing repeats // one image under several cards, so dedupe on the media url. let mut seen: HashSet = HashSet::new(); let mut parsed = Vec::with_capacity(strict.len() + moderate.len()); for card in strict.into_iter().chain(moderate) { if seen.insert(card.url.clone()) { parsed.push(card); } } parsed }) .buffered(HERO_SEARCH_WAVE_CONCURRENCY.min(wave_limit.max(1))); while let Some(parsed) = wave_pages.next().await { if out.len() >= total_limit { break 'queries; } if self.hero_diagnostics { eprintln!( "HERO_WAVE\t{}\tq{}\tparsed={}\t{}", artist, query_index, parsed.len(), query, ); } parsed_total += parsed.len(); for result in parsed { // Bing occasionally publishes a legacy http image URL even // when the same asset is available over TLS. Upgrade it so // the Masquerade press image (and similar results) is not // discarded or exposed as mixed content. let murl = result .url .strip_prefix("http://") .map(|rest| format!("https://{rest}")) .unwrap_or(result.url); let lower = murl.to_lowercase(); if !murl.starts_with("https://") || lower.ends_with(".gif") || lower.ends_with(".svg") { rejected_scheme += 1; continue; } let title = result.title; let purl = result.page_url; // Before identity, not after: reading the unfiltered bucket // puts explicit results in the pool, and the identity gate // is a relevance test with no notion of decency. if is_adult_result(&title, &purl, &murl) { rejected_unsuitable += 1; continue; } let Some(identity) = bing_identity_confidence(&title, &purl, &murl, artist) else { rejected_identity += 1; if self.hero_diagnostics && rejected_identity <= 6 { eprintln!( "HERO_REJECT_ID\t{}\ttitle={}\tpurl={}\tmurl={}", artist, title, purl, murl, ); } continue; }; // Result metadata can repeat the requested name while the // actual image filename names a different person with the // same surname. Bing did this for Billy Cobb with a // `jimmy-cobb` photo under a misleading Billy Cobb title. if image_path_conflicts_with_two_part_artist(&murl, artist) { rejected_surname += 1; continue; } // Roundups can name the requested artist while displaying // a different act's adjacent photo. When a clearly // multi-artist headline is paired with a filename naming // somebody else, wait for the artist-specific image. if multi_artist_result_mismatches_image(&title, &murl, artist) { rejected_multi_artist += 1; continue; } // A winner-release card on an artist-named host may prove // that host belongs to the selected namesake, while still // being release artwork that must never display. Retain // that narrow product shape as metadata-only evidence; // every other unsuitable result is discarded immediately. let product_result = is_product_image_result(&title, &purl, &murl); if !product_result && is_unsuitable_bing_result(&title, &purl, &murl, artist) { rejected_unsuitable += 1; continue; } let rank_in_query = out.len() - query_start; let candidate = BingImageCandidate { url: murl, title, page_url: purl, identity, display_eligible: !product_result, query_index, rank_in_query, }; if product_result && !bing_candidate_can_corroborate_artist_host(&candidate, policy, artist) { rejected_uncorroborated += 1; continue; } // Bing can repeat one murl under weaker and stronger cards. // Keep one transfer, but upgrade its attached identity and // source metadata rather than letting the first weak card // permanently suppress a later artist-profile result. if let Some(existing) = out .iter_mut() .find(|existing| existing.url == candidate.url) { if bing_candidate_metadata_is_better(&candidate, existing) { if self.hero_diagnostics { eprintln!( "HERO_UPGRADE\t{}\t{:?}->{:?}\t{}", artist, existing.identity, candidate.identity, candidate.url, ); } // Upgrade identity/source metadata, but keep the // best discovery position ever seen for this same // raster. A later trusted card must not inherit a // worse query/rank penalty merely because it was // parsed later. let best_position = (existing.query_index, existing.rank_in_query) .min((candidate.query_index, candidate.rank_in_query)); *existing = candidate; existing.query_index = best_position.0; existing.rank_in_query = best_position.1; } continue; } if self.hero_diagnostics { eprintln!( "HERO_ACCEPT\t{}\t{:?}\tq{}\tr{}\t{}\t{}\t{}", artist, identity, query_index, rank_in_query, diagnostic_field(&candidate.title), candidate.page_url, candidate.url, ); } out.push(candidate); if out.len() >= total_limit || out.len() - query_start >= per_query_limit { break; } } if out.len() - query_start >= per_query_limit { break; } } } if self.hero_diagnostics { eprintln!( "HERO_GATES\t{}\tparsed={}\taccepted={}\tscheme={}\tidentity={}\tsurname={}\tmulti={}\tunsuitable={}\tuncorroborated={}", artist, parsed_total, out.len(), rejected_scheme, rejected_identity, rejected_surname, rejected_multi_artist, rejected_unsuitable, rejected_uncorroborated, ); } // Strong-vs-Weak selection happens after download/decode/scoring. A // broken metadata-Strong URL must not erase the only usable fallback. out } /// A page's HTML fetched with a browser-ish identity — several providers /// serve a stub (or a wall) to unknown user agents. /// Drop a cached page so the next read re-fetches it. Bing can answer a /// perfectly valid image search with a page carrying no result cards, and /// `browser_html` would otherwise hold that answer for CATALOG_TTL — one bad /// response leaving an artist heroless for fifteen minutes. async fn forget_html(&self, url: &str) { self.html_cache.lock().await.remove(url); } /// A page's HTML fetched with a browser-ish identity, with an optional /// Cookie header. Bing's adult-content /// preference selects which cached result bucket answers a query, and any /// one bucket can be serving another query's results, so the hero search /// reads more than one. Callers must keep the cookie deterministic per URL: /// the cache is still keyed by URL alone. async fn browser_html_cookie(&self, url: &str, cookie: Option<&str>) -> Option { { let cache = self.html_cache.lock().await; if let Some((at, html)) = cache.get(url) { if at.elapsed() < CATALOG_TTL { return Some(html.clone()); } } } let fetch = { let mut inflight = self.html_inflight.lock().await; inflight .entry(url.to_string()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone() }; let html = fetch .get_or_init(|| async { let mut request = self .http .get(url) .header( "user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0", ) .header("accept-language", "en-US,en;q=0.9") .header( "accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", ) .header("cache-control", "no-cache"); if let Some(cookie) = cookie { request = request.header("cookie", cookie); } let response = tokio::time::timeout(Duration::from_secs(6), request.send()) .await .ok()? .ok()?; tokio::time::timeout(Duration::from_secs(6), response.text()) .await .ok()? .ok() }) .await .clone(); if let Some(html) = html.as_ref() { let mut cache = self.html_cache.lock().await; cache.retain(|_, (at, _)| at.elapsed() < CATALOG_TTL); if cache.len() >= 64 && !cache.contains_key(url) { if let Some(oldest) = cache .iter() .min_by_key(|(_, (at, _))| *at) .map(|(key, _)| key.clone()) { cache.remove(&oldest); } } cache.insert(url.to_string(), (Instant::now(), html.clone())); } let mut inflight = self.html_inflight.lock().await; if inflight .get(url) .is_some_and(|current| Arc::ptr_eq(current, &fetch)) { inflight.remove(url); } html } /// Download (into the shared image cache) and quality-score a candidate: /// usable-wide-window area × sharpness. Sharpness is the Laplacian /// variance of a small grayscale thumbnail, normalised so it deprioritises /// genuinely blurry photos without letting sharp-edged text cards dominate /// over larger real photos. Candidates additionally receive graded /// penalties for near-white product canvases, high-contrast text-like /// overlays, and video-thumbnail CDNs. These are priors rather than hard /// rejects so a sparse artist can still receive a usable hero. /// `None` = unusable (too small or undecodable). /// Perceptual hashes of the artist's OWN release artwork. /// /// A square raster is a hero risk because it is usually a sleeve, but shape /// cannot tell a sleeve from a square press crop. This can: a sleeve IS one /// of these images. Deliberately cheap -- `cover_medium` is 250px, the /// bytes come from the shared image cache, and hashing is microseconds -- /// so it costs far less than the multi-megabyte hero candidates alongside /// it. async fn artist_cover_dhashes(&self, deezer_id: u64) -> Vec { let Some(body) = self .deezer(&format!( "https://api.deezer.com/artist/{deezer_id}/albums?limit=25" )) .await else { return Vec::new(); }; let Some(items) = body.get("data").and_then(|data| data.as_array()) else { return Vec::new(); }; let urls = items .iter() .filter_map(|item| { item.get("cover_medium") .and_then(|cover| cover.as_str()) .map(str::to_string) }) .take(HERO_COVER_HASH_LIMIT) .collect::>(); let mut hashes = Vec::with_capacity(urls.len()); for url in urls { if let Some(bytes) = self.get_image_shared(&url).await { if let Some(image) = decode_hero_image_slice(bytes.as_slice()) { hashes.push(dhash64(&image)); } } } hashes } async fn hero_candidate_score( &self, candidate: &BingImageCandidate, artist: &str, cover_dhashes: &[u64], ) -> Option { let cache_key = format!( "{}\u{1f}{}\u{1f}{}:{}\u{1f}{}:{}\u{1f}{}:{}", HERO_SCORE_RECIPE, normalize(artist), candidate.url.len(), candidate.url, candidate.title.len(), candidate.title, candidate.page_url.len(), candidate.page_url, ); { let cache = self.hero_score_cache.lock().await; if let Some((at, score)) = cache.get(&cache_key) { if at.elapsed() < IMAGE_CACHE_TTL { return score.clone(); } } } // Consult the longer-lived pixel-score cache before the raw-byte cache: // after compressed bytes are evicted, a warm seed→full or revisit must // not redownload and decode an image whose score is still valid. // Transient network failures remain uncached so the exhaustive pass can // retry when a provisional download lost a race with the CDN. let bytes = self.get_image_shared(&candidate.url).await?; let fetch = { let mut inflight = self.hero_score_inflight.lock().await; inflight .entry(cache_key.clone()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone() }; let candidate = candidate.clone(); let artist = artist.to_string(); let diagnostics = self.hero_diagnostics; let cover_dhashes = cover_dhashes.to_vec(); let resolved = fetch .get_or_init(|| async move { tokio::task::spawn_blocking(move || { score_bing_hero_image(bytes, &candidate, &artist, &cover_dhashes, diagnostics) }) .await .ok() .flatten() }) .await .clone(); { let mut cache = self.hero_score_cache.lock().await; cache.retain(|_, (at, _)| at.elapsed() < IMAGE_CACHE_TTL); if cache.len() >= HERO_SCORE_CACHE_MAX_ENTRIES && !cache.contains_key(&cache_key) { if let Some(oldest) = cache .iter() .min_by_key(|(_, (at, _))| *at) .map(|(key, _)| key.clone()) { cache.remove(&oldest); } } cache.insert(cache_key.clone(), (Instant::now(), resolved.clone())); } let mut inflight = self.hero_score_inflight.lock().await; if inflight .get(&cache_key) .is_some_and(|current| Arc::ptr_eq(current, &fetch)) { inflight.remove(&cache_key); } resolved } /// Editorial album context: TheAudioDB first (description + genre/label), /// Wikipedia page summary as the description fallback. Best-effort with /// per-source timeouts; results ride the shared JSON cache, so repeated /// expander opens for the same album do no network work within the TTL. pub async fn album_info(&self, artist: &str, title: &str) -> AlbumInfo { self.album_info_for_artist_id(None, artist, title).await } /// Identity-scoped expander metadata. This stays off the page hot path, but /// lets the caller prevent a name-only TADB/Wikipedia result from crossing /// between exact-name Deezer artists. pub async fn album_info_for_artist_id( &self, artist_id: Option, artist: &str, title: &str, ) -> AlbumInfo { // A display title may carry an edition/variant bracket TheAudioDB does // not use ("Weezer (Blue Album)" is catalogued as "Weezer"), so retry // with the pre-bracket base when the verbose form finds nothing. let base = title .split(['(', '[']) .next() .map(str::trim) .filter(|s| !s.is_empty()) .unwrap_or(title); let required_tadb_artist_id = if let Some(artist_id) = artist_id { // Independent identity reads; joining them saves a round trip on // every cold call (MusicBrainz link + TheAudioDB record). let (linked_mbid, record) = tokio::join!( self.deezer_linked_mbid(artist_id), self.audiodb_artist(artist), ); let identified = match record.as_ref() { Some(record) => { self.audiodb_identity_ok(record, artist_id, linked_mbid.as_deref()) .await } None => false, }; record.filter(|_| identified).and_then(|record| { record .get("idArtist") .and_then(|id| id.as_str()) .map(str::trim) .filter(|id| !id.is_empty()) .map(String::from) }) } else { None }; let allow_name_only = match artist_id { None => true, Some(artist_id) => { required_tadb_artist_id.is_some() || self.editorial_fallback_allowed(artist, artist_id).await } }; // TheAudioDB may be consulted with a name match alone whenever the // artist owns the name (same trust level the hero pipeline grants): // its record must still carry the EXACT normalized artist name. The // linked-identity requirement only stands for contested names. let tadb_allowed = artist_id.is_none() || required_tadb_artist_id.is_some() || allow_name_only; net_trace( title, &format!( "album-info-gate tadb_id={:?} name_only={allow_name_only}", required_tadb_artist_id ), ); // TheAudioDB (verbose title + pre-bracket base) and the Wikipedia // full-title lookup are independent; the whole chain used to run // serially, which made the worst case the SUM of every source's // timeout. Result precedence is unchanged: TADB's record still wins, // the verbose title still outranks the base title, and the base-title // Wikipedia query still only runs after the full-title one misses. let tadb = async { if !tadb_allowed { return None; } let (full, base_form) = tokio::join!( self.audiodb_album_info(artist, title, required_tadb_artist_id.as_deref()), async { if base != title { self.audiodb_album_info(artist, base, required_tadb_artist_id.as_deref()) .await } else { None } }, ); full.or(base_form) }; // Search with the FULL display title first: for self-titled // variants the bracket ("Weezer (Blue Album)") IS the identity and // matches Wikipedia's own article naming; a bare base-title query // ranks whichever self-titled record is currently most edited. let wiki_full = async { if !allow_name_only { return None; } self.wikipedia_album_summary(artist, title).await }; // Overlap the two sources, but never make a TADB record that already // carries a description WAIT for the speculative Wikipedia chain — // that chain has no timeout of its own (it may politely sit out a // 429 cooldown) and its result would be discarded anyway, while // still spending Wikipedia pacer budget. Whichever source finishes // first decides whether the other is still needed; precedence is // unchanged (TADB's description outranks Wikipedia's). tokio::pin!(tadb); tokio::pin!(wiki_full); let (info, mut desc) = tokio::select! { info = &mut tadb => { if info.as_ref().is_some_and(|found| found.description.is_some()) { (info, None) } else { (info, wiki_full.await) } } desc = &mut wiki_full => (tadb.await, desc), }; let mut info = info.unwrap_or_default(); if info.description.is_none() && allow_name_only { if desc.is_none() && base != title { desc = self.wikipedia_album_summary(artist, base).await; } if let Some(desc) = desc { info.description = Some(desc); info.source = Some("wikipedia".to_string()); } } info.type_hint = info .description .as_deref() .and_then(description_type_hint) .map(String::from); info } /// One TheAudioDB album record, identity-checked: the artist must match /// exactly (normalized) and the album title loosely, otherwise the fuzzy /// first hit could describe a different record entirely. async fn audiodb_album_info( &self, artist: &str, title: &str, required_tadb_artist_id: Option<&str>, ) -> Option { let url = format!( "https://www.theaudiodb.com/api/v1/json/123/searchalbum.php?s={}&a={}", urlencode(artist), urlencode(title) ); // Longer leash than the hot-path lookups: this only runs when the // album expander opens and its result fades in asynchronously, while // TheAudioDB is routinely slow during a page-load network burst. let body = tokio::time::timeout(Duration::from_secs(8), self.deezer(&url)) .await .ok()??; let albums = body.get("album")?.as_array()?; let want_artist = normalize(artist); let want_title = normalize(title); let record = albums.iter().find(|a| { let got_artist = a.get("strArtist").and_then(|v| v.as_str()).unwrap_or(""); let got_title = a.get("strAlbum").and_then(|v| v.as_str()).unwrap_or(""); normalize(got_artist) == want_artist && loose_name_match(&normalize(got_title), &want_title) && required_tadb_artist_id.is_none_or(|required| { a.get("idArtist") .and_then(|id| id.as_str()) .is_some_and(|id| id.trim() == required) }) })?; let text = |key: &str| { record .get(key) .and_then(|v| v.as_str()) .map(str::trim) .filter(|s| !s.is_empty()) .map(str::to_string) }; let description = text("strDescriptionEN"); let info = AlbumInfo { source: description.as_ref().map(|_| "theaudiodb".to_string()), description, genre: text("strGenre"), label: text("strLabel"), type_hint: None, }; (info.description.is_some() || info.genre.is_some() || info.label.is_some()).then_some(info) } /// Wikipedia description fallback: search for the release's article, then /// take the page-summary extract. Guarded so a wrong article can't slip /// through: the summary must read like a release page and name the /// artist. Two search shapes run in order — the album form, then a song /// form — so singles and EPs get coverage too ("Mr. Brightside" has no /// album article, but a rich song one). async fn wikipedia_album_summary(&self, artist: &str, title: &str) -> Option { for kind in ["album", "song"] { if let Some(extract) = self.wikipedia_release_summary(artist, title, kind).await { return Some(extract); } } None } async fn wikipedia_release_summary( &self, artist: &str, title: &str, kind: &str, ) -> Option { // Ask for more candidates than we expect to need: the artist's // discography page and other albums often outrank the wanted one, and // the title guard below rejects them, so the correct article must still // be within the returned set to be found. let search_url = format!( "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={}&format=json&srlimit=8", urlencode(&format!("{title} {artist} {kind}")) ); // No extra timeout wrapper here: the shared fetch layer owns Wikipedia // pacing and 429 recovery (a 4s wrapper would kill the request while // it politely waits out a `Retry-After`), and the HTTP client's own // hard timeout still bounds a hung read. let body = self.deezer(&search_url).await?; let results = body.get("query")?.get("search")?.as_array()?; let want_artist = normalize(artist); let want_title = normalize(title); // The article must actually be ABOUT the requested album — its title // must contain the album's normalized title. Without this the search // ("Butterfly 3000 album") happily returns the artist's // *discography* page or another album ("Changes"), which mention the // artist and the word "album" and so passed the old guard. let candidates: Vec = results .iter() .filter_map(|result| result.get("title").and_then(|v| v.as_str())) .filter(|page_title| normalize(page_title).contains(&want_title)) .map(str::to_string) .collect(); // Judge candidates in the original search-rank order — the same // article the serial loop chose — but prefetch a small look-ahead // window so one slow summary no longer serializes the walk (the old // worst case was 8 candidates × a full timeout, back to back). The // buffer stays small on purpose: most albums accept the first // candidate, and an eager fetch of all eight tripled the Wikipedia // demand of a cold artist page, which the pacer then had to queue // into the next window (measured: ~10s stalls under a 14-album // burst). let mut summaries = stream::iter(candidates.into_iter().map(|page_title| { let summary_url = format!( "https://en.wikipedia.org/api/rest_v1/page/summary/{}", urlencode(&page_title.replace(' ', "_")) ); async move { (page_title, self.deezer(&summary_url).await) } })) .buffered(3); while let Some((page_title, summary)) = summaries.next().await { let Some(summary) = summary else { continue; }; let Some(extract) = summary.get("extract").and_then(|v| v.as_str()) else { continue; }; let extract = extract.trim(); let mentions_artist = normalize(extract).contains(&want_artist) || normalize(&page_title).contains(&want_artist); // A release page's lead names its own kind: albums say "album", // song/single articles say "song" or "single", EPs "extended // play". Bare "EP" is not tested — two letters match inside // ordinary words. let lower = extract.to_lowercase(); let reads_like_release = lower.contains("album") || lower.contains(" song") || lower.contains("single") || lower.contains("extended play") || lower.contains("soundtrack"); if !extract.is_empty() && reads_like_release && mentions_artist { return Some(extract.to_string()); } } None } /// TheAudioDB's first artist match (raw record), best-effort with timeouts. async fn audiodb_artist(&self, artist: &str) -> Option { let url = format!( "https://www.theaudiodb.com/api/v1/json/123/search.php?s={}", urlencode(artist) ); // Share the catalog cache: art and discography resolution ask for this // exact identity record during the same artist-page load. let body = tokio::time::timeout(Duration::from_secs(4), self.deezer(&url)) .await .ok()??; select_audiodb_artist(&body, artist).cloned() } /// Is a provider's IMAGE host actually serving bytes right now? /// /// A provider's API and its CDN are different machines, and the API being /// up is no promise the images are. Measured 2026-07-28: Fanart.tv's /// `webservice.fanart.tv` answered normally while `assets.fanart.tv` /// refused every TCP connection. Because the whole hero chain gates on /// "did the provider return URLs" and never on "do those URLs load", every /// artist Fanart.tv covers got eight dead banners and four dead logos — /// and the `banners.is_empty()` guards that pick TheAudioDB and then Bing /// were both satisfied, so neither fallback ever ran. Pages fell back to a /// blown-up square artist portrait; only artists Fanart.tv did NOT cover /// (Tame Impala) still had a working hero. /// /// One probe per host per [`IMAGE_HOST_PROBE_TTL`] turns "this provider's /// CDN is down" into "this provider returned nothing", which is precisely /// the state the existing fallback chain already knows how to handle. pub async fn image_host_alive(&self, url: &str) -> bool { let Some(host) = reqwest::Url::parse(url) .ok() .and_then(|parsed| parsed.host_str().map(String::from)) else { // Not a URL this can reason about; never block art on that. return true; }; let health = IMAGE_HOST_HEALTH.get_or_init(|| Mutex::new(HashMap::new())); if let Some((alive, checked)) = health.lock().await.get(&host).copied() { if checked.elapsed() < IMAGE_HOST_PROBE_TTL { return alive; } } match self.probe_image_host(url, IMAGE_HOST_PROBE_TIMEOUT).await { Ok(()) => { health.lock().await.insert(host, (true, Instant::now())); true } Err(why) => { // Say what was measured, not what it means. "The host is down" // is a conclusion this single 3-second sample cannot support, // and stating it as fact sent a later reader hunting an outage // at Fanart.tv that never happened. log::warn!( "image host {host} failed a liveness probe ({why}); \ falling through to next provider" ); health .lock() .await .insert(host.clone(), (false, Instant::now())); self.confirm_image_host_failure(host, url); false } } } /// One liveness request, with the failure kept instead of collapsed to a /// bool. A one-byte ranged GET is proof the host serves image bytes without /// pulling a 4K background to find out. Any HTTP answer means the host is /// up — a 404 for one image is not an outage — so only a transport failure /// or a server error counts as dead. async fn probe_image_host(&self, url: &str, budget: Duration) -> Result<(), String> { let started = Instant::now(); let sent = self .http .get(url) .header(reqwest::header::RANGE, "bytes=0-0") .send(); match tokio::time::timeout(budget, sent).await { Err(_) => Err(format!("no answer inside {}ms", budget.as_millis())), Ok(Err(err)) => { let mut why = format!("{err}"); let mut source = std::error::Error::source(&err); while let Some(inner) = source { why.push_str(&format!(": {inner}")); source = std::error::Error::source(inner); } Err(format!("{why} after {}ms", started.elapsed().as_millis())) } Ok(Ok(resp)) if resp.status().is_server_error() => Err(format!( "HTTP {} after {}ms", resp.status(), started.elapsed().as_millis() )), Ok(Ok(_)) => Ok(()), } } /// Ask a second time, off the page's critical path, before a host stays /// written off for [`IMAGE_HOST_PROBE_TTL`]. /// /// The probe's budget is a LATENCY rule — the page must not wait on it — /// and losing that race is not the same fact as the CDN being down. What /// ate the budget in the one case chased to the bottom was **name /// resolution**, before a single byte was sent. /// /// Measured 2026-08-03, on the machine that reported the warning: 213 /// `assets.fanart.tv` urls all answered in ~130ms (cache MISS included), /// from curl and from this exact client, cold, warm and 12-way concurrent — /// the host was never down. But the active adapter listed a dead resolver /// first — no ping, every direct query timing out at 12s — and eight cold /// lookups of the same name through the normal path came back **1028ms**, /// 20ms, 30ms, 24ms, 25ms, **2029ms**, 28ms, **1035ms**: the Windows /// resolver's 1s/2s retransmit schedule, paid before falling through to a /// server that answers in 21ms. /// The A record's own TTL is 300s, exactly [`IMAGE_HOST_PROBE_TTL`], so /// every probe lands on an expired name and pays that lottery again. A /// single unlucky draw then blacked out Fanart.tv for five minutes and gave /// every artist a blown-up square portrait instead of a hero. /// /// A slow resolver is not this app's to fix, and the fast verdict still /// stands for the page that paid for it — it falls through to the next /// provider immediately, as before. What changes is how long one lost race /// is believed: a background confirmation with a budget that can absorb a /// DNS retry decides whether the verdict persists, so a blip costs one page /// instead of five minutes. fn confirm_image_host_failure(&self, host: String, url: &str) { static CONFIRMING: std::sync::OnceLock>> = std::sync::OnceLock::new(); let confirming = CONFIRMING.get_or_init(|| Mutex::new(HashSet::new())); let http = self.http.clone(); let url = url.to_string(); tokio::spawn(async move { // One confirmation per host at a time: an artist page probes // banners and logos, twice each, and four identical retries would // add exactly the load that is the prime suspect. if !confirming.lock().await.insert(host.clone()) { return; } let outcome = tokio::time::timeout( IMAGE_HOST_CONFIRM_TIMEOUT, http.get(&url) .header(reqwest::header::RANGE, "bytes=0-0") .send(), ) .await; let recovered = matches!(outcome, Ok(Ok(ref resp)) if !resp.status().is_server_error()); if recovered { log::warn!( "image host {host} answered the confirming probe; \ the fast probe lost a race, not the host — restoring it" ); if let Some(health) = IMAGE_HOST_HEALTH.get() { health .lock() .await .insert(host.clone(), (true, Instant::now())); } } else { log::warn!("image host {host} failed the confirming probe too; staying written off"); } confirming.lock().await.remove(&host); }); } /// Can this already-chosen artwork actually be shown right now? /// /// A persisted manifest is a list of URLs, and the CDN behind them can go /// down long after it was written. Serving one then produces a hero that /// never loads AND no path to the fallback that would have worked, because /// the stored answer short-circuits discovery entirely. A manifest with no /// banners is not a liveness question, and a custom data: URL has no host. pub async fn art_is_servable(&self, art: &CatalogArtistArt) -> bool { let Some(url) = art.banners.first().or(art.banner.as_ref()) else { return true; }; self.image_host_alive(url).await } /// Drop a candidate list whose host cannot currently serve it, so an /// unreachable CDN reads downstream as "this provider had nothing". async fn drop_if_host_is_down(&self, urls: &mut Vec) { let Some(probe) = urls.first().cloned() else { return; }; if !self.image_host_alive(&probe).await { urls.clear(); } } /// Fanart.tv's most-liked 1920×1080 backgrounds and HD ClearLOGO candidates /// for a MusicBrainz id — one call, both best-effort. Returns /// `(banners, logos)`; either list is empty when the artist has none. async fn fanart_art(&self, mbid: &str) -> (Vec, Vec) { let url = format!("https://webservice.fanart.tv/v3/music/{mbid}?api_key={FANART_KEY}"); let Some(body) = tokio::time::timeout(Duration::from_secs(4), self.cached_json(&url, false)) .await .ok() .flatten() else { return (Vec::new(), Vec::new()); }; // Prefer HD logos; fall back to standard-res if that's all there is. Up // to 6 candidates so the UI can pick the cleanest to recolour. let mut logos = top_liked_urls(&body, "hdmusiclogo", 6); if logos.is_empty() { logos = top_liked_urls(&body, "musiclogo", 6); } // fanart.tv publishes newer uploads under `artist4kbackground` and only // older ones under `artistbackground`; a record can carry one, the // other, or both. Reading only the legacy key silently discarded the // ONLY background some artists have -- MSPAINT's record is literally // just {artist4kbackground: [3840x2160 band photo], hdmusiclogo: [...]}, // fetched and thrown away on every page load while the artist showed no // hero at all. 4K first: it outranks the 1920x1080 legacy tier. let mut backgrounds = top_liked_urls(&body, "artist4kbackground", 8); for url in top_liked_urls(&body, "artistbackground", 8) { if !backgrounds.contains(&url) { backgrounds.push(url); } } // Keep every useful candidate from this best-quality tier. Eight is // enough variety without making a long-lived artist page accumulate an // unbounded set of processed 4K images in the frontend cache. backgrounds.truncate(8); (backgrounds, logos) } /// Catalog search: artists, albums and tracks for the query, in parallel. pub async fn catalog_search(&self, query: &str) -> CatalogSearch { let q = urlencode(query); let artist_url = format!("https://api.deezer.com/search/artist?q={q}&limit=8"); let album_url = format!("https://api.deezer.com/search/album?q={q}&limit=12"); let track_url = format!("https://api.deezer.com/search/track?q={q}&limit=10"); let (artists, albums, tracks) = tokio::join!( self.deezer(&artist_url), self.deezer(&album_url), self.deezer(&track_url), ); // Deezer returns same-named artists in an order that can bury the // canonical one (its "Pond" search lists a 38-fan namesake before the // 26k-fan band). Rank by name relevance first, then followers, so the // artist a search actually means becomes results.artists[0] — the top // card and the target of name-based links. let query_key = normalize(query); let mut artist_list: Vec = artists .as_ref() .and_then(|b| b.get("data")) .and_then(|d| d.as_array()) .map(|a| a.iter().filter_map(parse_artist).collect()) .unwrap_or_default(); artist_list.sort_by(|a, b| { artist_name_relevance(&b.name, &query_key) .cmp(&artist_name_relevance(&a.name, &query_key)) .then(b.fans.unwrap_or(0).cmp(&a.fans.unwrap_or(0))) }); // The suggestion dropdown is the first place a fused record's wrong // face appears, and it is typed against per keystroke — so it reads the // remembered verdict rather than re-deriving one. for artist in &mut artist_list { self.apply_known_portrait(artist); } CatalogSearch { artists: artist_list, albums: albums .as_ref() .and_then(|b| b.get("data")) .and_then(|d| d.as_array()) .map(|a| a.iter().filter_map(parse_album).collect()) .unwrap_or_default(), tracks: tracks .as_ref() .and_then(|b| b.get("data")) .and_then(|d| d.as_array()) .map(|a| a.iter().filter_map(parse_track).collect()) .unwrap_or_default(), } } /// The two things the search hero can say about an artist that Deezer /// cannot: what they sound like, and how big they are. /// /// Both hang off the linked MBID the rest of the identity chain already /// resolves, so neither is a name guess. The genres are MusicBrainz's own /// most-voted tags, read from the SAME cached artist document the socials /// and bio tiers use — no extra request. The totals are one Last.fm /// `artist.getinfo` keyed by that MBID. /// /// Everything is optional and independently so: an artist MusicBrainz has /// no Deezer link for returns empty rather than wrong. pub async fn artist_profile(&self, artist_id: u64) -> CatalogArtistProfile { let mut profile = CatalogArtistProfile::default(); let Some(mbid) = self.deezer_linked_mbid(artist_id).await else { return profile; }; if let Some(facts) = self.artist_identity_facts(&mbid).await { profile.tags = facts.tags; } let url = format!( "https://ws.audioscrobbler.com/2.0/?method=artist.getinfo\ &mbid={mbid}&autocorrect=1&api_key={LASTFM_KEY}&format=json" ); if let Some(body) = self.lastfm_json(&url).await { // Last.fm sends these as strings, like every other count it serves. let stat = |key: &str| { body.pointer(&format!("/artist/stats/{key}")) .and_then(|v| v.as_str()) .and_then(|v| v.parse::().ok()) .filter(|v| *v > 0) }; profile.listeners = stat("listeners"); profile.plays = stat("playcount"); } profile } /// Last.fm listener counts for the songs a SEARCH returned. /// /// `catalog_search` itself is three Deezer calls and nothing else — it is /// typed against per keystroke by the suggestion dropdown, so it may never /// wait on MusicBrainz's 1 req/s gate. This is the deferred half, fired once /// the results are already on screen, exactly like `catalog_artist_socials`. /// /// Identity, not names. Each row carries the performing artist's DEEZER ID /// (`CatalogTrack::artist_id`), which resolves through the same /// `deezer_linked_mbid` → `lastfm_listener_ranks` chain the artist page /// uses. Asking Last.fm for "Daft Punk" by name would re-open the namesake /// hole that chain exists to close. /// /// A search spans many artists, so the ranks are fetched once per DISTINCT /// artist and every row of theirs is answered from that one lookup. The /// title key strips featured-artist clauses, which is what lets Daft Punk's /// "Doin' it Right (feat. Panda Bear)" match the scrobbled "Doin' it Right". /// /// Returns one entry per input row, in order. `None` means "no count" — a /// track Last.fm has never seen, an artist MusicBrainz has no Deezer link /// for, or a row past the artist budget below. Never a fabricated zero. pub async fn track_listeners(&self, tracks: &[(Option, String)]) -> Vec> { // Distinct artists, in first-seen order. Capped: a pathological result // set spanning ten artists would otherwise spend ten MusicBrainz slots // (one second each, shared with every other page) on the last few rows // of one list. Six covers every real search measured. const ARTIST_BUDGET: usize = 6; let mut wanted: Vec = Vec::new(); let mut dropped = 0_usize; for (artist_id, _) in tracks { let Some(id) = artist_id else { continue }; if wanted.contains(id) { continue; } if wanted.len() < ARTIST_BUDGET { wanted.push(*id); } else { dropped += 1; } } // Said out loud rather than silently truncated: a bounded answer that // reads as a complete one is how "these rows just have no count" hides // a budget. if dropped > 0 { log::debug!( "track_listeners: {dropped} artist(s) past the {ARTIST_BUDGET}-artist budget \ get no counts" ); } if wanted.is_empty() { return vec![None; tracks.len()]; } // Concurrent by artist. Both hops are individually rate-limited and // cached upstream, so this is a fan-out over the gates rather than // around them. let ranks: Vec<(u64, Option)> = futures::future::join_all(wanted.into_iter().map(|artist_id| async move { let ranks = match self.deezer_linked_mbid(artist_id).await { Some(mbid) => self.lastfm_listener_ranks(&mbid).await, None => None, }; (artist_id, ranks) })) .await; let by_artist: HashMap = ranks .into_iter() .filter_map(|(id, ranks)| ranks.map(|r| (id, r))) .collect(); tracks .iter() .map(|(artist_id, title)| { let ranks = by_artist.get(&(*artist_id)?)?; let stats = ranks.by_key.get(&track_title_identity(title))?; (stats.listeners > 0).then_some(stats.listeners) }) .collect() } /// Artist profile plus their full discography. The artist-albums /// endpoint only returns Deezer's curated "main" discography (e.g. 30 /// entries for an artist with 60+ releases), so it's merged with a paged /// album *search* by artist name, which surfaces the rest — each hit /// verified against the artist id so tributes and namesakes stay out. /// Related artists ("fans also like") for the Now Playing discovery panel. pub async fn catalog_related(&self, artist_id: u64) -> Vec { let Some(body) = self .deezer(&format!( "https://api.deezer.com/artist/{artist_id}/related?limit=12" )) .await else { return Vec::new(); }; body.get("data") .and_then(|d| d.as_array()) .map(|items| { items .iter() .filter_map(parse_artist) .map(|mut artist| { self.apply_known_portrait(&mut artist); artist }) .collect() }) .unwrap_or_default() } /// Independent artist-art stage. This intentionally excludes albums and /// popular tracks so a slow provider cannot hold back the identity shell or /// usable catalog overview. `catalog_artist` reuses the populated provider /// caches when it completes the full page response. pub async fn catalog_artist_art(&self, artist_id: u64) -> Option { let profile = self .deezer(&format!("https://api.deezer.com/artist/{artist_id}")) .await?; let mut artist = parse_artist(&profile)?; // This stage's artist object is spread over the page's in // `mergeCatalogArtistArt`, so without this it would reinstate the // portrait the ownership pass just disproved. self.apply_known_portrait(&mut artist); let art = self.artist_art(&artist.name, artist_id).await; Some(CatalogArtistArt { artist, banner: art.banners.first().cloned(), banners: art.banners, banner_custom: false, logos: art.logos, }) } /// Fast provider-logo status for the deferred cover-logo decision. It runs /// the exact Fanart.tv/TheAudioDB identity path used by `artist_art`, but /// never waits for the unrelated Bing hero fallback. pub async fn catalog_artist_logos(&self, artist_id: u64) -> Option> { let profile = self .deezer(&format!("https://api.deezer.com/artist/{artist_id}")) .await?; let artist = parse_artist(&profile)?; Some(self.provider_art(&artist.name, artist_id).await.logos) } /// First identity-safe hero source for progressive display. It deliberately /// returns only a URL: the caller already has a synchronous artist shell, /// and the complete artwork command will supply the final provider tier, /// carousel and logos independently. pub async fn catalog_artist_hero_seed(&self, artist_id: u64) -> Option { let profile = self .deezer(&format!("https://api.deezer.com/artist/{artist_id}")) .await?; let artist = parse_artist(&profile)?; self.artist_hero_seed(&artist.name, artist_id).await } /// Fast, useful catalog stage. All three endpoints are independent once the /// artist id is known, so a cold page pays one network round rather than /// waiting for provider artwork or the exhaustive detail/classification /// pass used by `catalog_artist`. pub async fn catalog_artist_overview( self: &Arc, artist_id: u64, ) -> Option { let overview_started = Instant::now(); let profile_url = format!("https://api.deezer.com/artist/{artist_id}"); let albums_url = format!("https://api.deezer.com/artist/{artist_id}/albums?limit=50&index=0"); let popular_url = format!("https://api.deezer.com/artist/{artist_id}/top?limit={POPULAR_POOL_LIMIT}"); // The supplements chain (linked MBID → Last.fm ranks + original // release dates) rides this join under a BUDGET. Warm — memory or // same-day disk cache, every visit after the first — it answers in // single-digit milliseconds and rides for free, and the page is the // page it always was. Cold it is a MusicBrainz chain serialized // behind the 1 req/s gate, measured 2.2–2.6s on a THREE-album artist // against ~300ms for everything else in this join — it had quietly // become the whole critical path of the stage that exists to paint // fast. A missed budget cancels nothing: the task is spawned, keeps // filling the exact caches the full page is about to read, and that // page — which genuinely needs the dates for classification and // replaces this stage's content wholesale — collects the answer // moments later. The discography FLIP already glides that correction. // Overview: no recordings browse (the `false`). It paints fast and // the full page replaces its Popular list wholesale. let supplements_task = tokio::spawn({ let client = Arc::clone(self); async move { client.artist_supplements(artist_id, false).await } }); // Candidate discovery needs only the name; the identity SELECTION // needs this join's album evidence and is pure CPU. Split exactly the // way the full page splits it, so the Apple round trip rides the join // instead of queueing serially behind it. let apple_fut = async { let name = self .deezer(&profile_url) .await .and_then(|profile| Some(profile.get("name")?.as_str()?.to_string())); match name { Some(name) => self.apple_artist_candidates(&name).await, None => Arc::new(Vec::new()), } }; let (profile, album_page, popular, apple_candidates, supplements) = tokio::join!( self.deezer(&profile_url), self.deezer(&albums_url), self.deezer(&popular_url), apple_fut, async { let supplements_started = Instant::now(); match tokio::time::timeout(OVERVIEW_SUPPLEMENTS_BUDGET, supplements_task).await { Ok(Ok(supplements)) => { stage_trace("overview.supplements", supplements_started, "within-budget"); supplements } Ok(Err(_)) => ArtistSupplements::default(), Err(_) => { stage_trace( "overview.supplements", supplements_started, "budget-missed, detached", ); ArtistSupplements::default() } } }, ); stage_trace("overview.deezer_join", overview_started, ""); let mut artist = parse_artist(&profile?)?; self.apply_known_portrait(&mut artist); let album_page = album_page?; let album_items = album_page.get("data")?.as_array()?; let evidence: Vec = album_items .iter() .filter_map(deezer_release_evidence) .collect(); // Select among the exact-name Apple performers fetched in the join, // using the requested Deezer id's weighted release evidence to // identify the right one. This adds no MusicBrainz traffic and shares // its Apple I/O with the full page when both frontend stages overlap. let apple_started = Instant::now(); let apple = self .apple_catalog_for_deezer_candidates(artist_id, &artist.name, apple_candidates, &evidence) .await; stage_trace("overview.apple", apple_started, ""); stage_trace("overview.total", overview_started, ""); if apple.identity_unavailable { return None; } let mut foreign_album_ids = HashSet::new(); let mut albums: Vec = album_items .iter() .filter_map(|item| { let album = parse_album(item)?; let genre_id = item.get("genre_id").and_then(|id| id.as_u64()); if apple_release_is_foreign(&apple, &album.title, genre_id) { foreign_album_ids.insert(album.id); None } else { Some(album) } }) .collect(); for album in &mut albums { if album.artist.is_empty() { album.artist = artist.name.clone(); } } apply_original_release_dates(&mut albums, &supplements.first_released); // Sort here too, not just on the full page: with corrected dates the two // stages would otherwise disagree about chronology and the discography // would visibly reshuffle underneath the reader when the full page lands. albums.sort_by(|a, b| b.release_date.cmp(&a.release_date)); let pool = deezer_popular_pool(popular.as_ref(), artist_id, &foreign_album_ids, &apple); let popular_tracks = apply_lastfm_order(pool, &supplements.listeners, POPULAR_KEEP); Some(CatalogArtistOverview { artist, popular_tracks, albums, }) } /// Popular's fast lane. The overview paints the section immediately under /// its supplements BUDGET — cold, that means Deezer's order with no /// listener counts — and the REAL list (Last.fm order + counts) used to /// arrive only with the full page, queued behind the dates and recordings /// walks it never reads. This waits on exactly what Popular consumes: the /// linked MBID and one Last.fm call. Every Deezer/Apple input rides the /// caches the overview just filled, and the ranks chain singleflights /// with the overview's detached supplements task, so this usually JOINS /// in-flight work rather than adding any. Cold it answers in ~2s against /// the full page's 6–9s; warm it is the overview's own list. /// /// The full page still owns the final word — re-attribution and the /// Last.fm-ranked tracks Deezer does not sell arrive with it. pub async fn catalog_artist_popular( self: &Arc, artist_id: u64, ) -> Option> { let popular_started = Instant::now(); let profile_url = format!("https://api.deezer.com/artist/{artist_id}"); let albums_url = format!("https://api.deezer.com/artist/{artist_id}/albums?limit=50&index=0"); let popular_url = format!("https://api.deezer.com/artist/{artist_id}/top?limit={POPULAR_POOL_LIMIT}"); let ranks_fut = async { let mbid = self.deezer_linked_mbid(artist_id).await?; self.lastfm_listener_ranks(&mbid).await }; let apple_fut = async { let name = self .deezer(&profile_url) .await .and_then(|profile| Some(profile.get("name")?.as_str()?.to_string())); match name { Some(name) => self.apple_artist_candidates(&name).await, None => Arc::new(Vec::new()), } }; let (profile, album_page, popular, apple_candidates, ranks) = tokio::join!( self.deezer(&profile_url), self.deezer(&albums_url), self.deezer(&popular_url), apple_fut, ranks_fut, ); let artist = parse_artist(&profile?)?; let album_page = album_page?; let album_items = album_page.get("data")?.as_array()?; let evidence: Vec = album_items .iter() .filter_map(deezer_release_evidence) .collect(); let apple = self .apple_catalog_for_deezer_candidates( artist_id, &artist.name, apple_candidates, &evidence, ) .await; if apple.identity_unavailable { return None; } // The same foreign filter the overview applies to its pool. let mut foreign_album_ids = HashSet::new(); for item in album_items { let Some(album) = parse_album(item) else { continue }; let genre_id = item.get("genre_id").and_then(|id| id.as_u64()); if apple_release_is_foreign(&apple, &album.title, genre_id) { foreign_album_ids.insert(album.id); } } let pool = deezer_popular_pool(popular.as_ref(), artist_id, &foreign_album_ids, &apple); let ranks = ranks.unwrap_or_default(); let tracks = apply_lastfm_order(pool, &ranks.by_key, POPULAR_KEEP); stage_trace( "popular.total", popular_started, &format!("{} rows, ranked={}", tracks.len(), !ranks.by_key.is_empty()), ); (!tracks.is_empty()).then_some(tracks) } pub async fn catalog_artist(self: &Arc, artist_id: u64) -> Option { let page_started = Instant::now(); let profile_url = format!("https://api.deezer.com/artist/{artist_id}"); // Provider artwork is an independent frontend stage. Do not make the // exhaustive release list wait for its Bing hero fallback: that used to // delay the final cover list (and therefore cover-logo generation) by // 10–20 seconds even though the logo decision needs albums, not a hero. // MusicBrainz is NOT on this path any more. Apple candidate discovery // runs beside the Deezer chains below, then cheap weighted title // corroboration selects the exact namesake tied to this Deezer id. The // network overlap keeps the identity gate out of the critical path. // Curated albums and search-only extras are independent Deezer chains. // Fetch them together, then replay search candidates through the same // curated-first de-duplication below so ordering and quality stay exact. let curated_fut = async { let mut albums = Vec::new(); let mut evidence = Vec::new(); let first = self .deezer(&format!( "https://api.deezer.com/artist/{artist_id}/albums?limit=50&index=0" )) .await?; // A first page without `data` is a failed read, not an empty // catalog — same bail-out the serial walk had. first.get("data").and_then(|d| d.as_array())?; let mut pages = vec![first]; // The first page carries `total`, so the remaining pages (same // 4-page cap as the old serial walk) can be fetched together // instead of one round trip per page. Appending in page order // below keeps the album list byte-identical to the serial walk. // A missing/odd `total` beside a `next` link must not truncate // the walk to one page (the old serial walk would have kept // following `next`): assume the cap and let empty pages // contribute nothing. let total = pages[0] .get("total") .and_then(|t| t.as_u64()) .map(|t| t.min(200)) .unwrap_or(200) as usize; let extra_indices: Vec = (1..4).map(|p| p * 50).filter(|&i| i < total).collect(); if pages[0].get("next").and_then(|n| n.as_str()).is_some() && !extra_indices.is_empty() { let extra = futures::future::join_all(extra_indices.into_iter().map(|index| { let url = format!( "https://api.deezer.com/artist/{artist_id}/albums?limit=50&index={index}" ); async move { self.deezer(&url).await } })) .await; pages.extend(extra.into_iter().flatten()); } for page in &pages { let Some(items) = page.get("data").and_then(|d| d.as_array()) else { continue; }; albums.extend(items.iter().filter_map(|item| { Some(( parse_album(item)?, item.get("genre_id").and_then(|id| id.as_u64()), )) })); evidence.extend(items.iter().filter_map(deezer_release_evidence)); } Some::<(Vec<(CatalogAlbum, Option)>, Vec)>(( albums, evidence, )) }; // Only the search and Apple legs need the NAME. They ride the same // singleflighted profile fetch inside the join instead of queueing the // whole gather behind a serial profile round trip — the id-keyed legs // (curated, popular, and above all the cold supplements chain, which // is the join's long pole) start one Deezer round trip sooner. let search_fut = async { let Some(name) = self .deezer(&profile_url) .await .and_then(|profile| Some(profile.get("name")?.as_str()?.to_string())) else { return Vec::new(); // no profile, no page — the join's own bail }; let q = urlencode(&format!("artist:\"{name}\"")); let mut candidates = Vec::new(); let mut index = 0; for _ in 0..2 { let Some(page) = self .deezer(&format!( "https://api.deezer.com/search/album?q={q}&limit=100&index={index}" )) .await else { break; // search is a bonus — curated results still render }; let Some(items) = page.get("data").and_then(|d| d.as_array()) else { break; }; candidates.extend( items .iter() .filter(|item| { item.pointer("/artist/id").and_then(|v| v.as_u64()) == Some(artist_id) }) .filter_map(|item| { Some(( parse_album(item)?, item.get("genre_id").and_then(|id| id.as_u64()), )) }), ); if page.get("next").and_then(|n| n.as_str()).is_none() || items.is_empty() { break; } index += 100; } candidates }; let apple_fut = async { let name = self .deezer(&profile_url) .await .and_then(|profile| Some(profile.get("name")?.as_str()?.to_string())); match name { Some(name) => self.apple_artist_candidates(&name).await, None => Arc::new(Vec::new()), } }; let popular_url = format!("https://api.deezer.com/artist/{artist_id}/top?limit={POPULAR_POOL_LIMIT}"); let gather_started = Instant::now(); // The supplements chain (cold: several MusicBrainz requests behind // the mandatory 1 req/s gate — the longest thing this function does) // is SPAWNED here and collected beside the album-detail wave below, // the only other multi-second leg. Its output is first consumed after // the details land (dates, attribution, popular), so holding the // gather for it bought nothing — audited on Queen, the details wave // cost 5.3s serialized after a gather that was itself pinned to the // supplements chain. Spawned, not merely deferred: an // identity-unavailable bail below must leave it warming the caches // the identity pass reads next, exactly like the overview's budget // miss does. let supplements_task = tokio::spawn({ let client = Arc::clone(self); async move { client.artist_supplements(artist_id, true).await } }); let (profile, curated, search_candidates, popular, apple_candidates) = tokio::join!( async { let profile = self.deezer(&profile_url).await; stage_trace("page.profile", page_started, ""); profile }, curated_fut, search_fut, self.deezer(&popular_url), apple_fut, ); stage_trace("page.gather_join", gather_started, ""); let mut artist = parse_artist(&profile?)?; self.apply_known_portrait(&mut artist); let (curated, evidence) = curated?; let apple_started = Instant::now(); let apple = self .apple_catalog_for_deezer_candidates( artist_id, &artist.name, apple_candidates, &evidence, ) .await; stage_trace("page.apple_select", apple_started, ""); if apple.identity_unavailable { return None; } let mut foreign_album_ids = HashSet::new(); let mut albums: Vec = curated .into_iter() .filter_map(|(album, genre_id)| { if apple_release_is_foreign(&apple, &album.title, genre_id) { foreign_album_ids.insert(album.id); None } else { Some(album) } }) .collect(); // Artist-albums payloads omit the artist name; fill it in. for a in &mut albums { if a.artist.is_empty() { a.artist = artist.name.clone(); } } // Search extras: same album under a different id (regional dupes) // are dropped via a (title, record_type) key; curated entries win. let mut seen_ids: std::collections::HashSet = albums.iter().map(|a| a.id).collect(); let mut seen_titles: std::collections::HashSet<(String, String)> = albums .iter() .map(|a| { ( normalize(&a.title), a.record_type.clone().unwrap_or_default(), ) }) .collect(); for (album, genre_id) in search_candidates { if apple_release_is_foreign(&apple, &album.title, genre_id) { foreign_album_ids.insert(album.id); continue; } let key = ( normalize(&album.title), album.record_type.clone().unwrap_or_default(), ); if seen_ids.contains(&album.id) || seen_titles.contains(&key) { continue; } seen_ids.insert(album.id); seen_titles.insert(key); albums.push(album); } // Deezer sometimes lists one single twice, with and without its // featured-artist credit ("Dancing In Babylon" / "Dancing In Babylon // (feat. Christine and the Queens)", both 2024-02-20). Collect the // collisions now so the details pass below fetches their track counts, // then each same-day group folds to its fullest row after it. let mut featless_groups: HashMap> = HashMap::new(); for album in &albums { if album.record_type.as_deref() != Some("single") { continue; } featless_groups .entry(featless_title_identity(&album.title)) .or_default() .push(album.id); } featless_groups.retain(|_, ids| ids.len() > 1); // Exact-title duplicates among CURATED rows are a separate hole: the // (title, record_type) key below only guards search extras, so Deezer // carrying one release twice ("Changes (Remixes)", 2012-03-06 and // 2012-03-11, 7 tracks each) rendered two cards. These fold by year — // looser than the feat fold's same-day rule because an identical title // AND type inside one year is the same work under regional ids. let mut exact_groups: HashMap<(String, String), Vec> = HashMap::new(); for album in &albums { exact_groups .entry(( normalize(&album.title), album.record_type.clone().unwrap_or_default(), )) .or_default() .push(album.id); } exact_groups.retain(|_, ids| ids.len() > 1); let feat_collision_ids: HashSet = featless_groups .values() .chain(exact_groups.values()) .flatten() .copied() .collect(); // Normalized non-edition album titles, used to spot bespoke reissues // (a release whose title is built on an existing album's, e.g. "OK // Computer OKNOTOK 1997 2017") both for the details fetch below and the // edition-folding loop further down. let non_edition_titles: Vec = albums .iter() .filter(|a| a.record_type.as_deref() == Some("album") && !is_edition_title(&a.title)) .map(|a| normalize(&a.title)) .collect(); // One capped, cached album-details pass fixes three gaps at once: // search hits lack release_date, curated hits lack nb_tracks, and // Deezer types old CD-singles (a title track plus B-side remixes) // as "album" — the tracklist gives those away. Bespoke reissues are // pulled in too so their tracklist can anchor the fold below. // Albums and EPs first: the details cap below can starve a prolific // artist's later rows, and dates/tracklists on full releases feed the // edition folding — a single missing its B-side count costs far less // than a base album missing the tracklist its remaster folds against. let mut need: Vec<(bool, u64)> = albums .iter() .filter(|a| { a.release_date.is_none() || (a.track_count.is_none() && feat_collision_ids.contains(&a.id)) // EPs are pulled in for the ownership check: the list // endpoint carries no artist object, so details are the only // place a release's PRIMARY artist is visible, and a guest // appearance is typed "ep" as often as "album" — the four // Rooby Jeantal records in Mid-Air Thief's discography are // all EPs, which is why an album-only pass never saw them. // They already sort ahead of singles for the cap below. || a.record_type.as_deref() == Some("ep") || (a.record_type.as_deref() == Some("album") && (a.track_count.is_none_or(|n| n <= 14) || is_edition_title(&a.title) || contains_other_album_title(&a.title, &non_edition_titles))) }) .map(|a| { ( !matches!(a.record_type.as_deref(), Some("album") | Some("ep")), a.id, ) }) .collect(); need.sort_by_key(|(low_priority, _)| *low_priority); // The cap cuts from the TAIL of Deezer's newest-first curated order, // which is where the OLDEST records sit — and on a big catalogue those // are exactly the albums a popular track originally came out on. // Measured on Bob Dylan (82 album+EP rows), positions 60+ are Blonde On // Blonde, Highway 61 Revisited, Bringing It All Back Home, Another // Side, The Times They Are A-Changin', Freewheelin' and the debut. Nine // measured catalogues overflow (Sinatra 88, Cash 138, Miles 165, Dead // 200), and the failure mode is silent: the rows just stay // un-re-attributed. So top the selection back up from the oldest end. // Zero extra requests for everyone who already fits — Queen is 59. const DETAIL_CAP: usize = 60; const DETAIL_CEILING: usize = 80; let mut need: Vec = need .into_iter() .map(|(_, id)| id) .take(DETAIL_CAP) .collect(); if need.len() == DETAIL_CAP { let selected: HashSet = need.iter().copied().collect(); let mut oldest: Vec<&CatalogAlbum> = albums .iter() .filter(|a| { matches!(a.record_type.as_deref(), Some("album") | Some("ep")) && !selected.contains(&a.id) // A dateless row cannot be ordered by age, and it is // already unconditionally in `need` above (clause 1) -- // these are the search-only extras, the last rows this // top-up is for. && a.release_date.is_some() }) .collect(); oldest.sort_by(|a, b| a.release_date.cmp(&b.release_date)); need.extend( oldest .into_iter() .take(DETAIL_CEILING - DETAIL_CAP) .map(|a| a.id), ); } let details_started = Instant::now(); let detail_count = need.len(); let details_fut = stream::iter(need.iter().copied()) .map(|id| { let url = format!("https://api.deezer.com/album/{id}"); async move { self.deezer(&url).await } }) // Keep the same polite cap, but refill each slot as soon as one // request completes. Fixed 25-item waves made every later album // wait for the single slowest response in the preceding wave. .buffer_unordered(25) .collect::>(); // Collected together: the Deezer detail wave and the MusicBrainz // supplements tail are the two remaining multi-second legs, and // neither reads the other. Each leg stamps its own trace, so an audit // still sees which of the two the join was actually waiting on. let (fetched, supplements) = tokio::join!( async { let fetched = details_fut.await; stage_trace( "page.details", details_started, &format!("count={detail_count}"), ); fetched }, async { let supplements = supplements_task.await.unwrap_or_default(); stage_trace("page.supplements_wait", details_started, ""); supplements } ); let mut details: HashMap = HashMap::new(); for d in fetched.into_iter().flatten() { if let Some(id) = d.get("id").and_then(|v| v.as_u64()) { details.insert(id, d); } } for a in &mut albums { let Some(d) = details.get(&a.id) else { continue; }; if a.release_date.is_none() { a.release_date = d .get("release_date") .and_then(|v| v.as_str()) .map(String::from); } let nb = d .get("nb_tracks") .and_then(|v| v.as_u64()) .map(|n| n as u32); if a.track_count.is_none() { a.track_count = nb; } a.is_soundtrack = detail_is_soundtrack(d); // The list endpoint carries no artist object at all, so this pass is // the only place the release's PRIMARY artist is visible. A mismatch // means the artist is a contributor on someone else's record. a.appears_on = d .pointer("/artist/id") .and_then(|id| id.as_u64()) .is_some_and(|primary| primary != artist_id); // These albums arrived from search (no tracklist); the details pass // is the first look at their tracks, so read live-ness here. Titles // like "Ballet Slippers" give no hint — the tracks do. a.is_live = detail_is_live(d); // Album-typed release whose tracks are mostly the title track // and remixes/versions of it → really a single. (Deezer's 2009 // "The Girl and the Robot" has 13 tracks, all the title track, // so the gate is generous — the majority rule below decides.) if a.record_type.as_deref() == Some("album") && nb.unwrap_or(99) <= 14 { let names: Vec<&str> = d .pointer("/tracks/data") .and_then(|t| t.as_array()) .map(|arr| { arr.iter() .filter_map(|t| t.get("title").and_then(|s| s.as_str())) .collect() }) .unwrap_or_default(); if !names.is_empty() { let title_n = normalize(&a.title); let hits = names .iter() .filter(|name| { let ln = name.to_lowercase(); (!title_n.is_empty() && normalize(name).contains(&title_n)) || ln.contains("remix") || ln.contains(" mix") || ln.contains("mix)") || ln.contains("version") || ln.contains("radio edit") || ln.contains("extended") || ln.contains("instrumental") }) .count(); if hits * 2 >= names.len() { a.record_type = Some("single".into()); } } } } // Fold the duplicate groups detected above, now that the details pass // has track counts for them. Exact-title duplicates and featured-credit // variants ("Dancing In Babylon" beside "… (feat. Christine and the // Queens)", "Naughty Girl" two weeks before its credited twin) both // fold within a YEAR — the audited real-world pairs sit days to months // apart, never across years. Each group keeps its fullest row, so the // 4-track credited release survives its 1-track twin. A second exact // pass then collapses cross-year re-releases under the strict // same-material rules documented on `fold_exact_rereleases`. let exact_groups: Vec> = exact_groups.into_values().collect(); fold_duplicate_release_rows(&mut albums, &exact_groups, 4); let featless_groups: Vec> = featless_groups.into_values().collect(); fold_duplicate_release_rows(&mut albums, &featless_groups, 4); fold_exact_rereleases(&mut albums, &exact_groups); // Majority genre over the details fetched above — the artist's "top // genre" chip. "Films/Games" is a release property (soundtracks), not // an artist genre, so it never represents the artist. let mut genre_counts: HashMap = HashMap::new(); for d in details.values() { let Some(genres) = d.pointer("/genres/data").and_then(|g| g.as_array()) else { continue; }; for name in genres .iter() .filter_map(|g| g.get("name").and_then(|n| n.as_str())) { if name.is_empty() || name.eq_ignore_ascii_case("films/games") { continue; } *genre_counts.entry(name.to_string()).or_default() += 1; } } let top_genre = genre_counts .into_iter() .max_by_key(|(_, n)| *n) .map(|(name, _)| name); // Titles such as "Weezer (Deluxe Edition)" do not identify which of // several self-titled records they belong to. Deezer album details do // contain enough evidence: a real edition preserves the original core // track order. Attach a conservative work hint only when at least five // ordered tracks and 75% of the original's first ten tracks agree. let tracklists: HashMap> = details .iter() .filter_map(|(id, detail)| { let tracks = detail .pointer("/tracks/data")? .as_array()? .iter() .filter_map(|track| track.get("title").and_then(|title| title.as_str())) .map(track_identity) .collect::>(); (!tracks.is_empty()).then_some((*id, tracks)) }) .collect(); // A second, independent read of the same details: which songs does each // album actually CONTAIN? Deliberately not reusing `tracklists` above — // that map is ordered and consumed by `edition_track_score`, so // changing its normalizer or its `Vec` shape would silently move // edition links. This one is an unordered set under a normalizer that // also folds `feat.` clauses, which the containment match needs and // edition folding does not. let containment: HashMap> = details .iter() .filter_map(|(id, detail)| { let titles: HashSet = detail .pointer("/tracks/data")? .as_array()? .iter() .filter_map(|track| track.get("title").and_then(|title| title.as_str())) .map(contained_track_identity) .filter(|key| !key.is_empty()) .collect(); (!titles.is_empty()).then_some((*id, titles)) }) .collect(); let mut edition_links = Vec::new(); for (edition_index, edition) in albums.iter().enumerate() { if edition.record_type.as_deref() != Some("album") { continue; } // A keyword edition ("… Deluxe/Remaster/Anniversary") folds on track // overlap alone; a bespoke reissue (no keyword, but its title is // built on an existing album's) additionally requires that original's // title to be a substring of it, so the strong "named after it" // signal is tied to the actual pair. let is_keyword_edition = is_edition_title(&edition.title); let is_bespoke_reissue = !is_keyword_edition && contains_other_album_title(&edition.title, &non_edition_titles); if !is_keyword_edition && !is_bespoke_reissue { continue; } let Some(edition_tracks) = tracklists.get(&edition.id) else { continue; }; let edition_norm = normalize(&edition.title); if let Some((original_index, original_id)) = best_edition_original( edition_index, &edition_norm, edition_tracks, &albums, &tracklists, is_bespoke_reissue, ) { edition_links.push((edition_index, original_index, original_id)); } } for (edition_index, original_index, original_id) in edition_links { albums[edition_index].edition_group = Some(original_id); albums[original_index].edition_group = Some(original_id); } // Supplement only from the exact Apple performer corroborated against // THIS Deezer id. Ambiguous same-name searches fail closed: Deezer's // filtered catalogue still renders, but no guessed Apple rows enter it. let mb_studio = self .apple_supplement_albums(artist_id, &artist.name, &apple) .await; let socials: Vec = Vec::new(); // Apple is a GAP FILLER, not a second catalogue. Anything Deezer // already lists — under any record type — is dropped before the merge, // because Apple's edition titles ("Hurley (Deluxe Edition)") and // format tails would otherwise sail past the exact-title matcher below // and duplicate rows that are already on the page. let existing_titles: std::collections::HashSet = albums.iter().map(|a| normalize(&a.title)).collect(); let existing_releases: Vec<(String, Option)> = albums .iter() .map(|a| (normalize(&a.title), a.release_date.clone())) .collect(); let mb_studio: Vec = mb_studio .into_iter() .filter(|candidate| { let n = normalize(&candidate.title); !existing_titles.contains(&n) // "X (Deluxe Edition)" when plain "X" is already present. && !(is_edition_title(&candidate.title) && existing_titles.iter().any(|t| n.starts_with(t.as_str()))) // Same release day, and one title is the other plus a tail // ("Late Night Tales" / "Late Night Tales: Mgmt") — the // same work under a storefront naming variant, not a gap. // Length-guarded so short titles can't collide by accident. && !existing_releases.iter().any(|(t, date)| { let (short, long) = if t.len() <= n.len() { (t.as_str(), n.as_str()) } else { (n.as_str(), t.as_str()) }; short.len() >= 8 && long.starts_with(short) && same_original_date( date.as_deref(), candidate.release_date.as_deref(), ) }) }) .collect(); // Deezer remains the preferred catalog record (cover, track counts and // previews). The supplement fills only titles absent in this region and // corrects obvious type/date errors on exact-title matches. let mb_title_counts = mb_studio.iter().fold(HashMap::new(), |mut counts, album| { *counts.entry(normalize(&album.title)).or_insert(0_usize) += 1; counts }); for fallback in mb_studio { let fallback_title = normalize(&fallback.title); let fallback_title_is_unique = mb_title_counts .get(&fallback_title) .copied() .unwrap_or_default() == 1; let exact: Vec = albums .iter() .enumerate() // MusicBrainz supplied a studio album, so only album records // can satisfy this match. A same-title single must not make an // otherwise unique album look ambiguous (for example, Phantom // Island has both a 2024 single and a 2025 album). .filter(|(_, album)| exact_studio_album_title(album, &fallback_title)) .map(|(index, _)| index) .collect(); let matching_date = exact.iter().copied().find(|index| { same_original_date( albums[*index].release_date.as_deref(), fallback.release_date.as_deref(), ) }); // A single current match is not enough when MusicBrainz itself has // multiple distinct works with this title. Otherwise whichever // self-titled release happens to arrive first absorbs all others. let unique_exact = (exact.len() == 1 && fallback_title_is_unique).then(|| exact[0]); let self_titled_alias = (fallback_title == normalize(&artist.name)) .then(|| { albums.iter().position(|album| { is_self_titled_album_alias(&album.title, &artist.name) && same_original_date( album.release_date.as_deref(), fallback.release_date.as_deref(), ) }) }) .flatten(); if let Some(index) = matching_date.or(unique_exact).or(self_titled_alias) { let existing = &mut albums[index]; if existing.record_type.as_deref() == Some("single") { existing.record_type = Some("album".into()); } if fallback.release_date.is_some() && (existing.release_date.is_none() || fallback.release_date < existing.release_date) { existing.release_date = fallback.release_date; } } else { albums.push(fallback); } } // Second edition pass — fold Apple-supplemented editions (e.g. "Weezer // (2024 Remaster)") under their Deezer original. Apple albums carry no // Deezer tracklist, so the first pass could not score them; fetch their // iTunes tracklists now and match against the SAME Deezer `tracklists` // originals. Scoring against the Deezer-only map keeps this Apple→Deezer // (an Apple album never folds into another Apple album). let apple_candidates: Vec<(usize, u64, String, bool)> = { let registry = self.apple_catalog.lock().await; albums .iter() .enumerate() .filter_map(|(index, a)| { if a.edition_group.is_some() || a.record_type.as_deref() != Some("album") { return None; } let entry = registry.get(&a.id)?; let is_keyword_edition = is_edition_title(&a.title); let is_bespoke_reissue = !is_keyword_edition && contains_other_album_title(&a.title, &non_edition_titles); if !is_keyword_edition && !is_bespoke_reissue { return None; } Some(( index, entry.collection_id, normalize(&a.title), is_bespoke_reissue, )) }) .collect() }; if !apple_candidates.is_empty() { let scored = stream::iter(apple_candidates) .map(|(index, collection_id, norm, bespoke)| async move { ( index, self.apple_album_track_titles(collection_id).await, norm, bespoke, ) }) .buffer_unordered(8) .collect::>() .await; let mut apple_links = Vec::new(); let mut redundant_ids = Vec::new(); for (index, tracks, norm, bespoke) in &scored { let Some((original_index, original_id)) = best_edition_original(*index, norm, tracks, &albums, &tracklists, *bespoke) else { continue; }; // A pure remaster/reissue that duplicates the matched original's // tracklist adds nothing Deezer doesn't already carry — the Apple // supplement is a gap filler, so drop it instead of showing a // redundant edition ("Weezer (2024 Remaster)" == the Blue Album). if tracklists .get(&original_id) .is_some_and(|original_tracks| is_duplicate_tracklist(tracks, original_tracks)) { redundant_ids.push(albums[*index].id); } else { apple_links.push((*index, original_index, original_id)); } } for (edition_index, original_index, original_id) in apple_links { albums[edition_index].edition_group = Some(original_id); albums[original_index].edition_group = Some(original_id); } if !redundant_ids.is_empty() { albums.retain(|a| !redundant_ids.contains(&a.id)); } } // Same-work VARIANT LINKING, across every source. A release whose // title is an existing album's title plus a variant tail — "Revolver // (2022 Mix)", "THE TORTURED POETS DEPARTMENT: THE ANTHOLOGY", "The // Life of a Showgirl + Acoustic Collection" — is the same work in // different clothes. Tracklist folding catches these only when both // tracklists were fetched; this title/date rule needs neither. The // rows are LINKED as one edition family, never dropped, so every // variant stays reachable behind one card. Guards, audited against 44 // real catalogues: the base title must be ≥8 normalized chars; a // sequel/volume tail never links ("Greatest Hits II", "Fleetwood Mac: // Live 1975"); and the pair must share a release day, or sit within // ~45 days with a tail from the known variant vocabulary — "Greatest // Hits In Japan" (years apart, no variant word) stays its own card. { let mut variant_links: Vec<(usize, usize, u64)> = Vec::new(); for (variant_index, variant) in albums.iter().enumerate() { if variant.edition_group.is_some() || variant.record_type.as_deref() != Some("album") { continue; } let variant_norm = normalize(&variant.title); let base = albums .iter() .enumerate() .filter(|(base_index, base)| { if *base_index == variant_index || base.record_type.as_deref() != Some("album") { return false; } let base_norm = normalize(&base.title); if base_norm.len() < 8 || base_norm.len() >= variant_norm.len() || !variant_norm.starts_with(base_norm.as_str()) { return false; } let residual = &variant_norm[base_norm.len()..]; if residual_is_sequel_marker(residual) { return false; } if same_original_date( base.release_date.as_deref(), variant.release_date.as_deref(), ) { return true; } let close = match ( base.release_date.as_deref().and_then(release_date_days), variant.release_date.as_deref().and_then(release_date_days), ) { (Some(a), Some(b)) => (a - b).abs() <= 45, _ => false, }; close && residual_is_release_variant(residual) }) // The LONGEST matching base is the nearest work: "1989 // (Taylor's Version) (Deluxe)" belongs to "1989 (Taylor's // Version)", not to "1989". .max_by_key(|(_, base)| normalize(&base.title).len()); if let Some((base_index, base)) = base { variant_links.push((variant_index, base_index, base.id)); } } for (variant_index, base_index, base_id) in variant_links { let group = albums[base_index].edition_group.unwrap_or(base_id); albums[variant_index].edition_group = Some(group); albums[base_index].edition_group = Some(group); } } // The profile's nb_album reflects only Deezer's curated regional list; // show the real merged count and sort by original release chronology -- // which is only actually chronology once the reissue dates Deezer // reports have been replaced by the works' first-release dates. let dates_started = Instant::now(); let corrected = apply_original_release_dates(&mut albums, &supplements.first_released); // Then let the tracklist-verified edition links settle what titles // cannot: a pressing takes the date of the work it is a pressing of. let inherited = apply_edition_group_dates(&mut albums); stage_trace( "page.original_dates", dates_started, &format!( "{corrected}/{} corrected, {inherited} inherited{}", albums.len(), // Without this the trace reads "0/47 corrected" whether // MusicBrainz said the dates are already right or never // answered at all — which is precisely the confusion that sent // an hour chasing a regression in Eagles' dates that turned out // to be this machine having throttled itself with probes. if supplements.musicbrainz_unavailable { " — MUSICBRAINZ UNAVAILABLE, dates are unverified" } else { "" } ), ); artist.album_count = Some(albums.len() as u32); albums.sort_by(|a, b| b.release_date.cmp(&a.release_date)); let pool = deezer_popular_pool(popular.as_ref(), artist_id, &foreign_album_ids, &apple); let mut popular_tracks = build_popular( &supplements.ranked, pool, &supplements.recordings, &artist.name, POPULAR_KEEP, ); // Runs LAST, on the ten rows that survived: after the album pipeline // has foreign-filtered, ownership-checked, classified and folded, and // after `apply_original_release_dates` so the date tiebreak compares // first-release dates rather than the reissue dates Deezer reports. // Ordering is keyed on `track_title_identity`, not on ids, so moving a // row's id here cannot disturb the Last.fm order above it. let attribution_started = Instant::now(); let reattributed = apply_original_album_attribution( &mut popular_tracks, &albums, &containment, &details, &supplements.first_released, ); // Last, and only for songs whose original record Deezer has no copy of // at all — the case attribution structurally cannot reach. let relabelled = apply_musicbrainz_track_albums( &mut popular_tracks, &supplements.recordings, &albums, &supplements.first_released, ); // After the albums are settled, so the lookup asks Apple for the record // the row actually ends up on. self.apply_apple_previews(&mut popular_tracks, &apple).await; stage_trace( "page.attribution", attribution_started, &format!( "{reattributed}/{} re-attributed, {relabelled} relabelled{}", popular_tracks.len(), // Attribution reads the same facts, so it fails the same way: // with no secondary types the rule is a no-op and every popular // row keeps whatever pressing Deezer licensed. if supplements.musicbrainz_unavailable { " — MUSICBRAINZ UNAVAILABLE, rows keep Deezer's attribution" } else { "" } ), ); stage_trace( "page.total", page_started, &format!("albums={}", albums.len()), ); Some(CatalogArtistPage { artist, // Artwork is merged by the independent `catalog_artist_art` stage. // Only a user override added by the Tauri command populates these // fields on this content response. banner: None, banners: Vec::new(), banner_custom: false, logos: Vec::new(), popular_tracks, albums, top_genre, socials, }) } /// The MusicBrainz "last pass": socials, plus the few releases only /// MusicBrainz carries (practice tapes, BBC sessions, regional oddities /// Apple and Deezer both omit). /// /// Deliberately its own call. MusicBrainz is rate-limited to 1 request per /// second by policy, and this needs two or three serialized requests — so /// keeping it out of `catalog_artist` is what stops the release grid /// waiting ~2-3s on a throttle. The frontend merges the result in when it /// arrives; nothing on the page blocks for it. pub async fn catalog_artist_identity( &self, artist_id: u64, artist: &str, ) -> CatalogArtistIdentity { let identity_started = Instant::now(); let Some(mbid) = self.deezer_linked_mbid(artist_id).await else { stage_trace("identity.total", identity_started, "no-mbid"); return empty_catalog_artist_identity(artist_id); }; stage_trace("identity.linked_mbid", identity_started, ""); let join_started = Instant::now(); // Socials are footer content, not sorting evidence, yet every request // in this join serializes behind MusicBrainz's 1 req/s gate — so the // socials lookup was ~1 s of extra time-to-correctly-sorted on every // cold artist. They now ride [`Self::catalog_artist_socials`], which // the frontend calls only after this merge-critical response lands. // Ownership rides the join too: its MusicBrainz legs share the gate // with the three walks either way (the request count is what the // policy prices), but serialized ahead of them its non-MusicBrainz // leg — the Apple candidate fetch — added a tail the gate never // required. let (ownership, albums, mixtapes, type_hints) = tokio::join!( async { let ownership_started = Instant::now(); let ownership = self .artist_release_ownership(artist_id, artist, &mbid) .await; stage_trace( "identity.ownership", ownership_started, &format!( "owned={} foreign={} fused={}", ownership.owned.len(), ownership.foreign.len(), ownership.cotenancy_proven ), ); ownership }, self.musicbrainz_studio_albums(artist, &mbid), self.musicbrainz_mixtapes(artist, &mbid), self.musicbrainz_type_hints(&mbid) ); // One list from here on: a mixtape is a supplemented release like any // other and carries its own `is_mixtape` flag, so everything // downstream — the name-credit scope check, the seen/variant filters, // the grid — treats it exactly as it treats an injected album. let mixtape_count = mixtapes.len(); let albums: Vec = albums.into_iter().chain(mixtapes).collect(); stage_trace( "identity.mb_join", join_started, &format!( "albums={} mixtapes={} hints={}", albums.len() - mixtape_count, mixtape_count, type_hints.len() ), ); stage_trace("identity.total", identity_started, ""); CatalogArtistIdentity { artist_id, albums, socials: Vec::new(), type_hints, owned_album_ids: ownership.owned.into_iter().collect(), foreign_album_ids: ownership.foreign.into_iter().collect(), fused_record: ownership.cotenancy_proven, } } /// Every release group MusicBrainz files under this artist, all primary /// types. [`Self::musicbrainz_studio_albums`] deliberately asks for studio /// albums only because it SUPPLEMENTS the page; ownership has to recognise /// singles, EPs and live records too, or the artist's own promo singles read /// as somebody else's. async fn musicbrainz_release_titles(&self, mbid: &str) -> HashSet { let mut titles = HashSet::new(); for offset in [0usize, 100] { let url = format!( "https://musicbrainz.org/ws/2/release-group?artist={mbid}&limit=100&offset={offset}&fmt=json" ); let Some(body) = self.musicbrainz_json(&url).await else { break; }; let Some(groups) = body.get("release-groups").and_then(|g| g.as_array()) else { break; }; let returned = groups.len(); for group in groups { if let Some(title) = group.get("title").and_then(|t| t.as_str()) { let key = release_title_identity(title); if !key.is_empty() { titles.insert(key); } } } let total = body .get("release-group-count") .and_then(|c| c.as_u64()) .unwrap_or(returned as u64) as usize; if offset + returned >= total || returned == 0 { break; } } titles } /// The labels MusicBrainz attaches to this artist's releases. Curated /// per-release, so one sighting is enough to trust — unlike Apple's ℗ line, /// which inherits whatever else got filed under the performer. async fn musicbrainz_release_labels(&self, mbid: &str) -> HashSet { let url = format!( "https://musicbrainz.org/ws/2/release?artist={mbid}&limit=100&inc=labels&fmt=json" ); let Some(body) = self.musicbrainz_json(&url).await else { return HashSet::new(); }; let Some(releases) = body.get("releases").and_then(|r| r.as_array()) else { return HashSet::new(); }; let mut labels = HashSet::new(); for release in releases { let Some(infos) = release.get("label-info").and_then(|l| l.as_array()) else { continue; }; for info in infos { let Some(name) = info.pointer("/label/name").and_then(|n| n.as_str()) else { continue; }; // MusicBrainz's placeholder for a self-released record. It is // not a label and would match every unlabelled row. if name == "[no label]" { continue; } let key = label_identity(name); if key.len() >= 3 { labels.insert(key); } } } labels } /// Decide, release by release, which of this Deezer artist's rows are /// actually this artist's. See [`ReleaseOwnership`] for what the two output /// sets mean and why `foreign` is gated behind proof. async fn artist_release_ownership( &self, artist_id: u64, artist: &str, mbid: &str, ) -> ReleaseOwnership { let started = Instant::now(); let (mb_titles, mb_labels, candidates) = tokio::join!( self.musicbrainz_release_titles(mbid), self.musicbrainz_release_labels(mbid), self.apple_identity_candidates(artist), ); if mb_titles.is_empty() { // No identity to reason from. Saying nothing is the correct answer: // the caller keeps every row. stage_trace("ownership.total", started, "no-mb-titles"); return ReleaseOwnership::default(); } // The Apple performer that reproduces this MBID's catalogue is ours; // the rest are namesakes, and their exclusive titles are the proof that // a Deezer row was filed under the wrong act. let mut winner: Option<(usize, usize)> = None; for (index, candidate) in candidates.iter().enumerate() { let hits = candidate .releases .iter() .filter(|release| mb_titles.contains(&release.title_norm)) .map(|release| release.title_norm.clone()) .collect::>() .len(); if winner.is_none_or(|(_, best)| hits > best) { winner = Some((index, hits)); } } let winner = winner.filter(|(_, hits)| *hits >= APPLE_IDENTITY_MIN_TITLE_HITS); let mut owned_titles = mb_titles.clone(); let mut labels = mb_labels; // Collaborations Apple files under their own artist id while still // crediting ours. Held apart from `owned_titles` because on a FUSED // record the credit is ambiguous by construction — "rafftheraff, O mc // vini, Alec & Wand" names a Wand, just not this one — so these are // only conclusive where the record holds a single act, and otherwise // fall through to the label test below. let credited_titles: HashSet = winner .map(|(index, _)| candidates[index].credited_titles.clone()) .unwrap_or_default(); if let Some((index, _)) = winner { let mut apple_label_hits: HashMap = HashMap::new(); for release in &candidates[index].releases { if !release.title_norm.is_empty() { owned_titles.insert(release.title_norm.clone()); } if let Some(copyright) = release.copyright.as_deref() { let key = label_identity(&label_from_copyright(copyright)); if key.len() >= 3 { *apple_label_hits.entry(key).or_insert(0) += 1; } } } // Apple leaks: two rafftheraff singles sit under the LA band's // artist id. One sighting must not admit a co-tenant's whole label. labels.extend( apple_label_hits .into_iter() .filter(|(_, hits)| *hits >= 2) .map(|(key, _)| key), ); } let foreign_titles: HashSet<&String> = candidates .iter() .enumerate() .filter(|(index, _)| winner.is_none_or(|(win, _)| *index != win)) .flat_map(|(_, candidate)| candidate.releases.iter().map(|r| &r.title_norm)) .filter(|title| !title.is_empty() && !owned_titles.contains(*title)) .collect(); // Deezer's own list for this id. Cached from the page load, so on the // normal path this costs nothing. let mut rows: Vec<(u64, String)> = Vec::new(); for index in (0..200).step_by(50) { let url = format!( "https://api.deezer.com/artist/{artist_id}/albums?limit=50&index={index}" ); let Some(body) = self.deezer(&url).await else { break; }; let Some(items) = body.get("data").and_then(|d| d.as_array()) else { break; }; for item in items { let Some(id) = item.get("id").and_then(|i| i.as_u64()) else { continue; }; let Some(title) = item.get("title").and_then(|t| t.as_str()) else { continue; }; rows.push((id, release_title_identity(title))); } if body.get("next").and_then(|n| n.as_str()).is_none() { break; } } let owned: HashSet = rows .iter() .filter(|(_, title)| owned_titles.contains(title)) .map(|(id, _)| *id) .collect(); // The gate. A row carried by a DIFFERENT same-named Apple performer, // and by neither MusicBrainz nor ours, is a co-tenant we can name. let proven: Vec = rows .iter() .filter(|(_, title)| foreign_titles.contains(title)) .map(|(id, _)| *id) .collect(); let cotenancy_proven = candidates.len() >= 2 && !proven.is_empty(); if !cotenancy_proven { // One act on this record, so a credit naming them is decisive. // This is what returns VULTURES 1, VULTURES 2 and KIDS SEE GHOSTS // to Ye's page: MusicBrainz files all three under ¥$ and Kids See // Ghosts rather than under his own MBID, so nothing else corroborates // them and the guest guard was deleting all three. let owned = rows .iter() .filter(|(id, title)| owned.contains(id) || credited_titles.contains(title)) .map(|(id, _)| *id) .collect::>(); stage_trace( "ownership.total", started, &format!("gate=closed owned={} rows={}", owned.len(), rows.len()), ); return ReleaseOwnership { owned, foreign: HashSet::new(), cotenancy_proven: false, }; } // Only now is the per-row label worth buying: this is the one artist // shape where it decides anything, and details are one request each. // Wand's genuine rows are all Drag City / In The Red and its co-tenants // span twenty-one other labels — without this leg nine real releases // (JJ, Smile, The Leap, Goldfish, the live singles…) have no evidence // at all, because neither provider carries them by title. let unresolved: Vec = rows .iter() .filter(|(id, _)| !owned.contains(id)) .map(|(id, _)| *id) .collect(); let details = stream::iter(unresolved) .map(|id| { let url = format!("https://api.deezer.com/album/{id}"); async move { (id, self.deezer(&url).await) } }) .buffer_unordered(12) .collect::>() .await; let mut foreign = HashSet::new(); let mut owned = owned; for (id, detail) in details { let label = detail .as_ref() .and_then(|d| d.get("label")) .and_then(|l| l.as_str()) .unwrap_or_default(); if !label.is_empty() && label_matches(label, &labels) { owned.insert(id); } else { foreign.insert(id); } } stage_trace( "ownership.total", started, &format!( "gate=open owned={} foreign={} rows={}", owned.len(), foreign.len(), rows.len() ), ); ReleaseOwnership { owned, foreign, cotenancy_proven, } } /// Short artist biography for the Now Playing "About the artist" card. /// /// **Identity first.** A blind name search is how the panel came to show /// Mr. Whitekeys for Camping in Alaska, Crazy Horse for Crazy P and /// Måneskin's bassist for Partiboi69: Wikipedia's top hit for /// " musician band" is frequently a different act entirely, and /// nothing checked it. The article is now resolved through the artist's /// OWN MusicBrainz identity — Deezer id → linked MBID → Wikidata → English /// sitelink — which cannot name the wrong subject. /// /// Only when no such link exists does a name search run, and then every /// candidate must prove itself: the title must BE the artist's name and /// the lead must both name them and read like a music biography. An /// unprovable article yields no bio at all — a blank card is honest, a /// confident wrong one is not. /// Wikipedia covers famous acts and nobody else, so the underground /// artists this app exists to browse got a blank card. Every tier below is /// anchored to the artist's MusicBrainz page — the links are ones the /// artist's own MB entry declares, never a name guess: /// /// 1. **Wikipedia** via MB → Wikidata → English sitelink (richest prose). /// 2. **Discogs** artist profile, addressed by the id in MB's `discogs` /// relation. This is the tier that rescues small artists: Camping in /// Alaska has a full band history there and no Wikipedia article. /// 3. **Guarded Wikipedia name search** (title must BE the artist). /// 4. **TheAudioDB** biography, identity-checked the same way album /// lookups check it. /// 5. **MusicBrainz's own facts** — the disambiguation line, kind, area /// and genre tags, composed into one honest sentence. Hermeth's MB /// entry alone yields "David Ameal, techno / breakbeat producer". pub async fn artist_bio(&self, artist_id: Option, artist: &str) -> Option { let facts = match artist_id { Some(artist_id) => match self.deezer_linked_mbid(artist_id).await { Some(mbid) => self.artist_identity_facts(&mbid).await, None => None, }, None => None, }; if let Some(facts) = facts.as_ref() { if let Some(title) = self.wikipedia_title_from_facts(facts).await { if let Some(extract) = self.wikipedia_extract(&title).await { return Some(extract); } } if let Some(discogs_id) = facts.discogs_id.as_deref() { if let Some(profile) = self.discogs_profile(discogs_id).await { return Some(profile); } } } if let Some(extract) = self.wikipedia_artist_search_bio(artist).await { return Some(extract); } if let Some(biography) = self.audiodb_biography(artist_id, artist).await { return Some(biography); } facts.as_ref().and_then(|facts| facts.compose_bio(artist)) } /// One MusicBrainz artist read serving both the bio tiers and /// [`Self::artist_socials`] — same URL, so they share cache and the /// 1 req/s gate is paid once. async fn artist_identity_facts(&self, mbid: &str) -> Option { let body = self.musicbrainz_json(&artist_detail_url(mbid)).await?; let mut facts = ArtistIdentityFacts { disambiguation: body .get("disambiguation") .and_then(|value| value.as_str()) .map(str::trim) .filter(|value| !value.is_empty()) .map(str::to_string), kind: body .get("type") .and_then(|value| value.as_str()) .map(str::to_string), area: body .pointer("/area/name") .and_then(|value| value.as_str()) .map(str::to_string), tags: top_musicbrainz_tags(&body), ..ArtistIdentityFacts::default() }; for relation in body.get("relations")?.as_array()? { let Some(target) = relation.pointer("/url/resource").and_then(|v| v.as_str()) else { continue; }; match relation.get("type").and_then(|v| v.as_str()).unwrap_or("") { "wikipedia" => { facts.wikipedia_title = target .split("/wiki/") .nth(1) .map(|title| title.replace('_', " ")); } "wikidata" => { facts.wikidata_id = target.rsplit('/').next().map(str::to_string); } "discogs" => { facts.discogs_id = target .trim_end_matches('/') .rsplit('/') .next() .filter(|id| id.chars().all(|c| c.is_ascii_digit())) .map(str::to_string); } _ => {} } } Some(facts) } /// The English article title MusicBrainz points at, resolving a Wikidata /// entity through its `enwiki` sitelink when there is no direct link. async fn wikipedia_title_from_facts(&self, facts: &ArtistIdentityFacts) -> Option { if let Some(title) = facts.wikipedia_title.clone() { return Some(title); } let wikidata_id = facts.wikidata_id.as_deref()?; let url = format!( "https://www.wikidata.org/w/api.php?action=wbgetentities&ids={wikidata_id}&props=sitelinks&sitefilter=enwiki&format=json" ); let body = self.deezer(&url).await?; body.pointer(&format!("/entities/{wikidata_id}/sitelinks/enwiki/title")) .and_then(|value| value.as_str()) .map(str::to_string) } /// Discogs artist profile, addressed by id — so it is the right artist by /// construction. Discogs allows 25 unauthenticated requests a minute; this /// is paced under that and, being slow-moving prose, is disk cached. async fn discogs_profile(&self, discogs_id: &str) -> Option { let url = format!("https://api.discogs.com/artists/{discogs_id}"); self.discogs_throttle().await; let body = self.deezer(&url).await?; let profile = body.get("profile").and_then(|value| value.as_str())?; let cleaned = strip_discogs_markup(profile); (!cleaned.is_empty()).then_some(cleaned) } /// TheAudioDB's biography, held to the same identity test the album /// lookups use so a namesake's write-up can never appear. async fn audiodb_biography(&self, artist_id: Option, artist: &str) -> Option { let record = self.audiodb_artist(artist).await?; if let Some(artist_id) = artist_id { let linked_mbid = self.deezer_linked_mbid(artist_id).await; if !self .audiodb_identity_ok(&record, artist_id, linked_mbid.as_deref()) .await { return None; } } record .get("strBiographyEN") .and_then(|value| value.as_str()) .map(str::trim) .filter(|value| !value.is_empty()) .map(str::to_string) } /// Page summary for an exact article title, rejecting disambiguation stubs. async fn wikipedia_extract(&self, title: &str) -> Option { let url = format!( "https://en.wikipedia.org/api/rest_v1/page/summary/{}", urlencode(&title.replace(' ', "_")) ); let body = self.deezer(&url).await?; if body.get("type").and_then(|value| value.as_str()) == Some("disambiguation") { return None; } let extract = body.get("extract").and_then(|value| value.as_str())?.trim(); (!extract.is_empty()).then(|| extract.to_string()) } /// Guarded name search, used only for artists MusicBrainz cannot anchor. async fn wikipedia_artist_search_bio(&self, artist: &str) -> Option { let search_url = format!( "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={}&format=json&srlimit=6", urlencode(&format!("{artist} musician band")) ); let body = self.deezer(&search_url).await?; let results = body.get("query")?.get("search")?.as_array()?; let want = normalize(artist); // The title must BE the artist's name, give or take a parenthetical // qualifier ("Pond (band)"). "Mr. Whitekeys" cannot pass for "Camping // in Alaska", nor "Crazy Horse (band)" for "Crazy P". let titles: Vec = results .iter() .filter_map(|result| result.get("title").and_then(|value| value.as_str())) .filter(|title| { let base = title.split('(').next().unwrap_or(title); normalize(base) == want }) .map(str::to_string) .collect(); for title in titles { let Some(extract) = self.wikipedia_extract(&title).await else { continue; }; // The lead must name the artist AND describe a musical act, so a // same-named footballer or village cannot fill the card. if !normalize(&extract).contains(&want) { continue; } let lower = extract.to_lowercase(); let musical = [ "band", "musician", "singer", "rapper", "producer", "songwriter", "composer", "dj", "duo", "trio", "music", "album", ] .iter() .any(|marker| lower.contains(marker)); if musical { return Some(extract); } } None } /// Official links for the artist header, deferred OFF the identity pass's /// critical path (see [`Self::catalog_artist_identity`]). The linked-MBID /// read is already cached by the identity pass, so this costs one /// MusicBrainz request, made only after sorting has settled. pub async fn catalog_artist_socials(&self, artist_id: u64) -> Vec { let Some(mbid) = self.deezer_linked_mbid(artist_id).await else { return Vec::new(); }; self.artist_socials(&mbid).await } /// Classification hints for release groups MusicBrainz marks with a /// secondary type while still calling them primary-type Album — precisely /// the compilations, soundtracks and remix albums Deezer and Apple return /// tagged as a plain "album". The frontend matches these by title and moves /// the release to the right tab. /// /// Paginated because a prolific catalog overflows MusicBrainz's 100-per-page /// cap (Radiohead has 373 such groups) and search results are unsorted, so a /// single page can miss the one release that actually matters. Capped so an /// artist with hundreds of live/compilation entries can't turn this deferred /// pass into a long run of serialized 1 req/s calls. async fn musicbrainz_type_hints(&self, mbid: &str) -> Vec { let query = urlencode(&format!( "arid:{mbid} AND primarytype:album AND secondarytype:*" )); let mut hints = Vec::new(); for offset in (0..500).step_by(100) { let url = format!( "https://musicbrainz.org/ws/2/release-group/?query={query}&limit=100&offset={offset}&fmt=json" ); let Some(body) = self.musicbrainz_json(&url).await else { break; }; let Some(groups) = body .get("release-groups") .and_then(|value| value.as_array()) else { break; }; if groups.is_empty() { break; } let page_len = groups.len(); for group in groups { let Some(title) = group.get("title").and_then(|value| value.as_str()) else { continue; }; let secondary_types = group .get("secondary-types") .and_then(|value| value.as_array()) .map(|arr| { arr.iter() .filter_map(|value| value.as_str().map(String::from)) .collect() }) .unwrap_or_default(); hints.push(ReleaseTypeHint { title: title.to_string(), primary_type: group .get("primary-type") .and_then(|value| value.as_str()) .map(String::from), secondary_types, }); } // A short page means the result set is exhausted. if page_len < 100 { break; } } hints } /// The MusicBrainz id that MusicBrainz itself links to this Deezer artist, /// resolved through the Deezer artist URL's `artist` relationship. This is /// an *identity-anchored* id: unlike a name search (TheAudioDB), it cannot /// return a different same-named artist, so it is safe to key the studio /// supplement to. `None` when MusicBrainz has no Deezer link for the artist. async fn deezer_linked_mbid(&self, deezer_id: u64) -> Option { let resource = urlencode(&format!("https://www.deezer.com/artist/{deezer_id}")); let url = format!( "https://musicbrainz.org/ws/2/url?resource={resource}&inc=artist-rels&fmt=json" ); let mut body = self.musicbrainz_json(&url).await; // THE anchor request: dates, attribution, listeners and the identity // supplement all hang off this one lookup, so a single transient // failure here used to orphan the whole visit from MusicBrainz with // no second ask — observed live: a 242ms failure at open left a page // with 0/24 dates and an unranked Popular, while the identical // lookup answered normally two seconds later. Retried on the same // ladder the dates fetch runs — but ONLY when the miss was not a // 4xx: "MusicBrainz has no Deezer link for this artist" is a real // answer, and an artist without one must not pay two gated requests // and 3.6s of backoff on every cold visit to hear it again. if body.is_none() && !self.definitive_miss.lock().await.contains(&url) { for attempt in 0..MB_RETRY_ATTEMPTS { tokio::time::sleep(MB_RETRY_BACKOFF * (attempt + 1)).await; net_trace(&url, &format!("mb-retry {}", attempt + 1)); body = self.musicbrainz_json(&url).await; if body.is_some() { break; } if self.definitive_miss.lock().await.contains(&url) { break; } } } let body = body?; body.get("relations")? .as_array()? .iter() .find_map(|relation| { relation .pointer("/artist/id") .and_then(|value| value.as_str()) .map(String::from) }) } /// Official homepage + social links from MusicBrainz's url relationships, /// for a Deezer-linked MBID. Uses the same `inc=url-rels` artist lookup the /// Links MusicBrainz marks `ended` (dead accounts) are skipped, and only /// the first link of each kind is kept, in a fixed display order. async fn artist_socials(&self, mbid: &str) -> Vec { const ORDER: [&str; 8] = [ "website", "instagram", "x", "facebook", "tiktok", "youtube", "bandcamp", "soundcloud", ]; let Some(body) = self.musicbrainz_json(&artist_detail_url(mbid)).await else { return Vec::new(); }; let Some(relations) = body.get("relations").and_then(|v| v.as_array()) else { return Vec::new(); }; let mut by_kind: HashMap<&str, String> = HashMap::new(); for relation in relations { if relation.get("ended").and_then(|v| v.as_bool()) == Some(true) { continue; } let Some(target) = relation.pointer("/url/resource").and_then(|v| v.as_str()) else { continue; }; let rel_type = relation .get("type") .and_then(|v| v.as_str()) .unwrap_or_default(); let host = target.to_lowercase(); // The MB relation type is authoritative for homepage/bandcamp/ // soundcloud/youtube; "social network" needs the domain to tell // Instagram from X from Facebook. let kind = match rel_type { "official homepage" => Some("website"), "bandcamp" => Some("bandcamp"), "soundcloud" => Some("soundcloud"), "youtube" | "youtube music" => Some("youtube"), "social network" => { if host.contains("instagram.com") { Some("instagram") } else if host.contains("twitter.com") || host.contains("//x.com") || host.contains("www.x.com") { Some("x") } else if host.contains("facebook.com") { Some("facebook") } else if host.contains("tiktok.com") { Some("tiktok") } else if host.contains("youtube.com") { Some("youtube") } else { None } } _ => None, }; if let Some(kind) = kind { by_kind.entry(kind).or_insert_with(|| target.to_string()); } } ORDER .iter() .filter_map(|kind| { by_kind.remove(kind).map(|url| ArtistLink { kind: (*kind).to_string(), url, }) }) .collect() } /// Distinctive people attached to the Deezer-linked MusicBrainz artist. /// Only durable group-identity relationships are admitted; collaborators, /// supporting performers, teachers and other broad graph edges would not /// prove that a Bing card depicts this exact same-named act. async fn artist_identity_people(&self, mbid: &str, artist: &str) -> HashSet { let url = format!("https://musicbrainz.org/ws/2/artist/{mbid}?inc=artist-rels&fmt=json"); let Some(body) = self.musicbrainz_json(&url).await else { return HashSet::new(); }; musicbrainz_identity_people(&body, artist) } /// Studio release groups missing from Deezer's regional artist catalog, for /// a Deezer-linked MusicBrainz id (see [`Self::deezer_linked_mbid`]). The /// filtered query excludes every secondary type, so live bootlegs, /// compilations, remixes and soundtracks cannot flood the normal Albums /// section. Because the caller passes an identity-anchored id, this can /// never inject a different same-named artist's discography. async fn musicbrainz_studio_albums(&self, artist: &str, mbid: &str) -> Vec { // status:official matters as much as the type filters: without it the // supplement returned bootleg-only release groups as studio albums — // Queen's "Zoom 1979", "In the Lap of the Queen", Taylor Swift's // "Lover The Secret Studio Sessions". Verified live: with the filter // Queen's supplement is exactly its 14 studio albums. let query = format!("arid:{mbid} AND primarytype:album AND -secondarytype:* AND status:official"); self.musicbrainz_release_groups(artist, mbid, &query, false).await } /// Release groups MusicBrainz types `Mixtape/Street`, for the same /// identity-anchored id. See [`CatalogAlbum::is_mixtape`] for why these are /// worth fetching and why they cannot join the Albums grid. /// /// A SEPARATE request rather than a relaxed filter on the query above, and /// the reason is the 100-item page cap. Asking one query for both /// (`-secondarytype:live AND -secondarytype:compilation AND …`, the only /// form MusicBrainz's Lucene actually honours — a bare /// `(-secondarytype:* OR secondarytype:"Mixtape/Street")` returns either /// nothing or mixtapes only) returned Gucci Mane's 87 mixtapes alongside /// just 13 of his 15 studio albums: the mixtapes ate the page and pushed /// two real records off it. Splitting them leaves the album supplement /// byte-for-byte what it was, and costs one request on the DEFERRED /// identity pass, disk cached for a day like every other MusicBrainz call. async fn musicbrainz_mixtapes(&self, artist: &str, mbid: &str) -> Vec { let query = format!( "arid:{mbid} AND primarytype:album AND secondarytype:\"Mixtape/Street\" AND status:official" ); self.musicbrainz_release_groups(artist, mbid, &query, true).await } /// Shared body of the two supplements above: run a release-group search, /// register each group so its tracklist can be resolved later, and return /// it as a catalog row. async fn musicbrainz_release_groups( &self, artist: &str, artist_mbid: &str, query: &str, is_mixtape: bool, ) -> Vec { let query = urlencode(query); let url = format!("https://musicbrainz.org/ws/2/release-group/?query={query}&limit=100&fmt=json"); let Some(body) = self.musicbrainz_json(&url).await else { return Vec::new(); }; let Some(groups) = body .get("release-groups") .and_then(|value| value.as_array()) else { return Vec::new(); }; let mut albums = Vec::new(); let mut registry = self.mb_catalog.lock().await; if registry.len() > 1_000 { registry.clear(); } for group in groups { let Some(release_group_id) = group.get("id").and_then(|value| value.as_str()) else { continue; }; let Some(title) = group.get("title").and_then(|value| value.as_str()) else { continue; }; let title = if normalize(title) == normalize(artist) { group .get("disambiguation") .and_then(|value| value.as_str()) .map(str::trim) .filter(|value| !value.is_empty()) .map(|value| format!("{title} ({value})")) .unwrap_or_else(|| title.to_string()) } else { title.to_string() }; let release_date = group .get("first-release-date") .and_then(|value| value.as_str()) .filter(|value| !value.is_empty()) .map(String::from); let id = synthetic_catalog_id(&format!("release-group:{release_group_id}")); let cover = Some(format!( "https://coverartarchive.org/release-group/{release_group_id}/front-500" )); let entry = MbCatalogAlbum { release_group_id: release_group_id.to_string(), artist_mbid: artist_mbid.to_string(), artist: artist.to_string(), title: title.clone(), release_date: release_date.clone(), cover: cover.clone(), }; // Disk as well as memory: this map is the ONLY way back from a // synthetic id, and it dies with the process (and with the 1000-entry // clear below). See `supplement_cache_path`. self.persist_supplement(id, &entry); registry.insert(id, entry); albums.push(CatalogAlbum { id, title, artist: artist.to_string(), cover, release_date, record_type: Some("album".into()), track_count: None, edition_group: None, is_soundtrack: false, is_live: false, is_mixtape, appears_on: false, }); } albums } /// Resolve an injected MusicBrainz release group to one official edition, /// then return its recording list through the same CatalogAlbumPage shape /// used by Deezer-backed cards. async fn musicbrainz_album( &self, album_id: u64, source: MbCatalogAlbum, ) -> Option { let group_url = format!( "https://musicbrainz.org/ws/2/release-group/{}?inc=releases&fmt=json", source.release_group_id ); let group = self.musicbrainz_json(&group_url).await?; let releases = group.get("releases")?.as_array()?; let release = releases.iter().min_by_key(|release| { let status_rank = u8::from( release.get("status").and_then(|value| value.as_str()) != Some("Official"), ); let country_rank = match release.get("country").and_then(|value| value.as_str()) { Some("US" | "AU" | "XW" | "XE" | "GB") => 0, Some(_) => 1, None => 2, }; ( status_rank, country_rank, release.get("date").and_then(|value| value.as_str()), ) })?; let release_id = release.get("id")?.as_str()?; let release_url = format!("https://musicbrainz.org/ws/2/release/{release_id}?inc=recordings&fmt=json"); let body = self.musicbrainz_json(&release_url).await?; let mut tracks = Vec::new(); for medium in body .get("media") .and_then(|value| value.as_array()) .into_iter() .flatten() { for track in medium .get("tracks") .and_then(|value| value.as_array()) .into_iter() .flatten() { let title = track .get("title") .or_else(|| track.pointer("/recording/title")) .and_then(|value| value.as_str()) .unwrap_or("Unknown track") .to_string(); let source_id = track .get("id") .or_else(|| track.pointer("/recording/id")) .and_then(|value| value.as_str()) .unwrap_or(&title); let duration_secs = track .get("length") .or_else(|| track.pointer("/recording/length")) .and_then(|value| value.as_u64()) .map(|milliseconds| (milliseconds / 1_000) as u32); tracks.push(CatalogTrack { id: synthetic_catalog_id(&format!("track:{source_id}")), title, artist: source.artist.clone(), artist_id: None, rank: None, album: source.title.clone(), album_id: Some(album_id), cover: source.cover.clone(), duration_secs, preview: None, track_no: Some(tracks.len() as u32 + 1), disk_number: None, unreleased: false, // An album tracklist is not a ranked popular list; the // scrobble figures are attached by `apply_track_stats`. listeners: None, plays: None, origin_track_id: None, }); } } let album = CatalogAlbum { id: album_id, title: source.title, artist: source.artist, cover: source.cover, release_date: source.release_date, record_type: Some("album".into()), track_count: Some(tracks.len() as u32), edition_group: None, is_soundtrack: false, is_live: false, // The opened page's own row; tab filing is decided by the grid row // this was opened FROM, which carries the flag from the supplement. is_mixtape: false, appears_on: false, }; Some(CatalogAlbumPage { album, tracks, album_plays: None, album_listeners: None, totals_summed: false }) } /// Fill in the 30-second previews Deezer has no pressing for, from Apple. /// /// Popular is Last.fm's list now, so it routinely contains songs Deezer does /// not sell — and a row built from MusicBrainz has no clip at all. Measured /// over Last.fm's top twenty for six artists, Deezer covers 76 of 120 rows; /// Apple takes that to 104. It is decisive where Deezer's catalogue is /// thin — Neutral Milk Hotel 1 → 19, Slowdive 13 → 20 — and costs nothing /// where it is not, since a page with no gaps makes no requests at all. /// /// BY ALBUM, not by track. iTunes Search is documented at roughly 20 calls /// a minute, and Neutral Milk Hotel alone would want 19 track lookups; one /// collection lookup returns the whole tracklist with a preview on every /// entry instead. The album is found in the Apple catalogue the page has /// ALREADY fetched for release ownership, so the search half costs nothing /// either and this is one request per record with a gap in it. /// /// Apple is the only provider worth adding here. Spotify needs OAuth and /// has been returning a null preview for much of its catalogue since 2024; /// SoundCloud closed app registration years ago; Jamendo is /// Creative-Commons only. Probed against the residual gap, Audius, the /// Internet Archive and Bandcamp returned nothing for any of it. async fn apply_apple_previews( &self, tracks: &mut [CatalogTrack], apple: &AppleArtistCatalog, ) -> usize { // Which records actually have a hole in them, most-wanted first. let mut wanted: Vec = Vec::new(); for track in tracks.iter() { if track.preview.is_some() || track.album.is_empty() { continue; } let key = release_title_identity(&track.album); if !key.is_empty() && !wanted.contains(&key) { wanted.push(key); } } if wanted.is_empty() { return 0; } let started = Instant::now(); let mut previews: HashMap = HashMap::new(); for album_key in wanted.into_iter().take(APPLE_PREVIEW_ALBUMS) { let Some(collection_id) = apple .releases .iter() .find(|release| release.title_norm == album_key) .and_then(|release| release.collection_id) else { continue; }; for song in self.apple_collection_songs(collection_id).await { let (Some(name), Some(preview)) = ( song.get("trackName").and_then(|v| v.as_str()), song.get("previewUrl").and_then(|v| v.as_str()), ) else { continue; }; let key = track_title_identity(name); if !key.is_empty() { previews.entry(key).or_insert_with(|| preview.to_string()); } } } let mut filled = 0; for track in tracks.iter_mut() { if track.preview.is_some() { continue; } if let Some(preview) = previews.get(&track_title_identity(&track.title)) { track.preview = Some(preview.clone()); filled += 1; } } stage_trace( "page.apple_previews", started, &format!("{filled} filled"), ); filled } /// The iTunes song lookup for a collection — the album's tracklist. Shared /// by `apple_album` (full track records for an opened card) and /// `apple_album_track_titles` (edition scoring). Returns the song entries in /// order; empty on any failure. async fn apple_collection_songs(&self, collection_id: u64) -> Vec { let url = format!("https://itunes.apple.com/lookup?id={collection_id}&entity=song&limit=200"); let Some(resp) = self.http.get(&url).send().await.ok() else { return Vec::new(); }; let Some(body) = resp.json::().await.ok() else { return Vec::new(); }; let Some(results) = body.get("results").and_then(|r| r.as_array()) else { return Vec::new(); }; results .iter() .filter(|r| { r.get("wrapperType").and_then(|w| w.as_str()) == Some("track") && r.get("kind").and_then(|k| k.as_str()) == Some("song") }) .cloned() .collect() } /// Resolve an Apple-supplemented album (large synthetic id) to its tracklist /// through the iTunes song lookup, in the same CatalogAlbumPage shape used by /// Deezer- and MusicBrainz-backed cards. iTunes gives a real 30s preview URL. async fn apple_album( &self, album_id: u64, source: AppleCatalogAlbum, ) -> Option { let songs = self.apple_collection_songs(source.collection_id).await; if songs.is_empty() { return None; } let mut tracks = Vec::new(); for song in &songs { let title = song .get("trackName") .and_then(|v| v.as_str()) .unwrap_or("Unknown track") .to_string(); let track_id = song .get("trackId") .and_then(|v| v.as_u64()) .map(|n| n.to_string()) .unwrap_or_else(|| title.clone()); tracks.push(CatalogTrack { id: synthetic_catalog_id(&format!("apple-track:{track_id}")), title, artist: source.artist.clone(), artist_id: None, rank: None, album: source.title.clone(), album_id: Some(album_id), cover: source.cover.clone(), duration_secs: song .get("trackTimeMillis") .and_then(|v| v.as_u64()) .map(|ms| (ms / 1_000) as u32), preview: song .get("previewUrl") .and_then(|v| v.as_str()) .map(String::from), track_no: song .get("trackNumber") .and_then(|v| v.as_u64()) .map(|n| n as u32), disk_number: song .get("discNumber") .and_then(|v| v.as_u64()) .map(|n| n as u32), unreleased: false, // An album tracklist is not a ranked popular list; the // scrobble figures are attached by `apply_track_stats`. listeners: None, plays: None, origin_track_id: None, }); } let album = CatalogAlbum { id: album_id, title: source.title, artist: source.artist, cover: source.cover, release_date: source.release_date, record_type: Some("album".into()), track_count: Some(tracks.len() as u32), edition_group: None, is_soundtrack: false, is_live: false, // The opened page's own row; tab filing is decided by the grid row // this was opened FROM, which carries the flag from the supplement. is_mixtape: false, appears_on: false, }; Some(CatalogAlbumPage { album, tracks, album_plays: None, album_listeners: None, totals_summed: false }) } /// The album's track titles, normalized via `track_identity` for edition /// matching — lets an Apple-supplemented edition be scored against a Deezer /// original the same way Deezer editions are. async fn apple_album_track_titles(&self, collection_id: u64) -> Vec { self.apple_collection_songs(collection_id) .await .iter() .filter_map(|s| s.get("trackName").and_then(|v| v.as_str())) .map(track_identity) .collect() } /// Album details with the full tracklist (and preview URLs). pub async fn catalog_album(&self, album_id: u64) -> Option { // The scrobble table is anchored on an artist MBID. Two ways to reach // one: a Deezer album resolves its artist id and looks the link up, and // a MusicBrainz-supplemented album already knows it outright — it was // fetched by that id. The second path is why Loveless, which Deezer // never carried, now has play counts at all. let mut deezer_artist_id: Option = None; let mut artist_mbid: Option = None; // Memory first, then disk. The registries are process-lifetime maps // written only while an artist page is built, so reaching a // supplemented record any other way — the sidebar's recent places, a // replayed page, or after the 1000-entry clear — used to fall through // to Deezer with a synthetic id it cannot resolve, and report "album // lookup failed". The disk copy makes the id resolvable on its own. let mb_entry = match self.mb_catalog.lock().await.get(&album_id).cloned() { Some(entry) => Some(entry), None => self.load_supplement::(album_id), }; let apple_entry = if mb_entry.is_some() { None } else { match self.apple_catalog.lock().await.get(&album_id).cloned() { Some(entry) => Some(entry), None => self.load_supplement::(album_id), } }; let mut page = if let Some(album) = mb_entry { artist_mbid = Some(album.artist_mbid.clone()); self.musicbrainz_album(album_id, album).await? } else if let Some(album) = apple_entry { // Apple has no MusicBrainz id of its own, but it does know whose // page it was supplementing, so it reaches the scrobble table by // the ordinary Deezer-id route. deezer_artist_id = Some(album.artist_deezer_id); self.apple_album(album_id, album).await? } else { let body = self .deezer(&format!("https://api.deezer.com/album/{album_id}")) .await?; deezer_artist_id = body.pointer("/artist/id").and_then(|id| id.as_u64()); let album = parse_album(&body)?; // Deezer's `/album/{id}` embeds only the first ~25 tracks in // `/tracks/data` and gives no pagination cursor there, so a 50-track // deluxe would show half its tracklist. The dedicated tracks // endpoint paginates properly AND carries `disk_number`, so fetch // the full, disc-aware tracklist from there. let mut tracks: Vec = Vec::new(); let mut next = Some(format!( "https://api.deezer.com/album/{album_id}/tracks?limit=200" )); while let Some(url) = next { let Some(page) = self.deezer(&url).await else { break; }; if let Some(arr) = page.get("data").and_then(|d| d.as_array()) { for t in arr { if let Some(mut track) = parse_track(t) { if track.album.is_empty() { track.album = album.title.clone(); } tracks.push(track); } } } next = page.get("next").and_then(|n| n.as_str()).map(String::from); } // Fallback to the embedded list if the tracks endpoint is empty. if tracks.is_empty() { tracks = body .pointer("/tracks/data") .and_then(|d| d.as_array()) .map(|a| { a.iter() .enumerate() .filter_map(|(i, t)| { let mut track = parse_track(t)?; track.track_no = Some(i as u32 + 1); if track.album.is_empty() { track.album = album.title.clone(); } Some(track) }) .collect() }) .unwrap_or_default(); } CatalogAlbumPage { album, tracks, album_plays: None, album_listeners: None, totals_summed: false } }; // A future-dated album may still have individual songs already out as // singles. Flag the rest unreleased so the UI can block streaming them // while keeping any pre-released single playable. if is_future_or_unresolved_date(page.album.release_date.as_deref()) { self.mark_unreleased_tracks(&mut page).await; } self.apply_track_stats(&mut page, deezer_artist_id, artist_mbid) .await; Some(page) } /// Attach Last.fm scrobble figures to an album's tracklist and totals. /// /// Costs no extra network on the track side: `artist.gettoptracks` is the /// very body the artist page already pulled, disk cached for a day, and at /// [`LASTFM_TRACK_LIMIT`] it reaches an album's deep cuts — measured 56 of /// 57 tracks across five 150k-390k-track catalogues. A track Last.fm has /// never seen keeps `None` and renders as a dash: unknown is not zero. async fn apply_track_stats( &self, page: &mut CatalogAlbumPage, deezer_artist_id: Option, artist_mbid: Option, ) { // A supplemented album carries the MBID already; a Deezer one has to // resolve it from the artist id. Either way the scrobble lookup below // is identical, and both share one cached Last.fm body per artist. let mbid = match artist_mbid { Some(mbid) => mbid, None => { let Some(artist_id) = deezer_artist_id else { return; }; let Some(mbid) = self.deezer_linked_mbid(artist_id).await else { return; }; mbid } }; let Some(ranks) = self.lastfm_listener_ranks(&mbid).await else { return; }; let ranks = ranks.by_key; let started = Instant::now(); let mut matched = 0; let mut summed_plays = 0_u64; let mut peak_plays = 0_u64; let mut peak_listeners = 0_u64; for track in page.tracks.iter_mut() { let key = track_title_identity(&track.title); let Some(stats) = ranks.get(&key).copied() else { continue; }; matched += 1; track.plays = (stats.plays > 0).then_some(stats.plays); track.listeners = (stats.listeners > 0).then_some(stats.listeners); summed_plays += stats.plays; peak_plays = peak_plays.max(stats.plays); peak_listeners = peak_listeners.max(stats.listeners); } // Last.fm's own album entry is the authoritative total when it exists // AND is describing the same population as the tracklist above it. // // Those two things come apart on editions. An album's `playcount` counts // scrobbles tagged to THAT release page, while a track's counts every // scrobble of the song across every release it appears on. Measured on // Weezer's Blue Album deluxe: the album page reports 89k plays while its // own opening track reports 7.2M — a rail total two orders of magnitude // below the first row beneath it. So a total that cannot even cover its // biggest track is an edition-page artifact, not this record's figure; // sum the tracklist instead and label it. let provider = self .lastfm_album_totals(&page.album.artist, &page.album.title) .await .filter(|(plays, _)| *plays >= peak_plays); match provider { Some((plays, listeners)) => { page.album_plays = Some(plays); page.album_listeners = Some(listeners); } None if matched > 0 => { page.album_plays = Some(summed_plays); page.album_listeners = Some(peak_listeners); page.totals_summed = true; } None => {} } stage_trace( "album.track_stats", started, &format!("{matched}/{} matched", page.tracks.len()), ); } /// Last.fm's scrobble and listener totals for one record, as /// `(playcount, listeners)`. Disk cached like every other Last.fm call. async fn lastfm_album_totals(&self, artist: &str, album: &str) -> Option<(u64, u64)> { if artist.is_empty() || album.is_empty() { return None; } let url = format!( "https://ws.audioscrobbler.com/2.0/?method=album.getinfo&artist={}&album={}\ &autocorrect=1&api_key={LASTFM_KEY}&format=json", urlencode(artist), urlencode(album) ); let body = self.lastfm_json(&url).await?; // Both arrive as JSON strings, and a record nobody has scrobbled is a // real zero rather than a missing field. let plays = body .pointer("/album/playcount") .and_then(|v| v.as_str()) .and_then(|v| v.parse().ok())?; let listeners = body .pointer("/album/listeners") .and_then(|v| v.as_str()) .and_then(|v| v.parse().ok())?; Some((plays, listeners)) } /// Flag each track on an upcoming album as unreleased unless it is already /// out — either it carries its own Deezer preview, or a released single by /// the same artist with the same title exists. Checks run concurrently. async fn mark_unreleased_tracks(&self, page: &mut CatalogAlbumPage) { let artist = page.album.artist.clone(); let flags = futures::future::join_all( page.tracks.iter().map(|track| { let artist = artist.clone(); let title = track.title.clone(); let has_preview = track.preview.is_some(); async move { !has_preview && !self.deezer_has_released_single(&artist, &title).await } }), ) .await; for (track, unreleased) in page.tracks.iter_mut().zip(flags) { track.unreleased = unreleased; } } /// Whether Deezer already serves a playable (previewable) track by this /// artist with this title — the signal that a song from an upcoming album /// has been released early as a single. async fn deezer_has_released_single(&self, artist: &str, title: &str) -> bool { let query = urlencode(&format!( "artist:\"{}\" track:\"{}\"", escape(artist), escape(title) )); let url = format!("https://api.deezer.com/search/track?q={query}&limit=5"); let Some(body) = self.deezer(&url).await else { return false; }; let want_title = normalize(title); let want_artist = normalize(artist); body.get("data") .and_then(|data| data.as_array()) .is_some_and(|items| { items.iter().any(|item| { let title_ok = item .get("title") .and_then(|value| value.as_str()) .map(normalize) .as_deref() == Some(want_title.as_str()); let artist_ok = item .pointer("/artist/name") .and_then(|value| value.as_str()) .map(normalize) .as_deref() == Some(want_artist.as_str()); let playable = item .get("preview") .and_then(|value| value.as_str()) .is_some_and(|preview| !preview.is_empty()); title_ok && artist_ok && playable }) }) } async fn mb_throttle(&self) { let mut last = self.mb_gate.lock().await; let since = last.elapsed(); if since < Duration::from_secs(1) { tokio::time::sleep(Duration::from_secs(1) - since).await; } *last = Instant::now(); } /// Hold a Deezer request until the sliding 5-second window has room. The /// window is re-checked under the lock after every sleep, so a burst that /// wakes together still leaves at most [`DEEZER_RATE_BUDGET`] requests per /// window rather than all of them racing back in at once. async fn deezer_throttle(&self) { loop { let wait = { let mut window = self.deezer_gate.lock().await; let now = Instant::now(); while window .front() .is_some_and(|at| now.duration_since(*at) >= DEEZER_RATE_WINDOW) { window.pop_front(); } if window.len() < DEEZER_RATE_BUDGET { window.push_back(now); return; } match window.front() { Some(oldest) => DEEZER_RATE_WINDOW - now.duration_since(*oldest), None => continue, } }; tokio::time::sleep(wait).await; } } /// Hold a Wikipedia request until the sliding window has room, mirroring /// [`Self::deezer_throttle`]. Returns `false` — do not send — when the IP /// is inside a server-imposed penalty (429 `Retry-After`) longer than the /// fail-fast bound: sending would extend the ban, and waiting a minute /// inside one album-info call would hold an open expander for nothing. /// Short cooldowns are waited out so a paced retry succeeds. async fn wiki_throttle(&self) -> bool { loop { let cooldown = { let cooldown = self.wiki_cooldown.lock().await; cooldown.and_then(|until| until.checked_duration_since(Instant::now())) }; if let Some(remaining) = cooldown { if remaining > WIKI_COOLDOWN_FAIL_FAST { return false; } tokio::time::sleep(remaining).await; continue; } let wait = { let mut window = self.wiki_gate.lock().await; let now = Instant::now(); while window .front() .is_some_and(|at| now.duration_since(*at) >= WIKI_RATE_WINDOW) { window.pop_front(); } if window.len() < WIKI_RATE_BUDGET { window.push_back(now); return true; } match window.front() { Some(oldest) => WIKI_RATE_WINDOW - now.duration_since(*oldest), None => continue, } }; tokio::time::sleep(wait).await; } } /// GET a Last.fm endpoint, paced, disk cached, and with its in-band errors /// treated as failures. /// /// Last.fm answers `HTTP 200` with `{"error":6,...}` in the body for a /// lookup that found nothing, exactly like Deezer signals quota inside a /// 200. Returning that body as success would hand the caller an object with /// no `toptracks` and cache it for a day. async fn lastfm_json(&self, url: &str) -> Option { let body = self.cached_json(url, false).await?; if let Some(code) = body.get("error") { net_trace(url, &format!("lastfm-error {code}")); return None; } Some(body) } /// Everything the page needs from the MBID-anchored providers: the MBID is /// resolved once, then both providers are fetched together. /// /// Deliberately takes only the artist id so it can ride in the same /// `tokio::join!` as the Deezer fetches instead of extending the critical /// path behind them. Every hop it makes is disk cached for a day, so a /// revisit pays nothing, and a cold page overlaps it with work that already /// takes longer. /// /// No MusicBrainz entry for this Deezer artist means no anchor for either /// provider, and the caller falls back to Deezer's own ordering and dates — /// which is exactly what the page did before any of this existed. async fn artist_supplements( &self, artist_id: u64, with_recordings: bool, ) -> ArtistSupplements { // Traced separately from the joined legs below: this hop is serial — // nothing downstream can start without the MBID — and it was invisible // in the stage audit that found the overview's critical path, because // every leg it gates IS traced and the ~1.5s cold gap between them and // the join total belonged to no stage. let mbid_started = Instant::now(); let Some(mbid) = self.deezer_linked_mbid(artist_id).await else { stage_trace("supplements.linked_mbid", mbid_started, "no-mbid"); return ArtistSupplements::default(); }; stage_trace("supplements.linked_mbid", mbid_started, ""); let (ranks, (first_released, musicbrainz_unavailable), recordings) = tokio::join!( self.lastfm_listener_ranks(&mbid), self.musicbrainz_first_release_dates(&mbid), // Only the full page pays for this: it is one or two more requests // behind the 1 req/s gate, and the overview exists to paint fast. // The overview keeps ordering Deezer's pool, and the full page // replaces that list wholesale a moment later. async { if with_recordings { self.musicbrainz_recordings(&mbid).await } else { HashMap::new() } }, ); let ranks = ranks.unwrap_or_default(); ArtistSupplements { listeners: ranks.by_key, ranked: ranks.ranked, recordings, first_released, musicbrainz_unavailable, } } /// First-release date of every release GROUP this artist has, keyed by /// [`album_title_identity`]. /// /// The browse endpoint is used rather than search: it takes the MBID /// directly (no query parsing, no relevance ranking) and returns the /// `first-release-date` this whole feature exists for. Compilations, live /// records and soundtracks are all fetched too — they are real releases /// whose dates are just as wrong on Deezer, and excluding them here would /// leave a compilation dated by its reissue. /// /// Returns `(facts, unavailable)`. The flag is the THIRD state, and it /// exists because the first two are indistinguishable without it: an /// artist MusicBrainz has nothing for answers with an empty /// `release-groups` ARRAY, while a failed request answers with nothing at /// all. Collapsing both into an empty map is what let one blip silently /// strip a page of its dates and its attribution. /// /// A `None` here is always a failure, never absence — the mbid was /// resolved FROM MusicBrainz moments earlier (see /// [`Self::deezer_linked_mbid`]), so the artist provably exists there. /// /// The first page is retried before that verdict is reached, because the /// failure this guards against is overwhelmingly transient: a 503 from the /// 1 req/s limiter, or our own probes having throttled the IP. A later page /// failing is reported as partial rather than unavailable — the dates /// already collected are real, and the discography is date-sorted so the /// tail matters least. async fn musicbrainz_first_release_dates( &self, mbid: &str, ) -> (HashMap, bool) { let started = Instant::now(); let mut dates: HashMap = HashMap::new(); for page in 0..MB_RELEASE_GROUP_PAGES { let offset = page * MB_RELEASE_GROUP_PAGE; // Byte-identical to the URL `musicbrainz_release_titles` builds, so // the two passes share one cached body instead of each paying the // 1 req/s gate for the same page. let url = format!( "https://musicbrainz.org/ws/2/release-group?artist={mbid}&limit=100&offset={offset}&fmt=json" ); let mut answer = self.musicbrainz_json(&url).await; if answer.is_none() && page == 0 { for attempt in 0..MB_RETRY_ATTEMPTS { tokio::time::sleep(MB_RETRY_BACKOFF * (attempt + 1)).await; net_trace(&url, &format!("mb-retry {}", attempt + 1)); answer = self.musicbrainz_json(&url).await; if answer.is_some() { break; } } } let Some(body) = answer else { if page == 0 { log::warn!( "musicbrainz did not answer for {mbid}; \ album dates and original-album attribution are unavailable, not absent" ); stage_trace("musicbrainz.first_release_dates", started, "UNAVAILABLE"); return (HashMap::new(), true); } break; }; let Some(groups) = body.get("release-groups").and_then(|v| v.as_array()) else { // A 200 whose shape we do not recognise is also not an answer // about the artist. Same reasoning as above on the first page. if page == 0 { stage_trace("musicbrainz.first_release_dates", started, "MALFORMED"); return (HashMap::new(), true); } break; }; let returned = groups.len(); let total = body .get("release-group-count") .and_then(|c| c.as_u64()) .unwrap_or(returned as u64) as usize; for group in groups { merge_release_group_facts(&mut dates, group); } if offset + returned >= total || returned == 0 { break; } } stage_trace( "musicbrainz.first_release_dates", started, &format!("{} groups", dates.len()), ); (dates, false) } /// Which record each of this artist's recordings comes from, keyed by /// [`track_title_identity`]. /// /// This is what lets Popular show a song Deezer does not sell. Deezer has /// none of Loveless — 8 of My Bloody Valentine's Last.fm top ten are absent /// from its catalogue outright — so a row for "When You Sleep" has no /// Deezer album, cover or duration to inherit. MusicBrainz has all three. /// /// One browse carries the whole discography: `inc=recordings` returns every /// release WITH its tracklist, and `release-groups` names the work each /// pressing belongs to. Measured on My Bloody Valentine, one 208 KB request /// covered 64 releases folding to 27 works — versus one request per album /// behind the 1 req/s gate, which is the reason this is a browse and not a /// lookup per supplemented row. /// /// A recording usually appears on several pressings, so the winner is /// chosen the way [`apply_original_album_attribution`] chooses one: a plain /// album outranks anything carrying a secondary type, and the earliest date /// breaks the tie. Without the type test the compilations win, because a /// hits package is dated earlier than nothing and later than everything. async fn musicbrainz_recordings(&self, mbid: &str) -> HashMap { let started = Instant::now(); let mut found: HashMap = HashMap::new(); for page in 0..MB_RECORDING_PAGES { let offset = page * MB_RECORDING_PAGE; // Filtered to official ALBUM releases, which is both cheaper and // more relevant: unfiltered, Radiohead has 1141 releases and the // two-page window saw 18% of them, mostly singles and bootleg // pressings. `type=album&status=official` leaves 274, so the same // two pages now cover 73% of the releases that could actually own a // popular song. // // It does not fix the tail — Queen still has 1119 official album // pressings — and that is accepted rather than paged around. This // map exists to place songs Deezer does not carry, and an artist // with a thousand pressings is one Deezer carries exhaustively: // measured, the relabel pass reports 0 for both Queen and // Radiohead. Paging to reach them would cost 11 requests behind the // 1 req/s gate to change nothing. let url = format!( "https://musicbrainz.org/ws/2/release?artist={mbid}&type=album&status=official\ &inc=recordings+release-groups&limit={MB_RECORDING_PAGE}&offset={offset}&fmt=json" ); let Some(body) = self.musicbrainz_json(&url).await else { break; }; let Some(releases) = body.get("releases").and_then(|v| v.as_array()) else { break; }; let returned = releases.len(); for release in releases { let Some(group) = release.get("release-group") else { continue; }; let Some(album_title) = group.get("title").and_then(|v| v.as_str()) else { continue; }; let Some(release_group_id) = group.get("id").and_then(|v| v.as_str()) else { continue; }; // Mixtape/Street counts as a studio record here for the same // reason it gets its own tab rather than being thrown away: // Exmilitary is Death Grips' debut and Acid Rap is Chance the // Rapper's best-scrobbled record. Excluding it left "Guillotine" // attributed to the SINGLE named after it. Every other secondary // type — Compilation, Live, Soundtrack, Remix — still disqualifies, // which is the whole point of the test. let secondary: Vec<&str> = group .get("secondary-types") .and_then(|v| v.as_array()) .map(|types| types.iter().filter_map(|t| t.as_str()).collect()) .unwrap_or_default(); let plain_album = group.get("primary-type").and_then(|v| v.as_str()) == Some("Album") && secondary .iter() .all(|kind| kind.eq_ignore_ascii_case("Mixtape/Street")); let release_date = group .get("first-release-date") .and_then(|v| v.as_str()) .filter(|v| !v.is_empty()) .map(String::from); let tracks = release .get("media") .and_then(|v| v.as_array()) .into_iter() .flatten() .filter_map(|medium| medium.get("tracks").and_then(|v| v.as_array())) .flatten(); for track in tracks { let Some(title) = track.get("title").and_then(|v| v.as_str()) else { continue; }; let key = track_title_identity(title); if key.is_empty() { continue; } let candidate = MbRecording { release_group_id: release_group_id.to_string(), album_title: album_title.to_string(), title: title.to_string(), release_date: release_date.clone(), length_ms: track.get("length").and_then(|v| v.as_u64()), plain_album, }; match found.get(&key) { Some(held) if !candidate.outranks(held) => {} _ => { found.insert(key, candidate); } } } } if returned < MB_RECORDING_PAGE { break; } } stage_trace( "musicbrainz.recordings", started, &format!("{} recordings", found.len()), ); found } /// All-time Last.fm listener counts for one artist's tracks, keyed by /// [`track_title_identity`]. /// /// Identity is ANCHORED, never name-matched, in two steps — the project /// rule is Deezer id → MBID → declared links, else show nothing: /// /// 1. `mbid=` — Last.fm's own MusicBrainz index. We send an identifier and /// Last.fm resolves it, so no name is involved and there is nothing to /// guard. This alone covers 36 of 52 sampled artists, including five of /// the six smallest in the local library. /// 2. If Last.fm's index has no entry (its MBID coverage is partial — it /// returns error 6 for King Gizzard even given the correct MBID), fall /// back to the last.fm URL **MusicBrainz itself declares** for this /// artist, and require the response's own `artist.url` to echo that same /// declared identifier back. That is a declared-link anchor, not a name /// guess: the identifier comes from MusicBrainz, and the guard proves /// Last.fm agrees it is the same page. /// /// Nothing else is attempted. Querying Last.fm with the artist's *name* was /// measured and rejected: it reaches only 2 more artists out of 52 while /// re-opening the exact namesake hole the identity rule exists to close. /// /// Step 2 costs no extra MusicBrainz traffic — it reads the same /// `artist_detail_url` body [`Self::artist_socials`] already fetches, which /// is memory and disk cached. async fn lastfm_listener_ranks(&self, mbid: &str) -> Option { let started = Instant::now(); let by_mbid = format!( "https://ws.audioscrobbler.com/2.0/?method=artist.gettoptracks\ &mbid={mbid}&limit={LASTFM_TRACK_LIMIT}&autocorrect=1&api_key={LASTFM_KEY}&format=json" ); let mut body = self.lastfm_json(&by_mbid).await; if body.is_none() { let declared = self.musicbrainz_declared_lastfm(mbid).await?; // The identifier is what MusicBrainz declared, decoded; querying with // it is not a name guess. Last.fm matches it case-insensitively, so // the same normalized form serves as both the query and the value the // guard below compares against. let identifier = lastfm_artist_identifier(&declared)?; let by_name = format!( "https://ws.audioscrobbler.com/2.0/?method=artist.gettoptracks\ &artist={}&limit={LASTFM_TRACK_LIMIT}&autocorrect=1&api_key={LASTFM_KEY}&format=json", urlencode(&identifier) ); let candidate = self.lastfm_json(&by_name).await?; // THE GUARD. `autocorrect=1` makes Last.fm answer with the page it // actually resolved to rather than echoing the query -- a misspelled // "Radiohed" comes back as .../music/Radiohead. So an echoed // identifier that matches what MusicBrainz declared is Last.fm // agreeing on identity. With autocorrect=0 it parrots the query and // this check would be worthless. let echoed = candidate .pointer("/toptracks/track/0/artist/url") .and_then(|url| url.as_str()) .and_then(lastfm_artist_identifier); if echoed.as_deref() != Some(identifier.as_str()) { net_trace(&by_name, "lastfm-identity-rejected"); return None; } body = Some(candidate); } // The ORDER is kept as well as the counts. Popular used to take // Deezer's fifty and merely re-sort them, so a song Last.fm ranks first // could not appear at all if Deezer did not sell it — which is the // normal case for an artist whose catalogue Deezer carries badly. Eight // of My Bloody Valentine's Last.fm top ten are absent from Deezer // entirely, so its Popular showed b-sides and soundtrack cuts. Keeping // this list means Last.fm decides WHAT is popular and Deezer is only // consulted for a playable pressing. let entries: Vec<(String, String, TrackStats)> = body? .pointer("/toptracks/track")? .as_array()? .iter() .filter_map(|track| { let title = track.get("name")?.as_str()?; // Counts arrive as JSON strings, not numbers. let listeners = track.get("listeners")?.as_str()?.parse().ok()?; // Plays ride the same body. Absent on a sparse entry, which is // a zero here and a dash in the UI, never a fabricated count. let plays = track .get("playcount") .and_then(|value| value.as_str()) .and_then(|value| value.parse().ok()) .unwrap_or(0); let key = track_title_identity(title); (!key.is_empty()).then_some(( key, title.to_string(), TrackStats { plays, listeners }, )) }) .collect(); let ranks = fold_lastfm_duplicates(entries); stage_trace("lastfm.ranks", started, &format!("{} tracks", ranks.by_key.len())); (!ranks.by_key.is_empty()).then_some(ranks) } /// The `last.fm` URL MusicBrainz declares for this artist, if any. Shares /// the cached `inc=url-rels` body the socials row already reads. async fn musicbrainz_declared_lastfm(&self, mbid: &str) -> Option { let body = self.musicbrainz_json(&artist_detail_url(mbid)).await?; body.get("relations")? .as_array()? .iter() .find(|relation| { relation.get("type").and_then(|kind| kind.as_str()) == Some("last.fm") }) .and_then(|relation| relation.pointer("/url/resource")) .and_then(|url| url.as_str()) .map(String::from) } /// Hold a Last.fm request until its sliding window has room. async fn lastfm_throttle(&self) { loop { let wait = { let mut window = self.lastfm_gate.lock().await; let now = Instant::now(); while window .front() .is_some_and(|at| now.duration_since(*at) >= LASTFM_RATE_WINDOW) { window.pop_front(); } if window.len() < LASTFM_RATE_BUDGET { window.push_back(now); return; } match window.front() { Some(oldest) => LASTFM_RATE_WINDOW - now.duration_since(*oldest), None => continue, } }; tokio::time::sleep(wait).await; } } /// Hold a Google News request until its sliding window has room. async fn news_throttle(&self) { loop { let wait = { let mut window = self.news_gate.lock().await; let now = Instant::now(); while window .front() .is_some_and(|at| now.duration_since(*at) >= NEWS_RATE_WINDOW) { window.pop_front(); } if window.len() < NEWS_RATE_BUDGET { window.push_back(now); return; } match window.front() { Some(oldest) => NEWS_RATE_WINDOW - now.duration_since(*oldest), None => continue, } }; tokio::time::sleep(wait).await; } } /// Headlines about one artist, from Google News RSS. /// /// The query is the QUOTED artist name plus the word `music` — measured on /// 2026-08-04: a bare ambiguous name scored 0/10 relevant items ("Girls"), /// the quoted name with a corroborating token scored 8/8 and an easy name /// 10/10. Keyless and undocumented, hence the conservative pacer and the /// default UA (the contact UA is for providers with a policy asking for /// it; scraped surfaces see the same UA a browser fetch would send). /// /// Sorting, the freshness window and the review filter are display policy /// and live in the frontend; this returns what the feed said, newest /// first, capped at a dozen. pub async fn artist_news(&self, artist: &str) -> Vec { let url = format!( "https://news.google.com/rss/search?q={}&hl=en-US&gl=US&ceid=US:en", urlencode(&format!("\"{artist}\" music")) ); let now_secs = || { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0) }; { let cache = self.news_cache.lock().await; if let Some((at, items)) = cache.get(&url) { if at.elapsed() < NEWS_TTL { return items.clone(); } } } // Disk: survives a restart. The generic reader enforces the 24h cache // TTL; news wants 6h, so the fetch time rides inside the body and is // checked here. if let Some(path) = self.disk_cache_path(&url) { if let Some(body) = read_disk_cache(&path, &url) { let fresh = body .get("at") .and_then(|v| v.as_u64()) .is_some_and(|at| now_secs().saturating_sub(at) < NEWS_TTL.as_secs()); if fresh { if let Some(items) = body .get("items") .and_then(|v| serde_json::from_value::>(v.clone()).ok()) { let mut cache = self.news_cache.lock().await; cache.insert(url.clone(), (Instant::now(), items.clone())); return items; } } } } self.news_throttle().await; let started = Instant::now(); let text = match self.http.get(&url).send().await { Ok(resp) if resp.status().is_success() => resp.text().await.unwrap_or_default(), Ok(resp) => { net_trace(&url, &format!("news-status {}", resp.status())); return Vec::new(); } Err(e) => { net_trace(&url, &format!("news-error {e}")); return Vec::new(); } }; let items = filter_news(parse_news_rss(&text), artist, started); if let Some(path) = self.disk_cache_path(&url) { let body = serde_json::json!({ "at": now_secs(), "items": items }); write_disk_cache(&path, &url, &body); } let mut cache = self.news_cache.lock().await; cache.insert(url, (Instant::now(), items.clone())); items } /// Hold a Discogs request until its sliding window has room. async fn discogs_throttle(&self) { loop { let wait = { let mut window = self.discogs_gate.lock().await; let now = Instant::now(); while window .front() .is_some_and(|at| now.duration_since(*at) >= DISCOGS_RATE_WINDOW) { window.pop_front(); } if window.len() < DISCOGS_RATE_BUDGET { window.push_back(now); return; } match window.front() { Some(oldest) => DISCOGS_RATE_WINDOW - now.duration_since(*oldest), None => continue, } }; tokio::time::sleep(wait).await; } } /// Canonicalize an (artist, album) guess via MusicBrainz release search. pub async fn lookup_album(&self, artist: &str, album: &str) -> Option { self.mb_throttle().await; let query = format!( "release:\"{}\" AND artist:\"{}\"", escape(album), escape(artist) ); let url = format!( "https://musicbrainz.org/ws/2/release/?query={}&fmt=json&limit=5", urlencode(&query) ); let body: serde_json::Value = self.http.get(&url).send().await.ok()?.json().await.ok()?; let releases = body.get("releases")?.as_array()?; let best = releases .iter() .filter(|r| r.get("score").and_then(|s| s.as_i64()).unwrap_or(0) >= 85) .max_by_key(|r| r.get("score").and_then(|s| s.as_i64()).unwrap_or(0))?; let canonical_album = best.get("title")?.as_str()?.to_string(); let canonical_artist = best .pointer("/artist-credit/0/name") .and_then(|v| v.as_str()) .unwrap_or(artist) .to_string(); let release_date = best.get("date").and_then(|d| d.as_str()).map(String::from); let year = release_date .as_deref() .and_then(|d| d.get(..4)) .and_then(|y| y.parse().ok()); let mbid = best.get("id").and_then(|v| v.as_str()).map(String::from); let rg_mbid = best .pointer("/release-group/id") .and_then(|v| v.as_str()) .map(String::from); Some(AlbumMeta { artist: canonical_artist, album: canonical_album, release_date, year, release_mbid: mbid, release_group_mbid: rg_mbid, }) } /// Fetch front-cover art at the highest real resolution available. /// /// Candidates are gathered from three sources, decoded, and ranked: /// - **Deezer** `cover_xl` (1000×1000) — the exact artwork the catalog /// browser displays, so it doubles as the *reference* image. /// - **Apple/iTunes** at original upload resolution (mzstatic renders any /// requested size; `100000x100000-999` returns the original, commonly /// 3000×3000+). Artist- and album-title-verified. /// - **Cover Art Archive** original `front` (not the 500px thumb) by /// release then release-group MBID. /// /// Selection: when the Deezer reference exists, only candidates whose /// perceptual hash matches it may win — the library/tags then always carry /// the SAME artwork the user saw while browsing, just at whichever source /// serves it biggest. Without a reference, the largest verified candidate /// wins. Deezer sizes are never inflated past `xl`: the CDN silently /// upscales larger requests, and fake pixels would outrank real ones. pub async fn fetch_art( &self, release_mbid: Option<&str>, release_group_mbid: Option<&str>, artist: &str, album: &str, ) -> Option> { let (deezer, apple, caa) = tokio::join!( self.deezer_cover_candidate(artist, album), self.apple_cover_candidate(artist, album), self.caa_cover_candidates(release_mbid, release_group_mbid), ); let mut candidates: Vec = Vec::new(); candidates.extend(deezer); candidates.extend(apple); candidates.extend(caa); if candidates.is_empty() { return None; } let best = pick_best_cover(&candidates)?; let chosen = &candidates[best]; log::info!( "cover art for {artist} - {album}: {} {}x{} ({} KB) from {} candidate(s)", chosen.source, chosen.width, chosen.height, chosen.bytes.len() / 1024, candidates.len() ); Some(chosen.bytes.as_ref().clone()) } /// Resolve the best same-artwork cover for a catalog release, ranking /// official sources (Apple, then Deezer) above Cover Art Archive and gating /// every candidate against the known-correct reference cover the catalog /// already displays. This is what makes remix/karaoke/wrong-artist art /// impossible to select: a candidate must both pass artist verification and /// be perceptually the SAME image as the reference. Returns a full report /// (reference + every candidate + the pick) for diagnostics and the UI. pub async fn cover_report( &self, artist: &str, title: &str, reference_url: &str, ) -> CoverReport { // The reference is whatever cover the catalog currently shows for this // release (correct edition, delivered with the album). Label its source // from the host so the report reads clearly. let ref_source: &'static str = if reference_url.contains("dzcdn") { "deezer" } else if reference_url.contains("coverartarchive") { "cover-art-archive" } else { "current" }; let reference = match self.get_image_shared(reference_url).await { Some(bytes) => { decode_cover_async(bytes, ref_source, true, reference_url.to_string()).await } None => None, }; let mut candidates: Vec = Vec::new(); // Deezer at its native max: request 1500 (the CDN caps at the real // source, commonly 1200–1500), never inflated beyond what exists. if reference_url.contains("dzcdn") { let max_url = reference_url.replace("/1000x1000-", "/1500x1500-"); if max_url != reference_url { if let Some(bytes) = self.get_image_shared(&max_url).await { if let Some(c) = decode_cover_async(bytes, "deezer-max", false, max_url).await { candidates.push(c); } } } } // Apple — artist- and title-verified inside apple_cover_candidate, so a // karaoke/tribute label with the same song title never qualifies. if let Some(c) = self.apple_cover_candidate(artist, title).await { candidates.push(c); } // Cover Art Archive via MusicBrainz-verified release/release-group MBIDs. // SKIP_MB models the proposed hot path: no per-album MusicBrainz search. if std::env::var("SKIP_MB").is_err() { if let Some(meta) = self.lookup_album(artist, title).await { let caa = self .caa_cover_candidates( meta.release_mbid.as_deref(), meta.release_group_mbid.as_deref(), ) .await; candidates.extend(caa); } } let ref_hash = reference.as_ref().map(|r| r.dhash); let distance_of = |c: &CoverCandidate| -> Option { ref_hash.map(|h| (c.dhash ^ h).count_ones()) }; // Only artwork that MATCHES the reference is eligible — so once the // perceptual gate has confirmed it's the SAME image, the largest is // simply the best copy of the correct art. Official source (Apple, then // Deezer) breaks ties at equal resolution. let chosen_idx = candidates .iter() .enumerate() .filter(|(_, c)| { distance_of(c) .map(|d| d <= COVER_SAME_ART_MAX_DISTANCE) .unwrap_or(true) }) .max_by_key(|(_, c)| { ( u64::from(c.width) * u64::from(c.height), cover_source_priority(c.source), ) }) .map(|(i, _)| i); let make_info = |c: &CoverCandidate, chosen: bool| -> CandInfo { let distance = distance_of(c); CandInfo { source: c.source.to_string(), url: c.url.clone(), width: c.width, height: c.height, distance, eligible: distance .map(|d| d <= COVER_SAME_ART_MAX_DISTANCE) .unwrap_or(true), chosen, } }; let candidate_infos: Vec = candidates .iter() .enumerate() .map(|(i, c)| make_info(c, Some(i) == chosen_idx)) .collect(); // If no candidate beats the reference, the reference itself is the pick. let (chosen_source, chosen_url) = match chosen_idx { Some(i) => ( Some(candidates[i].source.to_string()), Some(candidates[i].url.clone()), ), None => ( reference.as_ref().map(|r| r.source.to_string()), reference.as_ref().map(|r| r.url.clone()), ), }; CoverReport { artist: artist.to_string(), title: title.to_string(), reference: reference .as_ref() .map(|r| make_info(r, chosen_idx.is_none())), candidates: candidate_infos, chosen_source, chosen_url, } } /// Identify one album record against the providers — the import review's /// on-demand "which record is this?" (research/import/enrich.mjs /// providerLookup, ported). Deezer is asked down the cleaned try-list /// until something answers; when its answer is missing — or is a /// DIFFERENT record (Bastard → Cherry Bomb: free release, Deezer search /// happily answers with another album) — MusicBrainz answers for the /// album itself, the same supplement the artist page already leans on. /// A title that is merely an edition of ours does NOT read as wrong /// here: containment either way counts as a match, and the edition rule /// lives with the judge, not the fetcher. /// /// `folder` is the record's folder name, asked after its own claim — the /// second witness for a record whose tags are wrong. See /// [`identify_titles`]. `shape` is everything else the scan knows about the /// record, which is what the discography pass compares against when no /// phrase can find it ([`RecordShape`]). pub async fn identify_album( &self, artist: &str, raw_title: &str, folder: Option<&str>, shape: &RecordShape, ) -> IdentifyResult { let tracks = shape.tracks; let tries = identify_titles(artist, raw_title, folder); let mut candidate = None; let mut used = None; let mut mb_failed = false; for t in &tries { if let Some(c) = self.deezer_identify(artist, t).await { candidate = Some(c); used = Some(t.clone()); break; } } let deezer_wrong = candidate .as_ref() .is_some_and(|c| !tries.iter().any(|t| normx_overlap(&c.title, t))); if candidate.is_none() || deezer_wrong { let (found, failed) = self.mb_identify(artist, &tries, tracks).await; mb_failed = failed; if let Some((mb, t)) = found { candidate = Some(mb); used = Some(t); } // MB missing too leaves the wrong Deezer answer in place ON // PURPOSE: the judge shows it as "Wrong record" honestly, which // beats an empty card (Jenny Death stays wrong, deliberately). } // Still nothing: stop SEARCHING for the title and start COMPARING the // record. Both passes above ask a provider to find a phrase, so a // record whose name no release carries cannot be found by either, // however it is cleaned — "1989 Pauls Boutique", or "Rock Bottom 7\"" // for a single the label called "Rock Bottom / Octopus". Comparing the // record against the artist's own discography asks a different // question, and one that can use every fact we have about it rather // than just its name. // // Only when there is NOTHING: a wrong Deezer answer is left standing // for the judge to call wrong, exactly as above. let mut matched_by = None; if candidate.is_none() { if let Some((c, matched, why)) = self .discography_identify(artist, raw_title, &tries, shape) .await { candidate = Some(c); used = Some(matched); matched_by = Some(why.to_string()); } } IdentifyResult { tries, used, // Nothing found AND a request failed: say so, rather than letting // a 503 be filed as an answer about the record. incomplete: candidate.is_none() && mb_failed, candidate, deezer_wrong, matched_by, } } /// Deezer's answer for one cleaned query: first search hit, hydrated /// with the album detail (label/date/genre) and the full paginated /// disc-aware tracklist — `/album/{id}` embeds only the first ~25. async fn deezer_identify(&self, artist: &str, t: &str) -> Option { let q = urlencode(&format!("{artist} {t}")); let found = self .deezer(&format!("https://api.deezer.com/search/album?q={q}&limit=3")) .await?; let hit = found.get("data")?.as_array()?.first()?.clone(); let id = hit.get("id")?.as_u64()?; self.deezer_album_by_id(id, Some(&hit)).await } /// One album, fully hydrated: detail (label/date/genre) plus the full /// paginated disc-aware tracklist — `/album/{id}` embeds only the first /// ~25. `hit` is the search result when there was one; the album detail /// carries the same fields, so the catalogue path passes None. async fn deezer_album_by_id( &self, id: u64, hit: Option<&serde_json::Value>, ) -> Option { let full = self.deezer(&format!("https://api.deezer.com/album/{id}")).await; let hit = match hit { Some(h) => h.clone(), None => full.clone()?, }; let mut tracklist: Vec = Vec::new(); let mut next = Some(format!("https://api.deezer.com/album/{id}/tracks?limit=200")); while let Some(url) = next { let Some(page) = self.deezer(&url).await else { break }; for tr in page.get("data").and_then(|d| d.as_array()).into_iter().flatten() { tracklist.push(IdentifyTrack { n: tr.get("track_position").and_then(|v| v.as_u64()).map(|v| v as u32), title: tr .get("title") .and_then(|v| v.as_str()) .unwrap_or_default() .to_string(), secs: tr.get("duration").and_then(|v| v.as_u64()).map(|v| v as u32), }); } next = page .get("next") .and_then(|v| v.as_str()) .map(String::from); } let s = |v: &serde_json::Value, k: &str| { v.get(k).and_then(|x| x.as_str()).map(String::from) }; Some(IdentifyCandidate { id: id.to_string(), source: "Deezer".into(), title: s(&hit, "title").unwrap_or_default(), artist: hit .pointer("/artist/name") .and_then(|v| v.as_str()) .map(String::from), cover: s(&hit, "cover_xl").or_else(|| s(&hit, "cover_big")), cover_big: s(&hit, "cover_big"), cover_medium: s(&hit, "cover_medium"), label: full.as_ref().and_then(|f| s(f, "label")), release_date: full.as_ref().and_then(|f| s(f, "release_date")), genre: full .as_ref() .and_then(|f| f.pointer("/genres/data/0/name")) .and_then(|v| v.as_str()) .map(String::from), track_count: tracklist.len(), tracklist, }) } /// The artist's own catalogue, ranked against what we call this record. /// /// The last resort, and the one that answers "the search found nothing but /// I can see the album exists". A folder named `1989 Pauls Boutique` is a /// hopeless SEARCH — Deezer is asked for a phrase no release carries — but /// it is an easy COMPARISON once you have the fifty-odd titles Deezer files /// under Beastie Boys and can score each one. /// /// Anchored on an exactly-matching artist name, never a name-alike: the /// same guard `select_audiodb_artist` applies, because the whole value of /// this pass is that the candidate set is already known to be the right /// artist's. Every name we have for the record is scored, and the best /// title across all of them wins if it clears `MIN`. async fn discography_identify( &self, artist: &str, raw_title: &str, tries: &[String], shape: &RecordShape, ) -> Option<(IdentifyCandidate, String, &'static str)> { /// A name this strong stands on its own — the old single threshold, /// unchanged, so every record that matches today still matches. const NAME_ALONE: f32 = 0.72; /// Below this a name is not evidence at all, and only the artwork can /// speak for the record. const NAME_WITH_HELP: f32 = 0.34; /// Enough of the tracklist to be the same record. Two-track singles are /// common, so this is a ratio, not a count. const TRACKS_AGREE: f32 = 0.6; /// How many candidates are worth PAYING for (a release fetch each, at /// MusicBrainz's 1 req/s). Ranked before spending, and only spent where /// a tracklist could actually change the answer. Five, because a large /// discography puts real answers well down the name ranking — the /// Animal Collective white label scores 0.5 against 113 groups. const CONFIRM_TOP: usize = 5; let groups = self.mb_discography(artist).await?; if groups.is_empty() { return None; } // ---- free evidence: the name, the shape, the year ------------------ let mut names: Vec<&str> = vec![raw_title]; for t in tries { names.push(t.as_str()); } let mut ranked: Vec = groups .iter() .map(|g| { let name = names .iter() .map(|n| title_match_score(n, &g.title)) .fold(0.0_f32, f32::max); DiscoCandidate { group: g.clone(), name, type_fit: mb_type_fit(g.primary_type.as_deref(), shape.tracks), year_agrees: match (shape.year, g.year()) { (Some(a), Some(b)) => Some(a == b), _ => None, }, art_distance: None, tracks_agree: None, } }) .collect(); // ---- artwork: the discography's own covers, hashed and compared ---- // // This is what rescues a record whose NAME is useless — a folder called // "ACPP", a title the label never used. Thumbnails, not masters: CAA's // front-250 is ~20 KB and 11 of 12 groups have one, so a whole // discography costs a few hundred KB, cached per URL by the shared image // cache and per artist by the list above. if let Some(ours) = shape.art_dhash { let hashes = futures::future::join_all( ranked .iter() .map(|c| async { self.caa_group_dhash(&c.group.id).await }), ) .await; for (c, theirs) in ranked.iter_mut().zip(hashes) { c.art_distance = theirs.map(|h| (h ^ ours).count_ones()); } } // Best first: an image match outranks everything, then the name, then // how well the type fits the record's own size. ranked.sort_by(|a, b| { b.art_match() .cmp(&a.art_match()) .then(b.name.total_cmp(&a.name)) .then(a.type_fit.cmp(&b.type_fit)) }); // ---- paid evidence: the tracklist, for the few that could win ------- // // The strongest signal there is, and the one a name cannot carry: his // two files are "Octopus" and "Rock Bottom", and the release is called // "Rock Bottom / Octopus". // // Paid for ONLY where it can change the answer — a name already past // NAME_ALONE has won, an artwork match has won, and a name below // NAME_WITH_HELP has nothing for a tracklist to corroborate. Each of // these is a paced MusicBrainz request, so the window is small and // spent on the best-ranked candidates first. if !shape.track_titles.is_empty() { let mut spent = 0usize; for c in ranked.iter_mut() { if spent >= CONFIRM_TOP { break; } if c.art_match() == 1 || c.name >= NAME_ALONE || c.name < NAME_WITH_HELP { continue; } spent += 1; let listed = self.mb_group_tracks(&c.group.id).await; if !listed.is_empty() { c.tracks_agree = Some(tracklist_overlap(&shape.track_titles, &listed)); } } } // ---- the decision -------------------------------------------------- // // Several independent facts, not one loosened threshold. A weak name // with nothing behind it stays unmatched, which is what the old single // floor was really protecting. let winner = ranked.iter().find_map(|c| { let why = if c.art_match() == 1 { "the same artwork" } else if c.name >= NAME_ALONE { "its name" } else if c.name >= NAME_WITH_HELP && c.tracks_agree.is_some_and(|o| o >= TRACKS_AGREE) { "its tracklist" } else { return None; }; Some((c, why)) }); let (winner, why) = winner?; // A year that positively DISAGREES kills a name-only match: two records // can share a title, and the folder's year is the cheapest way to tell // them apart. Artwork and tracklists outrank it — they are about this // record, where a year is about a pressing. if why == "its name" && winner.year_agrees == Some(false) && winner.tracks_agree.is_none() { return None; } let candidate = self.mb_candidate_of(artist, &winner.group).await?; Some((candidate, winner.group.title.clone(), why)) } /// Every release group MusicBrainz files under this artist, by NAME — the /// candidate set the comparison passes rank against. /// /// MusicBrainz, not Deezer, because Deezer's artist listing is incomplete in /// exactly the places this matters: measured on King Gizzard it returns 31 /// entries (20 `album`, 11 `single`, not one `ep`), missing both "Murder of /// the Universe" and "Willoughby's Beach" — records Deezer itself sells and /// serves at `/album/{id}`. MusicBrainz returned all 24 of King Krule's, /// singles and EPs included. /// /// Anchored on an exactly-normalized artist name, never a name-alike: the /// whole value of the pass is that the candidate set is already known to be /// the right artist's. Cached per URL, memory and disk (24h), so one fetch /// serves every record by that artist in the session. async fn mb_discography(&self, artist: &str) -> Option> { let want = normalize(artist); if want.is_empty() { return None; } let query = urlencode(&format!("artist:\"{}\"", escape(artist))); let found = self .musicbrainz_json(&format!( "https://musicbrainz.org/ws/2/artist?query={query}&fmt=json&limit=5" )) .await?; let mbid = found .get("artists")? .as_array()? .iter() .find(|a| { a.get("name") .and_then(|v| v.as_str()) .is_some_and(|n| normalize(n) == want) }) .and_then(|a| a.get("id")) .and_then(|v| v.as_str())? .to_string(); // Bounded exactly as the Deezer walk was: a prolific artist must not // turn one unmatched record into an unbounded crawl. let mut out: Vec = Vec::new(); for offset in [0usize, 100, 200] { let url = format!( "https://musicbrainz.org/ws/2/release-group?artist={mbid}&limit=100&offset={offset}&fmt=json" ); let Some(body) = self.musicbrainz_json(&url).await else { break }; let Some(groups) = body.get("release-groups").and_then(|g| g.as_array()) else { break; }; let returned = groups.len(); for g in groups { let (Some(id), Some(title)) = ( g.get("id").and_then(|v| v.as_str()), g.get("title").and_then(|v| v.as_str()), ) else { continue; }; out.push(MbGroup { id: id.to_string(), title: title.to_string(), primary_type: g .get("primary-type") .and_then(|v| v.as_str()) .map(String::from), first_release_date: g .get("first-release-date") .and_then(|v| v.as_str()) .map(String::from), }); } let total = body .get("release-group-count") .and_then(|c| c.as_u64()) .unwrap_or(returned as u64) as usize; if returned == 0 || offset + returned >= total { break; } } Some(out) } /// The perceptual hash of a release group's front cover, from Cover Art /// Archive's 250px thumbnail. Small on purpose — the question is "is this /// the same picture", which survives scaling, and a discography of masters /// would be tens of megabytes. async fn caa_group_dhash(&self, group_mbid: &str) -> Option { let url = format!("https://coverartarchive.org/release-group/{group_mbid}/front-250"); let bytes = self.get_image_shared(&url).await?; decode_cover_async(bytes, "caa", false, url).await.map(|c| c.dhash) } /// One release group's tracklist, from its first release. Read through the /// shared cache, so confirming a candidate and then building it costs ONE /// paced request, not two — same URL both times. /// /// Official pressings first, then ANY pressing. The status filter alone is /// too strict for exactly the records that need this pass: Automine's /// "Padington Band" has no official release and one unfiltered release with /// the 3 tracks he has, and MusicBrainz files his Animal Collective white /// label as "[The Purple Bottle]" the same way. A tracklist that exists and /// is not returned reads as a record with no songs, which corroborates /// nothing and quietly loses the match. async fn mb_release_tracks(&self, group_mbid: &str) -> Vec { for status in ["&status=official", ""] { let url = format!( "https://musicbrainz.org/ws/2/release?release-group={group_mbid}\ &inc=recordings{status}&fmt=json&limit=1" ); let Some(body) = self.musicbrainz_json(&url).await else { continue }; let tracks = parse_mb_tracklist(&body); if !tracks.is_empty() { return tracks; } } Vec::new() } /// Just the titles, for the tracklist comparison. async fn mb_group_tracks(&self, group_mbid: &str) -> Vec { self.mb_release_tracks(group_mbid) .await .into_iter() .map(|t| t.title) .collect() } /// A discography candidate as the record the review will show. async fn mb_candidate_of(&self, artist: &str, group: &MbGroup) -> Option { let tracklist = self.mb_release_tracks(&group.id).await; Some(IdentifyCandidate { id: format!("mb:{}", group.id), source: "MusicBrainz".into(), title: group.title.clone(), artist: Some(artist.to_string()), cover: Some(format!( "https://coverartarchive.org/release-group/{}/front", group.id )), cover_big: None, cover_medium: None, label: None, release_date: group.first_release_date.clone(), genre: None, track_count: tracklist.len(), tracklist, }) } /// MusicBrainz release-group as the album RECORD, not just as cover art /// (the mock's mbCandidate). Exact-normalized title first, containment /// second; one official release with recordings gives the tracklist. /// Paced by the crate's own MusicBrainz gate. async fn mb_identify( &self, artist: &str, tries: &[String], tracks: Option, ) -> (Option<(IdentifyCandidate, String)>, bool) { let mut failed = false; for t in tries { let query = format!("releasegroup:\"{}\" AND artist:\"{}\"", escape(t), escape(artist)); let url = format!( "https://musicbrainz.org/ws/2/release-group?query={}&fmt=json&limit=3", urlencode(&query) ); // None here means the REQUEST failed, not that MB has nothing — // `mb_get` only returns None after its retry also failed. Carrying // that out is what stops a 503 being cached as "no such record". let Some(body) = self.mb_get(&url).await else { failed = true; continue; }; let empty = Vec::new(); let groups = body .get("release-groups") .and_then(|v| v.as_array()) .unwrap_or(&empty); // Several release groups can carry the SAME title, and MB's order // between equally-scored ones is not stable. "Murder of the // Universe" is both a 21-track Album and a 1-track Single, both // scored 100; taking whichever came first identified a 21-track // record as a one-track single, and differently between runs // (measured: Single first on one request, Album first on the next). // So the record's own shape breaks the tie. Stable sort, so equal // ranks keep MB's relevance order. let pick = |matches: &dyn Fn(&str) -> bool| -> Option<&serde_json::Value> { let mut hits: Vec<&serde_json::Value> = groups .iter() .filter(|g| { g.get("title") .and_then(|v| v.as_str()) .is_some_and(|title| matches(title)) }) .collect(); hits.sort_by_key(|g| { mb_type_fit(g.get("primary-type").and_then(|v| v.as_str()), tracks) }); hits.into_iter().next() }; let rg = pick(&|title| normx(title) == normx(t)) .or_else(|| pick(&|title| normx_overlap(title, t))); let Some(rg) = rg else { continue }; let Some(rgid) = rg.get("id").and_then(|v| v.as_str()) else { continue }; let rel_url = format!( "https://musicbrainz.org/ws/2/release?release-group={rgid}\ &inc=recordings&status=official&fmt=json&limit=1" ); let rel = self.mb_get(&rel_url).await; let tracklist: Vec = rel.as_ref().map(parse_mb_tracklist).unwrap_or_default(); let candidate = IdentifyCandidate { id: format!("mb:{rgid}"), source: "MusicBrainz".into(), title: rg .get("title") .and_then(|v| v.as_str()) .unwrap_or_default() .to_string(), artist: rg .pointer("/artist-credit/0/name") .and_then(|v| v.as_str()) .map(String::from) .or_else(|| Some(artist.to_string())), cover: Some(format!( "https://coverartarchive.org/release-group/{rgid}/front" )), cover_big: None, cover_medium: None, label: None, release_date: rg .get("first-release-date") .and_then(|v| v.as_str()) .map(String::from), genre: None, track_count: tracklist.len(), tracklist, }; return (Some((candidate, t.clone())), failed); } (None, failed) } /// Fetch a chosen cover's bytes for the import apply — a plain GET, the /// caller decides what the pixels are for. pub async fn fetch_cover_bytes(&self, url: &str) -> Option> { let resp = self.http.get(url).send().await.ok()?; if !resp.status().is_success() { return None; } resp.bytes().await.ok().map(|b| b.to_vec()) } /// Throttled MusicBrainz GET for identify, with one retry. Status-aware /// on purpose: a 503 from MB's rate limiter — routine right after app /// start, when the identity warmer shares the 1 req/s budget — must /// read as a retryable failure, never as "record absent" (the paced- /// probe-contamination lesson, now inside the app). The retry passes /// back through the throttle, so it is properly spaced, and a second /// failure surfaces as an honest miss the user can re-open. async fn mb_get(&self, url: &str) -> Option { for attempt in 0..2 { self.mb_throttle().await; match self.http.get(url).send().await { Ok(resp) if resp.status().is_success() => { if let Ok(v) = resp.json().await { return Some(v); } } Ok(resp) if attempt == 0 => { log::debug!("mb identify: {} from MusicBrainz, retrying", resp.status()); } Ok(_) | Err(_) => {} } } None } /// Production hot-path cover upgrade for a catalog release. Returns the URL /// of the highest-resolution image that is *verifiably the same artwork* as /// the cover the UI already shows (`reference_url`), or `None` to keep it. /// /// Deliberately does NO MusicBrainz search — that 1 req/s lookup is the /// artist-page bottleneck. Sources are Apple (artist-id lookup, cached per /// artist) and Deezer at native max, both of which parallelize freely. /// Cover Art Archive is consulted only when the reference URL already /// carries a release-group MBID (the MusicBrainz-supplemented cards), so it /// costs no extra lookup and still upgrades those `front-500` thumbnails. pub async fn resolve_cover( &self, artist: &str, title: &str, reference_url: &str, ) -> Option { self.resolve_cover_for_artist_id(None, artist, title, reference_url) .await } /// Identity-scoped variant for artist-page cover/logo work. Once overview /// or the full page has corroborated a same-name Apple performer, callers /// can reuse that `dz:{id}` catalogue and retain original-resolution Apple /// upgrades without falling back to an unsafe name-only choice. pub async fn resolve_cover_for_artist_id( &self, artist_id: Option, artist: &str, title: &str, reference_url: &str, ) -> Option { let reference_bytes = self.get_image_shared(reference_url).await?; let reference = decode_cover_async( reference_bytes, "reference", true, reference_url.to_string(), ) .await?; let mut candidates: Vec = Vec::new(); // Deezer at native max (URL rewrite only; capped at the real source). if reference_url.contains("dzcdn") { let max_url = reference_url.replace("/1000x1000-", "/1500x1500-"); if max_url != reference_url { if let Some(b) = self.get_image_shared(&max_url).await { if let Some(c) = decode_cover_async(b, "deezer-max", false, max_url).await { candidates.push(c); } } } } // Apple — verified, cached, no MusicBrainz. Artist pages reuse the // Deezer-id selection; standalone callers stay safely name-scoped. let apple = if let Some(artist_id) = artist_id { self.apple_cover_candidate_for_deezer(artist_id, artist, title) .await } else { self.apple_cover_candidate(artist, title).await }; if let Some(c) = apple { candidates.push(c); } // Cover Art Archive only when the MBID is already in hand (free). if let Some(mbid) = caa_release_group_mbid(reference_url) { candidates.extend(self.caa_cover_candidates(None, Some(&mbid)).await); } let ref_hash = reference.dhash; let ref_area = u64::from(reference.width) * u64::from(reference.height); let best = candidates .iter() .filter(|c| (c.dhash ^ ref_hash).count_ones() <= COVER_SAME_ART_MAX_DISTANCE) .max_by_key(|c| { ( u64::from(c.width) * u64::from(c.height), cover_source_priority(c.source), ) })?; // Only replace when we actually gained resolution. if u64::from(best.width) * u64::from(best.height) > ref_area { Some(best.url.clone()) } else { None } } /// Deezer's `cover_xl` for the album, artist+title verified. This is the /// artwork the catalog UI shows, so it anchors cover selection. async fn deezer_cover_candidate(&self, artist: &str, album: &str) -> Option { let query = urlencode(&format!( "artist:\"{}\" album:\"{}\"", escape(artist), escape(album) )); let url = format!("https://api.deezer.com/search/album?q={query}&limit=5"); let body = self.deezer(&url).await?; let want_artist = normalize(artist); let want_album = normalize(album); let hit = body.get("data")?.as_array()?.iter().find(|a| { let got_artist = a .pointer("/artist/name") .and_then(|v| v.as_str()) .map(normalize) .unwrap_or_default(); let got_album = a .get("title") .and_then(|v| v.as_str()) .map(normalize) .unwrap_or_default(); loose_name_match(&got_artist, &want_artist) && loose_name_match(&got_album, &want_album) })?; let cover_url = hit.get("cover_xl").and_then(|c| c.as_str())?; let bytes = self.get_image_shared(cover_url).await?; decode_cover_async(bytes, "deezer", true, cover_url.to_string()).await } /// Apple's artwork at original resolution for a specific album. iTunes text /// search cannot reliably surface one artist's one album (common titles /// return other artists; the artist's own album is often not in the top /// results), so we resolve the artist's iTunes id, list *their* albums, and /// match the title locally. The perceptual gate downstream still guarantees /// the winning image is the same artwork as the reference. async fn apple_cover_candidate(&self, artist: &str, album: &str) -> Option { let catalog = self.apple_artist_albums(artist).await; if catalog.identity_confirmed { return self .apple_cover_from_releases(&catalog.releases, album, true) .await; } // A shared name may still have an album title found under exactly one // Apple performer. That is safe for cover polish (and preserves 3000px // cover-logo input) because the caller additionally gates the raster by // perceptual hash against Deezer's known-correct reference. let candidates = self.apple_artist_candidates(artist).await; if candidates .iter() .any(|candidate| !candidate.lookup_complete) { return None; } let want = release_title_identity(album); let matching: Vec<&AppleArtistCandidate> = candidates .iter() .filter(|candidate| { candidate .releases .iter() .any(|release| release.title_norm == want) }) .collect(); if matching.len() != 1 { return None; } self.apple_cover_from_releases(&matching[0].releases, album, false) .await } async fn apple_cover_candidate_for_deezer( &self, deezer_id: u64, artist: &str, album: &str, ) -> Option { let key = format!("dz:{deezer_id}"); let catalog = self.apple_albums.lock().await.get(&key).cloned(); if let Some(catalog) = catalog.filter(|catalog| catalog.identity_confirmed) { return self .apple_cover_from_releases(&catalog.releases, album, true) .await; } self.apple_cover_candidate(artist, album).await } async fn apple_cover_from_releases( &self, releases: &[AppleRelease], album: &str, allow_loose: bool, ) -> Option { let want = release_title_identity(album); let thumb = releases .iter() .find(|release| release.title_norm == want) .or_else(|| { if allow_loose { releases .iter() .find(|release| loose_name_match(&release.title_norm, &want)) } else { None } }) .map(|release| release.artwork.clone()) .filter(|artwork| !artwork.is_empty())?; for url in apple_artwork_urls(&thumb) { if let Some(bytes) = self.get_image_shared(&url).await { if let Some(c) = decode_cover_async(bytes, "apple", false, url.clone()).await { return Some(c); } } } None } /// Name-only fallback used by standalone cover resolution. It deliberately /// fails closed when Apple exposes more than one exact-name performer: a /// cover request has no Deezer id/release evidence with which to choose. async fn apple_artist_albums(&self, artist: &str) -> Arc { let key = format!("name:{}", normalize(artist)); if let Some(v) = self.apple_albums.lock().await.get(&key) { return v.clone(); } let candidates = self.apple_artist_candidates(artist).await; let catalog = Arc::new(select_apple_artist_catalog(&candidates, &[], false)); let mut cache = self.apple_albums.lock().await; if cache.len() >= 512 { cache.clear(); } cache.insert(key, catalog.clone()); catalog } async fn apple_catalog_for_deezer_candidates( &self, deezer_id: u64, artist: &str, candidates: Arc>, evidence: &[DeezerReleaseEvidence], ) -> Arc { let key = format!("dz:{deezer_id}"); let catalog = Arc::new(select_apple_artist_catalog(&candidates, evidence, true)); log::debug!( "Apple identity for {artist} (Deezer {deezer_id}): candidates={}, winner={:?}, confirmed={}, releases={}, rejected_titles={}, rejected_genres={}", candidates.len(), catalog.apple_artist_id, catalog.identity_confirmed, catalog.releases.len(), catalog.foreign_titles.len(), catalog.foreign_genres.len(), ); // Candidate fetching is the expensive part and is already shared. Do // the tiny scoring pass for each caller's evidence so a first-50 // overview cannot poison the exhaustive result. Only confirmed choices // become reusable, and richer corroboration wins a cache race. let mut cache = self.apple_albums.lock().await; if catalog.identity_confirmed { if cache.len() >= 512 && !cache.contains_key(&key) { cache.clear(); } let replace = cache.get(&key).is_none_or(|current| { !current.identity_confirmed || (catalog.evidence_weight, catalog.evidence_count) > (current.evidence_weight, current.evidence_count) }); if replace { cache.insert(key, catalog.clone()); } } else if cache .get(&key) .is_some_and(|current| catalog.evidence_count > current.evidence_count) { // Richer evidence that becomes ambiguous invalidates a provisional // confirmed cache entry instead of leaving cover work attached to // the first-50 decision. cache.remove(&key); } catalog } /// Apple's discography as catalog records, for supplementing Deezer's /// regional list. This is the unthrottled stand-in for the MusicBrainz /// studio-album lookup: one `/lookup` request returns the artist's whole /// catalogue (61 releases for Weezer, including the Deezer-missing ones), /// where MusicBrainz costs a rate-limited request per artist at 1/second. async fn apple_supplement_albums( &self, deezer_id: u64, artist: &str, catalog: &AppleArtistCatalog, ) -> Vec { if !catalog.identity_confirmed { return Vec::new(); } let mut albums = Vec::new(); let mut registered = Vec::new(); for r in &catalog.releases { let raw = r.title.trim(); if raw.is_empty() { continue; } // Apple's promo pseudo-albums ("iTunes Originals: …", "iTunes Pass: // …", "iTunes Session/Festival/Live") are not part of the studio // discography. No genuine album is titled starting with the word // "iTunes", so drop them here — never in `fetch_apple_artist_albums`, // which also feeds the cover resolver. if is_itunes_promo(raw) { continue; } // Continuous DJ mixes are streaming-only: their song lookup returns // no tracks (verified live on the Defected/Yuksek Crazy P mixes), // so a supplemented card would open with a Play button and nothing // to play. Deezer's own playable mix compilations are unaffected — // this filter only guards what Apple contributes. if is_apple_dj_mix_title(raw) { continue; } // Apple encodes the format in the TITLE, not `collectionType` (which // says "Album" even for a single). Strip that tail so the title // matches Deezer's, and read the type from it. let (title, format_type) = strip_release_format_tail(raw); let record_type = if let Some(format_type) = format_type { format_type } else if r .collection_type .as_deref() .is_some_and(|t| t.eq_ignore_ascii_case("compilation")) { "compile" } else { "album" }; // Deezer's singles/EP coverage is already exhaustive (31 singles for // Weezer). Apple is here only to fill ALBUM gaps — releases missing // from Deezer's regional catalogue, such as Hurley. Contributing its // singles just duplicates the grid. if record_type != "album" { continue; } // An "album" with a couple of tracks is a promo single Apple simply // never tagged ("It's Working", 2 tracks, no " - Single" tail). // Deezer's single coverage is already exhaustive; the supplement // exists to fill ALBUM gaps. if r.track_count.is_some_and(|n| n <= 3) { continue; } // Apple ships an RFC3339 timestamp; the catalog wants a date. let release_date = r .release_date .as_deref() .map(|d| d.split('T').next().unwrap_or(d).to_string()); let id = apple_synthetic_catalog_id(deezer_id, r, title); // The FITTED size, not `apple_artwork_urls`' first entry — that one // is the speculative 100000x100000-999 rewrite the cover resolver // probes with fallbacks. A plain has no fallback, so handing it // that URL renders a placeholder. `resolveCovers` upgrades this later. let cover = (!r.artwork.is_empty()) .then(|| apple_artwork_urls(&r.artwork).into_iter().last()) .flatten(); // Route an opened card back to iTunes for its tracklist. Without a // collection id there is NOTHING to look up: `catalog_album` falls // through to Deezer with a synthetic id it cannot know, and the // card opens as an error. A card guaranteed dead on arrival does // not belong on the grid. let Some(collection_id) = r.collection_id else { continue; }; registered.push(( id, AppleCatalogAlbum { collection_id, artist_deezer_id: deezer_id, artist: artist.to_string(), title: title.to_string(), cover: cover.clone(), release_date: release_date.clone(), }, )); albums.push(CatalogAlbum { id, title: title.to_string(), artist: artist.to_string(), cover, release_date, record_type: Some(record_type.to_string()), track_count: r.track_count, edition_group: None, is_soundtrack: false, is_live: false, // Apple has no mixtape record type either; only the // MusicBrainz supplement sets this. is_mixtape: false, appears_on: false, }); } if !registered.is_empty() { let mut registry = self.apple_catalog.lock().await; if registry.len() > 1_000 { registry.clear(); } for (id, entry) in registered { // Same reasoning as the MusicBrainz registry above. self.persist_supplement(id, &entry); registry.insert(id, entry); } } albums } /// Resolve *all* exact-normalized Apple performers, not whichever happens /// to be the first search result. Lookups are concurrent but bounded; the /// cache/inflight maps make overview, full page and cover work share them. async fn apple_artist_candidates(&self, artist: &str) -> Arc> { let want = normalize(artist); if let Some(candidates) = self.apple_candidates.lock().await.get(&want) { return candidates.clone(); } let fetch = { let mut inflight = self.apple_candidates_inflight.lock().await; inflight .entry(want.clone()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone() }; let candidates = fetch .get_or_init(|| async { Arc::new( self.fetch_apple_artist_candidates(artist, APPLE_EXACT_ARTIST_CANDIDATE_LIMIT) .await, ) }) .await .clone(); // A complete zero-match search is a real, cacheable result. Transport // or partial lookup failure is marked non-cacheable so the next page // generation can retry instead of remaining unsafe until restart. if candidates.iter().all(|candidate| candidate.cacheable) { let mut cache = self.apple_candidates.lock().await; if cache.len() >= 256 { cache.clear(); } cache.insert(want.clone(), candidates.clone()); } let mut inflight = self.apple_candidates_inflight.lock().await; if inflight .get(&want) .is_some_and(|current| Arc::ptr_eq(current, &fetch)) { inflight.remove(&want); } candidates } /// The same resolution with a far higher candidate ceiling, for the /// MusicBrainz-anchored identity pass only. /// /// [`Self::apple_artist_candidates`] fails closed above /// [`APPLE_EXACT_ARTIST_CANDIDATE_LIMIT`] because at page-load time the only /// way to pick between same-named performers is to score them against the /// requested Deezer artist's own release list — and on a fused record that /// list is partly the co-tenants'. Circular evidence has to fail closed. /// Here the MBID has already named the artist, so more candidates are more /// evidence rather than more risk, and the losing catalogues are exactly /// what proves a Deezer row belongs to someone else. async fn apple_identity_candidates(&self, artist: &str) -> Arc> { let want = normalize(artist); if let Some(candidates) = self.apple_identity_candidates.lock().await.get(&want) { return candidates.clone(); } let fetch = { let mut inflight = self.apple_identity_candidates_inflight.lock().await; inflight .entry(want.clone()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone() }; let candidates = fetch .get_or_init(|| async { Arc::new( self.fetch_apple_artist_candidates( artist, APPLE_IDENTITY_ARTIST_CANDIDATE_LIMIT, ) .await, ) }) .await .clone(); if candidates.iter().all(|candidate| candidate.cacheable) { let mut cache = self.apple_identity_candidates.lock().await; if cache.len() >= 256 { cache.clear(); } cache.insert(want.clone(), candidates.clone()); } let mut inflight = self.apple_identity_candidates_inflight.lock().await; if inflight .get(&want) .is_some_and(|current| Arc::ptr_eq(current, &fetch)) { inflight.remove(&want); } candidates } async fn fetch_apple_artist_candidates( &self, artist: &str, candidate_limit: usize, ) -> Vec { let want = normalize(artist); let search = format!( "https://itunes.apple.com/search?term={}&entity=musicArtist&limit=25", urlencode(artist) ); let Some(resp) = self .http .get(&search) .send() .await .ok() .and_then(|response| response.error_for_status().ok()) else { return unavailable_apple_candidates(); }; let Some(body) = resp.json::().await.ok() else { return unavailable_apple_candidates(); }; let Some(results) = body.get("results").and_then(|r| r.as_array()) else { return unavailable_apple_candidates(); }; let mut seen = HashSet::new(); let artist_ids: Vec = results .iter() .filter(|r| { r.get("artistName") .and_then(|a| a.as_str()) .is_some_and(|a| normalize(a) == want) }) .filter_map(|r| r.get("artistId").and_then(|v| v.as_u64())) .filter(|artist_id| seen.insert(*artist_id)) .collect(); if artist_ids.len() > candidate_limit { log::debug!( "Apple identity for {artist} has {} exact-name candidates (cap {candidate_limit}); failing closed before lookups", artist_ids.len() ); // Preserve an explicit ambiguous shape. Returning [] would make the // Bing gate mistake "too many to resolve safely" for "Apple found // no namesake" and re-enable a name-only hero. return artist_ids .into_iter() .take(2) .map(|artist_id| AppleArtistCandidate { artist_id, releases: Vec::new(), credited_titles: HashSet::new(), // Intentional ambiguity, not transport failure: selection // and Bing still fail closed, but native Deezer rows render. lookup_complete: true, cacheable: true, }) .collect(); } let mut candidates = stream::iter(artist_ids) .map(|artist_id| async move { self.fetch_apple_artist_candidate(artist_id, artist).await }) .buffer_unordered(6) .collect::>() .await; candidates.sort_by_key(|candidate| candidate.artist_id); candidates } async fn fetch_apple_artist_candidate( &self, artist_id: u64, artist: &str, ) -> AppleArtistCandidate { let lookup = format!("https://itunes.apple.com/lookup?id={artist_id}&entity=album&limit=200"); let body = match self.http.get(&lookup).send().await { Ok(resp) => match resp.error_for_status() { Ok(resp) => resp.json::().await.ok(), Err(_) => None, }, Err(_) => None, }; let parsed = body .as_ref() .and_then(|body| body.get("results")) .and_then(|r| r.as_array()) .map(|results| { let mut releases = Vec::new(); let mut credited_titles = HashSet::new(); for entry in results { if entry.get("wrapperType").and_then(|w| w.as_str()) != Some("collection") { continue; } let Some(name) = entry.get("collectionName").and_then(|c| c.as_str()) else { continue; }; let owned_by_this_id = entry .get("artistId") .and_then(|id| id.as_u64()) .is_none_or(|id| id == artist_id); if !owned_by_this_id { // Filed under a collaboration's own Apple artist. Only // a display credit naming this artist makes it theirs; // "(feat. …)" in the title does not. let credited = entry .get("artistName") .and_then(|c| c.as_str()) .is_some_and(|credit| apple_credit_names_artist(credit, artist)); if credited { let key = release_title_identity(name); if !key.is_empty() { credited_titles.insert(key); } } continue; } releases.push(AppleRelease { title: name.to_string(), title_norm: release_title_identity(name), artwork: entry .get("artworkUrl100") .and_then(|c| c.as_str()) .unwrap_or_default() .to_string(), release_date: entry .get("releaseDate") .and_then(|c| c.as_str()) .map(String::from), track_count: entry .get("trackCount") .and_then(|c| c.as_u64()) .map(|n| n as u32), collection_type: entry .get("collectionType") .and_then(|c| c.as_str()) .map(String::from), collection_id: entry.get("collectionId").and_then(|c| c.as_u64()), copyright: entry .get("copyright") .and_then(|c| c.as_str()) .map(String::from), }); } (releases, credited_titles) }); let lookup_complete = parsed.is_some(); let (releases, credited_titles) = parsed.unwrap_or_default(); AppleArtistCandidate { artist_id, releases, credited_titles, lookup_complete, cacheable: lookup_complete, } } /// Cover Art Archive originals — often high-resolution scans. `front` /// redirects to the full uploaded image (front-500 is only a thumbnail). async fn caa_cover_candidates( &self, release_mbid: Option<&str>, release_group_mbid: Option<&str>, ) -> Vec { let urls = [ release_mbid.map(|m| format!("https://coverartarchive.org/release/{m}/front")), release_group_mbid .map(|m| format!("https://coverartarchive.org/release-group/{m}/front")), ]; let mut out = Vec::new(); for url in urls.into_iter().flatten() { if let Some(bytes) = self.get_image_shared(&url).await { if let Some(c) = decode_cover_async(bytes, "cover-art-archive", false, url.clone()).await { out.push(c); // The release image and the release-group image are almost // always the same upload; one original is enough. break; } } else { log::debug!("no CAA art at {url}"); } } out } /// Best-effort artist portrait from Deezer's public catalog. The image is /// cached locally by the app and is only used for the library UI. pub async fn fetch_artist_art(&self, artist: &str) -> Option> { // `limit=1` + `/data/0` took whatever Deezer happened to rank first, // with no name check and no prominence check. For a shared name that is // routinely a dead duplicate row: "GUM" resolved to the 90-fan entry // whose portrait is the empty-md5 placeholder, so the library shelf drew // a grey silhouette while the real 3707-fan GUM has a photo. // // Ask for a handful instead and break the tie the way the rest of the // crate does — exact name first, then followers (see // `deezer_exact_name_state`) — and never accept the placeholder. let url = format!( "https://api.deezer.com/search/artist?q={}&limit=10", urlencode(artist) ); // Library imports call this once per artist, so it shares the same // cache, single-flight and request-budget gate as every other Deezer // read instead of spending an unpaced request beside them. let body = self.deezer(&url).await?; let image = pick_artist_portrait(&body, artist)?.to_string(); self.get_image(&image).await } /// Fetch an image and return it as a data URL. Catalog covers live on /// CDNs without CORS headers, so the webview can't canvas-sample them /// directly — this proxies the bytes through Rust instead. pub async fn fetch_image_data_url(&self, url: &str) -> Option { let bytes = self.get_image(url).await?; let mime = if bytes.starts_with(b"\x89PNG") { "image/png" } else if bytes.starts_with(b"GIF8") { "image/gif" } else { "image/jpeg" }; Some(format!("data:{mime};base64,{}", base64(&bytes))) } /// Raw provider bytes for the app's hero-only resampling path. Kept /// separate from ordinary catalog art so covers and transparent logos are /// never resized or converted to JPEG. pub async fn fetch_image_bytes(&self, url: &str) -> Option> { self.get_image(url).await } /// Shared compressed bytes for the preview/GPU hero processors. These /// commands frequently follow Bing scoring for the same URL, so retaining /// the cache Arc avoids another multi-megabyte body copy at the IPC edge. pub async fn hero_bytes(&self, url: &str) -> Option>> { self.get_image_shared(url).await } /// Public wrapper over the cached, single-flight image fetch — used by the /// cover-logo extractor to pull album covers for OCR. pub async fn cover_bytes(&self, url: &str) -> Option>> { self.get_image_shared(url).await } async fn get_image(&self, url: &str) -> Option> { self.get_image_shared(url) .await .map(|bytes| bytes.as_ref().clone()) } /// Cached raw image body without cloning it. The logo pipeline keeps several /// compressed covers in flight, so sharing the cache's Arc avoids copying /// multi-megabyte hi-res bodies at each phase boundary. async fn get_image_shared(&self, url: &str) -> Option>> { { let cache = self.image_cache.lock().await; if let Some((at, bytes)) = cache.get(url) { if at.elapsed() < IMAGE_CACHE_TTL { return Some(Arc::clone(bytes)); } } } // The HD first-paint path and the 4K finisher intentionally race, but // both begin with the same provider URL. Share that network transfer so // the race costs one download rather than making the visible preview // compete with an identical request for bandwidth and CDN connections. let fetch = { let mut inflight = self.image_inflight.lock().await; inflight .entry(url.to_string()) .or_insert_with(|| Arc::new(OnceCell::new())) .clone() }; let bytes = tokio::time::timeout( Duration::from_secs(10), fetch.get_or_init(|| async { let mut resp = self.http.get(url).send().await.ok()?; if !resp.status().is_success() { return None; } if resp .content_length() .is_some_and(|length| length > IMAGE_DOWNLOAD_MAX_BYTES as u64) { return None; } let mut bytes = Vec::with_capacity( resp.content_length() .unwrap_or(0) .min(IMAGE_DOWNLOAD_MAX_BYTES as u64) as usize, ); while let Some(chunk) = resp.chunk().await.ok()? { if bytes.len().saturating_add(chunk.len()) > IMAGE_DOWNLOAD_MAX_BYTES { return None; } bytes.extend_from_slice(&chunk); } (bytes.len() > 1000).then(|| Arc::new(bytes)) }), ) .await .ok() .and_then(|bytes| bytes.clone()); if let Some(bytes) = bytes.as_ref() { let mut cache = self.image_cache.lock().await; cache.retain(|_, (at, _)| at.elapsed() < IMAGE_CACHE_TTL); if bytes.len() <= IMAGE_CACHE_MAX_BYTES { let mut total = cache.values().map(|(_, image)| image.len()).sum::(); while total.saturating_add(bytes.len()) > IMAGE_CACHE_MAX_BYTES { let oldest = cache .iter() .min_by_key(|(_, (at, _))| *at) .map(|(key, _)| key.clone()); let Some(oldest) = oldest else { break }; if let Some((_, removed)) = cache.remove(&oldest) { total = total.saturating_sub(removed.len()); } } cache.insert(url.to_string(), (Instant::now(), Arc::clone(bytes))); } } let mut inflight = self.image_inflight.lock().await; if inflight .get(url) .is_some_and(|current| Arc::ptr_eq(current, &fetch)) { inflight.remove(url); } bytes } } fn parse_artist(v: &serde_json::Value) -> Option { Some(CatalogArtist { id: v.get("id")?.as_u64()?, name: v.get("name")?.as_str()?.to_string(), picture: v .get("picture_xl") .or_else(|| v.get("picture_big")) .or_else(|| v.get("picture_medium")) .and_then(|p| p.as_str()) .map(String::from), album_count: v.get("nb_album").and_then(|n| n.as_u64()).map(|n| n as u32), fans: v.get("nb_fan").and_then(|n| n.as_u64()), }) } fn parse_album(v: &serde_json::Value) -> Option { Some(CatalogAlbum { id: v.get("id")?.as_u64()?, title: v.get("title")?.as_str()?.to_string(), artist: v .pointer("/artist/name") .and_then(|a| a.as_str()) .unwrap_or_default() .to_string(), cover: v .get("cover_xl") .or_else(|| v.get("cover_big")) .or_else(|| v.get("cover_medium")) .and_then(|c| c.as_str()) .map(String::from), release_date: v .get("release_date") .and_then(|d| d.as_str()) .map(String::from), record_type: v .get("record_type") .and_then(|r| r.as_str()) .map(String::from), track_count: v .get("nb_tracks") .and_then(|n| n.as_u64()) .map(|n| n as u32), edition_group: None, is_soundtrack: detail_is_soundtrack(v), // Search results carry no tracklist, so this is false there and is set // later from the album-details pass; a full album fetch (catalog_album) // does include /tracks/data and resolves it here directly. is_live: detail_is_live(v), // Deezer has no mixtape record type, so a Deezer-sourced row is never // one. Only the MusicBrainz supplement sets this. is_mixtape: false, // Needs the requested artist's id to mean anything, which this parser // does not have. Resolved in the album-details pass. appears_on: false, }) } /// Deezer classifies soundtrack releases as regular albums/singles. Its /// Films/Games genre (173) is the strongest structured signal, while checking /// the returned genre names makes this resilient if the primary genre differs. fn detail_is_soundtrack(v: &serde_json::Value) -> bool { if v.get("genre_id").and_then(|id| id.as_u64()) == Some(173) { return true; } v.pointer("/genres/data") .and_then(|genres| genres.as_array()) .is_some_and(|genres| { genres.iter().any(|genre| { let name = genre .get("name") .and_then(|name| name.as_str()) .unwrap_or_default() .to_lowercase(); name == "films/games" || name.contains("film score") || name.contains("soundtrack") }) }) } /// Whether a single track title carries a live-performance marker — "(Live)", /// "(Live at the …)", "[Live]", or a "- Live" suffix. Deliberately anchored to /// bracket/dash-prefixed forms so ordinary song titles that merely contain the /// word (e.g. "Long Live", "Come Alive") do not match; the majority rule in /// [`detail_is_live`] is the real guard, this just keeps individual matches /// honest. fn is_live_track_title(title: &str) -> bool { let ln = title.to_lowercase(); ln.contains("(live") || ln.contains("[live") || ln.contains("- live") || ln.contains("– live") || ln.contains("— live") || ln.contains("live at ") || ln.contains("live from ") || ln.contains("live in ") || ln.contains("live on ") } /// A release is a live album when a clear majority of its tracks are marked as /// live recordings. Reads the album detail's `/tracks/data`, so it only reports /// true once the tracklist has actually been fetched. Requires at least three /// tracks and ≥60% live-marked, so a studio album carrying a couple of live /// bonus cuts is never misfiled. fn detail_is_live(v: &serde_json::Value) -> bool { let Some(tracks) = v.pointer("/tracks/data").and_then(|d| d.as_array()) else { return false; }; let titles: Vec<&str> = tracks .iter() .filter_map(|t| t.get("title").and_then(|s| s.as_str())) .collect(); if titles.len() < 3 { return false; } let live = titles.iter().filter(|t| is_live_track_title(t)).count(); // live / total >= 3/5 (60%), without floating point. live * 5 >= titles.len() * 3 } fn parse_track(v: &serde_json::Value) -> Option { Some(CatalogTrack { id: v.get("id")?.as_u64()?, title: v.get("title")?.as_str()?.to_string(), artist: v .pointer("/artist/name") .and_then(|a| a.as_str()) .unwrap_or_default() .to_string(), artist_id: v.pointer("/artist/id").and_then(|a| a.as_u64()), rank: v.get("rank").and_then(|r| r.as_u64()).map(|r| r as u32), album: v .pointer("/album/title") .and_then(|a| a.as_str()) .unwrap_or_default() .to_string(), album_id: v.pointer("/album/id").and_then(|a| a.as_u64()), cover: v .pointer("/album/cover_medium") .or_else(|| v.pointer("/album/cover_big")) .and_then(|c| c.as_str()) .map(String::from), duration_secs: v.get("duration").and_then(|d| d.as_u64()).map(|d| d as u32), preview: v .get("preview") .and_then(|p| p.as_str()) .filter(|p| !p.is_empty()) .map(String::from), track_no: v .get("track_position") .and_then(|t| t.as_u64()) .map(|t| t as u32), disk_number: v .get("disk_number") .and_then(|t| t.as_u64()) .map(|t| t as u32), unreleased: false, // Filled in by `apply_lastfm_order`; Deezer carries no scrobble data. listeners: None, plays: None, // Set only when `apply_original_album_attribution` moves this row. origin_track_id: None, }) } /// Remove provider-added media format tails before any cross-provider title /// comparison. Apple appends these to collection names while Deezer exposes the /// format separately; comparing raw strings made namesake singles invisible to /// both corroboration and rejection. fn strip_release_format_tail(title: &str) -> (&str, Option<&'static str>) { let title = title.trim(); let lower = title.to_lowercase(); const TAILS: [(&str, &str); 12] = [ (" - single", "single"), (" – single", "single"), (" — single", "single"), (" (single)", "single"), (" - ep", "ep"), (" – ep", "ep"), (" — ep", "ep"), (" (ep)", "ep"), (" - album", "album"), (" – album", "album"), (" — album", "album"), (" (album)", "album"), ]; for (suffix, record_type) in TAILS { if lower.ends_with(suffix) { let stripped = title[..title.len() - suffix.len()].trim_end(); if !stripped.is_empty() { return (stripped, Some(record_type)); } } } (title, None) } fn release_title_identity(title: &str) -> String { normalize(strip_release_format_tail(title).0) } /// Does this bracketed/dashed segment name the SAME recording, just a different /// pressing of it? /// /// Deliberately absent: live, remix, reprise, interlude, acoustic, demo. Those /// name a genuinely different recording, and folding them into the parent would /// delete a real track from the list. Measured over 18 artists x Deezer's top /// 50: this curated set carries 74% of Last.fm's canonical top ten across to a /// Deezer track while merging 16 entries, where stripping EVERY parenthetical /// reached 79% but merged 61 — four times the collateral for five points. fn is_track_version_marker(segment: &str) -> bool { let segment = segment.trim().trim_matches(|c: char| !c.is_alphanumeric()); if segment.is_empty() { return false; } // "2011 Remaster", "Remastered 2009", "Taylor's Version", "Album Version". segment.contains("remaster") || segment.contains("version") || segment.contains("radio edit") || segment.contains("single edit") || segment.contains("original mix") || segment.contains("extended mix") || segment.contains("club mix") || segment.contains("bonus track") // A bracketed credit — "(feat. Reckz'Capo)" — never reaches the loop // above, which only sees a SPACE-prefixed clause. || segment.starts_with("feat.") || segment.starts_with("feat ") || segment.starts_with("featuring") || segment.starts_with("ft.") || segment.starts_with("ft ") || matches!(segment, "edit" | "deluxe" | "mono" | "stereo" | "explicit" | "clean") } /// The cross-provider key for one track title. /// /// Last.fm reports the name people scrobbled ("Shake It Off") while Deezer /// carries the pressing it sells ("Shake It Off (Taylor's Version)"), so a raw /// string comparison finds neither. Only ever applied WITHIN a single already /// anchored artist — it decides which of that artist's own tracks two providers /// are both describing, never which artist we are looking at, so it is not the /// kind of name match [`audiodb_artist_matches_identity`] exists to forbid. /// Drop every bracketed segment and trailing dash-clause that `is_marker` /// accepts, then normalize. /// /// Lowercased ONCE up front, so every index below refers to a single string. /// Comparing a lowercased copy while slicing the original desyncs the two the /// moment a character's lowercase form has a different byte length (İ -> i̇), /// which is a panic on a non-ASCII title rather than a wrong answer. fn title_identity_stripping(title: &str, is_marker: fn(&str) -> bool) -> String { let mut work = title.trim().to_lowercase(); loop { let found = work.char_indices().find_map(|(open, ch)| { let close_ch = match ch { '(' => ')', '[' => ']', _ => return None, }; let close = open + work[open..].find(close_ch)?; is_marker(&work[open + 1..close]).then_some((open, close)) }); let Some((open, close)) = found else { break }; work.replace_range(open..=close, " "); } if let Some(at) = [" - ", " – ", " — "] .iter() .filter_map(|dash| work.rfind(dash)) .max() { if is_marker(&work[at..]) { work.truncate(at); } } normalize(&work) } /// Fold Last.fm's top-tracks list, which arrives listeners-descending, into /// one entry per song. /// /// The FIRST entry for a title is the canonical song; every later one is a /// re-listing of it — an "(Album Version)", a festival recording, a remaster — /// carrying that pressing's own much smaller count. Keeping the first is the /// whole point, and it has to be done with a vacancy check: `HashMap::insert` /// returns the previous value but STILL overwrites, so a guard written as /// `if map.insert(..).is_some() { continue }` lets the duplicate win the map /// while correctly skipping the ranked list. /// /// That was the bug behind the album tracklist's counts. Last.fm lists Buddy /// Holly at rank 1 with 2,351,642 listeners and again at rank 679 as /// "(Album Version)" with 1,919 — and 1,919 is the number the album page /// printed. Every song popular enough to be re-listed showed its most obscure /// pressing; only songs Last.fm happens to list exactly once (Weezer's "Tired /// of Sex", "Across the Sea") were ever right, which is why the column looked /// plausible in places and absurd in others. fn fold_lastfm_duplicates(entries: Vec<(String, String, TrackStats)>) -> LastfmRanks { let mut by_key: HashMap = HashMap::new(); let mut ranked: Vec = Vec::new(); for (key, title, stats) in entries { let std::collections::hash_map::Entry::Vacant(slot) = by_key.entry(key.clone()) else { continue; }; slot.insert(stats); if ranked.len() < LASTFM_RANKED_KEEP { ranked.push(LastfmTrack { key, title, stats }); } } LastfmRanks { by_key, ranked } } fn track_title_identity(title: &str) -> String { let mut work = title.trim().to_lowercase(); // A featured-artist clause is Deezer packaging, never part of the scrobble. for marker in [" feat.", " feat ", " ft.", " ft ", " featuring "] { if let Some(at) = work.find(marker) { work.truncate(at); } } title_identity_stripping(&work, is_track_version_marker) } /// Does this segment name a re-PRESSING of an album rather than a different /// work? Mirrors the frontend's `EDITION_MARKER` vocabulary /// (`CatalogArtist.svelte`), so Rust and the UI fold the same titles. /// /// `"'s Version"` is deliberately exempt, for the same reason the /// frontend exempts it: a re-recorded catalogue (Taylor's Version) is its own /// work with its own first-release date, not a reissue of the original. fn is_album_edition_marker(segment: &str) -> bool { let segment = segment.trim().trim_matches(|c: char| !c.is_alphanumeric()); if segment.is_empty() { return false; } if segment.ends_with("'s version") || segment.ends_with("\u{2019}s version") { return false; } const MARKERS: [&str; 17] = [ "super deluxe", "deluxe", "expanded", "remaster", "anniversary", "legacy edition", "special edition", "collector", "bonus track", "complete edition", "definitive edition", "reissue", "redux", "mono edition", "stereo edition", "half-speed", "half speed", ]; MARKERS.iter().any(|marker| segment.contains(marker)) // "'s Edition", "Tour Edition", "2011 Edition" -- the frontend's // `\w+(?:['’]s)?\s+edition` arm, minus the exemption handled above. || segment.ends_with(" edition") || segment == "edition" } /// The cross-provider key for an ALBUM title, used only to line a Deezer /// release up with the MusicBrainz release group it is a pressing of. /// /// Deezer sells "Hot Space", "A Night at the Opera (2011 Remaster)" and /// "American Idiot (Deluxe)"; MusicBrainz calls all three by the work's name. /// Like [`track_title_identity`], this only ever runs INSIDE an artist already /// anchored by Deezer id -> MBID, so it decides which release two providers are /// both describing, never which artist we are looking at. fn album_title_identity(title: &str) -> String { let base = strip_release_format_tail(title).0.to_lowercase(); let base = keep_edition_disambiguators(&base); title_identity_stripping(&fold_volume_markers(&base), is_album_edition_marker) } /// Keep the identifying half of a parenthetical that mixes a pressing with a /// disambiguator. /// /// Deezer sells Weezer's colour records as "Weezer (Deluxe Edition / Red /// Album)". That single parenthetical carries BOTH the pressing ("Deluxe /// Edition") and the only thing saying WHICH self-titled album this is ("Red /// Album"), so dropping it whole reduces the 2008 Red Album to `weezer` — the /// same key as the 1994 debut. MusicBrainz titles all seven self-titled Weezer /// groups plain "Weezer", so that key resolves to the EARLIEST of them, and the /// Red Album deluxe was dated 1994-05-10 and filed fourteen years early. /// /// Splitting on the separator and keeping only the parts that are NOT edition /// markers leaves "weezer (red album)", which keeps its colour and so its own /// identity. A segment that is entirely edition markers is left exactly as it /// was, for [`title_identity_stripping`] to remove as before. fn keep_edition_disambiguators(lowercased: &str) -> String { let mut work = lowercased.to_string(); let mut from = 0; while from < work.len() { let found = work[from..].char_indices().find_map(|(offset, ch)| { let closer = match ch { '(' => ')', '[' => ']', _ => return None, }; let open = from + offset; let close = open + work[open..].find(closer)?; Some((open, close)) }); let Some((open, close)) = found else { break }; let parts: Vec<&str> = work[open + 1..close].split('/').map(str::trim).collect(); let kept: Vec<&str> = parts .iter() .copied() .filter(|part| !part.is_empty() && !is_album_edition_marker(part)) .collect(); // Only rewrite a segment that genuinely MIXES the two. A pure edition // clause still belongs to `title_identity_stripping`, and a segment // with no edition part at all is already kept verbatim. if parts.len() > 1 && !kept.is_empty() && kept.len() < parts.len() { let replacement = format!("({})", kept.join(" ")); work.replace_range(open..=close, &replacement); from = open + replacement.len(); } else { from = close + 1; } } work } /// Drop the word in front of a volume number, so "Volume 2", "Vol. 2" and a /// bare "2" all reduce alike. /// /// Deezer and MusicBrainz number a series differently — measured on Queen, /// Deezer sells "Deep Cuts 2 (1977-1982)" while MusicBrainz files "Deep Cuts, /// Volume 2 (1977–1982)". Only the word differs, so removing it leaves both /// sides at `deepcuts219771982` once [`normalize`] discards punctuation. /// /// Only applied when a digit actually follows, so a record named "Volume" or /// "The Volume of Noise" keeps its word. fn fold_volume_markers(lowercased: &str) -> String { const MARKERS: [&str; 3] = ["volume", "vol.", "vol"]; let mut out = String::with_capacity(lowercased.len()); let mut rest = lowercased; 'outer: while !rest.is_empty() { for marker in MARKERS { let Some(tail) = rest.strip_prefix(marker) else { continue; }; // A marker only counts on a word boundary, and only when the next // non-space character is the number it introduces. let starts_word = out.is_empty() || !out.ends_with(|c: char| c.is_alphanumeric()); if starts_word && tail.trim_start().starts_with(|c: char| c.is_ascii_digit()) { rest = tail.trim_start(); continue 'outer; } } let ch = rest.chars().next().unwrap_or_default(); out.push(ch); rest = &rest[ch.len_utf8()..]; } out } /// Decode a `%XX`/`+` encoded URL path segment. MusicBrainz stores a declared /// Last.fm link with spaces as `%20` while Last.fm echoes the same artist back /// with `+`, so the guard has to compare DECODED identifiers — a byte compare /// rejects 2 of every 18 real matches (measured: Inland Knights, Flight /// Facilities). fn percent_decode_plus(value: &str) -> String { let bytes = value.as_bytes(); let mut out: Vec = Vec::with_capacity(bytes.len()); let mut i = 0; while i < bytes.len() { match bytes[i] { b'+' => { out.push(b' '); i += 1; } b'%' if i + 2 < bytes.len() => { match u8::from_str_radix(&value[i + 1..i + 3], 16) { Ok(byte) => { out.push(byte); i += 3; } Err(_) => { out.push(b'%'); i += 1; } } } other => { out.push(other); i += 1; } } } String::from_utf8_lossy(&out).into_owned() } /// The artist identifier inside a `last.fm/music/` URL, decoded and /// lowercased so two encodings of one artist compare equal. fn lastfm_artist_identifier(url: &str) -> Option { let rest = url.split("/music/").nth(1)?; let segment = rest.split('/').next()?; if segment.is_empty() { return None; } Some(percent_decode_plus(segment).to_lowercase()) } /// Lookup key pairing a release's TYPE with its [`album_title_identity`]. /// /// The type belongs in the key because a title alone is not unique within one /// artist: Queen has a "The Miracle" album (1989) and a "The Miracle" single /// (1984), and keying on title alone dated the album to the single. fn release_date_key(primary_type: Option<&str>, title: &str) -> String { let identity = album_title_identity(title); if identity.is_empty() { return String::new(); } let kind = primary_type.unwrap_or("album").trim().to_lowercase(); format!("{kind}\u{1f}{identity}") } /// Fold one MusicBrainz release group into the first-release map. /// /// Two groups can share one key when an artist reissued a work under the same /// title and type; the earliest is the original. Different TYPES are already /// separated by [`release_date_key`] — Queen has both a "The Miracle" album /// (1989) and a "The Miracle" single (1984), and folding those together dated /// the album five years before it existed. /// /// The secondary types must come from WHICHEVER GROUP SUPPLIED THE SURVIVING /// DATE, never unioned across the key. Measured: MusicBrainz has three groups /// keyed `album/fleetwoodmac` — the real 1975 album with no secondary types, a /// 1979 Compilation, and a 2006 Compilation+Live. Unioning tags the 1975 album /// Compilation+Live; [`apply_original_album_attribution`] then excludes it and /// "Rhiannon" lands on `Greatest Hits`, which is the exact failure the type /// exclusion exists to prevent. fn merge_release_group_facts(out: &mut HashMap, group: &serde_json::Value) { let Some(title) = group.get("title").and_then(|v| v.as_str()) else { return; }; let Some(first) = group .get("first-release-date") .and_then(|v| v.as_str()) .filter(|value| !value.is_empty()) else { return; }; let primary_type = group.get("primary-type").and_then(|v| v.as_str()); let candidate = ReleaseFacts { first: first.to_string(), secondary: group .get("secondary-types") .and_then(|value| value.as_array()) .map(|types| { types .iter() .filter_map(|value| value.as_str().map(String::from)) .collect() }) .unwrap_or_default(), ambiguous: false, }; // Index the DISAMBIGUATED name first. MusicBrainz titles all seven // self-titled Weezer albums plain "Weezer" and separates them only by // `disambiguation` — "Blue Album", "Red Album", … "Gold Album" — which is // precisely what Deezer prints in its parenthetical ("Weezer (Red // Album)"). Keying on that turns a hopeless seven-way collision into seven // exact matches, and it is the only reason the 2026 Gold Album can be told // apart from the 1994 debut at all. if let Some(disambiguation) = group .get("disambiguation") .and_then(|value| value.as_str()) .map(str::trim) .filter(|value| !value.is_empty()) { merge_one_release_key( out, release_date_key(primary_type, &format!("{title} ({disambiguation})")), candidate.clone(), ); } merge_one_release_key(out, release_date_key(primary_type, title), candidate); } /// Fold one candidate into one key, tracking whether the key stays trustworthy. fn merge_one_release_key( out: &mut HashMap, key: String, candidate: ReleaseFacts, ) { if key.is_empty() { return; } let Some(held) = out.get_mut(&key) else { out.insert(key, candidate); return; }; // Two groups under one key with two different dates: the key can no longer // identify a record. A reissue duplicate (same date) is still fine, and so // is the disambiguated key, which stays one-to-one. let ambiguous = held.ambiguous || held.first != candidate.first; // Strictly earlier wins outright. On an equal date the plainer group wins: // when a studio album and a compilation of it share both title and date, // the album is the one the songs actually came out on. if candidate.first < held.first || (candidate.first == held.first && candidate.secondary.len() < held.secondary.len()) { *held = candidate; } held.ambiguous = ambiguous; } /// Replace Deezer's edition dates with the work's true first release. /// /// Deezer dates the PRESSING it licensed, not the record. It reports "A Night /// at the Opera" as 2005-11-22, "The Game" as 1994-10-17 and "Hot Space" as /// 1994-10-21 — CD reissue dates for albums from 1975, 1980 and 1982. Sorting a /// discography by those numbers puts a band's catalogue in the order its label /// happened to re-press it. /// /// MusicBrainz models this correctly: a release GROUP is the work, and its /// `first-release-date` is the original. Matching is by /// [`album_title_identity`], so an edition folds onto the work it is a pressing /// of; an unmatched album simply keeps whatever Deezer said. fn apply_original_release_dates( albums: &mut [CatalogAlbum], first_released: &HashMap, ) -> usize { let mut corrected = 0; for album in albums.iter_mut() { // Deezer's record_type and MusicBrainz's primary-type agree on the // vocabulary (album / ep / single), and matching on it is what keeps a // same-named single from dating its album. let key = release_date_key( Some(album.record_type.as_deref().unwrap_or("album")), &album.title, ); if key.is_empty() { continue; } // An ambiguous key is several different records wearing one name. It // dates nothing — the release keeps whatever its own provider said, // which is the same "anchor, else leave it alone" rule the rest of the // page follows. let Some(original) = first_released.get(&key).filter(|facts| !facts.ambiguous) else { continue; }; if album.release_date.as_deref() == Some(original.first.as_str()) { continue; } album.release_date = Some(original.first.clone()); corrected += 1; } corrected } /// Give every edition the first-release date of the work it is a pressing of. /// /// `edition_group` is built from TRACKLIST overlap, so it knows something no /// title comparison can: which record a pressing actually is. That matters most /// exactly where titles fail — Deezer sells "Weezer (Deluxe Edition / Red /// Album)" and "Weezer (Deluxe Edition / Blue Album)" while MusicBrainz titles /// all seven self-titled Weezer groups plain "Weezer", so a title join cannot /// tell 1994 from 2008 and both deluxes landed on the debut's date. The edition /// link had already resolved both correctly from their tracklists. /// /// Runs after [`apply_original_release_dates`] so an edition inherits the /// original's CORRECTED date, not Deezer's reissue date for it. A pressing of a /// work did, by definition, first come out when the work did — which is the /// same premise the whole date pass rests on. fn apply_edition_group_dates(albums: &mut [CatalogAlbum]) -> usize { // An original carries `edition_group == Some(own id)`, so it is its own // source here and is never rewritten. let originals: HashMap = albums .iter() .filter(|album| album.edition_group == Some(album.id)) .filter_map(|album| Some((album.id, album.release_date.clone()?))) .collect(); let mut corrected = 0; for album in albums.iter_mut() { let Some(group) = album.edition_group else { continue; }; if group == album.id { continue; } let Some(original) = originals.get(&group) else { continue; }; if album.release_date.as_deref() == Some(original.as_str()) { continue; } album.release_date = Some(original.clone()); corrected += 1; } corrected } /// The playable, id-verified popular pool for one artist, in Deezer's own /// order. Both the overview and the full page build it identically, and both /// then hand it to [`apply_lastfm_order`], so it lives here rather than as two /// copies kept in sync by hand. fn deezer_popular_pool( popular: Option<&serde_json::Value>, artist_id: u64, foreign_album_ids: &HashSet, apple: &AppleArtistCatalog, ) -> Vec { popular .and_then(|body| body.get("data")) .and_then(|data| data.as_array()) .map(|tracks| { tracks .iter() .filter(|track| track_belongs_to_artist(track, artist_id)) .filter_map(parse_track) .filter(|track| { !track .album_id .is_some_and(|id| foreign_album_ids.contains(&id)) && !apple_release_is_foreign(apple, &track.album, None) }) .collect() }) .unwrap_or_default() } /// Reorder Deezer's top tracks by Last.fm listener counts. /// /// **Last.fm supplies ORDER; Deezer supplies MEMBERSHIP.** Nothing is ever /// promoted into this list from Last.fm — only the tracks Deezer already /// verified belong to this artist id (`track_belongs_to_artist`) are ranked. /// That is what makes a merged Last.fm page harmless: Last.fm files several /// bands under `/music/Scarlet`, and its top track there ("Independent Love /// Song", 35,745 listeners) belongs to a different one — but it is not in /// Deezer's list for this artist, so it simply never appears. /// /// Tracks Last.fm has no count for keep their Deezer order BELOW the ranked /// ones rather than being dropped; a brand-new single nobody has scrobbled yet /// is still a real track. Same-recording pressings are collapsed to the entry /// Deezer ranks highest, because they would otherwise sort adjacently showing /// an identical count twice. /// Build Popular from Last.fm's ranking, using Deezer only for playable rows. /// /// The old rule took Deezer's `/top` fifty and re-sorted them by scrobbles, /// which silently capped the list at what Deezer happens to sell. For most /// artists that is invisible — measured across Queen, Radiohead, Weezer, Green /// Day, Fleetwood Mac and Tame Impala the two agree on 56 of 60 rows. For an /// artist Deezer carries badly it is the whole list: Deezer has no Loveless at /// all, so 8 of My Bloody Valentine's Last.fm top ten were unreachable and /// Popular filled up with b-sides, a compilation cut and a soundtrack cut. /// /// So Last.fm decides what is popular, in its order, and each entry is resolved /// in preference order: /// /// 1. A Deezer track of the same [`track_title_identity`] — the ideal case, /// since it brings a real album, cover, preview and duration, and stays /// eligible for [`apply_original_album_attribution`] afterwards. /// 2. Otherwise a MusicBrainz recording, which supplies the record it belongs /// to and the track length. There is no preview, and that is the accepted /// cost: playback resolves over Soulseek by artist and title, so the row /// plays exactly like any other. /// 3. Otherwise the entry is dropped — a title with no album behind it from /// either provider is not something the page can meaningfully show. /// /// Deezer rows that Last.fm has never seen still backfill from underneath, so a /// thin Last.fm artist keeps a full list rather than an empty one. /// Put a popular row on the record its song came from, when that record is one /// Deezer does not sell. /// /// [`apply_original_album_attribution`] can only ever move a track onto ANOTHER /// DEEZER ALBUM, because it swaps the row onto that album's real track. When /// the original is missing from Deezer entirely there is nothing to swap to, so /// the row keeps whatever pressing Deezer licensed — and the pressing it /// licensed is exactly the kind of record the rule exists to get rid of. My /// Bloody Valentine is the clean example: Deezer carries just two Loveless /// songs, "Only Shallow" on the compilation *Amateur* and "Sometimes" on the /// *Lost In Translation* soundtrack, so those two sat under a compilation and a /// soundtrack while the eighteen tracks Deezer does NOT carry were already /// showing Loveless. /// /// This is a RELABEL, not a swap: `album`, `album_id` and `cover` follow /// MusicBrainz while `id`, `preview` and `duration_secs` stay on the Deezer /// pressing, because that pressing is still the playable one. Nothing is /// misrepresented — it is the same recording, and the album named is the album /// the recording is from. /// /// Only a PLAIN album may claim a row, and only away from a record that is not /// itself one. Without the second half a legitimately-attributed row churns: /// MusicBrainz would happily relabel Queen's "Bohemian Rhapsody" from Deezer's /// *A Night At The Opera* pressing to its own, which is the same record under a /// different spelling. fn apply_musicbrainz_track_albums( tracks: &mut [CatalogTrack], recordings: &HashMap, albums: &[CatalogAlbum], mb: &HashMap, ) -> usize { let mut relabelled = 0; for track in tracks.iter_mut() { let Some(recording) = recordings.get(&track_title_identity(&track.title)) else { continue; }; if !recording.plain_album { continue; } // Already on that record, however the two providers spell it. if album_title_identity(&track.album) == album_title_identity(&recording.album_title) { continue; } // Only take a row AWAY from a record that is not a studio album. A row // already sitting on one is either correctly attributed or was just // placed there by `apply_original_album_attribution`, and overruling it // would trade a real Deezer album card for a synthetic id with no card // in the discography behind it. let current = track .album_id .and_then(|id| albums.iter().find(|album| album.id == id)); let displaceable = match current { Some(album) => { album.is_soundtrack || album.is_live || album.record_type.as_deref() != Some("album") || musicbrainz_excludes_original(mb, album) } // No album row on this page at all — nothing to protect. None => true, }; if !displaceable { continue; } track.album = recording.album_title.clone(); track.album_id = Some(synthetic_catalog_id(&format!( "release-group:{}", recording.release_group_id ))); track.cover = Some(format!( "https://coverartarchive.org/release-group/{}/front-500", recording.release_group_id )); relabelled += 1; } relabelled } fn build_popular( ranked: &[LastfmTrack], pool: Vec, recordings: &HashMap, artist: &str, keep: usize, ) -> Vec { let mut by_key: HashMap = HashMap::new(); let mut pool_order: Vec = Vec::new(); for track in pool { let key = track_title_identity(&track.title); if key.is_empty() { continue; } if !by_key.contains_key(&key) { pool_order.push(key.clone()); by_key.insert(key, track); } } let mut out: Vec = Vec::new(); let mut used: HashSet = HashSet::new(); for entry in ranked { if out.len() >= keep { break; } if let Some(mut track) = by_key.remove(&entry.key) { track.listeners = Some(entry.stats.listeners); track.plays = (entry.stats.plays > 0).then_some(entry.stats.plays); used.insert(entry.key.clone()); out.push(track); continue; } let Some(recording) = recordings.get(&entry.key) else { continue; }; used.insert(entry.key.clone()); out.push(CatalogTrack { // Stable across loads and distinct from every Deezer id, the same // way supplemented ALBUM rows are keyed. id: synthetic_catalog_id(&format!( "lastfm-track:{}:{}", recording.release_group_id, entry.key )), // Deezer's spelling is preferred wherever it exists; here there is // none, so MusicBrainz's is the canonical one rather than Last.fm's // scrobbled string, which carries whatever users typed. title: recording.title.clone(), artist: artist.to_string(), artist_id: None, rank: None, album: recording.album_title.clone(), // The SAME synthetic id the album supplement mints for this release // group, so the row points at the card the discography already // shows and the now-playing marker lines up. album_id: Some(synthetic_catalog_id(&format!( "release-group:{}", recording.release_group_id ))), track_no: None, disk_number: None, unreleased: false, cover: Some(format!( "https://coverartarchive.org/release-group/{}/front-500", recording.release_group_id )), // No Deezer pressing means no 30-second preview. The row still // plays: streaming resolves by artist and title over Soulseek. preview: None, duration_secs: recording.length_ms.map(|ms| (ms / 1000) as u32), listeners: Some(entry.stats.listeners), plays: (entry.stats.plays > 0).then_some(entry.stats.plays), origin_track_id: None, }); } // Backfill in Deezer's own order for anything Last.fm never ranked, so a // sparsely-scrobbled artist still gets a full list. for key in pool_order { if out.len() >= keep { break; } if used.contains(&key) { continue; } if let Some(track) = by_key.remove(&key) { out.push(track); } } out } fn apply_lastfm_order( tracks: Vec, ranks: &HashMap, keep: usize, ) -> Vec { let mut seen: HashSet = HashSet::new(); let mut ordered: Vec<(usize, CatalogTrack)> = Vec::with_capacity(tracks.len()); for track in tracks { let key = track_title_identity(&track.title); if !key.is_empty() && !seen.insert(key.clone()) { continue; } let mut track = track; let stats = ranks.get(&key).copied(); track.listeners = stats.map(|s| s.listeners); track.plays = stats.map(|s| s.plays).filter(|plays| *plays > 0); ordered.push((ordered.len(), track)); } // Stable by construction: ranked tracks descend by listeners, and anything // Last.fm has never seen keeps its Deezer position underneath them. ordered.sort_by(|(left_at, left), (right_at, right)| { match (left.listeners, right.listeners) { (Some(a), Some(b)) => b.cmp(&a), (Some(_), None) => std::cmp::Ordering::Less, (None, Some(_)) => std::cmp::Ordering::Greater, (None, None) => std::cmp::Ordering::Equal, } .then_with(|| left_at.cmp(right_at)) }); ordered.truncate(keep); ordered.into_iter().map(|(_, track)| track).collect() } /// Does MusicBrainz say this Deezer release is a compilation, a live record or /// a soundtrack rather than a studio album? /// /// Exclusion requires POSITIVE evidence. A key MusicBrainz has never heard of /// is not excluded, because "MusicBrainz is silent" and "MusicBrainz says /// compilation" are different facts — Deezer sells plenty of pressings whose /// exact title MusicBrainz has no group for, and refusing those would throw /// away the real album (the measured Weezer case). /// /// Only the three MEASURED types are listed. `Remix` and `DJ-mix` are the same /// class and would probably be fine, but they were never measured, and the /// Deezer side already refuses the `compile` record type independently. fn musicbrainz_excludes_original(mb: &HashMap, album: &CatalogAlbum) -> bool { let key = release_date_key( Some(album.record_type.as_deref().unwrap_or("album")), &album.title, ); if key.is_empty() { return false; } mb.get(&key).is_some_and(|facts| { // An ambiguous key describes several records at once, so it is not // evidence about this one. !facts.ambiguous && facts.secondary.iter().any(|kind| { matches!( kind.trim().to_lowercase().as_str(), "compilation" | "live" | "soundtrack" ) }) }) } /// Move each popular row onto the album the song is actually FROM. /// /// Deezer attributes a popular track to whichever release it LICENSED, not the /// record the song came out on: "Bohemian Rhapsody" comes back on the 2018 /// movie soundtrack, carrying that poster as its cover art, and 11 of Queen's /// top 12 are wrong the same way. The artist's own album details are already /// fetched for the discography and every one of them carries a full tracklist, /// so the song's original home is findable at ZERO extra network cost. /// /// Containment alone is nowhere near enough, and that is the whole difficulty: /// "Bohemian Rhapsody" is contained by FIFTEEN of Queen's albums — seven live /// records, four hits packages, the soundtrack, two box sets, plus the real /// album and its deluxe. The ranking is the feature, not the lookup. /// /// Measured across 8 artists / 80 tracks: **79/80 (98.8%)** against 58/80 for /// Deezer's own attribution. The MusicBrainz secondary-type exclusion is /// load-bearing — without it the score falls to 63/80 and 16 picks flip to /// WRONG, because compilations are dated EARLIER than the works they compile, /// so "earliest release wins" actively *selects* hits packages. That is how an /// earlier attempt landed "Boulevard of Broken Dreams" on the American Idiot /// Broadway cast recording. **The type exclusion is the feature; the date is /// only the tiebreak.** /// /// Returns the number of rows moved. Every failure path degrades to exactly /// today's Deezer attribution rather than to a guess. fn apply_original_album_attribution( tracks: &mut [CatalogTrack], albums: &[CatalogAlbum], containment: &HashMap>, details: &HashMap, mb: &HashMap, ) -> usize { // MusicBrainz evidence is the anchor, and without it the rule collapses to // "earliest release wins", which is measurably worse than doing nothing // (12 rows broken). An artist with no linked MBID, a failed browse, or a // quota-emptied details pass all land here and leave the list untouched. if mb.is_empty() || containment.is_empty() { return 0; } let mut moved = 0; for track in tracks.iter_mut() { let key = contained_track_identity(&track.title); if key.is_empty() { continue; } let winner = albums .iter() .enumerate() .filter(|(_, album)| { // Singles and `compile` rows are not where a song is from. matches!(album.record_type.as_deref(), Some("album") | Some("ep")) // A guest record, a live album and a soundtrack are all // releases the song is genuinely ON — none is where it is // FROM. These three are Deezer-side evidence, independent // of MusicBrainz, which is why the Broadway-cast class is // rejected three separate ways. && !album.appears_on && !album.is_live && !album.is_soundtrack && !is_future_or_unresolved_date(album.release_date.as_deref()) && containment .get(&album.id) .is_some_and(|titles| titles.contains(&key)) && !musicbrainz_excludes_original(mb, album) }) .min_by(|(left_at, left), (right_at, right)| { // An ALBUM outranks an EP, above everything else. A hit is // routinely issued as a lead single/EP months before the record // it belongs to, so on date alone the EP always wins: measured // in the running app, Radiohead's "Creep" moved off *Pablo // Honey* (1993-02-22, and Deezer's own correct answer) onto a // "Creep EP" (1992-09-21). That is this feature making a right // row wrong, and it is the same packaging-versus-work confusion // the edition tiebreak below settles. An EP still wins whenever // no album contains the song, so EP-only tracks are unaffected. let is_ep = |album: &CatalogAlbum| u8::from(album.record_type.as_deref() == Some("ep")); // A plain title outranks an edition title ABOVE the date, not // after it. Measured: `album_title_identity` reduces "Weezer // (Deluxe Edition / Blue Album)" to `weezer`, which collides // with seven self-titled Weezer groups and inherits 1994-05-10, // while the plain "Weezer (Blue Album)" matches no group at all // and keeps a `None` date. Date-first therefore picks the // deluxe on every Weezer track; edition-first fixed all five // and took the measured pass from 74/80 to 79/80. An album // MusicBrainz has never heard of always loses to a wrongly- // dated album it has, so this has to sit above the date too. is_ep(left) .cmp(&is_ep(right)) .then_with(|| is_edition_title(&left.title).cmp(&is_edition_title(&right.title))) // Only now the date, and only as a tiebreak. .then_with(|| { left.release_date .as_deref() .unwrap_or("9999") .cmp(right.release_date.as_deref().unwrap_or("9999")) }) // A title one candidate BUILDS ON is the original of it. // // Above the track-count rule below, and Radiohead is why. // Deezer carries "OK Computer" (12 tracks) and "OK Computer // OKNOTOK 1997 2017" (23) and dates BOTH 1997-05-21, so EP, // edition-keyword and date all tie and the sampler rule // then picked the 23-track reissue for "No Surprises", // "Karma Police" and "Let Down". `is_edition_title` cannot // catch it because "OKNOTOK 1997 2017" is a bespoke name // with none of the usual keywords in it. // // Only ever fires when both records actually contain the // song, which is what makes it safe: a sequel does not // share its predecessor's tracklist, so "Greatest Hits II" // is never in the running against "Greatest Hits" for the // same track. The numeral guard is belt and braces. // `false` sorts before `true`, and the winner is the // SMALLEST, so the one that expands on the other loses. .then_with(|| expands_on(&left.title, &right.title) .cmp(&expands_on(&right.title, &left.title))) // An original album over a truncated sampler of it. .then_with(|| { right .track_count .unwrap_or(0) .cmp(&left.track_count.unwrap_or(0)) }) // Curated-Deezer order, so the pick is deterministic. .then_with(|| left_at.cmp(right_at)) }); let Some((_, album)) = winner else { continue; }; // Deezer already licensed the record the song is from. This has to be // an OUTPUT test rather than a candidate filter: excluding the current // album from the ranking hands every already-correct row to the // RUNNER-UP, which is how "Dreams" — correctly attributed to Rumours — // moved onto "Rumours (Super Deluxe)". 57 of the 80 measured tracks // are in this "already right, left alone" state. if Some(album.id) == track.album_id { continue; } // Everything below is the ATOMIC half: the row moves completely or not // at all. Swapping `album_id` while leaving `id` on the licensed // pressing is the one combination that silently kills album // auto-continue (`maybeExtendAlbumQueue` finds no matching track) and // suppresses the now-playing -> song jump, so a missing piece here has // to abandon the whole swap rather than write part of it. let Some(detail) = details.get(&album.id) else { continue; }; let matched = detail .pointer("/tracks/data") .and_then(|rows| rows.as_array()) .and_then(|rows| { rows.iter().find(|row| { row.get("title") .and_then(|title| title.as_str()) .is_some_and(|title| contained_track_identity(title) == key) }) }); let Some(matched) = matched else { continue; }; if matched.get("readable").and_then(|v| v.as_bool()) == Some(false) { continue; } let Some(new_id) = matched.get("id").and_then(|v| v.as_u64()) else { continue; }; // Take a SIZED cover. Deezer's bare `album.cover` has no `250x250` // segment, and both the preview reel and the player upgrade a cover by // rewriting that segment to 1000x1000 — handed a bare url the rewrite // is a silent no-op and the reel shows a 120px thumbnail blown up. let Some(cover) = detail .get("cover_medium") .or_else(|| detail.get("cover_big")) .and_then(|c| c.as_str()) else { continue; }; track.origin_track_id = Some(track.id); track.id = new_id; track.album = album.title.clone(); track.album_id = Some(album.id); track.cover = Some(cover.to_string()); // Carried, not preserved: the preview modal treats this as one-bit // evidence that a clip exists, and it must describe the row that is // actually there now. track.preview = matched .get("preview") .and_then(|p| p.as_str()) .filter(|p| !p.is_empty()) .map(String::from); if let Some(duration) = matched.get("duration").and_then(|d| d.as_u64()) { track.duration_secs = Some(duration as u32); } moved += 1; } moved } fn deezer_release_evidence(value: &serde_json::Value) -> Option { let title_norm = release_title_identity(value.get("title")?.as_str()?); if title_norm.is_empty() { return None; } // Cap one viral release so arithmetic remains bounded. The base point lets // several exact low-traffic matches still corroborate one another. let fans = value .get("fans") .and_then(|fans| fans.as_u64()) .unwrap_or_default() .min(1_000_000); Some(DeezerReleaseEvidence { title_norm, weight: fans + 1, genre_id: value.get("genre_id").and_then(|id| id.as_u64()), }) } fn unavailable_apple_candidates() -> Vec { // Two non-cacheable sentinels preserve the only safe facts after a failed // search: identity is unavailable and may be ambiguous. The Bing gate sees // len > 1, while overview/full return no stage and a later call retries. [0_u64, u64::MAX] .into_iter() .map(|artist_id| AppleArtistCandidate { artist_id, releases: Vec::new(), credited_titles: HashSet::new(), lookup_complete: false, cacheable: false, }) .collect() } /// Select one Apple performer by weighted title overlap with the releases /// returned for the requested Deezer id. Multiple exact-name candidates with a /// tied/no-overlap score are deliberately unresolved: supplements and name-only /// cover upgrades then fail closed rather than choosing result zero. fn select_apple_artist_catalog( candidates: &[AppleArtistCandidate], evidence: &[DeezerReleaseEvidence], require_evidence: bool, ) -> AppleArtistCatalog { if candidates.is_empty() { return AppleArtistCatalog::default(); } let mut evidence_by_title: HashMap)> = HashMap::new(); for item in evidence { let entry = evidence_by_title .entry(item.title_norm.clone()) .or_insert_with(|| (0, HashSet::new())); entry.0 = entry.0.max(item.weight); if let Some(genre_id) = item.genre_id { entry.1.insert(genre_id); } } if candidates.len() > 1 && candidates .iter() .any(|candidate| !candidate.lookup_complete) { return AppleArtistCatalog { evidence_count: evidence_by_title.len(), identity_unavailable: true, ..AppleArtistCatalog::default() }; } let mut scores: Vec<(usize, u64, usize, HashMap)> = candidates .iter() .enumerate() .map(|(index, candidate)| { let mut score = 0_u64; let mut overlap_count = 0_usize; let mut genres = HashMap::new(); let mut seen_titles = HashSet::new(); for release in &candidate.releases { if release.title_norm.is_empty() || !seen_titles.insert(&release.title_norm) { continue; } if let Some((weight, matched_genres)) = evidence_by_title.get(&release.title_norm) { score = score.saturating_add(*weight); overlap_count += 1; for genre_id in matched_genres { *genres.entry(*genre_id).or_insert(0) += 1; } } } (index, score, overlap_count, genres) }) .collect(); scores.sort_by(|left, right| { right .1 .cmp(&left.1) .then_with(|| right.2.cmp(&left.2)) .then_with(|| { candidates[left.0] .artist_id .cmp(&candidates[right.0].artist_id) }) }); let winner_index = if candidates.len() == 1 { (!require_evidence || scores[0].1 > 0).then_some(0) } else { let best = &scores[0]; let runner_up = &scores[1]; ((best.1, best.2) > (runner_up.1, runner_up.2) && best.1 > 0).then_some(best.0) }; let Some(winner_index) = winner_index else { return AppleArtistCatalog { evidence_count: evidence_by_title.len(), ..AppleArtistCatalog::default() }; }; let evidence_weight = scores .iter() .find(|score| score.0 == winner_index) .map(|score| score.1) .unwrap_or_default(); let winner_titles: HashSet = candidates[winner_index] .releases .iter() .map(|release| release.title_norm.clone()) .filter(|title| !title.is_empty()) .collect(); let winner_genres = scores .iter() .find(|score| score.0 == winner_index) .map(|score| score.3.clone()) .unwrap_or_default(); let mut foreign_titles = HashSet::new(); let mut foreign_genres = HashSet::new(); for (index, candidate) in candidates.iter().enumerate() { if index == winner_index { continue; } foreign_titles.extend( candidate .releases .iter() .map(|release| release.title_norm.clone()) .filter(|title| !title.is_empty() && !winner_titles.contains(title)), ); if let Some(score) = scores.iter().find(|score| score.0 == index) { foreign_genres.extend( score .3 .iter() // One exact losing title is already removed by title. Only // repeated same-candidate corroboration can expand that to // a whole losing-only genre cluster. .filter(|(_, count)| **count >= 2) .map(|(genre_id, _)| *genre_id), ); } } // A genre shared by winner and loser is not identity evidence. This keeps // legitimate collaborations and broad styles intact. foreign_genres.retain(|genre_id| !winner_genres.contains_key(genre_id)); AppleArtistCatalog { releases: candidates[winner_index].releases.clone(), foreign_titles, foreign_genres, apple_artist_id: Some(candidates[winner_index].artist_id), evidence_weight, evidence_count: evidence_by_title.len(), identity_unavailable: false, identity_confirmed: true, } } /// A record label reduced to a comparison key. MusicBrainz writes full legal /// names ("In the Red Records"), Deezer writes trade names ("In The Red") and /// Apple writes whatever sits after the ℗ year, so the corporate furniture has /// to go before two spellings of one label can meet. Punctuation is dropped /// entirely rather than collapsed to spaces — "20/20 Vision" and "2020Vision" /// are the same label and were not matching while a space survived between /// them. fn label_identity(label: &str) -> String { // Whole words only. A substring pass turns "southern" into "sou|rn" and // "rafftheraff" into "raff|raff" by eating an inner "the", which is how a // co-tenant's imprint could come to collide with an unrelated one. const NOISE: [&str; 15] = [ "records", "record", "recordings", "recording", "music", "musik", "entertainment", "group", "limited", "ltd", "llc", "inc", "gmbh", "the", "division", ]; label .to_lowercase() .replace('&', " and ") .split(|c: char| !c.is_alphanumeric()) .filter(|word| !word.is_empty() && !NOISE.contains(word)) .collect() } /// "℗ 2014 GOD?" → "GOD?". Apple states the label only inside the copyright /// line, and the leading symbol/year is the only reliably-shaped part of it. fn label_from_copyright(copyright: &str) -> String { let trimmed = copyright .trim_start_matches(['℗', '©', ' ']) .trim_start(); let without_year = trimmed .split_once(' ') .filter(|(head, _)| head.len() == 4 && head.chars().all(|c| c.is_ascii_digit())) .map(|(_, tail)| tail) .unwrap_or(trimmed); without_year.trim().to_string() } fn label_matches(label: &str, known: &HashSet) -> bool { let key = label_identity(label); if key.len() < 3 { return false; } known.iter().any(|candidate| { candidate.len() >= 3 && (candidate == &key || (key.len() >= 5 && candidate.contains(&key)) || (candidate.len() >= 5 && key.contains(candidate.as_str()))) }) } /// Per-release ownership verdicts for one artist, decided by that artist's own /// MusicBrainz identity rather than by Deezer's release list. /// /// Two different failures are answered by the same evidence: /// /// 1. **Deezer files several unrelated acts under one artist id.** There is /// only ONE Deezer artist called "Wand" (1213178) and it carries the LA /// psych band, a Brazilian funk MC, an Indonesian act and four more — 46 of /// its 72 releases belong to somebody else. Nothing about ranking search /// results or reading credit roles can help: there is one candidate to open, /// and every row is primary-credited to it. /// 2. **The guest-appearance guard deletes real collaborations.** That guard /// keeps an `appears_on` row only when MusicBrainz credits it to this /// artist's own MBID, which assumes MB files collaborations under both /// parties. It does for "Watch the Throne" (one artist credit naming both) /// and it does NOT when the collaboration is its own MB entity: neither /// VULTURES 1, VULTURES 2 nor KIDS SEE GHOSTS appears anywhere in Ye's 315 /// release groups, so all three were being erased from his page. /// /// `owned` answers (2) — it is every Deezer row this artist's identity /// corroborates, so a flagged collaboration survives on evidence rather than on /// MB happening to duplicate the credit. `foreign` answers (1), and is empty /// unless co-tenancy is PROVEN: some row on this page must be carried by a /// *different* same-named Apple performer and by neither of ours. Without that /// proof the pass observes and removes nothing, which is what keeps Weezer's 67 /// rows, Kanye's 73 and Pond's 30 untouched — measured, not assumed. #[derive(Debug, Clone, Default)] struct ReleaseOwnership { owned: HashSet, foreign: HashSet, cotenancy_proven: bool, } fn apple_release_is_foreign( catalog: &AppleArtistCatalog, title: &str, genre_id: Option, ) -> bool { catalog.identity_confirmed && (catalog .foreign_titles .contains(&release_title_identity(title)) || genre_id.is_some_and(|genre_id| catalog.foreign_genres.contains(&genre_id))) } /// True when an Apple display credit names this artist as one of the principals /// rather than merely mentioning them. /// /// The credit is a list, so it is read as one: "¥$, Kanye West & Ty Dolla $ign" /// names Kanye, "Estelle" does not — even though her record's TITLE says /// "American Boy (feat. Kanye West)". Matching the title instead would turn /// every guest spot into an owned release, which is the exact bug the /// `appears_on` guard exists to prevent. fn apple_credit_names_artist(credit: &str, artist: &str) -> bool { let want = normalize(artist); if want.is_empty() { return false; } credit .split([',', '&', ';']) .flat_map(|part| part.split(" and ")) .any(|part| normalize(part) == want) } /// Top-track payloads retain featured appearances through the contributors /// list. Missing identity fields are accepted (older provider shapes), but an /// explicit different primary/contributor set is not. fn track_belongs_to_artist(track: &serde_json::Value, artist_id: u64) -> bool { let primary = track.pointer("/artist/id").and_then(|id| id.as_u64()); let contributor_ids: Vec = track .get("contributors") .and_then(|contributors| contributors.as_array()) .into_iter() .flatten() .filter_map(|contributor| contributor.get("id").and_then(|id| id.as_u64())) .collect(); primary == Some(artist_id) || contributor_ids.contains(&artist_id) || (primary.is_none() && contributor_ids.is_empty()) } fn apple_synthetic_catalog_id(deezer_id: u64, release: &AppleRelease, stripped_title: &str) -> u64 { synthetic_catalog_id(&format!( "apple:dz:{deezer_id}:itunes:{}:{}", release.collection_id.unwrap_or_default(), normalize(stripped_title) )) } /// Apple promo pseudo-albums ("iTunes Originals: Weezer", "iTunes Pass: …", /// "iTunes Session/Festival/Live"). They are real iTunes collections but not /// part of an artist's studio discography, so the release supplement drops them. fn is_itunes_promo(title: &str) -> bool { title.trim_start().to_lowercase().starts_with("itunes ") } /// Apple lists continuous DJ mixes ("Defected: Crazy P at Glitterbox … (DJ /// Mix)", "Late Night Tales: MGMT (DJ Mix)") as ordinary albums, but the mix /// is one streaming-only file: the iTunes song lookup returns ZERO tracks even /// though collection metadata claims 14/30, so an injected card can never /// render a tracklist or play anything. The bracketed "DJ Mix" tag is Apple's /// own naming convention for exactly these releases. fn is_apple_dj_mix_title(title: &str) -> bool { let lower = title.to_lowercase(); lower.contains("(dj mix)") || lower.contains("[dj mix]") || lower.contains("(continuous mix)") || lower.contains("[continuous mix]") } /// Collapse groups of same-work release rows down to their fullest member — /// most tracks, then the longer (credited) title. Rows fold only when they /// carry a release date and only inside one `date_prefix`-length bucket /// (4 = same year for exact-title duplicates, 10 = same day for /// featured-credit variants), so a genuine same-name re-release from another /// year keeps its own card. fn fold_duplicate_release_rows( albums: &mut Vec, groups: &[Vec], date_prefix: usize, ) { let mut drop_ids: HashSet = HashSet::new(); for ids in groups { let mut buckets: HashMap<&str, Vec> = HashMap::new(); for (index, album) in albums.iter().enumerate() { if !ids.contains(&album.id) { continue; } let Some(date) = album.release_date.as_deref() else { continue; }; buckets .entry(&date[..date.len().min(date_prefix)]) .or_default() .push(index); } for indices in buckets.values() { if indices.len() < 2 { continue; } // Most tracks wins; on a tie the SHORTER title is the canonical // base release ("All My Favorite Songs" over its later credited // re-release), not whichever variant is wordier. let keep = *indices .iter() .max_by_key(|&&index| { ( albums[index].track_count.unwrap_or(0), std::cmp::Reverse(albums[index].title.len()), ) }) .expect("bucket has at least two members"); for &index in indices { if index != keep { drop_ids.insert(albums[index].id); } } } } if !drop_ids.is_empty() { albums.retain(|album| !drop_ids.contains(&album.id)); } } /// Collapse exact same-title re-releases ACROSS years — the label re-issuing /// one single under a fresh id years later ("Step Into Christmas" 2018 and /// 2022, "Today Was A Fairytale" 2010 and 2018). Unlike the year-bucketed /// fold, this may only fire when the rows are provably the same material: /// singles/EPs with equal or unknown track counts, or albums whose KNOWN /// track counts agree at ≥5 tracks (a self-titled catalogue with two distinct /// eponymous records must never merge on a guess). The OLDEST row survives — /// original chronology is what the grid sorts by — preferring a row that /// actually knows its track count. fn fold_exact_rereleases(albums: &mut Vec, groups: &[Vec]) { let mut drop_ids: HashSet = HashSet::new(); for ids in groups { let indices: Vec = albums .iter() .enumerate() .filter(|(_, album)| ids.contains(&album.id) && !drop_ids.contains(&album.id)) .map(|(index, _)| index) .collect(); if indices.len() < 2 { continue; } let record_type = albums[indices[0]].record_type.as_deref().unwrap_or(""); let counts: Vec> = indices.iter().map(|&i| albums[i].track_count).collect(); let known: Vec = counts.iter().filter_map(|c| *c).collect(); let counts_agree = known.windows(2).all(|w| w[0] == w[1]); let same_material = match record_type { "single" | "ep" => counts_agree, "album" => { counts_agree && known.len() == indices.len() && known.first().copied().unwrap_or(0) >= 5 } _ => false, }; if !same_material { continue; } let keep = *indices .iter() .min_by_key(|&&index| { ( albums[index] .release_date .clone() .unwrap_or_else(|| "9999".into()), albums[index].track_count.is_none(), ) }) .expect("group has at least two members"); for &index in &indices { if index != keep { drop_ids.insert(albums[index].id); } } } if !drop_ids.is_empty() { albums.retain(|album| !drop_ids.contains(&album.id)); } } /// Title with its featured-artist credit segment removed, normalized. Deezer /// often carries one single twice — "Dancing In Babylon" and "Dancing In /// Babylon (feat. Christine and the Queens)" — where the credit is the only /// difference between the titles. /// /// Only the feat BRACKET itself is removed, never the rest of the title: on /// "Beneath with Me (feat. Skylar Grey) (Kaskade's V.4)" the remix qualifier /// after the credit is what makes it a different release, and an early cut /// from the credit onward folded every such variant into the plain single. /// All finds and slices run on one lowercased copy, so multi-byte lowercasing /// cannot split a character; `normalize` lowercases anyway. fn featless_title_identity(title: &str) -> String { let lower = title.to_lowercase(); let mut out = String::with_capacity(lower.len()); let mut rest = lower.as_str(); while let Some(at) = [ "(feat.", "[feat.", "(ft.", "[ft.", "(featuring ", "[featuring ", ] .iter() .filter_map(|marker| rest.find(marker)) .min() { out.push_str(&rest[..at]); let closer = if rest[at..].starts_with('(') { ')' } else { ']' }; match rest[at..].find(closer) { Some(end) => rest = &rest[at + end + 1..], None => { rest = ""; break; } } } out.push_str(rest); if normalize(&out).is_empty() { normalize(&lower) } else { normalize(&out) } } /// Whether the tail a longer title adds over a shorter one is a sequel or /// volume marker ("Greatest Hits" → "Greatest Hits II", "Deep Cuts" → "Deep /// Cuts 2"). Those are DIFFERENT works that happen to extend the name, so no /// same-work rule may treat them as variants. Input is already normalized. fn residual_is_sequel_marker(residual: &str) -> bool { let residual = residual.trim_matches(|c: char| !c.is_alphanumeric()); if residual.is_empty() { return false; } matches!( residual, "i" | "ii" | "iii" | "iv" | "v" | "vi" | "vii" | "viii" | "ix" | "x" ) || residual.chars().all(|c| c.is_ascii_digit()) && residual.len() <= 2 || ["vol", "volume", "pt", "part", "chapter"] .iter() .any(|marker| { residual.strip_prefix(marker).is_some_and(|n| { !n.is_empty() && n.chars() .all(|c| c.is_ascii_digit() || matches!(c, 'i' | 'v' | 'x')) }) }) } /// Whether the tail a variant title adds over its base names a known release /// variant — an alternate mix, version, edition or companion cut of the SAME /// work ("Revolver (2022 Mix)", "Higher Power (Acoustic Version)", "Triplicate /// (Sampler)", "…: THE ANTHOLOGY"). Input is already normalized. fn residual_is_release_variant(residual: &str) -> bool { [ "version", "mix", "mixes", "edit", "edits", "acoustic", "instrumental", "demo", "mono", "stereo", "sampler", "extras", "deluxe", "remaster", "edition", "anniversary", "expanded", "anthology", "collection", "soundtrack", "trackbytrack", "reissue", "redux", ] .iter() .any(|marker| residual.contains(marker)) } /// Days-since-epoch for a `YYYY-MM-DD`-prefixed date, for cheap date-gap /// comparisons. Not a calendar library: month lengths are approximated, which /// is safely within tolerance for a "released about a month apart" test. fn release_date_days(date: &str) -> Option { let mut parts = date.splitn(3, '-'); let year: i64 = parts.next()?.parse().ok()?; let month: i64 = parts .next() .and_then(|p| p.get(..2)) .and_then(|p| p.parse().ok()) .unwrap_or(6); let day: i64 = parts .next() .and_then(|p| p.get(..2)) .and_then(|p| p.parse().ok()) .unwrap_or(15); Some(year * 365 + month * 30 + day) } fn is_edition_title(title: &str) -> bool { let title = title.to_lowercase(); [ "deluxe", "expanded", "remaster", "anniversary", "special edition", "collector's edition", "collectors edition", "legacy edition", "bonus track", "complete edition", "definitive edition", "reissue", "mono edition", "stereo edition", // " edition" (space-prefixed so "Expedition" can't match) covers the // bespoke forms the fixed list kept missing: "Platinum Edition", // "Tour Edition", "3am Edition", "Drumless Edition". " edition", "half-speed", "half speed", "mono version", "stereo version", "track by track", ] .iter() .any(|marker| title.contains(marker)) // "…(2022 Mix)" / "(2010 Mono Version)" remix-remasters: a bracketed tail // that is a year plus mix/version wording is an edition of the base // record, not a new work. Kept narrow — a named remix package ("Kaskade's // V.4") is NOT an edition. || regex_lite_year_mix_tail(&title) } /// `(… 2022 mix)` / `(2010 mono …)`-shaped bracketed tail, without pulling in /// a regex dependency: a bracket segment containing a 4-digit year alongside /// "mix" or "version". fn regex_lite_year_mix_tail(lower_title: &str) -> bool { let Some(open) = lower_title.rfind(['(', '[']) else { return false; }; let segment = &lower_title[open + 1..]; let segment = segment .strip_suffix(|c| matches!(c, ')' | ']')) .unwrap_or(segment); let has_year = segment .split(|c: char| !c.is_ascii_digit()) .any(|run| run.len() == 4 && (run.starts_with("19") || run.starts_with("20"))); has_year && (segment.contains("mix") || segment.contains("version")) } /// Whether `title` contains one of `non_edition_titles` (already normalized) as a /// substring other than itself — the signal that a bespoke-named reissue /// ("OK Computer OKNOTOK 1997 2017") is built on an existing album ("OK /// Computer") despite carrying no edition keyword. Length-guarded so a very /// short album title cannot spuriously match. /// Does `title` look like `base` plus something — a reissue, an expansion, a /// bespoke anniversary name — rather than a different record? /// /// Returns `false` when the extra part is only a sequel number, because /// "Greatest Hits II" builds on nothing: it is its own record that happens to /// share a prefix. Same rule the frontend uses when it decides whether a /// MusicBrainz row is already covered by a Deezer variant. fn expands_on(title: &str, base: &str) -> bool { let (title, base) = (normalize(title), normalize(base)); if base.len() < 5 || title.len() <= base.len() || !title.starts_with(&base) { return false; } let tail = &title[base.len()..]; // A bare numeral or volume marker is a SEQUEL, not an edition. !matches!( tail, "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" | "ii" | "iii" | "iv" | "v" | "vi" | "vii" | "viii" | "ix" | "x" ) && !tail.starts_with("vol") && !tail.starts_with("part") && !tail.starts_with("pt") } fn contains_other_album_title(title: &str, non_edition_titles: &[String]) -> bool { let n = normalize(title); non_edition_titles .iter() .any(|t| t.len() >= 5 && n != *t && n.contains(t.as_str())) } /// Best-matching non-edition original for an edition, by ordered track overlap /// (`edition_track_score`). When `require_title_substring` is set, the original's /// normalized title must also be a substring of the edition's — the extra guard /// that lets bespoke reissues fold without absorbing an unrelated album. Returns /// `(original_index, original_id)`. fn best_edition_original( edition_index: usize, edition_norm_title: &str, edition_tracks: &[String], albums: &[CatalogAlbum], tracklists: &HashMap>, require_title_substring: bool, ) -> Option<(usize, u64)> { albums .iter() .enumerate() .filter(|(index, candidate)| { *index != edition_index && candidate.record_type.as_deref() == Some("album") && !is_edition_title(&candidate.title) }) .filter_map(|(index, candidate)| { let cand_norm = normalize(&candidate.title); let title_substring = cand_norm.len() >= 5 && edition_norm_title.contains(cand_norm.as_str()); if require_title_substring && !title_substring { return None; } let candidate_tracks = tracklists.get(&candidate.id)?; edition_track_score(edition_tracks, candidate_tracks) .map(|score| (title_substring, cand_norm.len(), score, index, candidate.id)) }) // The named base outranks a raw track-overlap winner, and the LONGEST // named base is the nearest work: "1989 (Taylor's Version) (Deluxe)" // belongs to "1989 (Taylor's Version)", never to "1989", even though // the re-recorded tracklist matches both once suffixes normalize away. .max_by_key(|(title_substring, base_len, score, _, _)| { (*title_substring, *base_len, *score) }) .map(|(_, _, _, index, id)| (index, id)) } /// Whether two normalized tracklists describe essentially the same release — /// same track count and ≥90% shared titles. The signal that an Apple /// "remaster"/reissue (e.g. "Weezer (2024 Remaster)") merely re-presents an /// album Deezer already carries in full, so the gap-filler supplement should /// drop it rather than show a redundant edition. fn is_duplicate_tracklist(a: &[String], b: &[String]) -> bool { if a.is_empty() || a.len() != b.len() { return false; } let b_set: std::collections::HashSet<&String> = b.iter().collect(); let shared = a.iter().filter(|t| b_set.contains(t)).count(); shared * 10 >= a.len() * 9 } /// Track identity for edition matching. Provider-added suffixes such as /// "(2011 Remaster)" and "(Original Mix)" describe the recording presentation, /// not a different song in the album's core sequence. fn track_identity(title: &str) -> String { let mut title = title.trim().to_lowercase(); for opener in [" (", " [", " - ", " – ", " — "] { if let Some(index) = title.rfind(opener) { let suffix = &title[index + opener.len()..]; if [ "remaster", "version", " mix", "mix)", "edit", "mono", "stereo", "live", "demo", "acoustic", "instrumental", ] .iter() .any(|marker| suffix.contains(marker)) { title.truncate(index); } } } normalize(&title) } /// Track identity for CONTAINMENT: does this album's tracklist hold this song? /// /// Neither existing normalizer answers that on both sides. /// [`track_title_identity`] deliberately keeps `(Live)` distinct — the test /// `track_identity_folds_pressings_but_not_performances` pins it — and is built /// on a different marker vocabulary than album tracklists use. /// [`track_identity`] folds `(Remastered 2011)` exactly the way pressings need, /// but has no `feat.` handling, so Deezer's `/top` row "Under Pressure (feat. /// David Bowie)" would never match the plain "Under Pressure" inside *Hot /// Space* — one of the twelve measured Queen rows. /// /// So: drop the featured-artist clause first, with the same markers /// [`track_title_identity`] uses, then fold pressing suffixes. Both sides of /// the comparison go through this one function. fn contained_track_identity(title: &str) -> String { let mut work = title.trim().to_lowercase(); // A featured-artist clause is Deezer packaging, never part of the song. It // arrives BOTH bare ("Song feat. X") and bracketed ("Song (feat. X)"), and // `track_title_identity` only lists the bare form because its bracket scan // catches the other one — `track_identity`, which this function ends in, // has no such scan. Each opener is spelled out rather than searched for // loosely, because the leading delimiter is the only thing stopping "ft " // from matching inside "soft " or "left ". for marker in [ " feat.", " feat ", " ft.", " ft ", " featuring ", "(feat.", "(feat ", "(ft.", "(ft ", "(featuring ", "[feat.", "[feat ", "[ft.", "[ft ", "[featuring ", ] { if let Some(at) = work.find(marker) { work.truncate(at); } } track_identity(&work) } fn edition_track_score(edition: &[String], original: &[String]) -> Option { let core = edition.len().min(original.len()).min(10); if core < 5 { return None; } let matches = edition .iter() .zip(original) .take(core) .filter(|(left, right)| !left.is_empty() && left == right) .count(); (matches >= 5 && matches * 4 >= core * 3).then_some(matches * 100 + core) } /// Prefer exact day precision when both sources provide it. A year-only date /// remains useful for older/obscure releases, where MusicBrainz sometimes has /// no more precise first-release date. /// Mirrors the frontend's upcoming-date test: a full date strictly after today, /// or a partial (year / year-month) date not yet established as past. Keeps both /// ends of the app in agreement about what "upcoming" means. fn is_future_or_unresolved_date(value: Option<&str>) -> bool { let Some(value) = value else { return false; }; let today = today_ymd(); if value.len() >= 10 { return value[..10] > today[..10]; } if value.len() >= 7 { return value[..7] >= today[..7]; } if value.len() >= 4 { return value[..4] >= today[..4]; } false } /// Today's UTC date as "YYYY-MM-DD". Day-level precision is all the /// upcoming-release check needs, so a wall clock without a timezone crate is /// fine here. fn today_ymd() -> String { let secs = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|elapsed| elapsed.as_secs()) .unwrap_or(0); let (year, month, day) = civil_from_days((secs / 86_400) as i64); format!("{year:04}-{month:02}-{day:02}") } /// Days since the Unix epoch to a civil (year, month, day), via Howard /// Hinnant's public-domain algorithm. fn civil_from_days(days: i64) -> (i64, u32, u32) { let z = days + 719_468; let era = if z >= 0 { z } else { z - 146_096 } / 146_097; let doe = z - era * 146_097; let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; let year = yoe + era * 400; let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); let mp = (5 * doy + 2) / 153; let day = (doy - (153 * mp + 2) / 5 + 1) as u32; let month = if mp < 10 { mp + 3 } else { mp - 9 } as u32; (if month <= 2 { year + 1 } else { year }, month, day) } fn same_original_date(left: Option<&str>, right: Option<&str>) -> bool { let (Some(left), Some(right)) = (left, right) else { return false; }; if left.len() >= 10 && right.len() >= 10 { left[..10] == right[..10] } else { left.get(..4) == right.get(..4) } } fn exact_studio_album_title(album: &CatalogAlbum, normalized_title: &str) -> bool { album.record_type.as_deref() == Some("album") && normalize(&album.title) == normalized_title } /// MusicBrainz uses the canonical self-titled name while stores often append a /// familiar disambiguator such as "(Blue Album)". Date + this shape safely /// joins those aliases without baking artist-specific color names into code. fn is_self_titled_album_alias(title: &str, artist: &str) -> bool { let title_lower = title.trim().to_lowercase(); let artist_lower = artist.trim().to_lowercase(); let Some(rest) = title_lower.strip_prefix(&artist_lower) else { return false; }; let rest = rest.trim(); (rest.starts_with('(') || rest.starts_with('[')) && rest.contains("album") } /// One decoded cover-art candidate considered by [`Metadata::fetch_art`]. /// One resolved cover option in a [`CoverReport`]. #[derive(serde::Serialize)] pub struct CandInfo { pub source: String, pub url: String, pub width: u32, pub height: u32, /// Perceptual (dHash) Hamming distance to the reference cover. `None` for /// the reference itself or when no reference decoded. pub distance: Option, /// Whether the candidate is the same artwork as the reference (distance /// within [`COVER_SAME_ART_MAX_DISTANCE`]). pub eligible: bool, pub chosen: bool, } /// Diagnostic result of resolving the best same-artwork cover for a catalog /// release across official (Apple, Deezer) and fallback (Cover Art Archive) /// sources, all gated against the known-correct reference cover. #[derive(serde::Serialize)] pub struct CoverReport { pub artist: String, pub title: String, pub reference: Option, pub candidates: Vec, pub chosen_source: Option, pub chosen_url: Option, } /* ---------------------------------------------------------------- identify -- The import review's record identification, ported from the approved mock (research/import/enrich.mjs). The folder name is not the album name: this kind of library writes rip descriptors into it — "Heartleap - WEB V0", "(1997.07.24) (album) Uchu Nippon Setagaya [128kbps]" — and sometimes the artist again. Sent verbatim, those queries miss records that are plainly in every catalogue (58 of 100 in the mock's first pass; the cleaned try-list + MB fallback resolved 50 of 65). The try-list mirrors the app's own simplifyTitle (state.svelte.js:3739); the judge still decides whatever comes back, so better recall cannot smuggle in a wrong record. */ /// One track of an identify candidate's tracklist. #[derive(Debug, Clone, serde::Serialize)] #[serde(rename_all = "camelCase")] pub struct IdentifyTrack { pub n: Option, pub title: String, pub secs: Option, } /// A provider's answer for "which record is this?", in the shape the mock's /// candidates.json established (the picker UI is built against it). `id` is /// a string on purpose: Deezer ids are numeric ("302127"), MusicBrainz ones /// are "mb:". #[derive(Debug, Clone, serde::Serialize)] #[serde(rename_all = "camelCase")] pub struct IdentifyCandidate { pub id: String, pub source: String, // "Deezer" | "MusicBrainz" pub title: String, pub artist: Option, /// Biggest artwork URL (Deezer xl / Cover Art Archive front). pub cover: Option, pub cover_big: Option, pub cover_medium: Option, pub label: Option, pub release_date: Option, pub genre: Option, pub track_count: usize, pub tracklist: Vec, } #[derive(Debug, Clone, serde::Serialize)] #[serde(rename_all = "camelCase")] pub struct IdentifyResult { /// The cleaned queries actually sent, least- to most-aggressive. pub tries: Vec, /// Which try produced the accepted candidate. pub used: Option, pub candidate: Option, /// Deezer answered with a record whose title matches no form of ours, /// so MusicBrainz was consulted. When MB also missed, `candidate` still /// carries Deezer's wrong answer — shown honestly as "Wrong record". pub deezer_wrong: bool, /// A provider REQUEST failed (503, timeout, unparseable) while nothing was /// found — so "nothing found" is not a fact about the record, it is a fact /// about the network. Caching it as an answer is what made "Murder of the /// Universe" a permanent deep cut: MusicBrainz holds it, scored 100, and /// answered on the next ask. /// /// MusicBrainz only. Its 1 req/s limiter makes a 503 routine — the shared /// Deezer helper has ~35 call sites and reports no failure reason, so a /// Deezer outage still reads as a miss. Honest about what it covers. pub incomplete: bool, /// Which piece of evidence carried a DISCOGRAPHY match — "its name", "its /// tracklist", "the same artwork". Absent when a phrase search answered, or /// when nothing did. /// /// Carried for the review to SAY so — a record identified by its songs or /// its sleeve rather than its name is a different kind of answer, and the /// verdict panel currently cannot tell the user which it was. Nothing /// renders it yet; the wording is a look change and belongs in a mockup /// first. pub matched_by: Option, } /// ASCII-normalize for title matching, exactly as the mock's normx: only /// lowercase ASCII alphanumerics survive. A fully non-Latin title /// normalizes to "" and therefore matches anything — the Kuchu Camp gap, /// kept faithfully (script-aware retry is a someday). fn normx(s: &str) -> String { s.to_lowercase() .chars() .filter(|c| c.is_ascii_alphanumeric()) .collect() } /// A title's words, punctuation folded INSIDE each word rather than split on. /// "Paul's Boutique" is two words, not three — splitting on the apostrophe /// would leave "paul" and "s" and stop it matching a folder spelled "Pauls". fn title_words(s: &str) -> Vec { s.split_whitespace() .map(|w| { w.chars() .filter(|c| c.is_alphanumeric()) .flat_map(char::to_lowercase) .collect::() }) .filter(|w| !w.is_empty()) .collect() } /// How strongly two album titles name the same record, 0.0 to 1.0. /// /// This is a COMPARISON, never a rewrite. Nothing is stripped from either /// side, which is the whole point: an album genuinely called "1989" still /// scores 1.0 against "1989", while a folder called "1989 Pauls Boutique" /// scores high against "Paul's Boutique" and zero against "Licensed to Ill". /// Stripping a leading year would have destroyed the first case to fix the /// second. /// /// Scored on the EDITION-FOLDED form, so a provider selling "Paul's Boutique /// (20th Anniversary Edition / Remastered)" is not penalised for the four /// words that say which pressing it is. Coverage is measured from BOTH sides /// and averaged: covering all of their title matters as much as covering all /// of ours, so a short title contained in a long one cannot score full marks. fn title_match_score(ours: &str, theirs: &str) -> f32 { if ours.trim().is_empty() || theirs.trim().is_empty() { return 0.0; } if normx(ours) == normx(theirs) || album_title_identity(ours) == album_title_identity(theirs) { return 1.0; } let a = comparable_words(ours); let b = comparable_words(theirs); if a.is_empty() || b.is_empty() { return 0.0; } // Multiset intersection: a word repeated in one title only counts as many // times as the other title actually repeats it. let mut pool: Vec<&String> = b.iter().collect(); let mut shared = 0usize; for w in &a { if let Some(at) = pool.iter().position(|x| *x == w) { pool.remove(at); shared += 1; } } if shared == 0 { return 0.0; } // Half for covering the shorter title, half for covering the longer. The // first half is what lets a folder carry an extra year and still match; // the second is what stops a two-word title matching everything that // contains it. let (small, large) = if a.len() <= b.len() { (a.len(), b.len()) } else { (b.len(), a.len()) }; let shared = shared as f32; 0.5 * (shared / small as f32) + 0.5 * (shared / large as f32) } /// Words that say which PRESSING this is rather than which record. Mirrors the /// frontend's `EDITION_WORD` vocabulary (`import-evidence.js`), so a provider /// selling "(20th Anniversary Edition / Remastered)" is compared on the four /// words that name the album and not the four that name the pressing. /// /// Deliberately narrower than that list: "original" and "special" are dropped /// from it because they are ordinary title words ("Original Pirate Material"). const EDITION_NOISE: [&str; 14] = [ "deluxe", "expanded", "remaster", "remastered", "anniversary", "edition", "editions", "reissue", "bonus", "version", "explicit", "clean", "mono", "stereo", ]; /// A bare ordinal — the "20th" of "20th Anniversary". Alone it identifies /// nothing, and left in it drags a real match below the threshold. fn is_ordinal(w: &str) -> bool { let (digits, suffix) = w.split_at(w.len().saturating_sub(2)); !digits.is_empty() && digits.chars().all(|c| c.is_ascii_digit()) && matches!(suffix, "st" | "nd" | "rd" | "th") } /// The words worth comparing. Falls back to the full set when stripping the /// pressing vocabulary would leave nothing — a record really called "Deluxe" /// still has to be comparable to itself. fn comparable_words(s: &str) -> Vec { let all = title_words(s); let kept: Vec = all .iter() .filter(|w| !EDITION_NOISE.contains(&w.as_str()) && !is_ordinal(w)) .cloned() .collect(); if kept.is_empty() { all } else { kept } } /// The mock's match rule: normalized equality or containment either way. /// Containment is what keeps an edition ("... 10th Anniversary Edition") /// from reading as a different record. fn normx_overlap(a: &str, b: &str) -> bool { let (na, nb) = (normx(a), normx(b)); na == nb || na.contains(nb.as_str()) || nb.contains(na.as_str()) } /// Case-insensitive prefix strip, char-boundary safe. fn strip_prefix_ci<'a>(s: &'a str, prefix: &str) -> Option<&'a str> { let mut end = 0; let mut sc = s.chars(); for p in prefix.chars() { let c = sc.next()?; if !c.to_lowercase().eq(p.to_lowercase()) { return None; } end += c.len_utf8(); } Some(&s[end..]) } /// `^[\[(]\d{4}[^\])]*[\])]\s*-?\s*` — a leading "[2015] - " or /// "(1997.07.24) " group, dates included. fn strip_leading_year_group(s: &str) -> String { if !(s.starts_with('[') || s.starts_with('(')) { return s.to_string(); } let bytes = s.as_bytes(); if bytes.len() < 6 || !bytes[1..5].iter().all(u8::is_ascii_digit) { return s.to_string(); } // The group runs to the FIRST close bracket of either kind, as in the // mock's character class. let Some(close) = s.find([']', ')']) else { return s.to_string(); }; if close < 5 { return s.to_string(); } let rest = s[close + 1..].trim_start(); let rest = rest.strip_prefix('-').unwrap_or(rest).trim_start(); rest.to_string() } /// Bracket groups that carry rip descriptors — "[V0]", "(320kbps)", /// "[WEB FLAC]" — removed whole (substring match inside the group, as the /// mock's alternation behaves). fn strip_rip_groups(s: &str) -> String { const TOKENS: [&str; 10] = [ "kbps", "v0", "v2", "web", "flac", "mp3", "320", "256", "192", "128", ]; let mut out = String::with_capacity(s.len()); let mut rest = s; while let Some(open) = rest.find(['[', '(']) { let after = &rest[open + 1..]; let Some(close) = after.find([']', ')']) else { break }; let low = after[..close].to_lowercase(); if TOKENS.iter().any(|t| low.contains(t)) { out.push_str(&rest[..open]); out.push(' '); } else { out.push_str(&rest[..open + 1 + close + 1]); } rest = &after[close + 1..]; } out.push_str(rest); out } /// Free-standing rip tokens (`\b`-bounded in the mock): whole words only, /// so "320kbps" the word survives while "320" alone goes. fn strip_rip_words(s: &str) -> String { const WORDS: [&str; 14] = [ "web", "v0", "v2", "flac", "mp3", "320", "320k", "256", "192", "128", "128kbps", "kbps", "vinyl", "rip", ]; fn flush(word: &mut String, out: &mut String) { if word.is_empty() { return; } let low = word.to_lowercase(); let user_n = low .strip_prefix("user") .is_some_and(|d| !d.is_empty() && d.bytes().all(|b| b.is_ascii_digit())); if WORDS.contains(&low.as_str()) || user_n { out.push(' '); } else { out.push_str(word); } word.clear(); } let mut out = String::with_capacity(s.len()); let mut word = String::new(); for c in s.chars() { if c.is_ascii_alphanumeric() || c == '_' { word.push(c); } else { flush(&mut word, &mut out); out.push(c); } } flush(&mut word, &mut out); out } /// Does this carry a word OUTSIDE brackets? /// /// Bracketed groups are decoration — catalogue numbers, rip descriptors, /// edition notes — and a "title" made only of them names nothing. Unicode /// alphanumerics count, so `ワールドイズユアーズ` is a word like any other. fn has_bare_word(s: &str) -> bool { let mut depth = 0i32; for c in s.chars() { match c { '(' | '[' | '{' => depth += 1, ')' | ']' | '}' => depth = (depth - 1).max(0), _ if depth == 0 && c.is_alphanumeric() => return true, _ => {} } } false } /// Release-type words a folder or a tag appends that no service files as part /// of the TITLE. /// /// Both providers require every word of the phrase, so one appended type word /// is enough to make a famous record unfindable. Measured on /// "Willoughby's Beach EP" (King Gizzard's 2011 debut, tagged with the suffix): /// Deezer answered `total=0` for `… Willoughby's Beach EP` and returned the /// record — 9 tracks, the exact release — for the same query without it, and /// MusicBrainz's `releasegroup:"Willoughby's Beach EP"` returned `count=0` /// while the plain title scored 100. MusicBrainz files that release group's /// `primary-type` AS `EP`, which is the whole argument: the type is metadata /// about the record, never part of its name. /// /// The artist's-own-catalogue pass cannot cover for this. Deezer's /// `/artist/3973121/albums` lists 31 releases for King Gizzard — 20 `album` /// and 11 `single`, not one `ep` — so the EP is reachable only by a search /// that spells its title the way Deezer does. /// /// Measured across the 497-record session: 14 records carry one of these as a /// trailing word (ep 5, single 3, soundtrack 3, album 1, lp 1, mixtape 1) and /// 13 of them already match, because the service's own title carries the word /// too. That is why the stripped form is ADDED AFTER the exact ones and never /// replaces them — the exact title is always asked first, so no record that /// works today changes its answer. /// /// "Live" and "Compilation" are deliberately absent: those name the work far /// more often than they annotate it ("Alchemy Live" is not "Alchemy"). const TYPE_WORDS: [&str; 7] = ["ep", "lp", "single", "album", "mixtape", "soundtrack", "ost"]; /// The title with a trailing release-type word removed, or None when there is /// no such word — or when removing it would leave nothing to search for. /// /// Punctuation folds into the word, so "E.P." is "ep". A separator left behind /// goes with it, which is what handles the store convention /// "Willoughby's Beach - EP". fn strip_type_suffix(s: &str) -> Option { let words: Vec<&str> = s.split_whitespace().collect(); if words.len() < 2 { return None; } let last: String = words[words.len() - 1] .chars() .filter(|c| c.is_alphanumeric()) .flat_map(char::to_lowercase) .collect(); if !TYPE_WORDS.contains(&last.as_str()) { return None; } let kept = words[..words.len() - 1].join(" "); let kept = kept.trim_end(); let kept = kept.strip_suffix(['-', '–', '—']).unwrap_or(kept).trim_end(); // A title that is ONLY its type word ("EP", "Album") keeps it: stripping // leaves nothing, and nothing names no record. if kept.is_empty() || !has_bare_word(kept) { return None; } Some(kept.to_string()) } /// A MusicBrainz release lookup's first release, as a tracklist. Discs are /// flattened in media order, which is the order the review shows them. fn parse_mb_tracklist(body: &serde_json::Value) -> Vec { body.pointer("/releases/0/media") .and_then(|m| m.as_array()) .map(|media| { media .iter() .flat_map(|m| { m.get("tracks").and_then(|t| t.as_array()).cloned().unwrap_or_default() }) .map(|tr| IdentifyTrack { n: tr.get("position").and_then(|v| v.as_u64()).map(|v| v as u32), title: tr .get("title") .and_then(|v| v.as_str()) .unwrap_or_default() .to_string(), secs: tr .get("length") .and_then(|v| v.as_u64()) .map(|ms| ((ms as f64) / 1000.0).round() as u32), }) .collect() }) .unwrap_or_default() } /// One release group from an artist's MusicBrainz discography. #[derive(Debug, Clone)] struct MbGroup { id: String, title: String, primary_type: Option, first_release_date: Option, } impl MbGroup { /// The release year, when the date says one. MusicBrainz dates are /// `YYYY`, `YYYY-MM` or `YYYY-MM-DD`, so the year is the first four. fn year(&self) -> Option { let d = self.first_release_date.as_deref()?; d.get(..4)?.parse().ok() } } /// One candidate release group with every piece of evidence gathered about it. /// Nothing here is a verdict — [`MetadataClient::discography_identify`] weighs /// them together, because any single one of them can be innocently wrong. #[derive(Debug, Clone)] struct DiscoCandidate { group: MbGroup, /// Wording similarity, 0.0–1.0, best over every name we have. name: f32, /// How well the release TYPE fits our track count — 0 is best. type_fit: u8, /// Some(false) means the years positively disagree; None means one of them /// is unknown, which is not a disagreement. year_agrees: Option, /// Hamming distance between our cover and theirs, when both exist. art_distance: Option, /// How much of the tracklist is shared, 0.0–1.0. tracks_agree: Option, } impl DiscoCandidate { /// 1 when the artwork proves this is the same record. Sorted descending, so /// an image match outranks every name. fn art_match(&self) -> u8 { u8::from(self.art_distance.is_some_and(|d| d <= COVER_SAME_ART_MAX_DISTANCE)) } } /// How much of two tracklists is the same music, 0.0–1.0 — the larger side /// decides, so a 2-track single does not "fully match" a 20-track album that /// happens to contain both songs. /// /// Titles are compared through [`normx`], which discards punctuation and case: /// a tagger's "Rock Bottom" and MusicBrainz's "Rock Bottom" are one song. fn tracklist_overlap(ours: &[String], theirs: &[String]) -> f32 { if ours.is_empty() || theirs.is_empty() { return 0.0; } let mut pool: Vec = theirs.iter().map(|t| normx(t)).filter(|t| !t.is_empty()).collect(); let mut shared = 0usize; for t in ours { let key = normx(t); if key.is_empty() { continue; } if let Some(at) = pool.iter().position(|x| *x == key) { pool.remove(at); shared += 1; } } shared as f32 / ours.len().max(theirs.len()) as f32 } /// Everything the SCAN already knows about a local record, for the passes that /// COMPARE rather than search. /// /// A phrase search can only use the album name, which is why a record whose /// name is wrong, annotated, or simply not what the label called it has been /// unfindable. Every other fact about the record is evidence: how many files it /// has, what those files are called, what year the folder claims, and what its /// own cover looks like. See [`MetadataClient::discography_identify`]. #[derive(Debug, Clone, Default)] pub struct RecordShape { /// How many audio files the record has. pub tracks: Option, /// The track titles, in order — the strongest signal available, and free: /// "Rock Bottom 7\"" scores 0.667 on its name against the release actually /// called "Rock Bottom / Octopus", and 2 of 2 on its tracks. pub track_titles: Vec, /// The year the folder or tags claim, if any. pub year: Option, /// Perceptual hash of the record's OWN cover — embedded art, usually a real /// front sleeve. Compared against the discography's artwork with the same /// gate `cover_report` uses, so a match is the same IMAGE, not a similar /// name. See [`cover_dhash`] and [`COVER_SAME_ART_MAX_DISTANCE`]. pub art_dhash: Option, } /// The perceptual hash of one cover's bytes, for callers outside this crate /// (the import review hands over the record's own embedded art). pub fn cover_dhash(bytes: &[u8]) -> Option { image::load_from_memory(bytes).ok().map(|img| dhash64(&img)) } /// The Hamming distance at or below which two covers are the SAME artwork, /// re-exported so a caller can describe its own confidence in the same terms. pub const SAME_ART_MAX_DISTANCE: u32 = COVER_SAME_ART_MAX_DISTANCE; /// How well a MusicBrainz release group's TYPE fits the record in hand, as a /// sort key — 0 best. Only ever used to break a tie between groups that match /// the title equally well. /// /// The thresholds are the frontend's `localFallback` (`localReleaseType.js`) /// verbatim: 3 tracks or fewer is a single, 7 or fewer an EP, more an album. /// With no count to reason from, an Album is the safe general answer — and, /// unlike MusicBrainz's ordering, a deterministic one. fn mb_type_fit(primary_type: Option<&str>, tracks: Option) -> u8 { let ty = primary_type.unwrap_or_default().to_ascii_lowercase(); let want = tracks.map(|n| { if n <= 3 { "single" } else if n <= 7 { "ep" } else { "album" } }); if want.is_some_and(|w| ty == w) { 0 } else if ty == "album" { 1 } else { 2 } } /// Every name we have for one record, as queries, in the order they earn: the /// record's OWN claim first (what its tags say), then the folder it sits in /// when that says something different. /// /// Identification used to ask about ONE name — whatever `albumNameOf` resolved, /// which is the tag majority — so a record whose tags are wrong had no second /// witness. The Soulseek copy search has consulted both since it was written /// (`findQueries` rung 2, "a fallback for records whose tags are wrong or /// missing"); this is that same rung, for the providers. /// /// Measured on King Gizzard's "Nonagon Infinity": all nine files tag the album /// `Nonagon Infiniry`, and the folder is spelled correctly. MusicBrainz answers /// `count=0` for the typo and `score=100` for the folder's spelling. Deezer's /// search index does not answer for this record AT ALL — not the typo, not the /// correct title, not `artist:"…" album:"…"` — although `/album/12965126` is /// there, available, with the right 9 tracks. So for this record the folder is /// the only witness that can find it, and the catalogue pass is the only route. /// /// The folder's rungs come LAST, and for the same reason `findQueries` puts /// them second: it is the ripper's naming, carrying catalogue numbers, years /// and rip descriptors, so it asks the narrower question. pub fn identify_titles(artist: &str, raw_title: &str, folder: Option<&str>) -> Vec { let mut out = search_titles(artist, raw_title); for t in folder.map(|f| search_titles(artist, f)).unwrap_or_default() { if !out.contains(&t) { out.push(t); } } out } /// The cleaned query try-list for one folder title, least- to most- /// aggressive, deduped; may be empty when nothing survives cleaning. pub fn search_titles(artist: &str, raw_title: &str) -> Vec { // Invisible marks break matching. let t: String = raw_title .chars() .filter(|c| !matches!(c, '\u{200e}' | '\u{200f}' | '\u{feff}')) .collect(); let mut t = t.trim().to_string(); t = strip_leading_year_group(&t); for tag in ["(album)", "(single)", "(ep)"] { if let Some(rest) = strip_prefix_ci(&t, tag) { t = rest.trim_start().to_string(); break; } } // The artist repeated at the front of the album folder (one separator // dash at most, as the mock's regex). if let Some(rest) = strip_prefix_ci(&t, artist) { let rest = rest.trim_start(); let rest = rest.strip_prefix(['-', '–', '—']).unwrap_or(rest); let rest = rest.trim_start(); // A strip that leaves only decoration is not a title. Where the album // IS the artist's name — a self-titled record, or a folder named after // the band rather than the act — this takes the whole name away and // leaves the catalogue number behind it: "Band Of Gypsys (SICP 2643)" // became "(SICP 2643)", which no service has ever heard of and which // nothing downstream can rescue, because cutting at the first "(" // leaves nothing at all. Six files that all say "Jimi Hendrix" sat // unidentifiable behind that one line. if has_bare_word(rest) { t = rest.to_string(); } } // Least aggressive: rip tokens and their bracket groups stripped. let cleaned = { let g = strip_rip_groups(&t); let w = strip_rip_words(&g); let d = w.trim_end(); let d = d.strip_suffix(['-', '–', '—']).unwrap_or(d); d.split_whitespace().collect::>().join(" ") }; // The app's simplifyTitle: cut at the first ; : ( [ and cap at six words. let base = if cleaned.is_empty() { t.clone() } else { cleaned.clone() }; let first = base .split([';', ':', '(', '[']) .next() .unwrap_or("") .trim() .to_string(); let simp_src = if first.is_empty() { base } else { first }; let simplified = simp_src .split_whitespace() .take(6) .collect::>() .join(" "); let mut out = Vec::new(); for v in [cleaned, simplified] { if !v.is_empty() && !out.contains(&v) { out.push(v); } } // Type-suffix forms LAST, so the title as the record spells it is always // the first thing asked and a record that matches today cannot change its // answer. See [`TYPE_WORDS`]. for v in out.clone() { if let Some(stripped) = strip_type_suffix(&v) { if !out.contains(&stripped) { out.push(stripped); } } } out } struct CoverCandidate { bytes: Arc>, width: u32, height: u32, /// 64-bit difference hash for same-artwork comparison across sources. dhash: u64, /// True for the Deezer cover — the artwork the catalog UI displays. is_reference: bool, source: &'static str, /// The URL this candidate was fetched from (for diagnostics/selection). url: String, } // -------------------------------------------------------------------------- // Preview media fetch — the bounded, allowlisted transfer behind the preview // resolver (scrollable-track-previews Phase 2). Kept separate from the shared // catalog client because it needs a CUSTOM REDIRECT POLICY: every hop is // re-validated against the media allowlist, where the shared client follows // redirects freely (fine for JSON APIs, not for media bytes we will decode). // -------------------------------------------------------------------------- /// Hosts preview media may be fetched from. HTTPS-only, exact match — one /// stable CDN host served 71/71 Phase 0 samples. Mirrors the frontend /// allowlist in `src/lib/preview/redact.js`; a new provider host is a /// reviewed change in both places. pub const PREVIEW_MEDIA_HOSTS: [&str; 1] = ["cdnt-preview.dzcdn.net"]; /// Hard cap on a preview transfer. Real clips are ~480 KB (Phase 0, zero /// variance); 2 MB tolerates provider drift while still refusing anything /// that plainly is not a 30-second clip. pub const PREVIEW_MEDIA_MAX_BYTES: usize = 2_000_000; pub struct PreviewMediaResponse { pub bytes: Vec, pub content_type: Option, /// Provider `Date` header as epoch ms, for provider-clock lease math. pub date_ms: Option, } #[derive(Debug, PartialEq, Eq)] pub enum PreviewFetchError { /// Initial URL or a redirect target fell outside the https+allowlist gate. DisallowedUrl, /// The CDN answered with a non-success status (403 = stale lease). Status(u16), /// Body exceeded [`PREVIEW_MEDIA_MAX_BYTES`]. TooLarge, /// Transport-level failure (timeout, DNS, connection). Network, } pub fn preview_url_allowed(url: &str) -> bool { let Ok(u) = reqwest::Url::parse(url) else { return false; }; u.scheme() == "https" && u.host_str() .is_some_and(|h| PREVIEW_MEDIA_HOSTS.contains(&h)) } fn preview_media_client() -> &'static reqwest::Client { use std::sync::OnceLock; static CLIENT: OnceLock = OnceLock::new(); CLIENT.get_or_init(|| { reqwest::Client::builder() .user_agent(USER_AGENT) .timeout(Duration::from_secs(15)) .redirect(reqwest::redirect::Policy::custom(|attempt| { // Every hop passes the same gate as the initial URL; a // redirect off the allowlist is a hard stop, never a follow. let ok = attempt.url().scheme() == "https" && attempt .url() .host_str() .is_some_and(|h| PREVIEW_MEDIA_HOSTS.contains(&h)); if !ok { return attempt.error("redirect target off the preview media allowlist"); } if attempt.previous().len() > 4 { return attempt.error("too many redirects"); } attempt.follow() })) .build() .expect("build preview media client") }) } /// Fetch preview bytes with the full host/response policy applied. Callers /// never see the URL again after this returns — bytes, MIME and provider /// time are all a resolver needs. pub async fn fetch_preview_media(url: &str) -> Result { if !preview_url_allowed(url) { return Err(PreviewFetchError::DisallowedUrl); } let resp = preview_media_client().get(url).send().await.map_err(|e| { // A redirect rejected by the policy surfaces as a generic // reqwest error; classify it as the policy violation it is. if e.is_redirect() { PreviewFetchError::DisallowedUrl } else { PreviewFetchError::Network } })?; let status = resp.status(); if !status.is_success() { return Err(PreviewFetchError::Status(status.as_u16())); } if let Some(len) = resp.content_length() { if len as usize > PREVIEW_MEDIA_MAX_BYTES { return Err(PreviewFetchError::TooLarge); } } let content_type = resp .headers() .get(reqwest::header::CONTENT_TYPE) .and_then(|v| v.to_str().ok()) .map(String::from); let date_ms = resp .headers() .get(reqwest::header::DATE) .and_then(|v| v.to_str().ok()) .and_then(|s| httpdate::parse_http_date(s).ok()) .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) .map(|d| d.as_millis() as i64); // Stream with the cap enforced even when Content-Length lied or was // absent — the count is on received bytes, not the header's promise. let mut bytes = Vec::with_capacity(512 * 1024); let mut stream = resp; loop { match stream.chunk().await { Ok(Some(chunk)) => { if bytes.len() + chunk.len() > PREVIEW_MEDIA_MAX_BYTES { return Err(PreviewFetchError::TooLarge); } bytes.extend_from_slice(&chunk); } Ok(None) => break, Err(_) => return Err(PreviewFetchError::Network), } } Ok(PreviewMediaResponse { bytes, content_type, date_ms, }) } /// Decode candidate bytes and keep only plausible front covers: square-ish /// (grids/banners/booklet scans are not covers) and at least 300px. fn decode_cover( bytes: impl Into>>, source: &'static str, is_reference: bool, url: String, ) -> Option { let bytes = bytes.into(); let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes.as_slice())) .with_guessed_format() .ok()?; let mut limits = image::Limits::default(); limits.max_image_width = Some(COVER_MAX_DECODE_DIMENSION); limits.max_image_height = Some(COVER_MAX_DECODE_DIMENSION); limits.max_alloc = Some(COVER_MAX_DECODE_ALLOC_BYTES); reader.limits(limits); let img = reader.decode().ok()?; let (w, h) = (img.width(), img.height()); if w < 300 || h < 300 { return None; } let aspect = w as f64 / h as f64; if !(0.8..=1.25).contains(&aspect) { return None; } Some(CoverCandidate { dhash: dhash64(&img), bytes, width: w, height: h, is_reference, source, url, }) } async fn decode_cover_async( bytes: Arc>, source: &'static str, is_reference: bool, url: String, ) -> Option { static GATE: std::sync::OnceLock = std::sync::OnceLock::new(); let gate = GATE.get_or_init(|| tokio::sync::Semaphore::new(COVER_DECODE_CONCURRENCY)); let permit = gate.acquire().await.ok()?; let decoded = tokio::task::spawn_blocking(move || decode_cover(bytes, source, is_reference, url)) .await .ok() .flatten(); drop(permit); decoded } /// Standard 64-bit difference hash: 9×8 grayscale, one bit per /// brighter-than-right-neighbour comparison. Robust to scaling and /// recompression, which is exactly the difference between one artwork /// served by different CDNs. fn dhash64(img: &image::DynamicImage) -> u64 { let gray = img .resize_exact(9, 8, image::imageops::FilterType::Triangle) .to_luma8(); let mut bits = 0u64; for y in 0..8 { for x in 0..8 { bits <<= 1; if gray.get_pixel(x, y)[0] > gray.get_pixel(x + 1, y)[0] { bits |= 1; } } } bits } /// Hamming distance at or below which two covers count as the same artwork. /// Same-art-different-CDN pairs land at 0–6; different covers cluster ≥20. const COVER_SAME_ART_MAX_DISTANCE: u32 = 10; /// Pick the winning candidate index. With a Deezer reference present, only /// same-artwork candidates are eligible, largest real resolution wins, and /// the reference itself breaks ties (bytes already proven identical-looking). /// Without one, the largest candidate wins. fn pick_best_cover(candidates: &[CoverCandidate]) -> Option { if candidates.is_empty() { return None; } let reference = candidates.iter().find(|c| c.is_reference); let eligible: Vec = match reference { Some(r) => candidates .iter() .enumerate() .filter(|(_, c)| (c.dhash ^ r.dhash).count_ones() <= COVER_SAME_ART_MAX_DISTANCE) .map(|(i, _)| i) .collect(), None => (0..candidates.len()).collect(), }; eligible.into_iter().max_by_key(|&i| { let c = &candidates[i]; // Area first; the reference wins exact-size ties. (u64::from(c.width) * u64::from(c.height), c.is_reference) }) } /// Size variants for an Apple `artworkUrl100` thumbnail, best first. mzstatic /// renders any requested size: `100000x100000-999` returns the original /// upload (commonly 3000×3000+), with fitted fallbacks when a CDN edge /// rejects it. fn apple_artwork_urls(thumb_url: &str) -> Vec { let Some(slash) = thumb_url.rfind('/') else { return vec![thumb_url.to_string()]; }; let base = &thumb_url[..slash + 1]; [ "100000x100000-999.jpg", "3000x3000bb.jpg", "1200x1200bb.jpg", ] .iter() .map(|size| format!("{base}{size}")) .collect() } /// Selection tiebreak once resolution is equal: prefer official sources /// (Apple, then Deezer) over Cover Art Archive. fn cover_source_priority(source: &str) -> u8 { match source { "apple" => 4, "deezer-max" | "deezer" => 3, "cover-art-archive" => 2, _ => 1, } } /// Extract the release-group MBID embedded in a Cover Art Archive URL, e.g. /// `https://coverartarchive.org/release-group/{mbid}/front-500`. Returns `None` /// for any other host, so Deezer covers never trigger a CAA/MusicBrainz path. fn caa_release_group_mbid(url: &str) -> Option { let rest = url.split("coverartarchive.org/release-group/").nth(1)?; let mbid = rest.split('/').next()?; // MBIDs are 36-char UUIDs; guard against odd inputs. if mbid.len() == 36 && mbid.bytes().all(|b| b == b'-' || b.is_ascii_hexdigit()) { Some(mbid.to_string()) } else { None } } /// Loose title equality for provider verification: normalized exact match or /// containment either way ("Weezer (Deluxe Edition)" ↔ "Weezer"). fn loose_name_match(got: &str, want: &str) -> bool { !got.is_empty() && !want.is_empty() && (got == want || got.contains(want) || want.contains(got)) } fn base64(data: &[u8]) -> String { const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; let mut out = String::with_capacity(data.len().div_ceil(3) * 4); for chunk in data.chunks(3) { let b = [ chunk[0], *chunk.get(1).unwrap_or(&0), *chunk.get(2).unwrap_or(&0), ]; let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); out.push(TABLE[(n >> 18) as usize & 63] as char); out.push(TABLE[(n >> 12) as usize & 63] as char); out.push(if chunk.len() > 1 { TABLE[(n >> 6) as usize & 63] as char } else { '=' }); out.push(if chunk.len() > 2 { TABLE[n as usize & 63] as char } else { '=' }); } out } fn escape(s: &str) -> String { s.replace('"', "\\\"") } /// Loose artist comparison: lowercase, alphanumerics only. fn normalize(s: &str) -> String { s.to_lowercase() .chars() .filter(|c| c.is_alphanumeric()) .collect() } /// TheAudioDB can return several fuzzy/name-alike records. Never let its first /// item bypass Bing with artwork for a different artist; require an exact /// normalized display-name match and otherwise treat the provider as empty. fn select_audiodb_artist<'a>( body: &'a serde_json::Value, requested: &str, ) -> Option<&'a serde_json::Value> { let requested = normalize(requested); body.get("artists")?.as_array()?.iter().find(|artist| { artist .get("strArtist") .and_then(|name| name.as_str()) .is_some_and(|name| normalize(name) == requested) }) } fn audiodb_deezer_id(record: &serde_json::Value) -> Option { record .get("strDeezerID") .and_then(|value| value.as_str()) .and_then(|value| value.trim().parse().ok()) } /// Cross-provider identity gate for name-searched TheAudioDB records. Exact /// names are deliberately insufficient: namesakes are the bug this protects. /// Either TheAudioDB must carry the requested Deezer id itself, or its MBID /// must equal the MB artist that links back to that Deezer id. fn audiodb_artist_matches_identity( record: &serde_json::Value, deezer_id: u64, linked_mbid: Option<&str>, ) -> bool { // A linked MBID deliberately outranks TheAudioDB's own strDeezerID, which // can be stale -- see `linked_mbid_outranks_stale_audiodb_deezer_id`. The // MusicBrainz-duplicate rescue lives in `audiodb_identity_ok`, which asks a // question this cheap check cannot: whether TheAudioDB's MB entry is a // second entry for the SAME act. if let Some(linked_mbid) = linked_mbid { return audiodb_record_mbid(record) .is_some_and(|value| value.eq_ignore_ascii_case(linked_mbid)); } audiodb_deezer_id(record) == Some(deezer_id) } fn audiodb_record_mbid(record: &serde_json::Value) -> Option<&str> { record .get("strMusicBrainzID") .and_then(|value| value.as_str()) .map(str::trim) .filter(|value| !value.is_empty()) } #[cfg(test)] fn deezer_search_allows_bing( body: &serde_json::Value, requested_name: &str, requested_id: u64, ) -> bool { deezer_exact_name_state(body, requested_name, requested_id) == DeezerExactNameState::Unambiguous } /// A namesake only threatens a name-searched hero if the open web might mean /// *it* instead. Deezer follower counts are the available prominence proxy, and /// exact-name rows are dominated by dead duplicates and typo entries /// (`crazy_p`, `Pond` x6) that no search engine ranks for. Requiring a decisive /// margin keeps the real collisions — the several genuinely followed Ponds — /// on the strict catalogue-anchored path. /// Deezer's "this artist has no photo" portrait is the md5 of the EMPTY STRING. /// The frontend carries the same constant for the sidebar's portrait fallback /// (`DEEZER_EMPTY_PICTURE_HASH` in src/lib/state.svelte.js) — there the hash is /// only reachable through the CDN's redirect, but a search row's `picture_*` /// often carries it literally, which is enough to reject the row outright. const DEEZER_EMPTY_PICTURE_HASH: &str = "d41d8cd98f00b204e9800998ecf8427e"; const BING_NAME_OWNER_FAN_RATIO: u64 = 4; /// Floor so two obscure rows (5 fans against 1) cannot manufacture a ratio. const BING_NAME_OWNER_MIN_FANS: u64 = 25; /// Release-type evidence from a description's LEAD. Editorial prose declares /// the record's kind in its first sentence ("…is a compilation album by…", /// "…is a live album…", "…is the soundtrack to…"), which is direct evidence /// the title heuristics can't see — a compilation named "Direct Hits" says so /// here even when the title says nothing. Only the lead is scanned: later /// prose mentions other records ("toured before releasing a live album…"). /// Precedence mirrors the page's category order. fn description_type_hint(description: &str) -> Option<&'static str> { let lead: String = description.chars().take(260).collect(); let lead = lead.to_lowercase(); let any = |phrases: &[&str]| phrases.iter().any(|p| lead.contains(p)); // A song or single article describes a TRACK, and every release word in // its lead names a DIFFERENT record — "'Run for Cover' is a song by the // Killers from their fifth studio album, Wonderful Wonderful" is not a // studio-album confirmation for a release called "Run For Cover (Workout // Mix)". Tested first so none of the checks below can mine prose that was // never about the release being filed. if lead_declares(&lead, "song") || lead_declares(&lead, "single") { return Some("song"); } if any(&[ "soundtrack", "film score", "original score", "music to the film", ]) { return Some("soundtrack"); } if any(&[ "live album", "concert album", "recorded live", "live recording", ]) { return Some("live"); } if any(&[ "compilation album", "greatest hits", "compilation of", "best-of", "box set", "anthology", ]) { return Some("compilation"); } if any(&["remix album", "mix album", "album of remixes", "remix ep"]) { return Some("remix"); } if any(&["extended play", "(ep)", " ep by", "debut ep"]) { return Some("ep"); } if any(&["studio album"]) { return Some("album"); } None } /// True when the lead *declares itself* to be `noun` — "is a song by", "is the /// second single from", "is a 2019 promotional single". /// /// Only the FIRST copula is read, because that is the one whose subject is the /// article's own title; an encyclopedia lead opens by naming its subject's /// kind. Any later one belongs to a different subject ("the record's third /// single was a song written on tour" describes a track ON an album). The noun /// must also be a whole word inside that one clause, so ordinary prose is not /// read as a declaration and "song" is not found inside "songwriter". fn lead_declares(lead: &str, noun: &str) -> bool { const COPULAS: [&str; 5] = [" is a ", " is an ", " is the ", " was a ", " was the "]; // Leads stack qualifiers between the copula and the noun ("is a 2019 // promotional single"), but the noun always arrives early. const WORDS: usize = 8; let Some(clause) = COPULAS .iter() .filter_map(|copula| lead.find(copula).map(|at| at + copula.len())) .min() .map(|at| &lead[at..]) else { return false; }; clause .split(['.', ',', ';', '(', ':']) .next() .unwrap_or_default() .split(|c: char| !c.is_alphanumeric()) .filter(|word| !word.is_empty()) .take(WORDS) .any(|word| word == noun) } /// Deezer reports an exhausted request budget as `{"error":{"code":4,...}}` /// under HTTP 200. That is the one provider error worth retrying: the window is /// five seconds wide, and every other code describes the resource itself. fn deezer_quota_error(body: &serde_json::Value) -> bool { let Some(error) = body.get("error") else { return false; }; error.get("code").and_then(|code| code.as_u64()) == Some(4) || error .get("message") .and_then(|message| message.as_str()) .is_some_and(|message| message.to_lowercase().contains("quota")) } /// Choose which row of a Deezer artist search owns the portrait for `artist`. /// /// Exact name outranks any follower count — a bigger artist whose name merely /// CONTAINS the query is the wrong answer, not a better one — and rows carrying /// the empty-md5 placeholder are dropped entirely. Returning `None` (every /// candidate placeholder-only) is deliberate: the shelf head's fallback icon is /// honest, where a stored grey silhouette reads as a broken fetch. fn pick_artist_portrait<'a>(body: &'a serde_json::Value, artist: &str) -> Option<&'a str> { let results = body.get("data").and_then(|data| data.as_array())?; let wanted = normalize(artist); let mut ranked: Vec<(bool, u64, &str)> = results .iter() .filter_map(|row| { let name = row.get("name").and_then(|name| name.as_str())?; let picture = row .get("picture_xl") .or_else(|| row.get("picture_big")) .and_then(|picture| picture.as_str())?; if picture.contains(DEEZER_EMPTY_PICTURE_HASH) { return None; } let fans = row .get("nb_fan") .and_then(|fans| fans.as_u64()) .unwrap_or_default(); Some((normalize(name) == wanted, fans, picture)) }) .collect(); ranked.sort_by(|a, b| b.0.cmp(&a.0).then(b.1.cmp(&a.1))); ranked.first().map(|(_, _, picture)| *picture) } fn deezer_exact_name_state( body: &serde_json::Value, requested_name: &str, requested_id: u64, ) -> DeezerExactNameState { let wanted = normalize(requested_name); let Some(results) = body.get("data").and_then(|data| data.as_array()) else { return DeezerExactNameState::Missing; }; let mut exact: HashMap = HashMap::new(); for result in results { let Some(name) = result.get("name").and_then(|name| name.as_str()) else { continue; }; if normalize(name) != wanted { continue; } let Some(id) = result.get("id").and_then(|id| id.as_u64()) else { continue; }; let fans = result .get("nb_fan") .and_then(|fans| fans.as_u64()) .unwrap_or_default(); exact .entry(id) .and_modify(|current| *current = (*current).max(fans)) .or_insert(fans); } let Some(&requested_fans) = exact.get(&requested_id) else { return DeezerExactNameState::Missing; }; if exact.len() == 1 { return DeezerExactNameState::Unambiguous; } let rival_fans = exact .iter() .filter(|(id, _)| **id != requested_id) .map(|(_, fans)| *fans) .max() .unwrap_or_default(); let owns_the_name = requested_fans > 0 && (rival_fans == 0 || (requested_fans >= rival_fans.saturating_mul(BING_NAME_OWNER_FAN_RATIO) && requested_fans >= BING_NAME_OWNER_MIN_FANS)); if owns_the_name { DeezerExactNameState::Dominant } else { DeezerExactNameState::Ambiguous } } /// Rank an artist name against an already-normalized query: exact match beats a /// prefix beats a substring beats nothing. Keeps name relevance ahead of raw /// popularity, so a "Pond" search surfaces bands actually named Pond before a /// more-followed fuzzy match, then prefers the most-followed among equals. fn artist_name_relevance(name: &str, query_key: &str) -> u8 { if query_key.is_empty() { return 0; } let name_key = normalize(name); if name_key == query_key { 3 } else if name_key.starts_with(query_key) { 2 } else if name_key.contains(query_key) { 1 } else { 0 } } /// Stable FNV-1a id in the upper half of JavaScript's safe-integer range, /// safely outside Deezer's numeric ids while still round-tripping through IPC. fn synthetic_catalog_id(key: &str) -> u64 { let mut hash = 0xcbf29ce484222325_u64; for byte in key.bytes() { hash ^= u64::from(byte); hash = hash.wrapping_mul(0x100000001b3); } (hash & ((1_u64 << 52) - 1)) | (1_u64 << 52) } /// Parse each Bing image card's own `m="{...}"` attribute. Keeping this /// object boundary is critical: the old ±byte window around `murl` could read /// `t`/`purl` from a neighboring card and attach an unrelated photograph to /// the requested artist. fn parse_bing_image_results(html: &str) -> Vec { let mut results = Vec::new(); let bytes = html.as_bytes(); let mut at = 0; while let Some(relative) = html[at..].find('m') { let name_at = at + relative; at = name_at + 1; if name_at == 0 || !bytes[name_at - 1].is_ascii_whitespace() { continue; } let mut cursor = name_at + 1; while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() { cursor += 1; } if bytes.get(cursor) != Some(&b'=') { continue; } cursor += 1; while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() { cursor += 1; } let Some(quote @ (b'\'' | b'"')) = bytes.get(cursor).copied() else { continue; }; let start = cursor + 1; let Some(length) = bytes[start..].iter().position(|byte| *byte == quote) else { break; }; let encoded = &html[start..start + length]; at = start + length + 1; let decoded = decode_bing_html_attribute(encoded); let Ok(value) = serde_json::from_str::(&decoded) else { continue; }; let Some(url) = value.get("murl").and_then(|field| field.as_str()) else { continue; }; let title = value .get("t") .and_then(|field| field.as_str()) .unwrap_or_default(); let page_url = value .get("purl") .and_then(|field| field.as_str()) .unwrap_or_default(); results.push(ParsedBingImage { url: url.to_string(), title: title.to_string(), page_url: page_url.to_string(), }); } results } /// Bing HTML-encodes the JSON attribute but leaves JSON backslash escaping /// intact. Decode only the entities observed in that attribute; ampersand is /// last so an encoded URL query becomes a literal `&`, not a second decode. fn decode_bing_html_attribute(value: &str) -> String { value .replace(""", "\"") .replace(""", "\"") .replace("'", "'") .replace("'", "'") .replace("<", "<") .replace(">", ">") .replace("&", "&") } fn diagnostic_field(value: &str) -> String { value .replace(['\t', '\r', '\n'], " ") .chars() .take(180) .collect() } /// Whether `text` names the artist. Char-level matching that ignores /// separators but demands word boundaries at both ends: "Partiboi 69" matches /// the artist "Partiboi69" (pages space names freely), while "Billy Cobham" /// can never match "Billy Cobb" — the trailing 'h' continues the word, and /// plain substring matching is NOT identity ("billycobb" ⊂ "billycobham"). fn names_artist(text: &str, artist: &str) -> bool { let haystack: Vec = text.to_lowercase().chars().collect(); let needle: Vec = artist .to_lowercase() .chars() .filter(|c| c.is_alphanumeric()) .collect(); if needle.is_empty() { return false; } for start in 0..haystack.len() { if !haystack[start].is_alphanumeric() { continue; } // Word boundary before the match. if start > 0 && haystack[start - 1].is_alphanumeric() { continue; } let mut hay_at = start; let mut needle_at = 0; while hay_at < haystack.len() && needle_at < needle.len() { if !haystack[hay_at].is_alphanumeric() { hay_at += 1; continue; } if haystack[hay_at] != needle[needle_at] { break; } hay_at += 1; needle_at += 1; } // Word boundary after the match. if needle_at == needle.len() && (hay_at >= haystack.len() || !haystack[hay_at].is_alphanumeric()) { return true; } } false } /// Only the image file itself is independent identity evidence. A requested /// name in a parent folder or tracking query must not bless a leaf file that /// belongs to another act on a roundup/event page. fn image_filename_names_artist(image_url: &str, artist: &str) -> bool { reqwest::Url::parse(image_url) .ok() .and_then(|url| { url.path_segments() .and_then(|segments| segments.filter(|segment| !segment.is_empty()).last()) .map(str::to_owned) }) .is_some_and(|filename| names_artist(&filename, artist)) } /// Ticketmaster's DAM is used for both polished artist profile imagery and /// event graphics. Trust only the former: the result must come from an /// `/artist/` profile on Ticketmaster/Live Nation and use an artist DAM path. /// Exact requested-artist identity is enforced separately by /// `bing_identity_confidence`. fn is_ticketing_artist_profile_image(page_url: &str, image_url: &str) -> bool { let Ok(page) = reqwest::Url::parse(page_url) else { return false; }; let Ok(image) = reqwest::Url::parse(image_url) else { return false; }; let page_host = page.host_str().unwrap_or_default().to_lowercase(); let image_host = image.host_str().unwrap_or_default().to_lowercase(); let ticketing_page = host_is_or_subdomain(&page_host, "ticketmaster.com") || host_is_or_subdomain(&page_host, "livenation.com"); let artist_path = page .path_segments() .is_some_and(|mut segments| segments.any(|segment| segment == "artist")); ticketing_page && artist_path && host_is_or_subdomain(&image_host, "ticketm.net") && image.path().starts_with("/dam/a/") } fn host_is_or_subdomain(host: &str, root: &str) -> bool { host == root || host .strip_suffix(root) .is_some_and(|prefix| prefix.ends_with('.')) } fn is_ticketing_artist_profile_for_artist(page_url: &str, image_url: &str, artist: &str) -> bool { let path_names_artist = reqwest::Url::parse(page_url) .ok() .is_some_and(|page| names_artist(page.path(), artist)); is_ticketing_artist_profile_image(page_url, image_url) && path_names_artist } /// Reject an image whose path explicitly names a different person sharing the /// requested artist's surname. This intentionally applies only to simple /// two-part artist names and only when the full requested name is absent from /// the image URL, keeping the heuristic narrow enough for stage names and band /// names. Generic path components such as `portrait-cobb.jpg` are not treated /// as somebody's first name. fn image_path_conflicts_with_two_part_artist(image_url: &str, artist: &str) -> bool { let artist_tokens: Vec = artist .to_lowercase() .split(|c: char| !c.is_alphanumeric()) .filter(|token| !token.is_empty()) .map(str::to_owned) .collect(); if artist_tokens.len() != 2 || artist_tokens[0].len() < 3 || artist_tokens[1].len() < 3 { return false; } let Ok(url) = reqwest::Url::parse(image_url) else { return false; }; let path_tokens: Vec = url .path() .to_lowercase() .split(|c: char| !c.is_alphanumeric()) .filter(|token| !token.is_empty()) .map(str::to_owned) .collect(); const GENERIC_PREFIXES: [&str; 16] = [ "artist", "band", "banner", "concert", "cover", "hero", "image", "img", "live", "musician", "photo", "portrait", "press", "profile", "stage", "thumbnail", ]; path_tokens.windows(2).any(|pair| { let first = &pair[0]; let surname = &pair[1]; surname == &artist_tokens[1] && first != &artist_tokens[0] && first.len() >= 3 && first.chars().all(char::is_alphabetic) && !GENERIC_PREFIXES.contains(&first.as_str()) }) } /// Result galleries and stock-search pages can have an artist-specific title /// and URL while showing an adjacent namesake's image (the Billy Cobb search /// returned jazz drummer Jimmy Cobb this way). They are not identity evidence. fn is_unreliable_image_aggregate(title: &str, page_url: &str, image_url: &str) -> bool { let title = title.to_lowercase(); let page = page_url.to_lowercase(); const TITLE_MARKERS: [&str; 14] = [ "stock photo", "stock photos", "stock images", "premium photo", "premium ai image", "ai generated", "free photo", "photos and images", "pictures and images", "high-res pictures", "photo gallery", "image gallery", "wallpapers - wallpaper", "wallpaper gallery", ]; if TITLE_MARKERS.iter().any(|marker| title.contains(marker)) { return true; } const STOCK_HOSTS: [&str; 24] = [ "gettyimages.com", "gettyimages.co.uk", "shutterstock.com", "alamy.com", "dreamstime.com", "depositphotos.com", "freepik.com", "vecteezy.com", "istockphoto.com", "123rf.com", "wallpapercave.com", "wallpapersafari.com", "unsplash.com", "pxhere.com", "stockcake.com", "motionarray.com", "artlist.io", "pexels.com", "pixabay.com", "stock.adobe.com", "quotesgram.com", "quotefancy.com", "brainyquote.com", "azquotes.com", ]; let stock_host = [page_url, image_url].into_iter().any(|url| { reqwest::Url::parse(url) .ok() .and_then(|parsed| parsed.host_str().map(str::to_lowercase)) .is_some_and(|host| { STOCK_HOSTS .iter() .any(|root| host_is_or_subdomain(&host, root)) }) }); let image = image_url.to_lowercase(); let hosted_html_search_spam = reqwest::Url::parse(page_url).ok().is_some_and(|url| { url.host_str() .is_some_and(|host| host_is_or_subdomain(host, "storage.googleapis.com")) && url.path().to_lowercase().ends_with(".html") }); stock_host || hosted_html_search_spam || page.contains("/image-search") || image.contains("shutterstock_") || image.contains("shutterstock-") } fn bing_identity_confidence( title: &str, page_url: &str, image_url: &str, artist: &str, ) -> Option { let title_names = names_artist(title, artist); let ticketing_profile = is_ticketing_artist_profile_image(page_url, image_url); // Ticketing query strings and fragments are tracking-controlled metadata, // not performer identity. On an official profile, match the requested act // only against the route itself so `...?ref=billy-cobb` cannot bless a // Jimmy Cobb profile. let page_names = if ticketing_profile { reqwest::Url::parse(page_url) .ok() .is_some_and(|page| names_artist(page.path(), artist)) } else { names_artist(page_url, artist) }; if !title_names && !page_names { return None; } if is_unreliable_image_aggregate(title, page_url, image_url) { return None; } let trusted_profile = ticketing_profile && page_names; if !trusted_profile && ambiguous_artist_result_lacks_entity_anchor(title, page_url, image_url, artist) { return None; } let image_names = image_filename_names_artist(image_url, artist); // Ticketmaster/Live Nation DAM imagery is authoritative only when the // artist-profile URL itself names the requested artist. A misleading Bing // title must never turn another performer's profile into Strong evidence. if ticketing_profile && !page_names { return None; } if image_names || (title_names && page_names) || trusted_profile { Some(BingIdentityConfidence::Strong) } else { Some(BingIdentityConfidence::Weak) } } fn distinctive_bing_release_anchor(title: &str, artist: &str) -> Option { let anchor = normalized_word_text(strip_release_format_tail(title).0); let artist = normalized_word_text(artist); if anchor.is_empty() || normalized_phrase_present(&anchor, &artist) || anchor == artist { return None; } let tokens: Vec<&str> = anchor.split_whitespace().collect(); let alphanumeric_len = anchor .chars() .filter(|character| character.is_alphanumeric()) .count(); const GENERIC_SINGLE_TITLES: [&str; 15] = [ "album", "best", "changes", "collection", "compilation", "greatest", "hits", "later", "live", "remaster", "remastered", "remixed", "remixes", "single", "the", ]; let distinctive = if tokens.len() >= 2 { alphanumeric_len >= 8 } else { alphanumeric_len >= 8 && !GENERIC_SINGLE_TITLES.contains(&tokens[0]) }; distinctive.then_some(anchor) } fn musicbrainz_identity_people(body: &serde_json::Value, artist: &str) -> HashSet { let artist_name = normalized_word_text(artist); body.get("relations") .and_then(|relations| relations.as_array()) .into_iter() .flatten() .filter(|relation| relation.get("ended").and_then(|value| value.as_bool()) != Some(true)) .filter(|relation| { matches!( relation.get("type").and_then(|value| value.as_str()), Some("member of band" | "founder") ) }) .filter_map(|relation| { relation .pointer("/artist/name") .and_then(|value| value.as_str()) }) .map(normalized_word_text) .filter(|name| { name != &artist_name && name.split_whitespace().count() >= 2 && name .chars() .filter(|character| character.is_alphanumeric()) .count() >= 6 }) .collect() } /// Winner-only release titles are the bridge between a numeric Deezer id and /// name-searched Bing metadata. Shared/self-titled/generic releases are not an /// independent identity signal and are deliberately absent from this set. fn bing_catalog_release_anchors( candidates: &[AppleArtistCandidate], catalog: &AppleArtistCatalog, artist: &str, ) -> HashSet { let Some(winner_id) = catalog.apple_artist_id else { return HashSet::new(); }; let Some(winner) = candidates .iter() .find(|candidate| candidate.artist_id == winner_id) else { return HashSet::new(); }; let losing_titles: HashSet<&str> = candidates .iter() .filter(|candidate| candidate.artist_id != winner_id) .flat_map(|candidate| candidate.releases.iter()) .map(|release| release.title_norm.as_str()) .collect(); winner .releases .iter() .filter(|release| !losing_titles.contains(release.title_norm.as_str())) .filter_map(|release| distinctive_bing_release_anchor(&release.title, artist)) .collect() } fn normalized_phrase_present(context: &str, phrase: &str) -> bool { if context.is_empty() || phrase.is_empty() { return false; } let padded_context = format!(" {context} "); let padded_phrase = format!(" {phrase} "); padded_context.contains(&padded_phrase) } fn bing_candidate_anchor_context(candidate: &BingImageCandidate) -> String { let page_path = reqwest::Url::parse(&candidate.page_url) .ok() .map(|url| url.path().to_string()) .unwrap_or_default(); let image_leaf = reqwest::Url::parse(&candidate.url) .ok() .and_then(|url| { url.path_segments() .and_then(|segments| segments.filter(|segment| !segment.is_empty()).last()) .map(str::to_owned) }) .unwrap_or_default(); normalized_word_text(&format!("{} {page_path} {image_leaf}", candidate.title)) } fn bing_candidate_matches_anchors( candidate: &BingImageCandidate, anchors: &HashSet, ) -> bool { if anchors.is_empty() { return false; } let context = bing_candidate_anchor_context(candidate); anchors .iter() .any(|anchor| normalized_phrase_present(&context, anchor)) } fn bing_page_host(page_url: &str) -> Option { reqwest::Url::parse(page_url) .ok()? .host_str() .map(|host| host.trim_start_matches("www.").to_lowercase()) } fn bing_host_names_artist(host: &str, artist: &str) -> bool { let label = host.split('.').next().unwrap_or_default(); !label.is_empty() && normalize(label) == normalize(artist) } fn bing_candidate_can_corroborate_artist_host( candidate: &BingImageCandidate, policy: &BingArtistIdentityPolicy, artist: &str, ) -> bool { let BingArtistIdentityPolicy::CatalogAnchored { release_titles, .. } = policy else { return false; }; candidate.identity == BingIdentityConfidence::Strong && bing_candidate_matches_anchors(candidate, release_titles) && bing_page_host(&candidate.page_url) .is_some_and(|host| bing_host_names_artist(&host, artist)) } /// Unambiguous and name-owner artists keep the full per-card result set: the /// card rules already decided identity, and anything stricter throws away the /// press/portrait photography that makes the best heroes. fn filter_bing_candidates_for_policy( candidates: Vec, policy: &BingArtistIdentityPolicy, artist: &str, linked_people: &HashSet, ) -> Vec { let BingArtistIdentityPolicy::CatalogAnchored { release_titles } = policy else { return candidates .into_iter() .filter(|candidate| candidate.display_eligible) .collect(); }; // A product/release result cannot display, but when it names a distinctive // winner-only release on a host whose own label is the artist name, it can // corroborate sibling press-photo cards from that exact host. Never spread // this trust across generic editorial hosts or merely similar domains. let corroborated_hosts: HashSet = candidates .iter() .filter(|candidate| bing_candidate_can_corroborate_artist_host(candidate, policy, artist)) .filter_map(|candidate| bing_page_host(&candidate.page_url)) .collect(); candidates .into_iter() .filter(|candidate| candidate.display_eligible) .filter(|candidate| candidate.identity == BingIdentityConfidence::Strong) .filter(|candidate| { bing_candidate_matches_anchors(candidate, release_titles) || bing_candidate_matches_anchors(candidate, linked_people) || bing_page_host(&candidate.page_url) .is_some_and(|host| corroborated_hosts.contains(&host)) }) .collect() } fn bing_candidate_metadata_is_better( candidate: &BingImageCandidate, existing: &BingImageCandidate, ) -> bool { let priority = |value: &BingImageCandidate| { ( value.display_eligible, is_ticketing_artist_profile_image(&value.page_url, &value.url), value.identity == BingIdentityConfidence::Strong, std::cmp::Reverse(value.query_index), std::cmp::Reverse(value.rank_in_query), ) }; priority(candidate) > priority(existing) } fn select_scored_hero_urls( mut scored: Vec<(String, f64, HeroFingerprint, BingIdentityConfidence)>, ) -> (Vec, Vec<(String, String)>) { // Metadata confidence is only meaningful after the URL has downloaded, // decoded, and passed quality checks. A broken Strong result is absent from // this list and therefore cannot erase a usable Weak fallback. let has_scored_strong = scored .iter() .any(|(_, _, _, identity)| *identity == BingIdentityConfidence::Strong); if has_scored_strong { scored.retain(|(_, _, _, identity)| *identity == BingIdentityConfidence::Strong); } scored.sort_by(|left, right| right.1.total_cmp(&left.1)); let limit = if has_scored_strong { 3 } else { 1 }; let mut selected: Vec<(String, HeroFingerprint)> = Vec::new(); let mut duplicates = Vec::new(); for (url, _, fingerprint, _) in scored { if let Some((kept_url, _)) = selected .iter() .find(|(_, kept_fingerprint)| hero_fingerprints_match(&fingerprint, kept_fingerprint)) { duplicates.push((kept_url.clone(), url)); continue; } selected.push((url, fingerprint)); if selected.len() == limit { break; } } ( selected.into_iter().map(|(url, _)| url).collect(), duplicates, ) } #[cfg(test)] fn select_strong_hero_seed( mut scored: Vec<(String, f64, HeroFingerprint, BingIdentityConfidence)>, ) -> Option { scored.retain(|(_, _, _, identity)| *identity == BingIdentityConfidence::Strong); select_scored_hero_urls(scored).0.into_iter().next() } fn normalized_word_text(value: &str) -> String { value .to_lowercase() .chars() .map(|character| { if character.is_alphanumeric() { character } else { ' ' } }) .collect::() .split_whitespace() .collect::>() .join(" ") } fn image_filename_starts_with_artist(image_url: &str, artist: &str) -> bool { let Some(filename) = reqwest::Url::parse(image_url).ok().and_then(|url| { url.path_segments() .and_then(|segments| segments.filter(|segment| !segment.is_empty()).last()) .map(str::to_owned) }) else { return false; }; let filename = normalized_word_text(&filename); let artist = normalized_word_text(artist); filename == artist || filename.starts_with(&format!("{artist} ")) } /// Some exact artist names are ordinary search nouns. "Equipment" is the /// verified stress fixture: quoted music queries otherwise return literal DJ /// gear, load-in stock photos, and recording hardware whose title, page slug, /// and filename all repeat the word. For this known collision, require grammar /// that treats the word as a named act, a structured artist route, or a leaf /// that begins with the act name. This keeps the real BrooklynVegan result while /// refusing endless replacement stock photos. fn ambiguous_artist_result_lacks_entity_anchor( title: &str, page_url: &str, image_url: &str, artist: &str, ) -> bool { let artist_key = normalized_word_text(artist); if artist_key == "wednesday" { let context = normalized_word_text(&format!("{title} {page_url} {image_url}")); let entity_token = ["album", "band", "concert", "tour"] .iter() .any(|marker| context.split_whitespace().any(|token| token == *marker)); const NAMED_MUSIC_CONTEXT: [&str; 8] = [ "alt country", "band interview", "elderberry wine", "indie rock", "karly hartzman", "new song", "new video", "newmusic", ]; return !entity_token && !NAMED_MUSIC_CONTEXT .iter() .any(|marker| context.contains(marker)); } if artist_key == "mspaint" { let raw_context = format!("{title} {page_url} {image_url}").to_lowercase(); if !raw_context.contains("mspaint") { return true; } let context = normalized_word_text(&raw_context); const MUSIC_ENTITY_CONTEXT: [&str; 10] = [ "album", "band", "concert", "ep", "hardcore", "interview", "musician", "performer", "song", "tour", ]; const NAMED_MUSIC_CONTEXT: [&str; 4] = ["audiotree", "post american", "synth punk", "artist mspaint"]; return !MUSIC_ENTITY_CONTEXT .iter() .any(|marker| context.split_whitespace().any(|token| token == *marker)) && !NAMED_MUSIC_CONTEXT .iter() .any(|marker| context.contains(marker)); } if artist_key == "weatherday" { let context = normalized_word_text(&format!("{title} {page_url} {image_url}")); const MUSIC_ENTITY_CONTEXT: [&str; 14] = [ "album", "band", "concert", "discography", "interview", "live", "music", "musician", "performance", "profile", "review", "single", "song", "tour", ]; return !MUSIC_ENTITY_CONTEXT .iter() .any(|marker| context.split_whitespace().any(|token| token == *marker)); } if artist_key != "equipment" { return false; } let title = normalized_word_text(title); let page = normalized_word_text(page_url); let leaf_anchor = image_filename_starts_with_artist(image_url, artist); let title_tokens: Vec<&str> = title.split_whitespace().collect(); const GENERIC_LEADING_FOLLOWS: [&str; 13] = [ "essentials", "for", "guide", "hire", "image", "list", "needed", "rental", "setup", "that", "tools", "used", "you", ]; let artist_leading_title = title_tokens.first() == Some(&"equipment") && title_tokens .get(1) .is_none_or(|next| !GENERIC_LEADING_FOLLOWS.contains(next)); const ENTITY_PHRASES: [&str; 15] = [ "band equipment announce", "band equipment interview", "band equipment release", "band equipment share", "equipment announce", "equipment band", "equipment concert", "equipment interview", "equipment live", "equipment release", "equipment releases", "equipment share", "equipment shares", "equipment tour", "meet equipment", ]; let entity_phrase = ENTITY_PHRASES.iter().any(|phrase| title.contains(phrase)); let structured_artist_page = [ " artist equipment", " artists equipment", " equipment bandcamp com", ] .iter() .any(|phrase| page.contains(phrase)); !(entity_phrase || structured_artist_page || (leaf_anchor && artist_leading_title)) } /// A one-word artist name can appear as the object in a generic music-photo /// description (for example, "sound equipment" or "bands equipment"). Such /// wording is not performer identity when the raster's own filename is also /// generic. This deliberately does not reject the inverse, "Equipment band", /// or an artist-named leaf such as `equipment-press.jpg`. fn single_word_artist_used_as_generic_noun( title: &str, page_url: &str, image_url: &str, artist: &str, ) -> bool { if image_filename_names_artist(image_url, artist) { return false; } let artist_tokens: Vec<&str> = artist .split(|character: char| !character.is_alphanumeric()) .filter(|token| !token.is_empty()) .collect(); if artist_tokens.len() != 1 { return false; } let artist_token = artist_tokens[0].to_lowercase(); let context = normalized_word_text(&format!("{title} {page_url}")); let tokens: Vec<&str> = context.split_whitespace().collect(); const GENERIC_PREFIXES: [&str; 10] = [ "audio", "bands", "dj", "music", "musical", "recording", "sound", "stage", "studio", "technical", ]; tokens .windows(2) .any(|pair| pair[1] == artist_token && GENERIC_PREFIXES.contains(&pair[0])) } /// For multilingual co-bill headlines, require corroboration that the image /// leaf names the *other* titled subject. This catches `... y Gerardo Niva` /// paired with `gerardo-niva.jpeg` without treating ordinary prose such as /// "Artist y su nuevo album" as a second performer. fn image_filename_names_other_title_subject(title: &str, image_url: &str) -> bool { let Some(filename) = reqwest::Url::parse(image_url).ok().and_then(|url| { url.path_segments() .and_then(|segments| segments.filter(|segment| !segment.is_empty()).last()) .map(str::to_owned) }) else { return false; }; const GENERIC_TOKENS: [&str; 14] = [ "asset", "banner", "hero", "image", "img", "jpeg", "jpg", "large", "original", "photo", "png", "press", "scaled", "webp", ]; let filename = normalized_word_text(&filename); let filename_tokens: Vec<&str> = filename .split_whitespace() .filter(|token| { token.len() >= 3 && token.chars().all(char::is_alphabetic) && !GENERIC_TOKENS.contains(token) }) .collect(); let title = normalized_word_text(title); filename_tokens .windows(2) .any(|pair| title.contains(&format!("{} {}", pair[0], pair[1]))) } /// Return the byte span of an artist name while ignoring punctuation and /// spacing inside it. This mirrors `names_artist` for the ASCII artist names /// used by the co-bill fixtures, while retaining offsets into the original /// title so the adjacent proper-name grammar can still be inspected. fn artist_name_span_in_title(title: &str, artist: &str) -> Option<(usize, usize)> { let needle: Vec = artist .chars() .filter(|character| character.is_alphanumeric()) .map(|character| character.to_ascii_lowercase()) .collect(); if needle.is_empty() { return None; } let characters: Vec<(usize, char)> = title.char_indices().collect(); for start in 0..characters.len() { if !characters[start].1.is_alphanumeric() || (start > 0 && characters[start - 1].1.is_alphanumeric()) { continue; } let mut title_at = start; let mut needle_at = 0; let mut end = characters[start].0; while title_at < characters.len() && needle_at < needle.len() { let (byte_at, character) = characters[title_at]; if !character.is_alphanumeric() { title_at += 1; continue; } if character.to_ascii_lowercase() != needle[needle_at] { break; } end = byte_at + character.len_utf8(); title_at += 1; needle_at += 1; } if needle_at == needle.len() && (title_at >= characters.len() || !characters[title_at].1.is_alphanumeric()) { return Some((characters[start].0, end)); } } None } /// Detect `Other Artist & Requested Artist` / `... and ...` even when the image /// CDN uses an opaque hash and the title spaces an artist name differently than /// the catalog. Two proper-name words remain the conservative default; a lone /// all-caps act is also sufficient, and `&` permits a lone title-cased act. /// Ordinary prose such as "Artist and the new album" therefore stays eligible. fn title_has_two_act_conjunction(title: &str, artist: &str) -> bool { fn name_edge_looks_like_act(segment: &str, from_end: bool, ampersand: bool) -> bool { const NON_NAME_WORDS: [&str; 12] = [ "a", "an", "the", "new", "live", "music", "album", "song", "release", "releases", "share", "shares", ]; let words: Vec<&str> = segment .split_whitespace() .map(|word| word.trim_matches(|character: char| !character.is_alphanumeric())) .filter(|word| !word.is_empty()) .collect(); let edge: Vec<&str> = if from_end { words.iter().rev().take(2).copied().collect() } else { words.iter().take(2).copied().collect() }; let proper_word = |word: &str| { let lower = word.to_lowercase(); word.chars().next().is_some_and(char::is_uppercase) && word .chars() .filter(|character| character.is_alphabetic()) .count() >= 2 && !NON_NAME_WORDS.contains(&lower.as_str()) }; if edge.len() >= 2 && edge.iter().take(2).all(|word| proper_word(word)) { return true; } let Some(adjacent) = edge.first() else { return false; }; let alphabetic: Vec = adjacent .chars() .filter(|character| character.is_alphabetic()) .collect(); proper_word(adjacent) && (ampersand || (alphabetic.len() >= 2 && alphabetic.iter().all(|character| character.is_uppercase()))) } let Some((artist_start, artist_end)) = artist_name_span_in_title(title, artist) else { return false; }; // ASCII-only folding preserves byte offsets even when the headline uses // typographic punctuation (for example `Queue’s Nala and Nikki Nair`). let lower = title.to_ascii_lowercase(); for conjunction in [" & ", " and "] { if lower[..artist_start].ends_with(conjunction) { let other_end = artist_start - conjunction.len(); if name_edge_looks_like_act(&title[..other_end], true, conjunction.trim() == "&") { return true; } } if lower[artist_end..].starts_with(conjunction) { let other_start = artist_end + conjunction.len(); if name_edge_looks_like_act(&title[other_start..], false, conjunction.trim() == "&") { return true; } } } false } /// Music roundups often put several artist names in one result title while /// Bing attaches just one of the article's photos. If the headline is clearly /// multi-artist and the image filename itself names another act, it is not a /// reliable identity match. Artist-specific filenames remain eligible. fn multi_artist_result_mismatches_image(title: &str, image_url: &str, artist: &str) -> bool { let lower = title.to_lowercase(); let listing_text = lower.replace(&artist.to_lowercase(), " "); let image_names_artist = image_filename_names_artist(image_url, artist); let image_names_other_subject = image_filename_names_other_title_subject(&listing_text, image_url); let two_act_conjunction = title_has_two_act_conjunction(title, artist); let plus_count = listing_text.matches(" + ").count(); let star_count = listing_text.matches(" * ").count(); let bullet_count = listing_text.matches(" • ").count() + listing_text.matches(" · ").count(); if plus_count >= 2 { return true; } if star_count >= 2 && !image_names_artist { return true; } if bullet_count >= 2 && !image_names_artist { return true; } // Even a two-act bill is ambiguous when the image filename/CDN gives no // artist identity. Artist-specific editorial filenames remain eligible. let event_with = listing_text.contains(" with ") && (listing_text.contains(" at ") || listing_text.contains(" tickets") || listing_text.contains(" live ")); let explicit_cobill = plus_count == 1 || listing_text.contains(" w/ ") || event_with || listing_text.contains(" feat. ") || listing_text.contains(" featuring ") || listing_text.contains(" vs. ") || listing_text.contains(" versus ") // `" x "` is deliberately NOT here. It reads as a co-bill between two // acts, but in practice it far more often joins a venue and a festival // or a brand: "Nikki Nair | Boiler Room x Bass Coast 2022". Because // every YouTube thumbnail is named `maxresdefault.jpg`, // `image_names_artist` is structurally false for the whole class, so // this one separator was rejecting every Boiler Room and festival set // photo -- the richest source of hero-shaped photography that exists // for DJs, and the only 1280x720 candidate Nikki Nair had. A genuine // collaboration graphic is still caught below by `named_collaboration`, // which requires the image itself to name the other subject. || two_act_conjunction || ([" y ", " con ", " junto a ", " et ", " und "] .iter() .any(|separator| listing_text.contains(separator)) && image_names_other_subject); // A collaboration graphic can name both the requested artist and the // collaborator in its filename, so `image_names_artist` alone is not // enough. This was the Nikki Nair + Hudson Mohawke illustrated release // card. Requiring a named second subject in the leaf keeps ordinary prose // such as "Artist and their new album" eligible. let named_collaboration = [ " + ", " & ", " and ", " with ", " feat. ", " featuring ", " x ", " y ", " con ", " junto a ", " et ", " und ", ] .iter() .any(|separator| listing_text.contains(separator)) && image_names_other_subject; if named_collaboration { return true; } if explicit_cobill && !image_names_artist { return true; } let separator_count = listing_text.matches(',').count() + listing_text.matches(" & ").count() + listing_text.matches(" with ").count() + listing_text.matches(" feat. ").count() + listing_text.matches(" featuring ").count() + listing_text.matches(" | ").count() + listing_text.matches(" / ").count() + listing_text.matches(';').count() + bullet_count; separator_count >= 2 && !image_names_artist } /// Explicit-content rejection, applied BEFORE the identity gate. /// /// The hero search reads Bing's unfiltered bucket as well as the strict one, /// because either can be serving another query's results. That is necessary for /// coverage, but it means pornography now enters the candidate pool: a direct /// read of one artist's unfiltered wave returned 24 hardcore cards. An audit /// found zero of ~67 such cards could clear `bing_identity_confidence` -- porn /// pages do not name obscure musicians -- but that is an empirical accident, /// not a guarantee, and it is one gate deep. This is the independent second /// layer that reading only the strict bucket used to provide. fn is_adult_result(title: &str, page_url: &str, image_url: &str) -> bool { let urls = format!("{page_url} {image_url}").to_lowercase(); const ADULT_HOSTS: [&str; 16] = [ "pornhub.", "xhamster.", "xhcdn.com", "xnxx.", "xvideos.", "eporner.", "redtube.", "youporn.", "spankbang.", "pictoa.", "thefappening", "onlyfans.", "fapello.", "hentai-foundry.", "rule34.", "e-hentai.", ]; if ADULT_HOSTS.iter().any(|host| urls.contains(host)) { return true; } // Title vocabulary, kept to terms that do not appear in music writing. // "Adult contemporary", "XXX" as a roman numeral and band names such as // Nude Beach or Pornography (The Cure) must NOT trip this, so match only // unambiguous phrases. let haystack = format!(" {} ", title.to_lowercase()); const ADULT_PHRASES: [&str; 10] = [ " porn ", " porno ", "hardcore porn", " nsfw ", " blowjob", " creampie", " anal sex", " sex tape", "nude leaks", " milf ", ]; ADULT_PHRASES.iter().any(|term| haystack.contains(term)) } /// True for Bing results that are unambiguously merchandise, a physical music /// format, or a shop listing rather than a photograph of the performer. Keep /// this conservative: page/image URL structure is the strongest signal, while /// titles must use explicit product vocabulary. A review that merely mentions /// an album is not rejected. fn is_product_image_result(title: &str, page_url: &str, image_url: &str) -> bool { let urls = format!("{} {}", page_url, image_url).to_lowercase(); const COMMERCE_URL_MARKERS: [&str; 29] = [ "/product/", "/products/", "/collections/", "/shop/", "/store/", "/merch/", "bandcamp.com/merch", "bigcartel.com", "cdn.shopify.com", "merchbar.com", "merchbucket.com", "product-images", "product_images", "productimage", "product-media", "product_media", "mockup", "bandcamp.com/album/", "bcbits.com/img/", "/upc/", "i.ebayimg.com/", "genius.com/albums/", "images.genius.com/", "albumoftheyear.org/album/", "cdn.albumoftheyear.org/album/", "mzstatic.com/image/thumb/music", "geo-media.beatport.com/image_size/", "/r/cassettereleases/", "/r/vinylreleases/", ]; if COMMERCE_URL_MARKERS .iter() .any(|marker| urls.contains(marker)) { return true; } // Bing titles sometimes describe an editorial announcement while the // attached raster is explicitly the physical release. Inspect only the // image leaf (never parent directories/query strings): `...-VINYL.jpg`, // `album-cover.png`, and similar product filenames cannot become heroes. let image_leaf = reqwest::Url::parse(image_url) .ok() .and_then(|url| { url.path_segments() .and_then(|segments| segments.filter(|segment| !segment.is_empty()).last()) .map(str::to_owned) }) .unwrap_or_default(); let leaf_words = normalized_word_text(&image_leaf); let leaf_tokens: HashSet<&str> = leaf_words.split_whitespace().collect(); if ["vinyl", "cassette", "mockup", "merch", "merchandise"] .iter() .any(|token| leaf_tokens.contains(token)) || ["album cover", "cover art", "record sleeve"] .iter() .any(|phrase| normalized_phrase_present(&leaf_words, phrase)) { return true; } let lower = title.to_lowercase(); const PRODUCT_PHRASES: [&str; 18] = [ "album cover", "album art", "album stream", "full album", "remastered album", "cover art", "colored vinyl", "coloured vinyl", "limited edition vinyl", "record pressing", "vinyl pressing", "pre-order", "preorder", "t-shirt", "tee shirt", "physical edition", "cassette release", "vinyl release", ]; if PRODUCT_PHRASES.iter().any(|phrase| lower.contains(phrase)) { return true; } // Token matching avoids traps such as `cd` inside "record" or `lp` inside // "help". These formats are explicit enough that the result is product // imagery even when Bing points directly at a generic CDN. lower.split(|c: char| !c.is_alphanumeric()).any(|word| { matches!( word, "vinyl" | "cassette" | "merch" | "merchandise" | "tshirt" | "mockup" | "preorder" | "lp" | "cd" ) }) } /// Bing can label channel branding and music visualizers as "photos". Reject /// those deterministic false positives while still allowing a YouTube result /// when its title explicitly describes a human-oriented interview, live set, /// session, performance, concert, or podcast. This keeps Bing as the source — /// there is no direct social-provider scraping — but prevents a channel banner /// or full-album thumbnail from becoming the artist hero. fn is_unsuitable_bing_result(title: &str, page_url: &str, image_url: &str, artist: &str) -> bool { if is_product_image_result(title, page_url, image_url) { return true; } let page = page_url.to_lowercase(); let image = image_url.to_lowercase(); let lower = title.to_lowercase(); let trusted_ticketing_profile = is_ticketing_artist_profile_for_artist(page_url, image_url, artist); let image_names_artist = image_filename_names_artist(image_url, artist); if page.contains("music.youtube.com/channel/") || image.contains("yt3.googleusercontent.com/") || image.contains("yt3.ggpht.com/") || page.contains("://twitter.com/") || page.contains("://www.twitter.com/") || page.contains("://x.com/") || page.contains("://www.x.com/") || page.contains("://instagram.com/") || page.contains("://www.instagram.com/") || page.contains("://facebook.com/") || page.contains("://www.facebook.com/") || page.contains("://pinterest.com/") || page.contains("://www.pinterest.com/") || page.contains("://deviantart.com/") || page.contains("://www.deviantart.com/") || page.contains("://artstation.com/") || page.contains("://www.artstation.com/") || page.contains("://www.wikipedia.org/") || page.contains("://en.wikipedia.org/") || page.contains("://commons.wikimedia.org/") || page.contains("://reddit.com/") || page.contains("://www.reddit.com/") || image.contains("pbs.twimg.com/") || image.contains("lookaside.instagram.com/") || image.contains("lookaside.fbsbx.com/") || image.contains("i.pinimg.com/") || image.contains("upload.wikimedia.org/") || image.contains("preview.redd.it/") || image.contains("external-preview.redd.it/") || image.contains("/production/artworks/") || page.contains("ra.co/events/") || page.contains("ra.co/event/") || ((page.contains("://www.ticketmaster.com/") || page.contains("://ticketmaster.com/") || page.contains("://www.livenation.com/") || page.contains("://livenation.com/")) && page.contains("/event/")) || page.contains("/tickets/") || (page.contains("/podcast/") && image.contains("podcast")) || (page.contains("soundcloud.com/") && image.contains("sndcdn.com/artworks-")) { return true; } // Mixmag's recurring "The Mix" assets carry a baked-in series label and // corner logo even when the raster is otherwise photographic. Prefer the // unbranded press portrait from the same result set. if (lower.starts_with("the mix ") || lower.starts_with("the mix:")) && image.contains("the_mix__") { return true; } if single_word_artist_used_as_generic_noun(title, page_url, image_url, artist) { return true; } // Event listings overwhelmingly contribute designed flyers rather than // artist photography. Keep this title-based instead of rejecting every // `/events/` page: some venues use genuine press photos on event pages. const EVENT_GRAPHIC_CONTEXT: [&str; 11] = [ " present:", " presents:", " presents ", "takeover", " tickets", " w/ ", "tour poster", "concert poster", "event poster", "show flyer", "lineup poster", ]; if EVENT_GRAPHIC_CONTEXT .iter() .any(|context| lower.contains(context)) && !trusted_ticketing_profile && !image_names_artist { return true; } // A venue/event page naming multiple acts often attaches the headliner's // photo or a shared bill graphic. It is not safe identity evidence for the // requested support artist unless the image URL itself names that artist. let event_listing = page.contains("/event/") || page.contains("/events/"); let listing_text = lower.replace(&artist.to_lowercase(), " "); let multilingual_cobill = [" y ", " con ", " junto a ", " et ", " und "] .iter() .any(|separator| listing_text.contains(separator)) && image_filename_names_other_title_subject(&listing_text, image_url); let multi_act_listing = listing_text.contains(" w/ ") || listing_text.matches(" * ").count() >= 2 || listing_text.matches(" • ").count() >= 2 || listing_text.matches(" · ").count() >= 2 || listing_text.contains(" x ") || multilingual_cobill || (listing_text.contains(" with ") && (listing_text.contains(" at ") || listing_text.contains(" live "))); if event_listing && multi_act_listing && !trusted_ticketing_profile && !image_names_artist { return true; } // Exact artist names can collide with companies (the Hermeth musician vs // Hermeth Corporation is the fixture). These terms are strong business or // industrial context and do not describe an artist photograph. let result_context = normalized_word_text(&format!("{lower} {page} {image}")); const NON_MUSIC_BUSINESS_CONTEXT: [&str; 8] = [ "corporation", "gmbh", "machinery", "maschinenbau", "manufacturing", "industrial equipment", "engineering company", "cnc machining", ]; if NON_MUSIC_BUSINESS_CONTEXT .iter() .any(|context| result_context.contains(context)) { return true; } // "Wednesday" is both an exact artist name and a cluster of much larger // film/TV and music entities. Bing frequently satisfies quoted musician // searches with those namesakes, so require explicit disambiguation here // rather than letting a generic `wednesday.jpg` leaf become identity. if normalized_word_text(artist) == "wednesday" { const WEDNESDAY_COLLISIONS: [&str; 46] = [ "wednesday addams", "addams family", "nevermore academy", "jenna ortega", "hunter doohan", "fielder jewett", "luyanda lewis nyawo", "lady gaga", "dead dance", "soundtrack", "netflix", "wednesday season", "wednesday show dance scene", "wednesday dance scene", "dance scene", "wednesday campanella", "wednesday 13", "scratchcard wednesday", "ra co dj wednesday", "special wednesday dj set", "dj for event", "wednesday profile icons", "wednesday profil simgeleri", "wednesday tv series", "film producer", "hollywood first look", "music supervisor", "executive producer", "exec producer", "second season", "co showrunner", "coshowrunner", "peaky blinders", "psychological thriller", "portrait that sees back", "concept poster", "charcoal drawing", "portrait drawing", "behance net gallery", "dribbble com shots", "etsy com listing", "hueforge", "makerworld", "move to ireland", "digital art", "wednesday dj", ]; if WEDNESDAY_COLLISIONS .iter() .any(|context| result_context.contains(context)) { return true; } } // MSPAINT the band collides with Microsoft's Paint application. Keep the // band spelling fully eligible, but refuse deterministic operating-system, // app, and UI contexts even when the image leaf itself says `MS-Paint`. if normalized_word_text(artist) == "mspaint" { const MSPAINT_APP_COLLISIONS: [&str; 10] = [ "microsoft paint", "ms paint for windows", "paint for windows", "paint app", "paint application", "windows 10", "windows 11", "windows insider", "windowslatest com", "untitled paint", ]; if MSPAINT_APP_COLLISIONS .iter() .any(|context| result_context.contains(context)) { return true; } } let video_result = page.contains("youtube.com/watch") || page.contains("youtube.com/live") || page.contains("youtu.be/") || page.contains("vimeo.com/") || image.contains("vimeocdn.com/"); if video_result && !has_human_video_context(&lower) { return true; } false } fn has_human_video_context(lower_title: &str) -> bool { const HUMAN_VIDEO_CONTEXT: [&str; 7] = [ "interview", " live", "session", "performance", "concert", "podcast", "behind the scenes", ]; HUMAN_VIDEO_CONTEXT .iter() .any(|context| lower_title.contains(context)) } /// Search-only visual prior against isolated products on white/light-gray /// canvases. It is deliberately a penalty, not a rejection: a legitimate /// white-studio portrait can still win when it is the artist's only usable /// image. Sampling only the outer eighth avoids mistaking pale faces/clothing /// in the subject area for a product background. fn search_product_background_penalty(image: &image::DynamicImage) -> f64 { let rgb = image.thumbnail(192, 192).to_rgb8(); let (w, h) = rgb.dimensions(); if w == 0 || h == 0 { return 1.0; } let band = (w.min(h) / 8).max(1); let mut sampled = 0_u64; let mut near_white = 0_u64; for y in 0..h { for x in 0..w { if x >= band && x < w - band && y >= band && y < h - band { continue; } sampled += 1; let [r, g, b] = rgb.get_pixel(x, y).0; let high = r.max(g).max(b); let low = r.min(g).min(b); if low >= 225 && high - low <= 20 { near_white += 1; } } } let fraction = near_white as f64 / sampled.max(1) as f64; if fraction >= 0.80 { 0.14 } else if fraction >= 0.65 { 0.25 } else if fraction >= 0.45 { 0.50 } else if fraction >= 0.30 { 0.75 } else { 1.0 } } /// Conservative visual prior for text/logos burned into a search image. The /// thumbnail is scanned for near-white-on-dark and near-black-on-light strokes, /// then only strokes concentrated into consecutive horizontal rows count. That /// row structure is common to captions and poster typography but much less /// common to ordinary photographic detail. This intentionally does not attempt /// OCR: it is fast, dependency-free, and language/letterform agnostic. fn search_text_overlay_penalty(image: &image::DynamicImage) -> f64 { let rgb = image.thumbnail(384, 216).to_rgb8(); let (w, h) = rgb.dimensions(); if w < 8 || h < 8 { return 1.0; } let mut row_hits = vec![0_u32; h as usize]; for y in 2..h - 2 { for x in 2..w - 2 { let [r, g, b] = rgb.get_pixel(x, y).0; let center = (u32::from(r) * 299 + u32::from(g) * 587 + u32::from(b) * 114) / 1_000; let neutral = r.max(g).max(b) - r.min(g).min(b) <= 28; let neighbors = [ rgb.get_pixel(x - 2, y).0, rgb.get_pixel(x + 2, y).0, rgb.get_pixel(x, y - 2).0, rgb.get_pixel(x, y + 2).0, ]; let neighbor_luma = neighbors.map(|[nr, ng, nb]| { (u32::from(nr) * 299 + u32::from(ng) * 587 + u32::from(nb) * 114) / 1_000 }); let darkest = neighbor_luma.iter().copied().min().unwrap_or(center); let lightest = neighbor_luma.iter().copied().max().unwrap_or(center); let bright_stroke = neutral && center >= 220 && darkest <= 100; let dark_stroke = neutral && center <= 35 && lightest >= 180; if bright_stroke || dark_stroke { row_hits[y as usize] += 1; } } } // Ignore isolated high-contrast rows (eyes, jewelry, stage lights). Text // produces at least three adjacent busy rows and usually one compact band. let busy_threshold = (w / 75).max(3); let mut structured_hits = 0_u64; let mut structured_rows = 0_u32; let mut row = 0_usize; while row < row_hits.len() { if row_hits[row] < busy_threshold { row += 1; continue; } let start = row; while row < row_hits.len() && row_hits[row] >= busy_threshold { row += 1; } let band_rows = row - start; // Dense contrast covering most of the frame is photographic texture // (murals, foliage, stage rigs), not a caption band. if band_rows >= 3 && band_rows <= (h as usize / 3).max(3) { structured_rows += (row - start) as u32; structured_hits += row_hits[start..row] .iter() .map(|hits| u64::from(*hits)) .sum::(); } } let pixel_fraction = structured_hits as f64 / (u64::from(w) * u64::from(h)) as f64; let row_fraction = f64::from(structured_rows) / f64::from(h); if pixel_fraction >= 0.015 || row_fraction >= 0.16 { 0.55 } else if pixel_fraction >= 0.008 || row_fraction >= 0.08 { 0.70 } else if pixel_fraction >= 0.004 || row_fraction >= 0.04 { 0.84 } else { 1.0 } } /// Whether the raster contains enough near-uniform neighboring pixels to look /// like a designed card rather than a photograph. On its own this is not a /// rejection signal (studio portraits often have flat backdrops); the scorer /// combines it with a strong text-overlay signal. fn search_has_flat_graphic_layout(image: &image::DynamicImage) -> bool { let rgb = image.thumbnail(192, 192).to_rgb8(); let (w, h) = rgb.dimensions(); if w < 2 || h < 2 { return false; } let mut flat = 0_u64; let mut compared = 0_u64; for y in 0..h - 1 { for x in 0..w - 1 { let [r, g, b] = rgb.get_pixel(x, y).0; for [nr, ng, nb] in [rgb.get_pixel(x + 1, y).0, rgb.get_pixel(x, y + 1).0] { compared += 1; let difference = r.abs_diff(nr) as u16 + g.abs_diff(ng) as u16 + b.abs_diff(nb) as u16; if difference <= 18 { flat += 1; } } } } flat as f64 / compared.max(1) as f64 >= 0.65 } /// Stronger flat-art signal used only alongside release/headline metadata. /// Entropy keeps dark studio photos with a flat backdrop from being rejected /// merely because most neighboring pixels are similar. fn search_has_low_palette_graphic(image: &image::DynamicImage) -> bool { let rgb = image .resize_exact(192, 108, image::imageops::FilterType::Triangle) .to_rgb8(); let (w, h) = rgb.dimensions(); let mut histogram = [0_u32; 512]; let mut flat = 0_u64; let mut compared = 0_u64; for y in 0..h { for x in 0..w { let [r, g, b] = rgb.get_pixel(x, y).0; let bin = (usize::from(r >> 5) << 6) | (usize::from(g >> 5) << 3) | usize::from(b >> 5); histogram[bin] += 1; for [nr, ng, nb] in [ (x + 1 < w).then(|| rgb.get_pixel(x + 1, y).0), (y + 1 < h).then(|| rgb.get_pixel(x, y + 1).0), ] .into_iter() .flatten() { compared += 1; let difference = r.abs_diff(nr) as u16 + g.abs_diff(ng) as u16 + b.abs_diff(nb) as u16; if difference <= 18 { flat += 1; } } } } let pixels = f64::from(w * h); let entropy = histogram.iter().fold(0.0, |sum, count| { if *count == 0 { sum } else { let probability = f64::from(*count) / pixels; sum - probability * probability.log2() } }); flat as f64 / compared.max(1) as f64 >= 0.82 && entropy <= 3.0 } /// Detect contact-sheet/album-art mosaics by looking for several strong cuts /// that span nearly a full axis. Ordinary photo edges are locally strong but /// average out across the frame; a panel boundary remains strong for almost /// every row or column. Outer five-percent borders are ignored so letterboxing /// and scanned-photo frames do not count as panels. fn search_has_panel_collage_layout(image: &image::DynamicImage) -> bool { let rgb = image .resize_exact(384, 216, image::imageops::FilterType::Triangle) .to_rgb8(); fn cut_groups(rgb: &image::RgbImage, vertical: bool) -> usize { let (width, height) = rgb.dimensions(); let axis_len = if vertical { width } else { height }; let span_len = if vertical { height } else { width }; if axis_len < 20 || span_len == 0 { return 0; } let margin = (axis_len / 20).max(1); let mut groups = 0_usize; let mut previous_hit: Option = None; for position in 1..axis_len { if position <= margin || position >= axis_len.saturating_sub(margin) { continue; } let mut difference = 0_u64; for offset in 0..span_len { let (before, after) = if vertical { ( rgb.get_pixel(position - 1, offset).0, rgb.get_pixel(position, offset).0, ) } else { ( rgb.get_pixel(offset, position - 1).0, rgb.get_pixel(offset, position).0, ) }; difference += before .into_iter() .zip(after) .map(|(left, right)| u64::from(left.abs_diff(right))) .sum::(); } let mean_difference = difference as f64 / f64::from(span_len * 3); if mean_difference < 35.0 { continue; } if previous_hit.is_none_or(|previous| position > previous + 2) { groups += 1; } previous_hit = Some(position); } groups } cut_groups(&rgb, true) >= 2 && cut_groups(&rgb, false) >= 1 } fn has_release_art_context(title: &str) -> bool { let lower = title.to_lowercase(); [ "new track", "new song", "new single", "premiere:", "track premiere", "single premiere", ] .iter() .any(|context| lower.contains(context)) } fn hero_fingerprint(image: &image::DynamicImage) -> HeroFingerprint { let mut dhashes = Vec::with_capacity(19); dhashes.push(dhash64(image)); for scale_percent in [85_u32, 70] { let crop_w = (image.width() * scale_percent / 100).max(9); let crop_h = (image.height() * scale_percent / 100).max(8); let x_positions = [0, (image.width() - crop_w) / 2, image.width() - crop_w]; let y_positions = [0, (image.height() - crop_h) / 2, image.height() - crop_h]; for y in y_positions { for x in x_positions { dhashes.push(dhash64(&image.crop_imm(x, y, crop_w, crop_h))); } } } let rgb = image .resize_exact(64, 64, image::imageops::FilterType::Triangle) .to_rgb8(); let mut color_histogram = [0_u16; 512]; for pixel in rgb.pixels() { let [r, g, b] = pixel.0; let bin = (usize::from(r >> 5) << 6) | (usize::from(g >> 5) << 3) | usize::from(b >> 5); color_histogram[bin] += 1; } HeroFingerprint { dhashes, color_histogram, } } fn hero_fingerprints_match(left: &HeroFingerprint, right: &HeroFingerprint) -> bool { let color_intersection: u32 = left .color_histogram .iter() .zip(&right.color_histogram) .map(|(left_count, right_count)| u32::from((*left_count).min(*right_count))) .sum(); if color_intersection < HERO_COLOR_INTERSECTION_MIN { return false; } let full_distance = (left.dhashes[0] ^ right.dhashes[0]).count_ones(); if full_distance <= HERO_EXACT_HASH_MAX_DISTANCE { return true; } let crop_distance = left .dhashes .iter() .flat_map(|left_hash| { right .dhashes .iter() .map(move |right_hash| (left_hash ^ right_hash).count_ones()) }) .min() .unwrap_or(u32::MAX); if crop_distance > HERO_CROP_HASH_MAX_DISTANCE { return false; } true } fn score_bing_hero_image( bytes: Arc>, candidate: &BingImageCandidate, artist: &str, cover_dhashes: &[u64], diagnostics: bool, ) -> Option { let url = &candidate.url; let decoded = decode_hero_image_slice(bytes.as_slice())?; let (w, h) = (decoded.width(), decoded.height()); // Wide profile banners (X is natively 1500×500) are intrinsically short, // so the height floor stays permissive. if w < 720 || h < 260 { if diagnostics { eprintln!("HERO_REJECT\tsize\t{}x{}\t{}", w, h, url); } return None; } if let Some((source_w, source_h)) = embedded_raster_dimensions(url) { let small_source = source_w < 720 || source_h < 400; let visibly_upscaled = w >= source_w.saturating_mul(2) || h >= source_h.saturating_mul(2); if small_source && visibly_upscaled { if diagnostics { eprintln!( "HERO_REJECT\tproxy-upscale\t{}x{} from {}x{}\t{}", w, h, source_w, source_h, url, ); } return None; } } let ratio = f64::from(w) / f64::from(h); let crop_w = f64::from(w).min(f64::from(h) * (16.0 / 9.0)); let crop_h = f64::from(h).min(f64::from(w) * (9.0 / 16.0)); let usable = crop_w * crop_h; let gray = decoded.thumbnail(384, 384).to_luma8(); let sharp_norm = (laplacian_variance(&gray) / 120.0).clamp(0.25, 1.0); let wide_bonus = if ratio >= 1.5 { 1.25 } else { 1.0 }; let product_penalty = search_product_background_penalty(&decoded); let text_penalty = search_text_overlay_penalty(&decoded); let source_penalty = search_thumbnail_source_penalty(url); let flat_graphic = search_has_flat_graphic_layout(&decoded); let low_palette_graphic = search_has_low_palette_graphic(&decoded); let panel_collage = text_penalty <= 0.70 && search_has_panel_collage_layout(&decoded); let trusted_ticketing_profile = is_ticketing_artist_profile_for_artist(&candidate.page_url, url, artist); if is_lossless_graphic_url(url) && text_penalty <= 0.84 && !trusted_ticketing_profile { if diagnostics { eprintln!( "HERO_REJECT\ttext-graphic\t{}x{}\ttext={:.2}\t{}", w, h, text_penalty, url, ); } return None; } if low_palette_graphic && has_release_art_context(&candidate.title) && !trusted_ticketing_profile { if diagnostics { eprintln!("HERO_REJECT\trelease-graphic\t{}x{}\t{}", w, h, url); } return None; } if text_penalty <= 0.70 && flat_graphic && !trusted_ticketing_profile { if diagnostics { eprintln!( "HERO_REJECT\ttext+flat\t{}x{}\ttext={:.2}\t{}", w, h, text_penalty, url, ); } return None; } if panel_collage && !trusted_ticketing_profile { if diagnostics { eprintln!( "HERO_REJECT\ttext-collage\t{}x{}\ttext={:.2}\t{}", w, h, text_penalty, url, ); } return None; } // A hero is a WIDE banner. Square rasters are release artwork far more // often than press photography — and album art passes every other gate: // a sleeve that is itself a photo collage is not flat, not low-palette, // and its title text can be small enough to clear the text penalty (the // collage detector only runs at `text_penalty <= 0.70`). Camping in // Alaska's square "lost and found" sleeve reached a live hero slot that // way. Real press photos measured across the fixtures run 1.33-1.9, so // the floor sits below every one of them. Trusted ticketing profiles // keep their existing exemption — that tier is identity-verified art. // // PORTRAIT is admitted, though, because the guard is really an ALBUM-ART // guard and sleeves are square: a 0.75-0.85 press portrait is not a sleeve. // The app already crops tall sources to 16:9 around the detected faces // (`hero_crop.rs`, recipe `vertical-yoloxnano-...-faces-first`), so these // were being discarded one crate before the machinery built to use them. // This is the only route to a hero for artists whose entire press corpus is // portrait: Hana Vu (2000x2500, 1705x2129) and Nikki Nair (1080x1350, // 1024x1365) both pass identity and died here. Text-heavy posters and // flyers cannot sneak in this way — the text, flat-graphic and collage // gates all run BEFORE this one. // // SQUARE is admitted only when we can prove it is not a sleeve. Shape // cannot distinguish a 1:1 press crop (Nikki Nair's Rinse FM and Corsica // Studios photographs) from release artwork, but the artist's own covers // can: a sleeve IS one of them, so a perceptual-hash match against the // catalogue answers the question the aspect ratio was only guessing at. // With no cover hashes available -- no Deezer id, or the lookup failed -- // this falls back to the old blanket rejection rather than guessing. let usable_portrait = ratio <= HERO_PORTRAIT_MAX_ASPECT && w >= HERO_PORTRAIT_MIN_WIDTH; let usable_square = ratio > HERO_PORTRAIT_MAX_ASPECT && w >= HERO_PORTRAIT_MIN_WIDTH && !cover_dhashes.is_empty() && { let hash = dhash64(&decoded); let matches_release = cover_dhashes .iter() .any(|cover| (cover ^ hash).count_ones() <= COVER_SAME_ART_MAX_DISTANCE); if matches_release && diagnostics { eprintln!( "HERO_REJECT\talbum-art\t{}x{}\tratio={:.2}\t{}", w, h, ratio, url, ); } !matches_release }; if ratio < HERO_MIN_ASPECT && !usable_portrait && !usable_square && !trusted_ticketing_profile { if diagnostics { eprintln!( "HERO_REJECT\taspect\t{}x{}\tratio={:.2}\t{}", w, h, ratio, url, ); } return None; } if source_penalty <= 0.68 && text_penalty < 1.0 { if diagnostics { eprintln!( "HERO_REJECT\tcaptioned-video\t{}x{}\ttext={:.2}\tsource={:.2}\t{}", w, h, text_penalty, source_penalty, url, ); } return None; } let score = usable.sqrt() * sharp_norm * wide_bonus * product_penalty * text_penalty * source_penalty; if diagnostics { eprintln!( "HERO_SCORE\t{}x{}\tusable={:.0}\tsharp={:.3}\twide={:.2}\tproduct={:.2}\ttext={:.2}\tsource={:.2}\tflat={}\tlow_palette={}\tpanel_collage={}\ttrusted={}\tbase={:.2}\t{}", w, h, usable, sharp_norm, wide_bonus, product_penalty, text_penalty, source_penalty, flat_graphic, low_palette_graphic, panel_collage, trusted_ticketing_profile, score, url, ); } Some(HeroVisualScore { score, fingerprint: hero_fingerprint(&decoded), }) } fn decode_hero_image_slice(bytes: &[u8]) -> Option { let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)) .with_guessed_format() .ok()?; reader.limits(hero_decode_limits()); reader.decode().ok() } #[cfg(test)] fn decode_hero_image(bytes: Vec) -> Option { decode_hero_image_slice(&bytes) } fn hero_decode_limits() -> image::Limits { let mut limits = image::Limits::default(); limits.max_image_width = Some(HERO_MAX_DECODE_DIMENSION); limits.max_image_height = Some(HERO_MAX_DECODE_DIMENSION); limits.max_alloc = Some(HERO_MAX_DECODE_ALLOC_BYTES); limits } /// Designed web graphics are commonly retained as PNG while ordinary press /// photographs are normally JPEG/WebP. This is deliberately only a companion /// signal for the text detector; a clean PNG photograph is still accepted. fn is_lossless_graphic_url(url: &str) -> bool { let path = url.split(['?', '#']).next().unwrap_or(url).to_lowercase(); path.ends_with(".png") } /// Extract the smallest `WIDTHxHEIGHT` token embedded in a source URL. Image /// proxies sometimes serve a nominal 2048px response whose filename reveals it /// was generated from a 300×168 thumbnail; treating the decoded dimensions as /// native would let that visibly smeared copy outrank a real press image. fn embedded_raster_dimensions(url: &str) -> Option<(u32, u32)> { let bytes = url.as_bytes(); let mut dimensions = Vec::new(); for (x_at, byte) in bytes.iter().enumerate() { if !matches!(byte, b'x' | b'X') || x_at == 0 || x_at + 1 >= bytes.len() { continue; } let mut start = x_at; while start > 0 && bytes[start - 1].is_ascii_digit() && x_at - start < 5 { start -= 1; } let mut end = x_at + 1; while end < bytes.len() && bytes[end].is_ascii_digit() && end - x_at <= 5 { end += 1; } if start == x_at || end == x_at + 1 { continue; } // Size tokens are delimited (`-300x168.jpg`, `size=1200x800`). Do not // reinterpret an arbitrary alphanumeric id containing digits+x+digits // as the raster's native resolution. if start > 0 && bytes[start - 1].is_ascii_alphanumeric() { continue; } if end < bytes.len() && bytes[end].is_ascii_alphanumeric() { continue; } let Ok(width) = std::str::from_utf8(&bytes[start..x_at]) .ok()? .parse::() else { continue; }; let Ok(height) = std::str::from_utf8(&bytes[x_at + 1..end]) .ok()? .parse::() else { continue; }; if width >= 16 && height >= 16 { dimensions.push((width, height)); } } dimensions .into_iter() .min_by_key(|(w, h)| u64::from(*w) * u64::from(*h)) } /// Video thumbnails are allowed when their result title proves a human-facing /// interview/performance, but they frequently contain baked-in titles, logos, /// and player branding. Prefer editorial/venue press photography when both are /// available; keep thumbnails as a sparse-artist fallback. fn search_thumbnail_source_penalty(url: &str) -> f64 { let lower = url.to_lowercase(); if lower.contains("i.ytimg.com/") || lower.contains("img.youtube.com/") { 0.58 } else if lower.contains("vimeocdn.com/") { 0.68 } else if lower.contains("lookaside.fbsbx.com/") { 0.85 } else { 1.0 } } /// Variance of the 4-neighbour Laplacian over a grayscale thumbnail — the /// standard cheap focus/sharpness measure. Blurry photos score low. fn laplacian_variance(gray: &image::GrayImage) -> f64 { let (w, h) = gray.dimensions(); if w < 3 || h < 3 { return 0.0; } let mut sum = 0.0; let mut sum_sq = 0.0; let mut count = 0.0; for y in 1..h - 1 { for x in 1..w - 1 { let center = f64::from(gray.get_pixel(x, y).0[0]); let lap = 4.0 * center - f64::from(gray.get_pixel(x - 1, y).0[0]) - f64::from(gray.get_pixel(x + 1, y).0[0]) - f64::from(gray.get_pixel(x, y - 1).0[0]) - f64::from(gray.get_pixel(x, y + 1).0[0]); sum += lap; sum_sq += lap * lap; count += 1.0; } } let mean = sum / count; sum_sq / count - mean * mean } fn urlencode(s: &str) -> String { s.bytes() .map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { (b as char).to_string() } b' ' => "+".to_string(), other => format!("%{other:02X}"), }) .collect() } #[cfg(test)] mod tests { /// The try-list port (`search_titles`) against the exact folder shapes /// the mock proved on: rip descriptors, date/type prefixes, doubled /// artists (research/import/enrich.mjs searchTitles). #[test] fn search_titles_cleans_rip_descriptors() { use super::search_titles; assert_eq!( search_titles("Vashti Bunyan", "Heartleap - WEB V0"), vec!["Heartleap".to_string()] ); assert_eq!( search_titles("Vashti Bunyan", "Just Another Diamond Day [V0]"), vec!["Just Another Diamond Day".to_string()] ); assert_eq!( search_titles("Fishmans", "(1997.07.24) (album) Uchu Nippon Setagaya [128kbps]"), vec!["Uchu Nippon Setagaya".to_string()] ); assert_eq!( search_titles("Smog", "Smog - The Doctor Came At Dawn (1996)"), vec![ "The Doctor Came At Dawn (1996)".to_string(), "The Doctor Came At Dawn".to_string(), ] ); // The artist strip must not eat the whole title. Where the folder is // named after the band, taking the "artist" off the front left only // the catalogue number — the providers were asked for "(SICP 2643)" // and a famous record read as a deep cut nothing had. assert_eq!( search_titles("Band Of Gypsys", "Band Of Gypsys (SICP 2643)"), vec![ "Band Of Gypsys (SICP 2643)".to_string(), "Band Of Gypsys".to_string(), ] ); // A self-titled record, same shape, nothing but the name. assert_eq!( search_titles("Weezer", "Weezer"), vec!["Weezer".to_string()] ); // …while a strip that leaves a real title still happens. assert_eq!( search_titles("Battles", "Battles - Mirrored"), vec!["Mirrored".to_string()] ); } /// An appended release type is not part of the title, and one extra word /// is enough to make a record unfindable at both providers (see /// [`super::TYPE_WORDS`] for the measured Deezer/MusicBrainz evidence). #[test] fn search_titles_offers_the_title_without_its_release_type() { use super::search_titles; // The record that found this: 9 tracks, tagged with the suffix, and // Deezer sells it as "Willoughby's Beach". assert_eq!( search_titles("King Gizzard & The Lizard Wizard", "Willoughby's Beach EP"), vec![ "Willoughby's Beach EP".to_string(), "Willoughby's Beach".to_string(), ] ); // The store convention, separator and all. assert_eq!( search_titles("Burial", "Street Halo - EP"), vec!["Street Halo - EP".to_string(), "Street Halo".to_string()] ); // Punctuation folds into the word. assert_eq!( search_titles("Alvvays", "Antisocialites E.P."), vec![ "Antisocialites E.P.".to_string(), "Antisocialites".to_string(), ] ); // The exact title is always first, so the 13 records whose service // title carries the word too are unaffected. assert_eq!( search_titles("Ratatat", "LP3")[0], "LP3".to_string() ); // A title that is ONLY its type word keeps it — stripping leaves // nothing, and nothing names no record. assert_eq!(search_titles("Ratatat", "EP"), vec!["EP".to_string()]); assert_eq!( search_titles("Public Image Ltd", "Album"), vec!["Album".to_string()] ); // Bracketed decoration is not a title either, so the type word stays // rather than reducing this to "(WARP 24)". assert_eq!( search_titles("Nightmares On Wax", "(WARP 24) EP"), vec!["(WARP 24) EP".to_string()] ); // Words that name the work far more often than they annotate it are // NOT stripped. assert_eq!( search_titles("Dire Straits", "Alchemy Live"), vec!["Alchemy Live".to_string()] ); // And a title merely ENDING in one of these letters is untouched. assert_eq!( search_titles("Sonic Youth", "Sister"), vec!["Sister".to_string()] ); } /// The folder as the SECOND witness — a record whose tags are wrong. #[test] fn identify_titles_asks_the_folder_after_the_records_own_claim() { use super::identify_titles; const A: &str = "King Gizzard & The Lizard Wizard"; // The record that found this: nine files tagged "Nonagon Infiniry", // sitting in a correctly-spelled folder. MusicBrainz answers count=0 // for the first and score=100 for the second. assert_eq!( identify_titles(A, "Nonagon Infiniry", Some("(2016) - Nonagon Infinity")), vec![ "Nonagon Infiniry".to_string(), "Nonagon Infinity".to_string(), ] ); // The claim always leads, so a record that matches on its tags today // asks exactly what it asked before. assert_eq!( identify_titles(A, "Nonagon Infiniry", Some("(2016) - Nonagon Infinity"))[0], "Nonagon Infiniry".to_string() ); // Agreement adds nothing — no second query for the same words. assert_eq!( identify_titles("Battles", "Mirrored", Some("Battles - Mirrored")), vec!["Mirrored".to_string()] ); // No folder is the old behaviour, exactly. assert_eq!( identify_titles("Battles", "Mirrored", None), super::search_titles("Battles", "Mirrored") ); // The folder's own rip descriptors are cleaned before it is asked, and // it still contributes the rungs its cleaning produces. assert_eq!( identify_titles("Vashti Bunyan", "Heartleep", Some("Heartleap - WEB V0")), vec!["Heartleep".to_string(), "Heartleap".to_string()] ); } /// The tracklist comparison that identifies a record its NAME cannot. #[test] fn tracklist_overlap_reads_the_same_music_through_different_names() { use super::tracklist_overlap as overlap; let ours = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::>(); // The record that argued for this pass: his two files are "Octopus" and // "Rock Bottom", filed as `Rock Bottom 7"`; MusicBrainz calls the // release "Rock Bottom / Octopus". The NAME scores 0.667, under the // 0.72 floor. The music is identical. assert_eq!( overlap(&ours(&["Octopus", "Rock Bottom"]), &ours(&["Rock Bottom", "Octopus"])), 1.0 ); // Punctuation and case are a tagger's business, not the record's. assert_eq!( overlap(&ours(&["Alter Me I"]), &ours(&["alter me i."])), 1.0 ); // The LARGER side decides, so a 2-track single does not read as a full // match for the 20-track album that happens to contain both songs. let single = ours(&["Rock Bottom", "Octopus"]); let album: Vec = (0..20) .map(|i| if i == 0 { "Rock Bottom".into() } else if i == 1 { "Octopus".into() } else { format!("Filler {i}") }) .collect(); assert!(overlap(&single, &album) < 0.2); // Different records share nothing. assert_eq!(overlap(&ours(&["Robot Stop"]), &ours(&["Easy Easy"])), 0.0); // Nothing to compare is not agreement. assert_eq!(overlap(&[], &ours(&["Rock Bottom"])), 0.0); assert_eq!(overlap(&ours(&["Rock Bottom"]), &[]), 0.0); // A repeated title counts once per actual occurrence, both ways. assert_eq!( overlap(&ours(&["Intro", "Intro"]), &ours(&["Intro"])), 0.5 ); } /// The artwork gate, in the terms the import review reasons about. #[test] fn same_art_distance_is_the_covers_gate() { // Re-exported so a caller can describe its confidence in one vocabulary // rather than inventing a second threshold. assert_eq!(super::SAME_ART_MAX_DISTANCE, super::COVER_SAME_ART_MAX_DISTANCE); // Identical artwork is distance 0 — the same picture from two CDNs // lands at 0-6, different covers cluster at 20 and above. let h: u64 = 0x0f1e_2d3c_4b5a_6978; assert_eq!((h ^ h).count_ones(), 0); assert!((h ^ h).count_ones() <= super::SAME_ART_MAX_DISTANCE); } /// A release group's year, from MusicBrainz's three date shapes. #[test] fn mb_group_year_reads_partial_dates() { let g = |d: Option<&str>| super::MbGroup { id: "x".into(), title: "t".into(), primary_type: None, first_release_date: d.map(String::from), }; assert_eq!(g(Some("2012-09-24")).year(), Some(2012)); assert_eq!(g(Some("2016-04")).year(), Some(2016)); assert_eq!(g(Some("2017")).year(), Some(2017)); assert_eq!(g(Some("")).year(), None); assert_eq!(g(None).year(), None); } /// One title, several release groups: the record's own shape breaks the /// tie, because MusicBrainz's order between equal scores does not. #[test] fn mb_type_fit_prefers_the_group_shaped_like_the_record() { use super::mb_type_fit as fit; // The case that found this: "Murder of the Universe" is a 21-track // Album AND a 1-track Single, both scored 100 by MB. assert!(fit(Some("Album"), Some(21)) < fit(Some("Single"), Some(21))); // …and the same collision from the other side: a real one-track single // must not be answered with the album of the same name. assert!(fit(Some("Single"), Some(1)) < fit(Some("Album"), Some(1))); // An EP-sized record prefers the EP. assert!(fit(Some("EP"), Some(6)) < fit(Some("Single"), Some(6))); // Type casing is MusicBrainz's, not ours. assert_eq!(fit(Some("album"), Some(21)), fit(Some("Album"), Some(21))); // With no count to reason from, an Album is the deterministic answer. assert!(fit(Some("Album"), None) < fit(Some("Single"), None)); assert!(fit(Some("Album"), None) < fit(Some("EP"), None)); // A missing type is the worst rank, never a coincidental best. assert!(fit(Some("Album"), Some(21)) < fit(None, Some(21))); } /// The ranking rule that replaced "strip the leading year". /// /// The point of scoring rather than stripping: a title that IS a year has /// to survive. `1989`, `1984`, `2001` and `1999` are all real albums, and /// any cleaner aggressive enough to rescue "1989 Pauls Boutique" would /// have reduced every one of them to nothing. #[test] fn title_match_score_ranks_without_stripping() { use super::title_match_score as score; const MIN: f32 = 0.72; // `discography_identify`'s NAME_ALONE // The case this exists for: a year the folder added, an apostrophe the // folder dropped, and a provider selling the anniversary pressing. assert!(score("1989 Pauls Boutique", "Paul's Boutique") >= MIN); assert!( score( "1989 Pauls Boutique", "Paul's Boutique (20th Anniversary Edition / Remastered)" ) >= MIN ); // …and it must not drag in the rest of the same artist's catalogue. assert!(score("1989 Pauls Boutique", "Licensed to Ill") < MIN); assert!(score("1989 Pauls Boutique", "Check Your Head") < MIN); assert!(score("1989 Pauls Boutique", "Ill Communication") < MIN); // A title that is a year keeps its own identity, and does not become a // free match for every record that happens to carry one. assert_eq!(score("1989", "1989"), 1.0); assert!(score("1989", "Paul's Boutique") < MIN); assert!(score("1984", "1989") < MIN); assert!(score("2001", "2001") >= MIN); // The re-recording is a different work and must not beat the original. assert!(score("1989", "1989") > score("1989", "1989 (Taylor's Version)")); // Rip descriptors. A short title buried in a long descriptor is the // try-list's job, not the scorer's — `search_titles` hands its cleaned // forms in as extra names to score, so "Heartleap - WEB V0" is matched // through the try "Heartleap" rather than by loosening this. assert!(score("Sung Tongs [FLAC]", "Sung Tongs") >= MIN); // The canonical wrong record stays wrong. assert_eq!(score("Bastard", "Cherry Bomb"), 0.0); // A shared word is not a match. assert!(score("Greatest Hits", "Greatest Hits II") < 1.0); assert!(score("Kid A", "Kid Amnesiae") < MIN); // Nothing to compare is not a match with everything. assert_eq!(score("", "Paul's Boutique"), 0.0); assert_eq!(score("Paul's Boutique", ""), 0.0); } /// The edition rule's foundation: containment matches, disjoint titles /// don't (Bastard → Cherry Bomb is the canonical wrong record). #[test] fn normx_overlap_editions_match_wrong_records_do_not() { assert!(super::normx_overlap( "Random Access Memories (10th Anniversary Edition)", "Random Access Memories" )); assert!(!super::normx_overlap("Cherry Bomb", "Bastard")); } /// Is a provider's image host actually failing, or is this machine? /// /// Run this the next time a `failed a liveness probe` warning appears. It /// reproduces [`super::MetadataClient::image_host_alive`]'s exact request /// four ways — cold client, warm client, the concurrent burst an artist /// page produces, and a probe interleaved with full-size hero downloads — /// and prints the reqwest error chain the probe otherwise reduces to a /// bool. /// /// 2026-08-03, the run that answered the fanart.tv report: every one of /// these came back 206 in ~30-170ms, and 4.3MB of hero downloaded /// alongside them in 340ms. The host was innocent; the machine's first DNS /// server was dead and name resolution alone was spending up to 3.6s. When /// this test is clean, look at `Resolve-DnsName` timings before suspecting /// the provider. /// /// ```text /// cargo test -p metadata fanart_probe_live -- --ignored --nocapture /// ``` #[tokio::test] #[ignore = "hits assets.fanart.tv"] async fn fanart_probe_live() { use std::time::{Duration, Instant}; // A real url out of this machine's own artist-art manifest. const URL: &str = "https://assets.fanart.tv/fanart/justice-4e1c8af0c25fc.jpg"; fn chain(err: &reqwest::Error) -> String { let mut out = format!( "{err} [timeout={} connect={} request={} body={}]", err.is_timeout(), err.is_connect(), err.is_request(), err.is_body(), ); let mut src = std::error::Error::source(err); while let Some(e) = src { out.push_str(&format!("\n caused by: {e}")); src = std::error::Error::source(e); } out } fn client() -> reqwest::Client { reqwest::Client::builder() .user_agent(super::USER_AGENT) .timeout(Duration::from_secs(15)) .build() .unwrap() } async fn probe(http: &reqwest::Client, label: String) { let started = Instant::now(); let outcome = tokio::time::timeout( super::IMAGE_HOST_PROBE_TIMEOUT, http.get(URL) .header(reqwest::header::RANGE, "bytes=0-0") .send(), ) .await; let ms = started.elapsed().as_millis(); match outcome { Err(_) => println!("{label:<18} {ms:>6}ms DEAD — probe timeout elapsed"), Ok(Err(err)) => println!("{label:<18} {ms:>6}ms DEAD — {}", chain(&err)), Ok(Ok(resp)) => println!( "{label:<18} {ms:>6}ms {} {:?} (server_error={})", resp.status(), resp.version(), resp.status().is_server_error(), ), } } println!("-- cold client per request (first TLS handshake every time)"); for i in 0..4 { probe(&client(), format!("cold #{i}")).await; } println!("-- one warm client, sequential"); let warm = client(); for i in 0..4 { probe(&warm, format!("warm #{i}")).await; } println!("-- 12 concurrent on one client (an artist page's shape)"); let burst = client(); let mut tasks = Vec::new(); for i in 0..12 { let http = burst.clone(); tasks.push(tokio::spawn( async move { probe(&http, format!("burst #{i:<2}")).await }, )); } for t in tasks { t.await.unwrap(); } // The suspect: the probe does not race the CDN, it races THIS APP. A // cold artist page has several multi-megabyte hero candidates in flight // on this same client while the one-byte probe is queued behind them. println!("-- probing every 250ms while 8 full-size heroes download on the same client"); let loaded = client(); let bulk: Vec<&str> = vec![ "https://assets.fanart.tv/fanart/acdc-4fcad35eb2643.jpg", "https://assets.fanart.tv/fanart/acdc-5467b70af2ff0.jpg", "https://assets.fanart.tv/fanart/acdc-4e19b4d51609c.jpg", "https://assets.fanart.tv/fanart/acdc-54575601f4146.jpg", "https://assets.fanart.tv/fanart/tame-impala-515a1f8fc9345.jpg", "https://assets.fanart.tv/fanart/tame-impala-4fd09eab05fc1.jpg", "https://assets.fanart.tv/fanart/fleetwood-mac-4e7ec8bebeba1.jpg", "https://assets.fanart.tv/fanart/queen-4fc9a7ea0e0f0.jpg", ]; let mut downloads = Vec::new(); for url in bulk { let http = loaded.clone(); downloads.push(tokio::spawn(async move { let started = Instant::now(); match http.get(url).send().await { Ok(resp) => { let bytes = resp.bytes().await.map(|b| b.len()).unwrap_or(0); (bytes, started.elapsed().as_millis()) } Err(_) => (0, started.elapsed().as_millis()), } })); } for i in 0..16 { probe(&loaded, format!("under-load #{i:<2}")).await; tokio::time::sleep(Duration::from_millis(250)).await; } let mut total = 0; for d in downloads { let (bytes, ms) = d.await.unwrap(); total += bytes; print!("[{bytes}B in {ms}ms] "); } println!("\n {total} bytes of hero pulled alongside the probes"); } /// A host that stalls once must not stay written off for /// [`super::IMAGE_HOST_PROBE_TTL`]. /// /// This is the fanart.tv failure reduced to its mechanism: the first /// request gets no answer inside the probe's 3-second budget (on the real /// machine, a dead first resolver eating 1-3.6s of name resolution), while /// the host itself is fine and answers the very next request instantly. /// Before the confirming probe, that single sample cost five minutes of /// every artist page falling back to a blown-up square portrait. /// /// Runs against a local socket, so it asserts the policy and never the /// weather. #[tokio::test(flavor = "multi_thread")] async fn one_stalled_probe_does_not_write_off_a_host() { use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let seen = Arc::new(AtomicUsize::new(0)); let served = seen.clone(); tokio::spawn(async move { loop { let Ok((mut socket, _)) = listener.accept().await else { return; }; let nth = served.fetch_add(1, Ordering::SeqCst); tokio::spawn(async move { let mut scratch = [0u8; 1024]; let _ = socket.read(&mut scratch).await; // Only the first caller is made to wait, and for longer // than the fast probe will ever wait for it. if nth == 0 { tokio::time::sleep(Duration::from_secs(5)).await; } let _ = socket .write_all( b"HTTP/1.1 206 Partial Content\r\n\ Content-Range: bytes 0-0/511115\r\n\ Content-Length: 1\r\n\r\nX", ) .await; }); } }); let client = super::MetadataClient::new(); let url = format!("http://127.0.0.1:{port}/fanart/justice-4e1c8af0c25fc.jpg"); // The page still gets its fast answer and falls through to the next // provider — that part is deliberately unchanged. assert!( !client.image_host_alive(&url).await, "a host that misses the probe budget must not hold up the page", ); // ...but the verdict is confirmed off the critical path, and the host // comes back without waiting out the five-minute TTL. let mut restored = false; for _ in 0..40 { tokio::time::sleep(Duration::from_millis(100)).await; if client.image_host_alive(&url).await { restored = true; break; } } assert!( restored, "the confirming probe answered, so the host must be usable again \ long before {:?}", super::IMAGE_HOST_PROBE_TTL, ); assert!( seen.load(Ordering::SeqCst) >= 2, "the confirmation has to actually ask the host a second time", ); } /// The search page's listener bars, end to end against the live providers. /// /// Ignored by default — it spends real Deezer, MusicBrainz and Last.fm /// requests. Run it deliberately: /// /// ```text /// cargo test -p metadata track_listeners_live -- --ignored --nocapture /// ``` /// /// What it proves, and why each half matters: /// /// 1. `catalog_search` now carries a per-row `artist_id`. Without it the /// counts can only be resolved for one artist, and every row credited to /// somebody else shows a dash. /// 2. A row by a DIFFERENT artist than the one searched for still gets its /// count. Searching "panda bear" returns Daft Punk's "Doin' it Right /// (feat. Panda Bear)", which the album page shows ~464k listeners for; /// it must not come back empty here. /// /// Deliberately loose on the numbers: listener counts move every day, and a /// test that pins them would fail for the wrong reason. It asserts the /// SHAPE — ids present, counts resolved, cross-artist rows answered — and /// prints the table so a human can sanity-check the figures. #[tokio::test] #[ignore = "hits Deezer, MusicBrainz and Last.fm"] async fn track_listeners_live() { let client = super::MetadataClient::new(); let results = client.catalog_search("panda bear").await; assert!(!results.tracks.is_empty(), "search returned no songs"); assert!( results.tracks.iter().all(|t| t.artist_id.is_some()), "every Deezer search row carries an artist id", ); let rows: Vec<(Option, String)> = results .tracks .iter() .map(|t| (t.artist_id, t.title.clone())) .collect(); let counts = client.track_listeners(&rows).await; assert_eq!(counts.len(), rows.len(), "one answer per row, in order"); for (track, listeners) in results.tracks.iter().zip(&counts) { println!( "{:>12} {:<52} {}", listeners .map(|n| n.to_string()) .unwrap_or_else(|| "—".into()), track.title, track.artist, ); } assert!( counts.iter().any(|c| c.is_some()), "no row resolved a listener count at all", ); // The point of the whole change: a row the searched-for artist only // guests on is still answered, from ITS OWN artist's Last.fm page. let searched = results.tracks.first().map(|t| t.artist_id).unwrap(); let cross: Vec<_> = results .tracks .iter() .zip(&counts) .filter(|(t, _)| t.artist_id != searched) .collect(); if let Some((track, _)) = cross.first() { assert!( cross.iter().any(|(_, c)| c.is_some()), "no cross-artist row resolved a count (e.g. {} by {})", track.title, track.artist, ); } } /// Last.fm returns a popular song many times over — once as itself and /// again for every "(Album Version)", live cut and remaster — listeners /// descending. The canonical first entry must win. /// /// Real figures, `artist.gettoptracks` for Weezer: Buddy Holly at rank 1 /// with 2,351,642 listeners, and again at rank 679 as "(Album Version)" /// with 1,919. The album page printed 1,919 until this was fixed — the /// map was being overwritten by every later duplicate. #[test] fn a_songs_canonical_entry_beats_its_re_listings() { let stats = |listeners: u64, plays: u64| TrackStats { plays, listeners }; let entries = vec![ ( track_title_identity("Buddy Holly"), "Buddy Holly".to_string(), stats(2_351_642, 24_083_192), ), ( track_title_identity("El Scorcho"), "El Scorcho".to_string(), stats(654_071, 5_368_072), ), // …hundreds of ranks later, the same songs re-listed. ( track_title_identity("Buddy Holly (Album Version)"), "Buddy Holly (Album Version)".to_string(), stats(1_919, 7_043), ), ( track_title_identity("El Scorcho (Album Version)"), "El Scorcho (Album Version)".to_string(), stats(636, 2_051), ), ]; let folded = fold_lastfm_duplicates(entries); let buddy = folded.by_key[&track_title_identity("Buddy Holly")]; assert_eq!(buddy.listeners, 2_351_642, "the re-listing won the map"); assert_eq!(buddy.plays, 24_083_192); let scorcho = folded.by_key[&track_title_identity("El Scorcho")]; assert_eq!(scorcho.listeners, 654_071); // And a duplicate never takes a second seat in the ranked list, which // is what Popular renders. assert_eq!(folded.ranked.len(), 2); assert_eq!(folded.ranked[0].title, "Buddy Holly"); assert_eq!(folded.ranked[1].title, "El Scorcho"); } /// A song Last.fm lists exactly once is unaffected — these were the only /// rows that were ever correct. #[test] fn a_song_listed_once_is_unchanged() { let entries = vec![( track_title_identity("Tired of Sex"), "Tired of Sex".to_string(), TrackStats { plays: 4_511_406, listeners: 606_324 }, )]; let folded = fold_lastfm_duplicates(entries); assert_eq!( folded.by_key[&track_title_identity("Tired of Sex")].listeners, 606_324 ); } use super::*; use serde_json::json; #[test] fn musicbrainz_album_match_ignores_a_same_title_single() { let release = |id: u64, record_type: &str| CatalogAlbum { id, title: "Phantom Island".into(), artist: "King Gizzard & the Lizard Wizard".into(), cover: None, release_date: Some(if record_type == "album" { "2025-05-16".into() } else { "2024-10-29".into() }), record_type: Some(record_type.into()), track_count: Some(if record_type == "album" { 10 } else { 1 }), edition_group: None, is_soundtrack: false, is_live: false, is_mixtape: false, appears_on: false, }; let releases = [release(1, "single"), release(2, "album")]; let matches = releases .iter() .filter(|album| exact_studio_album_title(album, "phantomisland")) .collect::>(); assert_eq!(matches.len(), 1); assert_eq!(matches[0].id, 2); } #[test] fn apple_artwork_urls_rewrite_to_original_then_fitted_fallbacks() { let urls = apple_artwork_urls( "https://is1-ssl.mzstatic.com/image/thumb/Music/v4/ab/cd/ef/x/100x100bb.jpg", ); assert_eq!( urls[0], "https://is1-ssl.mzstatic.com/image/thumb/Music/v4/ab/cd/ef/x/100000x100000-999.jpg" ); assert_eq!( urls[1], "https://is1-ssl.mzstatic.com/image/thumb/Music/v4/ab/cd/ef/x/3000x3000bb.jpg" ); assert_eq!( urls[2], "https://is1-ssl.mzstatic.com/image/thumb/Music/v4/ab/cd/ef/x/1200x1200bb.jpg" ); // A URL without slashes passes through untouched rather than panicking. assert_eq!(apple_artwork_urls("nonsense"), vec!["nonsense"]); } /// Deterministic gradient art, optionally inverted, at any square size — /// scaled copies must hash alike, the inverted one must not. fn cover_fixture(size: u32, invert: bool) -> CoverCandidate { let img = image::DynamicImage::ImageRgb8(image::RgbImage::from_fn(size, size, |x, y| { let v = ((x * 251 / size) ^ (y * 197 / size)) as u8; let v = if invert { 255 - v } else { v }; image::Rgb([v, v.wrapping_add(40), v.wrapping_mul(3)]) })); CoverCandidate { dhash: dhash64(&img), bytes: Arc::new(Vec::new()), width: size, height: size, is_reference: false, source: "test", url: String::new(), } } #[test] fn cover_picker_prefers_largest_same_artwork_as_reference() { let mut reference = cover_fixture(1000, false); reference.is_reference = true; reference.source = "deezer"; let same_art_bigger = cover_fixture(3000, false); // apple original let different_art_biggest = cover_fixture(4000, true); // wrong edition let candidates = vec![reference, same_art_bigger, different_art_biggest]; // The 4000px different artwork must lose to the 3000px same artwork. assert_eq!(pick_best_cover(&candidates), Some(1)); // Without a reference, sheer size wins. let candidates: Vec = candidates .into_iter() .map(|mut c| { c.is_reference = false; c }) .collect(); assert_eq!(pick_best_cover(&candidates), Some(2)); assert_eq!(pick_best_cover(&[]), None); } #[test] fn cover_decode_rejects_non_cover_shapes() { let encode = |w: u32, h: u32| { let img = image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel( w, h, image::Rgb([120, 90, 60]), )); let mut buf = std::io::Cursor::new(Vec::new()); img.write_to(&mut buf, image::ImageFormat::Png).unwrap(); buf.into_inner() }; // A banner-shaped image is not a front cover. assert!(decode_cover(encode(1500, 500), "test", false, String::new()).is_none()); // Tiny thumbnails are rejected outright. assert!(decode_cover(encode(200, 200), "test", false, String::new()).is_none()); // A real square cover decodes with its dimensions. let ok = decode_cover(encode(1000, 990), "test", false, String::new()) .expect("square cover accepted"); assert_eq!((ok.width, ok.height), (1000, 990)); } #[test] fn artist_naming_is_token_exact_not_substring() { // The trap this exists for: Cobb is a prefix of Cobham. assert!(!names_artist("Billy Cobham live at Montreux", "Billy Cobb")); assert!(names_artist("Billy Cobb — Zerwee vinyl", "Billy Cobb")); assert!(names_artist( "https://www.concertarchives.org/concerts/billy-cobb-2022", "Billy Cobb" )); assert!(!names_artist("Cobb Billy backwards", "Billy Cobb")); assert!(!names_artist("", "Billy Cobb")); // Pages space names freely: "Partiboi 69" must match "Partiboi69"... assert!(names_artist("Partiboi 69 at Circus Tokyo", "Partiboi69")); assert!(names_artist("PARTIBOI69 presents", "Partiboi69")); // ...but a run-on continuation is still a different word. assert!(!names_artist("Partiboi699 fan page", "Partiboi69")); } #[test] fn bing_parser_keeps_each_result_cards_metadata_together() { let html = concat!( r#""#, r#""#, r#""#, ); let parsed = parse_bing_image_results(html); assert_eq!(parsed.len(), 3); assert_eq!(parsed[0].title, "Billy Cobb press photo"); assert_eq!(parsed[0].url, "https://img.test/billy-cobb.jpg?x=1&y=2"); assert_eq!(parsed[0].page_url, "https://music.test/billy-cobb"); assert_eq!(parsed[1].title, "Jimmy Cobb with Miles Davis"); assert_eq!(parsed[1].url, "https://img.test/jimmy-cobb.jpg"); assert!(!names_artist(&parsed[1].title, "Billy Cobb")); assert_eq!(parsed[2].title, "Spacing variant"); } #[test] fn bing_identity_prefers_independent_artist_anchors() { assert_eq!( bing_identity_confidence( "Billy Cobb press photo", "https://music.test/features/billy-cobb", "https://cdn.test/generic-123.jpg", "Billy Cobb", ), Some(BingIdentityConfidence::Strong) ); assert_eq!( bing_identity_confidence( "Billy Cobb tour dates", "https://www.ticketmaster.com/jimmy-cobb-tickets/artist/999", "https://s1.ticketm.net/dam/a/abc/generic-hero.jpg", "Billy Cobb", ), None, "a misleading Bing title cannot bless another artist's profile", ); assert_eq!( bing_identity_confidence( "Billy Cobb tour dates", "https://www.ticketmaster.com/jimmy-cobb-tickets/artist/999?ref=billy-cobb", "https://s1.ticketm.net/dam/a/abc/generic-hero.jpg", "Billy Cobb", ), None, "query parameters cannot poison an official ticketing profile", ); assert_eq!( bing_identity_confidence( "Billy Cobb Live", "https://www.youtube.com/watch?v=generic", "https://i.ytimg.com/vi/generic/maxresdefault.jpg", "Billy Cobb", ), Some(BingIdentityConfidence::Weak) ); assert_eq!( bing_identity_confidence( "162 Billy Cobb Stock Photos, High-Res Pictures and Images", "https://www.gettyimages.co.uk/photos/billy-cobb", "https://media.gettyimages.com/id/jimmy-cobb.jpg", "Billy Cobb", ), None ); assert_eq!( bing_identity_confidence( "A stunning photo of a musician in a studio with sound equipment", "https://www.freepik.com/premium-ai-image/sound-equipment_247923856.htm", "https://img.freepik.com/premium-photo/sound-equipment_1294789-838.jpg", "Equipment", ), None, "stock/AI hosts are not performer identity evidence", ); assert_eq!( bing_identity_confidence( "Band setting up equipment on a stage photo", "https://unsplash.com/photos/band-setting-up-equipment-on-a-stage", "https://images.unsplash.com/photo-1768264312526-786bf6087043", "Equipment", ), None, ); assert_eq!( bing_identity_confidence( "Beginner DJ equipment: what you need to start DJing", "https://blog.native-instruments.com/dj-equipment/", "https://blog.native-instruments.com/beginner-dj-equipment-hero.jpg", "Equipment", ), None, "a repeated common noun is not a named performer", ); assert_eq!( bing_identity_confidence( "Ohio emo band Equipment announce new album", "https://www.brooklynvegan.com/ohio-emo-band-equipment-announce-new-album/", "https://media.brooklynvegan.com/uploads/equipment.jpg", "Equipment", ), Some(BingIdentityConfidence::Strong), ); assert_eq!( bing_identity_confidence( "Interview: Wednesday music supervisors", "https://awardsradar.com/interview-wednesday-music-supervisors/", "https://awardsradar.com/images/Wednesday_S1_E3.jpg", "Wednesday", ), None, "a TV interview is not the band merely because it discusses music", ); assert_eq!( bing_identity_confidence( "Wednesday Share New Video for Quarry", "https://pitchfork.com/news/wednesday-share-new-video-quarry-watch/", "https://media.pitchfork.com/photos/wednesday.jpg", "Wednesday", ), Some(BingIdentityConfidence::Strong), ); assert_eq!( bing_identity_confidence( "Wednesday Events", "https://www.livenation.com/artist/K8vZ917-example/wednesday-events", "https://s1.ticketm.net/dam/a/abc/wednesday-hero.jpg", "Wednesday", ), Some(BingIdentityConfidence::Strong), "a verified official artist profile is its own entity anchor", ); assert_eq!( bing_identity_confidence( "New MS Paint for Windows 11", "https://www.windowslatest.com/microsoft-paint-update/", "https://www.windowslatest.com/MS-Paint-for-Windows-11.jpg", "MSPAINT", ), None, ); assert_eq!( bing_identity_confidence( "MSPAINT is now live in your browser", "https://software.example/post/mspaint-live-browser", "https://software.example/images/MSPAINT.jpg", "MSPAINT", ), None, "generic app wording is not performer identity", ); assert_eq!( bing_identity_confidence( "MS Paint Review: a classic drawing program", "https://software.example/mspaint-review", "https://software.example/images/MSPAINT.jpg", "MSPAINT", ), None, "an app review is not music-entity evidence", ); assert_eq!( bing_identity_confidence( "MSPaint Announce New EP No Separation", "https://pitchfork.com/news/mspaint-announce-new-ep-no-separation/", "https://media.pitchfork.com/photos/MSPAINT.jpeg", "MSPAINT", ), Some(BingIdentityConfidence::Strong), ); assert_eq!( bing_identity_confidence( "MSPAINT Events", "https://www.ticketmaster.com/mspaint-tickets/artist/1234567", "https://s1.ticketm.net/dam/a/abc/mspaint-hero.jpg", "MSPAINT", ), Some(BingIdentityConfidence::Strong), "trusted ticketing profiles do not need article-style music wording", ); assert_eq!( bing_identity_confidence( "Equipment - Interview", "https://music.example/artists/equipment", "https://cdn.example/equipment-press.jpg", "Equipment", ), Some(BingIdentityConfidence::Strong), ); assert_eq!( bing_identity_confidence( "Home Is Where Quotes. QuotesGram", "https://quotesgram.com/home-is-where-quotes/", "https://cdn.quotesgram.com/pico-iyer-quote-home.jpg", "Home Is Where", ), None, ); assert_eq!( bing_identity_confidence( "Home.is Where Band at Kenneth Kane blog", "https://storage.googleapis.com/example/home-is-where-band.html", "https://images.example/Band-4_1.jpg", "Home Is Where", ), None, "Google Storage HTML search-spam pages are not identity evidence", ); assert_eq!( bing_identity_confidence( "Kilcullen Diary: Weatherday: Disaster enters, stage left", "https://kilcullenbridge.blogspot.com/weatherday-disaster-enters-stage-left.html", "https://blogger.googleusercontent.com/s4659/barleynight.JPG", "Weatherday", ), None, ); assert_eq!( bing_identity_confidence( "Weatherday – Hornet Disaster | Album Review", "https://swimintothesound.com/blog/weatherday-hornet-disaster-album-review", "https://images.squarespace-cdn.com/sputnik+3.jpeg", "Weatherday", ), Some(BingIdentityConfidence::Strong), ); assert_eq!( bing_identity_confidence( "Billy Cobb", "https://music.test/feature", "https://img.test/billy-cobb-portrait.jpg", "Billy Cobb", ), Some(BingIdentityConfidence::Strong) ); } #[test] fn audiodb_requires_the_exact_artist_not_result_zero() { let body = serde_json::json!({ "artists": [ { "strArtist": "Billy Cobham", "strArtistFanart": "wrong" }, { "strArtist": "Billy Cobb", "strArtistFanart": "right" } ] }); let selected = select_audiodb_artist(&body, "Billy Cobb").unwrap(); assert_eq!(selected["strArtistFanart"], "right"); assert!(select_audiodb_artist(&body, "Billy Cob").is_none()); } /// Three spellings of one label have to meet: MusicBrainz writes the full /// legal name, Deezer the trade name, Apple whatever follows the ℗ year. #[test] fn label_identity_reconciles_provider_spellings() { assert_eq!(label_identity("In the Red Records"), label_identity("In The Red")); // Crazy P's "Beatbox" was being dropped purely because a space survived // between these two. assert_eq!(label_identity("20/20 Vision"), label_identity("2020Vision")); assert_eq!(label_identity("Drag City"), "dragcity"); assert_ne!(label_identity("Drag City"), label_identity("Explosão Music")); // Whole-word noise removal only. assert_eq!(label_identity("rafftheraff"), "rafftheraff"); assert_eq!(label_identity("Southern"), "southern"); } #[test] fn label_from_copyright_drops_the_symbol_and_year() { assert_eq!(label_from_copyright("℗ 2014 GOD?"), "GOD?"); assert_eq!(label_from_copyright("℗ 2019 Drag City Records"), "Drag City Records"); // No year to strip, and a four-letter first word is not one. assert_eq!(label_from_copyright("Drag City"), "Drag City"); } /// The label leg is what keeps nine genuine Wand releases — JJ, Smile, The /// Leap, Goldfish, the live singles — which neither MusicBrainz nor Apple /// carries by title, while refusing every co-tenant imprint on that record. #[test] fn label_matches_anchored_set_but_not_co_tenants() { let anchored: HashSet = ["In the Red Records", "Drag City", "God?"] .into_iter() .map(label_identity) .collect(); assert!(label_matches("Drag City", &anchored)); assert!(label_matches("In The Red", &anchored)); for co_tenant in [ "WMD Distribuidora", "Explosão Music", "Netrilis Music", "MWR Records Retro", "https://www.instagram.com/wandbarbosas", "Ecstatic Peace/Universal Records", "Klap Label", "Tropicália Records", ] { assert!( !label_matches(co_tenant, &anchored), "{co_tenant} must not match the anchored set", ); } // A bare imprint equal to the artist's own name proves nothing on a // CONTESTED name — the co-tenant self-releases as "Wand" too. assert!(!label_matches("Wand", &anchored)); } /// A display credit is a LIST of principals. Reading the title instead /// would make every guest spot an owned release — the exact thing the /// `appears_on` guard exists to stop. #[test] fn apple_credit_names_only_actual_principals() { for credit in [ "¥$, Kanye West & Ty Dolla $ign", "KIDS SEE GHOSTS, Kanye West & Kid Cudi", "JAŸ-Z & Kanye West", "Rihanna and Kanye West and Paul McCartney", ] { assert!( apple_credit_names_artist(credit, "Kanye West"), "{credit} names Kanye West", ); } // "American Boy (feat. Kanye West)" is Estelle's record; the credit is // hers alone and that is the only thing consulted. assert!(!apple_credit_names_artist("Estelle", "Kanye West")); assert!(!apple_credit_names_artist("Clipse", "Kanye West")); // Substring of a longer name is not a credit. assert!(!apple_credit_names_artist("Kanye Western", "Kanye West")); } fn test_apple_release(title: &str, collection_id: u64) -> AppleRelease { AppleRelease { title: title.into(), title_norm: release_title_identity(title), artwork: format!("https://img.test/{collection_id}.jpg"), release_date: None, track_count: None, collection_type: Some("Album".into()), collection_id: Some(collection_id), copyright: None, } } fn test_apple_candidate(artist_id: u64, releases: Vec) -> AppleArtistCandidate { AppleArtistCandidate { artist_id, releases, credited_titles: HashSet::new(), lookup_complete: true, cacheable: true, } } fn test_deezer_evidence(title: &str, fans: u64, genre_id: u64) -> DeezerReleaseEvidence { deezer_release_evidence(&json!({ "title": title, "fans": fans, "genre_id": genre_id, })) .unwrap() } fn test_bing_candidate( url: &str, title: &str, page_url: &str, identity: BingIdentityConfidence, display_eligible: bool, ) -> BingImageCandidate { BingImageCandidate { url: url.into(), title: title.into(), page_url: page_url.into(), identity, display_eligible, query_index: 0, rank_in_query: 0, } } /// A square sleeve must never reach a hero slot, and the same photographic /// content in banner proportions must still pass. Camping in Alaska's /// "lost and found" cover — a photo collage whose only text is a small /// typewriter caption — cleared every other gate and was served as a live /// hero, because nothing tested proportions. #[test] fn discogs_markup_becomes_plain_prose() { let profile = "Duo of [a=Austin Davis] and [a=Jacob Stewart], released on \ [l=Broken World Media]. See [url=https://example.com]site[/url].\n\n\ [b]Emo[/b] from Alabama."; let cleaned = strip_discogs_markup(profile); assert_eq!( cleaned, "Duo of Austin Davis and Jacob Stewart, released on Broken World Media. \ See site. Emo from Alabama." ); assert!(!cleaned.contains('[')); } /// The last-resort tier must read as a sentence and never assert more /// than the MusicBrainz entry actually says. #[test] fn musicbrainz_facts_compose_an_honest_bio() { let disambiguated = ArtistIdentityFacts { disambiguation: Some("David Ameal, techno / breakbeat producer".into()), tags: vec!["acid techno".into(), "breakbeat".into()], ..ArtistIdentityFacts::default() }; assert_eq!( disambiguated.compose_bio("Hermeth").unwrap(), "Hermeth: David Ameal, techno / breakbeat producer. \ Associated genres: acid techno, breakbeat." ); let group = ArtistIdentityFacts { kind: Some("Group".into()), area: Some("Huntsville".into()), tags: vec!["emo".into()], ..ArtistIdentityFacts::default() }; assert_eq!( group.compose_bio("Camping In Alaska").unwrap(), "Camping In Alaska is a group from Huntsville. Associated genres: emo." ); // Nothing known at all yields nothing, not a hollow sentence. assert!(ArtistIdentityFacts::default() .compose_bio("Nobody") .is_none()); } #[test] fn square_release_artwork_is_rejected_but_wide_photography_survives() { // Photographic-looking noise: not flat, not low-palette, no text — so // only the aspect rule can reject it. let textured = |w: u32, h: u32| { let image = image::DynamicImage::ImageRgb8(image::RgbImage::from_fn(w, h, |x, y| { let n = |seed: u32| { let v = (x.wrapping_mul(2654435761) ^ y.wrapping_mul(2246822519) ^ seed) >> 13; (v % 200) as u8 + 30 }; image::Rgb([n(7), n(101), n(9001)]) })); let mut bytes = std::io::Cursor::new(Vec::new()); image .write_to(&mut bytes, image::ImageFormat::Png) .expect("encode test raster"); Arc::new(bytes.into_inner()) }; let candidate = test_bing_candidate( "https://example.com/artwork.jpg", "Camping In Alaska interview", "https://example.com/interview", BingIdentityConfidence::Strong, true, ); assert!( score_bing_hero_image( textured(1200, 1200), &candidate, "Camping In Alaska", &[], false ) .is_none(), "square artwork must be rejected when the catalogue is unknown", ); assert!( score_bing_hero_image( textured(1600, 900), &candidate, "Camping In Alaska", &[], false ) .is_some(), "wide press photography must still score", ); // 4:3 press photography is common and sits above the floor. assert!( score_bing_hero_image( textured(1600, 1200), &candidate, "Camping In Alaska", &[], false ) .is_some(), "4:3 press photography must still score", ); // A square candidate is adjudicated against the artist's own releases: // matching one is a sleeve, matching none is a press crop. let square = textured(1200, 1200); let square_hash = dhash64( &decode_hero_image_slice(square.as_slice()).expect("decode square test raster"), ); assert!( score_bing_hero_image( square.clone(), &candidate, "Camping In Alaska", &[square_hash], false ) .is_none(), "a square that IS one of the artist's covers must be rejected", ); assert!( score_bing_hero_image( square, &candidate, "Camping In Alaska", &[square_hash ^ u64::MAX], false ) .is_some(), "a square press crop matching no release must be admitted", ); // Portrait press photography is cropped to 16:9 around the face by the // app, so it is admitted when it carries enough width to survive that. assert!( score_bing_hero_image( textured(1600, 2000), &candidate, "Camping In Alaska", &[], false ) .is_some(), "portrait press photography must be admitted for face-aware cropping", ); assert!( score_bing_hero_image( textured(800, 1000), &candidate, "Camping In Alaska", &[], false ) .is_none(), "a portrait too narrow to crop into a hero must still be rejected", ); } #[test] fn crazy_p_namesakes_are_weighted_and_foreign_clusters_are_removed() { let candidates = vec![ test_apple_candidate( 83_401_183, vec![ test_apple_release("Walk Dance Talk Sing", 1), test_apple_release("Age of the Ego", 2), test_apple_release("The Wicked Is Music", 3), ], ), test_apple_candidate(1_735_763_718, vec![test_apple_release("Widderlich", 4)]), test_apple_candidate( 1_766_807_301, vec![ test_apple_release("Azote - Single", 5), test_apple_release("Castigo - Single", 6), test_apple_release("COMO TU NO HAY OTRA - Single", 7), ], ), ]; let evidence = vec![ test_deezer_evidence("Walk Dance Talk Sing", 2_847, 106), test_deezer_evidence("Age of the Ego", 771, 106), test_deezer_evidence("The Wicked Is Music", 638, 113), test_deezer_evidence("Widderlich", 83, 116), test_deezer_evidence("Azote", 0, 85), test_deezer_evidence("Castigo", 1, 85), test_deezer_evidence("COMO TU NO HAY OTRA", 0, 132), // Not present in the losing US Apple catalogue, but part of its // repeatedly corroborated losing-only Latin genre cluster. test_deezer_evidence("Mi esperanza", 0, 85), ]; let selected = select_apple_artist_catalog(&candidates, &evidence, true); assert!(selected.identity_confirmed); assert_eq!(selected.apple_artist_id, Some(83_401_183)); assert!(selected.foreign_titles.contains("widderlich")); assert!(selected.foreign_titles.contains("azote")); assert!(selected.foreign_genres.contains(&85)); assert!(!selected.foreign_genres.contains(&116)); assert!(!selected.foreign_genres.contains(&106)); assert!(apple_release_is_foreign( &selected, "Mi esperanza", Some(85) )); assert!(apple_release_is_foreign(&selected, "Widderlich", Some(116))); assert!(!apple_release_is_foreign( &selected, "A legitimate collaboration", Some(106) )); } #[test] fn apple_namesake_selection_fails_closed_when_ambiguous_or_uncorroborated() { let pond_namesakes = vec![ test_apple_candidate(1, vec![test_apple_release("Tasmania", 10)]), test_apple_candidate(2, vec![test_apple_release("Schwer", 20)]), ]; assert!(!select_apple_artist_catalog(&pond_namesakes, &[], true).identity_confirmed); let tied = vec![ test_deezer_evidence("Tasmania", 0, 106), test_deezer_evidence("Schwer", 0, 106), ]; assert!(!select_apple_artist_catalog(&pond_namesakes, &tied, true).identity_confirmed); let obscure_namesake = vec![test_apple_candidate( 99, vec![test_apple_release("Canonical Act Album", 30)], )]; assert!( !select_apple_artist_catalog( &obscure_namesake, &[test_deezer_evidence("Different Artist Album", 5, 1)], true, ) .identity_confirmed ); assert!( select_apple_artist_catalog(&obscure_namesake, &[], false).identity_confirmed, "the name-only cover path may use a sole candidate behind dHash verification" ); let mut incomplete = test_apple_candidate(2, Vec::new()); incomplete.lookup_complete = false; incomplete.cacheable = false; let partial = select_apple_artist_catalog( &[ test_apple_candidate(1, vec![test_apple_release("Tasmania", 1)]), incomplete.clone(), ], &[test_deezer_evidence("Tasmania", 100, 106)], true, ); assert!(partial.identity_unavailable); assert!(!partial.identity_confirmed); assert!( !select_apple_artist_catalog( &[incomplete], &[test_deezer_evidence("Native release", 1, 106)], true, ) .identity_unavailable, "one failed candidate need not hide Deezer-native releases" ); } #[test] fn apple_format_tails_and_synthetic_ids_preserve_identity() { assert_eq!(release_title_identity("Azote - Single"), "azote"); assert_eq!(release_title_identity("Azote – Single"), "azote"); assert_eq!(release_title_identity("Night Drive - EP"), "nightdrive"); let release = test_apple_release("Shared Title", 123); assert_ne!( apple_synthetic_catalog_id(1, &release, "Shared Title"), apple_synthetic_catalog_id(2, &release, "Shared Title"), "same-name artists must never overwrite each other's Apple registry row" ); let other_collection = test_apple_release("Shared Title", 456); assert_ne!( apple_synthetic_catalog_id(1, &release, "Shared Title"), apple_synthetic_catalog_id(1, &other_collection, "Shared Title") ); } #[test] fn linked_mbid_outranks_stale_audiodb_deezer_id() { let stale = json!({ "strDeezerID": "42", "strMusicBrainzID": "wrong-mbid", }); assert!(!audiodb_artist_matches_identity( &stale, 42, Some("right-mbid") )); assert!(audiodb_artist_matches_identity(&stale, 42, None)); let linked = json!({ "strDeezerID": "999", "strMusicBrainzID": "RIGHT-MBID", }); assert!(audiodb_artist_matches_identity( &linked, 42, Some("right-mbid") )); } #[test] fn artist_portrait_skips_the_placeholder_duplicate() { // The real GUM search, reduced: the first row Deezer returned had the // empty-md5 portrait, which `limit=1` stored as the library's shelf art. let gum = json!({ "data": [ { "name": "GUM", "nb_fan": 90, "picture_xl": "https://cdn-images.dzcdn.net/images/artist/d41d8cd98f00b204e9800998ecf8427e/1000x1000.jpg" }, { "name": "GUM", "nb_fan": 3707, "picture_xl": "https://cdn-images.dzcdn.net/images/artist/0728a4530cd8ee40f1212845423/1000x1000.jpg" }, { "name": "Gumball", "nb_fan": 900_000, "picture_xl": "https://cdn-images.dzcdn.net/images/artist/aaaa/1000x1000.jpg" }, ]}); assert_eq!( pick_artist_portrait(&gum, "GUM"), Some("https://cdn-images.dzcdn.net/images/artist/0728a4530cd8ee40f1212845423/1000x1000.jpg"), ); } #[test] fn artist_portrait_prefers_the_exact_name_over_a_bigger_namesake() { // A far more popular artist whose name merely contains the query must // never win — that is how a shelf ends up wearing the wrong face. let body = json!({ "data": [ { "name": "Weezer Rascal", "nb_fan": 5_000_000, "picture_xl": "https://x/rascal.jpg" }, { "name": "Weezer", "nb_fan": 725_694, "picture_xl": "https://x/weezer.jpg" }, ]}); assert_eq!(pick_artist_portrait(&body, "Weezer"), Some("https://x/weezer.jpg")); } #[test] fn artist_portrait_is_none_when_every_candidate_is_a_placeholder() { // Nothing beats the fallback icon here; a stored silhouette would read // as a failed fetch forever. let body = json!({ "data": [ { "name": "Nobody", "nb_fan": 3, "picture_xl": "https://cdn-images.dzcdn.net/images/artist/d41d8cd98f00b204e9800998ecf8427e/1000x1000.jpg" }, ]}); assert_eq!(pick_artist_portrait(&body, "Nobody"), None); } #[test] fn bing_fallback_requires_an_unambiguous_exact_name() { let pond = json!({ "data": [ { "id": 10, "name": "Pond", "nb_fan": 50_000 }, { "id": 20, "name": "Pond", "nb_fan": 12 }, { "id": 30, "name": "Ponder", "nb_fan": 1_000_000 } ]}); // Name-only editorial lookups stay strict: dominance is enough for an // image search, not for an unanchored Wikipedia article match. assert!(!deezer_search_allows_bing(&pond, "Pond", 10)); assert!(!deezer_search_allows_bing(&pond, "Pond", 20)); assert_eq!( deezer_exact_name_state(&pond, "Pond", 10), DeezerExactNameState::Dominant ); assert_eq!( deezer_exact_name_state(&pond, "Pond", 20), DeezerExactNameState::Ambiguous ); assert_eq!( deezer_exact_name_state(&pond, "Pond", 99), DeezerExactNameState::Missing ); let tied = json!({ "data": [ { "id": 10, "name": "Pond", "nb_fan": 5 }, { "id": 20, "name": "Pond", "nb_fan": 5 } ]}); assert!(!deezer_search_allows_bing(&tied, "Pond", 10)); assert_eq!( deezer_exact_name_state(&tied, "Pond", 10), DeezerExactNameState::Ambiguous ); let unique = json!({ "data": [ { "id": 42, "name": "Home Is Where", "nb_fan": 5 }, { "id": 43, "name": "Home Is Here", "nb_fan": 500 } ]}); assert!(deezer_search_allows_bing(&unique, "Home Is Where", 42)); assert_eq!( deezer_exact_name_state(&unique, "Home Is Where", 42), DeezerExactNameState::Unambiguous ); } /// The regression this guards: a dead duplicate row (`crazy_p`, 0 fans) /// made Crazy P "ambiguous", which pushed a well-known artist onto the /// per-card catalogue-anchor tier and discarded nine of twelve real Bing /// press photos. Deezer is full of these rows, so this was universal. #[test] fn empty_duplicate_rows_do_not_contest_a_real_artists_name() { let crazy_p = json!({ "data": [ { "id": 206_927, "name": "Crazy P", "nb_fan": 11_065 }, { "id": 204_170_017, "name": "crazy_p", "nb_fan": 0 } ]}); assert_eq!( deezer_exact_name_state(&crazy_p, "Crazy P", 206_927), DeezerExactNameState::Dominant ); // A real contest still is one. Both Ponds have an audience, so the // strict path keeps protecting the smaller id. let contested = json!({ "data": [ { "id": 10, "name": "Pond", "nb_fan": 900 }, { "id": 20, "name": "Pond", "nb_fan": 400 } ]}); assert_eq!( deezer_exact_name_state(&contested, "Pond", 10), DeezerExactNameState::Ambiguous ); // Ratio alone cannot promote two obscure rows. let obscure = json!({ "data": [ { "id": 10, "name": "Pond", "nb_fan": 20 }, { "id": 20, "name": "Pond", "nb_fan": 1 } ]}); assert_eq!( deezer_exact_name_state(&obscure, "Pond", 10), DeezerExactNameState::Ambiguous ); // An unfollowed id never owns a name it shares with anyone. let silent = json!({ "data": [ { "id": 10, "name": "Pond", "nb_fan": 0 }, { "id": 20, "name": "Pond", "nb_fan": 0 } ]}); assert_eq!( deezer_exact_name_state(&silent, "Pond", 10), DeezerExactNameState::Ambiguous ); } /// Deezer answers an exhausted request budget with HTTP 200 and an error /// object. Only that one is worth retrying — a quota rejection says nothing /// about the artist, while every other code describes the resource and /// would burn the window repeating a settled answer. #[test] fn only_a_quota_rejection_is_retryable() { let quota = json!({ "error": { "type": "Exception", "message": "Quota limit exceeded", "code": 4 }}); assert!(deezer_quota_error("a)); // Deezer has published the message without the numeric code. let worded = json!({ "error": { "message": "quota limit exceeded" } }); assert!(deezer_quota_error(&worded)); let missing = json!({ "error": { "type": "DataException", "message": "no data", "code": 800 }}); assert!(!deezer_quota_error(&missing)); // MusicBrainz reports its failures as a plain string. let musicbrainz = json!({ "error": "Not Found" }); assert!(!deezer_quota_error(&musicbrainz)); let ok = json!({ "data": [{ "id": 10, "name": "Pond" }] }); assert!(!deezer_quota_error(&ok)); } fn test_release( id: u64, title: &str, record_type: &str, date: &str, tracks: Option, ) -> CatalogAlbum { CatalogAlbum { id, title: title.into(), artist: "Artist".into(), cover: None, release_date: (!date.is_empty()).then(|| date.to_string()), record_type: Some(record_type.into()), track_count: tracks, edition_group: None, is_soundtrack: false, is_live: false, is_mixtape: false, appears_on: false, } } /// Same-work duplicate rows fold to the fullest member inside their date /// bucket; different-date works with the same name keep their own cards. #[test] fn duplicate_release_rows_fold_to_the_fullest_member() { // Exact-title year bucket: "Changes (Remixes)" twice, five days apart. let mut albums = vec![ test_release(1, "Changes (Remixes)", "single", "2012-03-06", Some(7)), test_release(2, "Changes (Remixes)", "single", "2012-03-11", Some(7)), // Same title, different YEAR — a real re-release, must survive. test_release(3, "Changes (Remixes)", "single", "2015-03-11", Some(7)), ]; fold_duplicate_release_rows(&mut albums, &[vec![1, 2, 3]], 4); let ids: Vec = albums.iter().map(|a| a.id).collect(); assert_eq!(ids.len(), 2); assert!(ids.contains(&3)); // Featured-credit day bucket: the credited 4-track row wins. let mut albums = vec![ test_release(10, "Dancing In Babylon", "single", "2024-02-20", Some(1)), test_release( 11, "Dancing In Babylon (feat. Christine and the Queens)", "single", "2024-02-20", Some(4), ), ]; fold_duplicate_release_rows(&mut albums, &[vec![10, 11]], 10); assert_eq!(albums.len(), 1); assert_eq!(albums[0].id, 11); // Undated rows never fold in the DATE-bucketed pass — but the exact // re-release pass may still collapse them (see the test below). let mut albums = vec![ test_release(20, "Mystery", "single", "", None), test_release(21, "Mystery", "single", "", None), ]; fold_duplicate_release_rows(&mut albums, &[vec![20, 21]], 4); assert_eq!(albums.len(), 2); } /// Cross-year exact-title re-releases: singles/EPs fold on compatible /// track counts, albums only on provably identical known counts, and a /// mismatch always keeps both rows. #[test] fn exact_rereleases_fold_only_same_material() { // "Step Into Christmas" re-released under a new id: fold, keep oldest. let mut albums = vec![ test_release(1, "Step Into Christmas", "single", "2022-11-22", None), test_release(2, "Step Into Christmas", "single", "2018-12-07", None), ]; fold_exact_rereleases(&mut albums, &[vec![1, 2]]); assert_eq!(albums.iter().map(|a| a.id).collect::>(), vec![2]); // Different track counts = different material ("Christmas Tree Farm" // 1-track 2019 vs 3-track 2024): keep both. let mut albums = vec![ test_release(3, "Christmas Tree Farm", "single", "2024-11-01", Some(3)), test_release(4, "Christmas Tree Farm", "single", "2019-12-06", Some(1)), ]; fold_exact_rereleases(&mut albums, &[vec![3, 4]]); assert_eq!(albums.len(), 2); // Albums fold only when every count is known and identical ≥5 — // "Anthology 2" and its remastered re-release. let mut albums = vec![ test_release(5, "Anthology 2", "album", "2025-08-21", Some(45)), test_release(6, "Anthology 2", "album", "2016-04-04", Some(45)), ]; fold_exact_rereleases(&mut albums, &[vec![5, 6]]); assert_eq!(albums.iter().map(|a| a.id).collect::>(), vec![6]); // An unknown count on an album is NOT proof: two eponymous records // must never merge on a guess. let mut albums = vec![ test_release(7, "Elton John", "album", "2008-06-02", None), test_release(8, "Elton John", "album", "2007-04-30", Some(10)), ]; fold_exact_rereleases(&mut albums, &[vec![7, 8]]); assert_eq!(albums.len(), 2); } /// The variant/sequel residual split is what keeps "Revolver (2022 Mix)" /// foldable while "Greatest Hits II" stays its own record. #[test] fn residual_classification_separates_variants_from_sequels() { assert!(residual_is_sequel_marker("ii")); assert!(residual_is_sequel_marker("2")); assert!(residual_is_sequel_marker("vol2")); assert!(residual_is_sequel_marker("pt1")); assert!(!residual_is_sequel_marker("theanthology")); assert!(!residual_is_sequel_marker("2022mix")); assert!(!residual_is_sequel_marker("injapan")); assert!(!residual_is_sequel_marker("1975")); // 4-digit year ≠ sequel number… // …but it is not a release variant either, so "Fleetwood Mac: Live // 1975" never links to "Fleetwood Mac: Live" (gap + no variant word). assert!(!residual_is_release_variant("1975")); assert!(residual_is_release_variant("2022mix")); assert!(residual_is_release_variant("acousticcollection")); assert!(residual_is_release_variant("theanthology")); assert!(residual_is_release_variant("trackbytrackversion")); assert!(!residual_is_release_variant("injapan")); assert!(!residual_is_release_variant( "narratesprokofievspeterandthewolf" )); } /// The lead sentence decides; later prose must not reclassify, and the /// declared kind wins over vaguer words around it. #[test] fn description_leads_map_to_type_hints() { assert_eq!( description_type_hint( "Direct Hits is a compilation album by American rock band the Killers, released in 2013." ), Some("compilation"), ); assert_eq!( description_type_hint( "Late Night Tales: MGMT is a mix album compiled by the band MGMT's members." ), Some("remix"), ); assert_eq!( description_type_hint( "Live Around the World is a live album by Queen and Adam Lambert." ), Some("live"), ); assert_eq!( description_type_hint( "Flash Gordon is the soundtrack to the film of the same name by Queen." ), Some("soundtrack"), ); assert_eq!( description_type_hint( "Hot Fuss is the debut studio album by American rock band the Killers." ), Some("album"), ); assert_eq!( description_type_hint("SZNZ: Spring is an extended play by Weezer."), Some("ep"), ); // Later-prose mentions don't reclassify: the lead names a studio album // and the live-album mention sits beyond the lead window. assert_eq!( description_type_hint( "Vauxhall and I is the fourth studio album by Morrissey, released in 1994 to critical acclaim and commercial success in both the United Kingdom and the United States of America. After extensive touring through the previous years, the band then released a live album." ), Some("album"), ); assert_eq!( description_type_hint("An enigmatic release with no stated kind."), None ); } /// A song article's lead is about a TRACK. Every release it names is a /// different record, so the lead must classify as "song" and never leak /// that record's type — the "Run For Cover (Workout Mix)" misfile. #[test] fn song_leads_never_report_the_album_they_mention() { assert_eq!( description_type_hint( "\"Run for Cover\" is a song by American rock band the Killers from their fifth studio album, Wonderful Wonderful (2017)." ), Some("song"), ); assert_eq!( description_type_hint( "\"Mr. Brightside\" is the debut single by American rock band the Killers." ), Some("song"), ); // Qualifiers stack between the copula and the noun. assert_eq!( description_type_hint( "\"Blue Christmas\" is a 1948 promotional single later recorded by Elvis Presley for his live album." ), Some("song"), ); // Album leads must be untouched: their singles/songs words never sit // in the declaration, so the studio-album reading still wins. assert_eq!( description_type_hint( "Hot Fuss is the debut studio album by the Killers. Its lead single, \"Mr. Brightside\", became a standard." ), Some("album"), ); assert_eq!( description_type_hint( "Sawdust is a compilation album by the Killers collecting b-sides, rarities and new songs." ), Some("compilation"), ); // Only the lead's OWN copula is read: a later sentence describing a // track on the record must not reclassify the record. assert_eq!( description_type_hint( "Wonderful Wonderful is the fifth studio album by American rock band the Killers, released through Island Records. The record's third single was a song written on tour." ), Some("album"), ); } /// Apple's continuous DJ mixes can never render a tracklist (their song /// lookup is empty), so the supplement must not inject them; ordinary /// remix albums must survive the same filter. #[test] fn apple_dj_mix_titles_are_recognized() { assert!(is_apple_dj_mix_title( "Defected: Crazy P at Glitterbox, Hï Ibiza, Jun 12, 2022 (DJ Mix)" )); assert!(is_apple_dj_mix_title( "Yuksek Presents Dance 'O' Drome 006: Danielle Moore (Crazy P) [DJ Mix]" )); assert!(is_apple_dj_mix_title("Late Night Tales: MGMT (DJ Mix)")); assert!(is_apple_dj_mix_title("Early Hours (DJ Mix)")); assert!(!is_apple_dj_mix_title("Remixed")); assert!(!is_apple_dj_mix_title("Age of the Ego (Remixes)")); assert!(!is_apple_dj_mix_title("Kids (Soulwax Mix)")); } /// The credit is the only difference between Deezer's duplicate single /// rows; anything else must keep its own identity. #[test] fn featless_identity_strips_only_the_featured_credit() { assert_eq!( featless_title_identity("Dancing In Babylon (feat. Christine and the Queens)"), featless_title_identity("Dancing In Babylon"), ); assert_eq!( featless_title_identity("Party Woman [feat. Crazy P]"), featless_title_identity("Party Woman"), ); // A remix credit is part of the work, not a featured-artist tail. assert_ne!( featless_title_identity("Kids (Soulwax Mix)"), featless_title_identity("Kids"), ); // Only the credit bracket is removed — a qualifier AFTER it survives, // so a remix variant never collides with the plain single. assert_ne!( featless_title_identity("Beneath with Me (feat. Skylar Grey) (Kaskade's V.4)"), featless_title_identity("Beneath with Me (feat. Skylar Grey)"), ); assert_eq!( featless_title_identity("Beneath with Me (feat. Skylar Grey)"), normalize("Beneath with Me"), ); // Different credits on the same song still collide with each other. assert_eq!( featless_title_identity("Check On It (feat. Voltio)"), featless_title_identity("Check On It (feat. Bun B & Slim Thug)"), ); // A title that IS a feat marker from position zero stays whole. assert_eq!( featless_title_identity("(feat. Nobody)"), normalize("(feat. Nobody)") ); } /// A name owner must keep the same candidate set an unambiguous name gets. /// Press photography cites no release title, so any narrowing here is the /// hero regression itself. #[test] fn name_owner_keeps_the_full_card_identity_result_set() { let candidates = vec![ test_bing_candidate( "https://images.squarespace-cdn.test/press-shot.jpeg", "GET TO KNOW: Crazy P — YoungOnes", "http://youngonesuk.test/crazy-p-get-to-know", BingIdentityConfidence::Strong, true, ), test_bing_candidate( "https://media.timeout.test/image.jpg", "Crazy P | Music in London", "https://timeout.test/london/music/crazy-p", BingIdentityConfidence::Strong, true, ), test_bing_candidate( "https://crazy-p.test/CRAZY-P-Any-Signs-Of-Love-VINYL.jpg", "New Album Announcement", "https://crazy-p.test/2024/11/20/announcement", BingIdentityConfidence::Strong, false, ), ]; let filtered = filter_bing_candidates_for_policy( candidates, &BingArtistIdentityPolicy::NameOwner, "Crazy P", &HashSet::new(), ); assert_eq!( filtered .iter() .map(|candidate| candidate.url.as_str()) .collect::>(), [ "https://images.squarespace-cdn.test/press-shot.jpeg", "https://media.timeout.test/image.jpg" ], "only the product/release card may be dropped" ); } #[test] fn bing_catalog_anchors_are_winner_only_distinctive_and_not_self_titled() { let candidates = vec![ test_apple_candidate( 1, vec![ test_apple_release("Any Signs of Love", 1), test_apple_release("Shared Release", 2), test_apple_release("Crazy P", 3), test_apple_release("Later", 4), ], ), test_apple_candidate( 2, vec![ test_apple_release("Shared Release", 5), test_apple_release("Widderlich", 6), ], ), ]; let catalog = AppleArtistCatalog { releases: candidates[0].releases.clone(), apple_artist_id: Some(1), identity_confirmed: true, ..AppleArtistCatalog::default() }; let anchors = bing_catalog_release_anchors(&candidates, &catalog, "Crazy P"); assert!(anchors.contains("any signs of love")); assert!(!anchors.contains("shared release")); assert!(!anchors.contains("crazy p")); assert!(!anchors.contains("later")); } #[test] fn unambiguous_bing_keeps_existing_strong_and_weak_display_candidates() { let candidates = vec![ test_bing_candidate( "https://img.test/strong.jpg", "Home Is Where band portrait", "https://music.test/home-is-where", BingIdentityConfidence::Strong, true, ), test_bing_candidate( "https://img.test/weak.jpg", "Home Is Where live", "https://video.test/watch/1", BingIdentityConfidence::Weak, true, ), test_bing_candidate( "https://img.test/vinyl.jpg", "Home Is Where vinyl", "https://homeiswhere.test/store", BingIdentityConfidence::Strong, false, ), ]; let filtered = filter_bing_candidates_for_policy( candidates, &BingArtistIdentityPolicy::Unambiguous, "Home Is Where", &HashSet::new(), ); assert_eq!( filtered .iter() .map(|candidate| candidate.url.as_str()) .collect::>(), ["https://img.test/strong.jpg", "https://img.test/weak.jpg"] ); } #[test] fn ambiguous_bing_requires_per_card_catalog_or_linked_identity() { let release_titles = HashSet::from(["any signs of love".to_string()]); let policy = BingArtistIdentityPolicy::CatalogAnchored { release_titles }; let linked_people = HashSet::from(["danielle moore".to_string()]); let candidates = vec![ // Metadata-only physical release proves the exact artist-named // host, but can never display itself. test_bing_candidate( "https://crazy-p.com/images/Any-Signs-of-Love-VINYL.jpg", "Crazy P announces Any Signs of Love", "https://crazy-p.com/news/any-signs-of-love", BingIdentityConfidence::Strong, false, ), test_bing_candidate( "https://crazy-p.com/images/live-press-shot.jpg", "Crazy P Curate", "https://crazy-p.com/news/curate", BingIdentityConfidence::Strong, true, ), test_bing_candidate( "https://cdn.fatsoma.test/danielle.jpg", "Danielle Moore (Crazy P) DJ set", "https://fatsoma.test/danielle-moore-crazy-p", BingIdentityConfidence::Strong, true, ), test_bing_candidate( "https://editorial.test/crazy-p.jpg", "Crazy P discuss Any Signs of Love", "https://editorial.test/interview/any-signs-of-love", BingIdentityConfidence::Strong, true, ), test_bing_candidate( "https://namesake.test/crazy-p.jpg", "Crazy P live", "https://namesake.test/artists/crazy-p", BingIdentityConfidence::Strong, true, ), test_bing_candidate( "https://weak.test/any-signs.jpg", "Crazy P Any Signs of Love", "https://weak.test/post/1", BingIdentityConfidence::Weak, true, ), // A release card on a generic publisher cannot bless unrelated // sibling pages across that publisher's entire host. test_bing_candidate( "https://publisher.test/vinyl.jpg", "Crazy P Any Signs of Love vinyl", "https://publisher.test/releases/any-signs-of-love", BingIdentityConfidence::Strong, false, ), test_bing_candidate( "https://publisher.test/other.jpg", "Crazy P live gallery", "https://publisher.test/gallery/crazy-p", BingIdentityConfidence::Strong, true, ), ]; let filtered = filter_bing_candidates_for_policy(candidates, &policy, "Crazy P", &linked_people); let urls: HashSet<&str> = filtered .iter() .map(|candidate| candidate.url.as_str()) .collect(); assert_eq!(urls.len(), 3); assert!(urls.contains("https://crazy-p.com/images/live-press-shot.jpg")); assert!(urls.contains("https://cdn.fatsoma.test/danielle.jpg")); assert!(urls.contains("https://editorial.test/crazy-p.jpg")); assert!(!urls.contains("https://crazy-p.com/images/Any-Signs-of-Love-VINYL.jpg")); assert!(!urls.contains("https://namesake.test/crazy-p.jpg")); assert!(!urls.contains("https://weak.test/any-signs.jpg")); assert!(!urls.contains("https://publisher.test/other.jpg")); } #[test] fn linked_identity_people_accept_only_current_members_and_founders() { let body = json!({ "relations": [ { "type": "member of band", "ended": false, "artist": { "name": "Danielle Moore" } }, { "type": "founder", "ended": false, "artist": { "name": "Chris Todd" } }, { "type": "member of band", "ended": true, "artist": { "name": "Former Person" } }, { "type": "collaboration", "ended": false, "artist": { "name": "Unrelated Guest" } }, { "type": "member of band", "ended": false, "artist": { "name": "Crazy P" } } ]}); let people = musicbrainz_identity_people(&body, "Crazy P"); assert_eq!( people, HashSet::from(["danielle moore".to_string(), "chris todd".to_string()]) ); } #[test] fn top_tracks_keep_real_collaborations_but_reject_other_explicit_ids() { let collaboration = json!({ "artist": { "id": 9 }, "contributors": [{ "id": 42 }, { "id": 9 }] }); assert!(track_belongs_to_artist(&collaboration, 42)); assert!(!track_belongs_to_artist(&collaboration, 77)); assert!(track_belongs_to_artist(&json!({}), 42)); } #[test] fn empty_deferred_identity_echoes_the_requested_artist() { let empty = empty_catalog_artist_identity(20_6927); assert_eq!(empty.artist_id, 20_6927); assert!(empty.albums.is_empty()); assert!(empty.socials.is_empty()); assert!(empty.type_hints.is_empty()); } #[test] fn bing_product_results_are_rejected_without_blocking_editorial_pages() { assert!(is_product_image_result( "Billy Cobb — Zerwee vinyl", "https://needlejuicerecords.com/products/zerwee", "https://cdn.shopify.com/files/zerwee-product.png", )); assert!(is_product_image_result( "Billy Cobb cassette reissue", "https://example.test/news/billy-cobb", "https://images.example.test/zerwee.jpg", )); assert!(is_product_image_result( "Billy Cobb official album cover", "https://music.example.test/billy-cobb", "https://images.example.test/cover.jpg", )); assert!(is_product_image_result( "Billy Cobb - Trash", "https://www.albumoftheyear.org/album/billy-cobb-trash.php", "https://cdn.albumoftheyear.org/album/161873-trash.jpg", )); assert!(is_product_image_result( "Billy Cobb - Bear Album", "https://genius.com/albums/Billy-cobb/Bear-album", "https://images.genius.com/cover.png", )); assert!(is_product_image_result( "Billy Cobb: Forgotten Words", "https://music.apple.com/us/album/forgotten-words/123", "https://is1-ssl.mzstatic.com/image/thumb/Music122/v4/artwork.jpg/1200x1200bf-60.jpg", )); assert!(is_product_image_result( "Electro Vibes featuring Hermeth", "https://www.beatport.com/release/electro-vibes/123", "https://geo-media.beatport.com/image_size/1400x1400/cover.jpg", )); assert!(is_product_image_result( "Billy Cobb - Halloween IV : r/CassetteReleases", "https://www.reddit.com/r/CassetteReleases/comments/example/", "https://external-preview.redd.it/billy-cobb-halloween-iv.jpg", )); assert!(is_product_image_result( "Crazy P announce Any Signs of Love", "https://crazy-p.com/news/any-signs-of-love", "https://crazy-p.com/images/CRAZY-P-Any-Signs-Of-Love-VINYL.jpg", )); assert!(is_product_image_result( "Crazy P release announcement", "https://crazy-p.com/news/new-record", "https://crazy-p.com/images/any-signs-of-love-album-cover.jpg", )); assert!(!is_product_image_result( "Billy Cobb discusses his new album", "https://music.example.test/interviews/billy-cobb", "https://images.example.test/billy-cobb-portrait.jpg", )); assert!(!is_product_image_result( "Camping in Alaska live at Songbyrd", "https://music.example.test/features/camping-in-alaska", "https://images.example.test/camping-in-alaska-band.jpg", )); } #[test] fn bing_channel_branding_and_album_videos_are_rejected() { assert!(is_unsuitable_bing_result( "Billy Cobb", "https://music.youtube.com/channel/example", "https://yt3.googleusercontent.com/channel-banner", "Billy Cobb", )); assert!(is_unsuitable_bing_result( "Billy Cobb (@COBBTIMUS_PRIME) / Twitter", "https://twitter.com/cobbtimus_prime", "https://pbs.twimg.com/media/product-photo.jpg", "Billy Cobb", )); assert!(is_unsuitable_bing_result( "Billy Cobb", "https://www.facebook.com/BillyCobbStaffie/", "https://lookaside.fbsbx.com/lookaside/crawler/media/", "Billy Cobb", )); assert!(is_unsuitable_bing_result( "Billy Cobb - Zerwee", "https://www.reddit.com/r/weezer/comments/example/", "https://preview.redd.it/physical-record.jpg", "Billy Cobb", )); assert!(is_unsuitable_bing_result( "Billy Cobb dance performance", "https://www.pinterest.com/pin/example/", "https://i.pinimg.com/736x/unrelated-namesake.jpg", "Billy Cobb", )); assert!(is_unsuitable_bing_result( "MSPaint self portrait by an artist", "https://www.deviantart.com/example/art/mspaint-portrait", "https://images.example/mspaint.png", "MSPAINT", )); assert!(is_unsuitable_bing_result( "MSPaint (band) - Wikipedia", "https://en.wikipedia.org/wiki/MSPAINT_(band)", "https://upload.wikimedia.org/mspaint.jpg", "MSPAINT", )); assert!(is_unsuitable_bing_result( "Wednesday Addams portrait", "https://entertainment.example/netflix-wednesday-addams-jenna-ortega", "https://images.example/wednesday.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "Media Tried And Failed To Make 'Wednesday' A Racism Scandal", "https://thefederalist.com/when-media-tried-to-turn-netflixs-wednesday-into-a-racism-controversy/", "https://thefederalist.com/wp-content/uploads/wednesday--scaled.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "Wednesday's Original Dance Scene Would Have Been Worse", "https://screenrant.com/wednesday-show-dance-scene-original-plan/", "https://static1.srcdn.com/jenna-ortega-wednesday-dancing-scene.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "‘Wednesday’ On Netflix: Season 2 News & More Updates", "https://hollywoodlife.com/feature/wednesday-season-2-updates/", "https://hollywoodlife.com/wp-content/uploads/wednesday-update-embed.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "Wednesday executive producer reveals plans for a second season", "https://entertainment.example/wednesday-coshowrunner-teases-second-season", "https://images.example/untitled-design.png", "Wednesday", )); assert!(!is_unsuitable_bing_result( "Wednesday Announce 2024 U.S. Tour and New Documentary", "https://pitchfork.com/news/wednesday-announce-2024-us-tour/", "https://media.pitchfork.com/photos/wednesday-band.jpg", "Wednesday", )); assert!(!is_unsuitable_bing_result( "Wednesday Interview - Rolling Stone Australia", "https://au.rollingstone.com/t/wednesday-interview/", "https://images.example/wednesday-band-interview.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "What is the song in Wednesday’s dance scene?", "https://www.nme.com/news/tv/what-song-wednesdays-dance-scene-3359077", "https://www.nme.com/wp-content/uploads/Wednesday-dance-main.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "Wednesday 13 — Just A Rock N Roll Junkie", "https://music.example/wednesday-13-interview/", "https://images.example/wednesday-13.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "Wednesday 13 Events", "https://www.ticketmaster.com/wednesday-13-tickets/artist/1234567", "https://s1.ticketm.net/dam/a/abc/wednesday-13-hero.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "Wednesday DJ for event", "https://fantasiaevents.gr/wednesday-party/", "https://fantasiaevents.gr/wp-content/uploads/dj-for-event.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "Lady Gaga released new song Dead Dance from Wednesday soundtrack", "https://music.example/lady-gaga-dead-dance", "https://images.example/lady-gaga-dead-dance-wednesday.jpg", "Wednesday", )); assert!(is_unsuitable_bing_result( "MS Paint for Windows 11 gets a new design", "https://www.windowslatest.com/2021/12/02/microsoft-paint-update/", "https://www.windowslatest.com/wp-content/uploads/2021/12/MS-Paint-for-Windows-11.jpg", "MSPAINT", )); assert!(!is_unsuitable_bing_result( "MSPaint Announce New EP No Separation", "https://pitchfork.com/news/mspaint-announce-new-ep-no-separation/", "https://media.pitchfork.com/photos/MSPAINT.jpeg", "MSPAINT", )); assert!(is_unsuitable_bing_result( "The Mix 013: Nikki Nair - Mixmag.net", "https://mixmag.net/feature/the-mix-013-nikki-nair", "https://cdn.craft.cloud/assets/THE_MIX__NIKKI_NAIR_WEB_2.jpg", "Nikki Nair", )); assert!(!is_unsuitable_bing_result( "The Mix 013: Nikki Nair - Mixmag.net", "https://mixmag.net/feature/the-mix-013-nikki-nair", "https://cdn.craft.cloud/assets/nikki-nair-press-photo.jpg", "Nikki Nair", )); assert!(is_unsuitable_bing_result( "Bands Equipment at Nathan Oneill blog", "https://storage.googleapis.com/example/bands-equipment.html", "https://argoargo.example/wp-content/uploads/IMG_8529.jpg", "Equipment", )); assert!(!is_unsuitable_bing_result( "Equipment band interview", "https://music.example/artists/equipment", "https://cdn.example/equipment-press.jpg", "Equipment", )); assert!(!is_unsuitable_bing_result( "Billy Cobb - Tour Dates & Concert Schedule", "https://www.livenation.com/artist/K8vZ917hLqV/billy-cobb-events", "https://s1.ticketm.net/dam/a/f7b/cc2a0ae7-77cb-4573-919e-323f2abd1f7b_TABLET_LANDSCAPE_LARGE_16_9.jpg", "Billy Cobb", )); assert!(!is_unsuitable_bing_result( "Billy Cobb Tickets, 2026 Concert Tour Dates", "https://www.ticketmaster.com/billy-cobb-tickets/artist/3025436", "https://s1.ticketm.net/dam/a/f7b/cc2a0ae7-77cb-4573-919e-323f2abd1f7b_RETINA_LANDSCAPE_16_9.jpg", "Billy Cobb", )); assert!(is_unsuitable_bing_result( "Billy Cobb + Other Band Tickets", "https://www.ticketmaster.com/event/12345", "https://s1.ticketm.net/dam/a/event-poster_TABLET_LANDSCAPE.jpg", "Billy Cobb", )); assert!(is_unsuitable_bing_result( "Billy Cobb - Zerwee (Full 2021 Remastered Album Stream)", "https://www.youtube.com/watch?v=album", "https://i.ytimg.com/vi/album/maxresdefault.jpg", "Billy Cobb", )); assert!(is_unsuitable_bing_result( "1955 - Billy Cobb Cover - YouTube", "https://www.youtube.com/watch?v=cover", "https://i.ytimg.com/vi/cover/maxresdefault.jpg", "Billy Cobb", )); assert!(is_unsuitable_bing_result( "NASTY! presents High Fidelity and Hermeth", "https://venue.example/events/hermeth", "https://static.example/event-flyer.webp", "Hermeth", )); assert!(is_unsuitable_bing_result( "Hermeth - Black Gojira (Weith Remix)", "https://vimeo.com/visualizer", "https://i.vimeocdn.com/video/visualizer", "Hermeth", )); assert!(is_unsuitable_bing_result( "Raw: Hermeth, Paris", "https://shotgun.live/events/hermeth", "https://res.cloudinary.com/shotgun/production/artworks/banner.jpg", "Hermeth", )); assert!(is_unsuitable_bing_result( "SLUMLUV and DR3 Sounds present: DJ MELL G & Hermeth", "https://ra.co/events/1612040", "https://imgproxy.ra.co/event-art.jpg", "Hermeth", )); assert!(is_unsuitable_bing_result( "Stream K-65 with Hermeth", "https://soundcloud.com/swufm/k-65-with-hermeth", "https://i1.sndcdn.com/artworks-example-t500x500.jpg", "Hermeth", )); assert!(is_unsuitable_bing_result( "Technopol Mix 063 | Hermeth", "https://www.technopol.net/podcast/technopol-mix-063-hermeth/", "https://www.technopol.net/uploads/Podcast-Hermeth-FormatCarre.png", "Hermeth", )); assert!(is_unsuitable_bing_result( "Hermeth (Brainwaves) – By Foggy, Freakuency & Gym Nasty", "https://www.secrettelaviv.com/tickets/hermeth-brainwaves", "https://www.secrettelaviv.com/uploads/hermeth-event.jpg", "Hermeth", )); assert!(is_unsuitable_bing_result( "Hermeth Corporation - Hermeth", "https://www.hermeth.de/hermeth-corporation/", "https://www.hermeth.de/uploads/hermeth.jpg", "Hermeth", )); assert!(!is_unsuitable_bing_result( "Camping In Alaska Interview (2022) - YouTube", "https://www.youtube.com/watch?v=interview", "https://i.ytimg.com/vi/interview/maxresdefault.jpg", "Camping in Alaska", )); assert!(!is_unsuitable_bing_result( "Billy Cobb Live 1/29/23 - YouTube", "https://www.youtube.com/watch?v=live", "https://i.ytimg.com/vi/live/maxresdefault.jpg", "Billy Cobb", )); assert!(!is_unsuitable_bing_result( "Hermeth live performance", "https://vimeo.com/live-set", "https://i.vimeocdn.com/video/live-set", "Hermeth", )); } #[test] fn multi_artist_roundups_require_an_artist_specific_image() { let title = "Die Groove DJ-Charts mit Bearcubs, FJAAK, Hermeth, Osmosis Jones, Revan und UU"; assert!(multi_artist_result_mismatches_image( title, "https://groove.de/images/Osmosis-Jones_Liam-Kavanagh.jpg", "Hermeth", )); assert!(!multi_artist_result_mismatches_image( title, "https://groove.de/images/Hermeth_Danny-Leal.jpg", "Hermeth", )); assert!(!multi_artist_result_mismatches_image( "Camping in Alaska: Millions of YouTube views, but they are still going", "https://www.al.com/images/band-photo.jpg", "Camping in Alaska", )); assert!(multi_artist_result_mismatches_image( "PARTIBOI69 + JUICY ROMANCE + CESOUS (50CL)", "https://venue.example/images/partiboi69-lineup.jpg", "Partiboi69", )); assert!(multi_artist_result_mismatches_image( "Billy Cobb + Other Artist", "https://venue.example/images/generic-stage.jpg", "Billy Cobb", )); assert!(!multi_artist_result_mismatches_image( "Billy Cobb + Other Artist", "https://venue.example/images/billy-cobb-stage.jpg", "Billy Cobb", )); assert!(multi_artist_result_mismatches_image( "Soccer Mommy with Hana Vu at The Truman", "https://cdn.example/images/soccer-mommy.jpg", "Hana Vu", )); assert!(multi_artist_result_mismatches_image( "Horse Jumper of Love - w/ Strange Mangers and Sapien Joyride", "https://venue.example/images/HJ-1920.png", "Horse Jumper of Love", )); assert!(multi_artist_result_mismatches_image( "Michael Cera Palin * Hey, ily * Bottom Bracket * Life Looks Good", "https://venue.example/images/generic-band.jpg", "Hey, ily", )); assert!(!multi_artist_result_mismatches_image( "Michael Cera Palin * Hey, ily * Bottom Bracket * Life Looks Good", "https://venue.example/images/hey-ily-band.jpg", "Hey, ily", )); assert!(multi_artist_result_mismatches_image( "HEY, ILY • Blind Equation • Flyover States • Red Sun • Endswell", "https://venue.example/images/red-sun-okc-900x667.jpeg", "Hey, ily", )); assert!(!multi_artist_result_mismatches_image( "HEY, ILY • Blind Equation • Flyover States • Red Sun • Endswell", "https://venue.example/images/hey-ily-press.jpeg", "Hey, ily", )); assert!(!is_unsuitable_bing_result( "HEY, ILY • Blind Equation • Flyover States • Red Sun • Endswell", "https://venue.example/events/hey-ily-and-friends", "https://venue.example/images/hey-ily-press.jpeg", "Hey, ily", )); assert!(is_unsuitable_bing_result( "HEY, ILY • Blind Equation • Flyover States • Red Sun • Endswell", "https://venue.example/events/hey-ily-and-friends", "https://venue.example/images/red-sun-okc.jpeg", "Hey, ily", )); assert!(!multi_artist_result_mismatches_image( "Earth, Wind & Fire band photo", "https://cdn.example/images/generic-press.jpg", "Earth, Wind & Fire", )); assert!(multi_artist_result_mismatches_image( "Earth, Wind & Fire • Chicago • Toto", "https://cdn.example/images/chicago.jpg", "Earth, Wind & Fire", )); assert!(multi_artist_result_mismatches_image( "Partiboi69 y Gerardo Niva nos sumergen en una noche de baile", "https://wololosound.com/wp-content/uploads/gerardo-niva.jpeg", "Partiboi69", )); assert!(!multi_artist_result_mismatches_image( "Partiboi69 y Gerardo Niva nos sumergen en una noche de baile", "https://press.example/images/partiboi69-press.jpeg", "Partiboi69", )); assert!(!multi_artist_result_mismatches_image( "Partiboi69 y su nuevo album llegan esta semana", "https://press.example/images/generic-press.jpeg", "Partiboi69", )); assert!(multi_artist_result_mismatches_image( "Hudson Mohawke and Nikki Nair Release Set The Roof EP", "https://dubiks.com/images/NIKKI-NAIR-HUDSON-MOHAWKE-2023.webp", "Nikki Nair", )); assert!(multi_artist_result_mismatches_image( "Hudson Mohawke & Nikki Nair", "https://cdn.example/images/abc123.webp", "Nikki Nair", )); assert!(multi_artist_result_mismatches_image( "Hudson Mohawke and Nikki Nair announce a tour", "https://cdn.example/images/abc123.webp", "Nikki Nair", )); assert!(multi_artist_result_mismatches_image( "FJAAK & Hermeth", "https://cdn.example/images/opaque-hash.webp", "Hermeth", )); assert!(multi_artist_result_mismatches_image( "FJAAK and Hermeth announce a show", "https://cdn.example/images/opaque-hash.webp", "Hermeth", )); assert!(multi_artist_result_mismatches_image( "Gerardo Niva & Partiboi 69", "https://cdn.example/images/opaque-hash.webp", "Partiboi69", )); assert!(multi_artist_result_mismatches_image( "Gerardo Niva and Partiboi 69 announce a show", "https://cdn.example/images/opaque-hash.webp", "Partiboi69", )); assert!(multi_artist_result_mismatches_image( "Aundy Crenshaw Queue’s Nala and Nikki Nair’s new collaboration", "https://cdn.example/images/opaque-hash.webp", "Nikki Nair", )); assert!(!multi_artist_result_mismatches_image( "Nikki Nair discusses music and his new label", "https://press.example/images/Nikki-Nair-Profile-Photo.jpg", "Nikki Nair", )); assert!(!multi_artist_result_mismatches_image( "Nikki Nair and the new album", "https://cdn.example/images/abc123.webp", "Nikki Nair", )); assert!(!multi_artist_result_mismatches_image( "Partiboi 69 and the new album", "https://cdn.example/images/abc123.webp", "Partiboi69", )); } #[test] fn conflicting_person_name_in_image_path_is_rejected() { assert!(image_path_conflicts_with_two_part_artist( "https://img.example/schlagzeuger-jimmy-cobb-von-links-mit-miles-davis.jpg", "Billy Cobb", )); assert!(!image_path_conflicts_with_two_part_artist( "https://s1.ticketm.net/dam/a/f7b/billy-cobb_TABLET_LANDSCAPE.jpg", "Billy Cobb", )); assert!(!image_path_conflicts_with_two_part_artist( "https://press.example/images/portrait-cobb.jpg", "Billy Cobb", )); assert!(!image_path_conflicts_with_two_part_artist( "https://press.example/images/a7cba5e4-hero.jpg", "Billy Cobb", )); assert!(!image_path_conflicts_with_two_part_artist( "https://press.example/images/not-the-band.jpg", "Camping in Alaska", )); assert!(image_path_conflicts_with_two_part_artist( "https://press.example/billy-cobb/jimmy-cobb.jpg", "Billy Cobb", )); assert!(image_path_conflicts_with_two_part_artist( "https://press.example/images/jimmy-cobb.jpg?ref=billy-cobb", "Billy Cobb", )); assert!(image_filename_names_artist( "https://press.example/roundups/other-act/hey-ily-press.jpg?ref=generic", "Hey, ily", )); assert!(!image_filename_names_artist( "https://press.example/hey-ily/red-sun.jpg?ref=hey-ily", "Hey, ily", )); } #[test] fn graphic_and_proxy_url_hints_are_conservative() { assert!(is_lossless_graphic_url( "https://press.example/interview-card.PNG?width=1200" )); assert!(!is_lossless_graphic_url( "https://press.example/artist-photo.webp?width=1200" )); assert_eq!( embedded_raster_dimensions( "https://cdn.example/c8b38bd6-300x168.jpg?rect=0,9,300,150&w=2048&h=1024" ), Some((300, 168)) ); assert_eq!( embedded_raster_dimensions("https://press.example/hero-credit-2000x1500-1068x801.jpg"), Some((1068, 801)) ); assert_eq!( embedded_raster_dimensions( "https://s1.ticketm.net/dam/a/id_TABLET_LANDSCAPE_LARGE_16_9.jpg" ), None ); assert_eq!( embedded_raster_dimensions("https://cdn.example/abc300x168def/photo.jpg"), None ); assert!(is_ticketing_artist_profile_image( "https://www.livenation.com/artist/K8vZ917hLqV/billy-cobb-events", "https://s1.ticketm.net/dam/a/id/hero.jpg", )); assert!(!is_ticketing_artist_profile_image( "https://evil.test/https://www.ticketmaster.com/artist/billy-cobb", "https://evil.test/ticketm.net/dam/a/hero.jpg", )); } #[test] fn white_product_canvas_is_penalized_but_not_rejected() { let white = image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel( 160, 90, image::Rgb([248, 248, 248]), )); let dark = image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel( 160, 90, image::Rgb([30, 28, 32]), )); assert_eq!(search_product_background_penalty(&white), 0.14); assert_eq!(search_product_background_penalty(&dark), 1.0); // A subject-colored center does not let a white product canvas evade // the border heuristic. let mut mockup = image::RgbImage::from_pixel(160, 90, image::Rgb([250, 250, 250])); for y in 20..70 { for x in 45..115 { mockup.put_pixel(x, y, image::Rgb([40, 60, 90])); } } let mockup = image::DynamicImage::ImageRgb8(mockup); assert_eq!(search_product_background_penalty(&mockup), 0.14); } #[test] fn structured_high_contrast_text_is_penalized() { let clean = image::DynamicImage::ImageRgb8(image::RgbImage::from_fn(384, 216, |x, y| { let shade = 55 + ((x + y) % 120) as u8; image::Rgb([shade, shade.saturating_add(12), shade.saturating_add(24)]) })); assert_eq!(search_text_overlay_penalty(&clean), 1.0); // Repeated narrow white glyph-like strokes form a caption band. This // avoids a font dependency while exercising the same row structure. let mut captioned = image::RgbImage::from_pixel(384, 216, image::Rgb([45, 48, 52])); for glyph_x in (24..360).step_by(8) { for y in 176..193 { for x in glyph_x..glyph_x + 4 { captioned.put_pixel(x, y, image::Rgb([248, 248, 248])); } } } let captioned = image::DynamicImage::ImageRgb8(captioned); assert!(search_text_overlay_penalty(&captioned) <= 0.70); assert!(search_has_flat_graphic_layout(&captioned)); let textured = image::DynamicImage::ImageRgb8(image::RgbImage::from_fn(192, 108, |x, y| { image::Rgb([ ((x * 47 + y * 13) % 256) as u8, ((x * 17 + y * 61) % 256) as u8, ((x * 79 + y * 29) % 256) as u8, ]) })); assert!(!search_has_flat_graphic_layout(&textured)); assert!(!search_has_low_palette_graphic(&textured)); let illustration = image::DynamicImage::ImageRgb8(image::RgbImage::from_fn(384, 216, |x, y| { if x < 192 && y < 108 { image::Rgb([12, 20, 92]) } else if x >= 192 && y < 108 { image::Rgb([245, 178, 92]) } else { image::Rgb([128, 24, 142]) } })); assert!(search_has_low_palette_graphic(&illustration)); assert!(has_release_art_context( "NEW TRACK: Artist - Under Your Spell" )); assert!(!has_release_art_context("Artist press photo and interview")); let panel_colors = [ image::Rgb([180, 25, 30]), image::Rgb([20, 130, 210]), image::Rgb([235, 190, 35]), image::Rgb([35, 170, 80]), image::Rgb([95, 35, 155]), image::Rgb([225, 80, 170]), image::Rgb([25, 185, 185]), image::Rgb([225, 225, 215]), ]; let panel_grid = image::DynamicImage::ImageRgb8(image::RgbImage::from_fn(800, 400, |x, y| { panel_colors[((y / 200) * 4 + (x / 200)) as usize] })); assert!(search_has_panel_collage_layout(&panel_grid)); assert!(!search_has_panel_collage_layout(&textured)); let one_architectural_edge = image::DynamicImage::ImageRgb8(image::RgbImage::from_fn(800, 400, |x, y| { let base = ((x * 11 + y * 7) % 90) as u8; if x < 400 { image::Rgb([40 + base, 55 + base, 70 + base]) } else { image::Rgb([125 + base, 115 + base, 105 + base]) } })); assert!(!search_has_panel_collage_layout(&one_architectural_edge)); } #[test] fn hero_selection_waits_for_scored_identity_and_deduplicates_crops() { let original = image::DynamicImage::ImageRgb8(image::RgbImage::from_fn(800, 600, |x, y| { match ((x / 40) + (y / 30)) % 4 { 0 => image::Rgb([26, 48, 84]), 1 => image::Rgb([174, 92, 58]), 2 => image::Rgb([62, 132, 108]), _ => image::Rgb([216, 184, 126]), } })); let cropped = original.crop_imm(120, 90, 560, 420).resize_exact( 1200, 900, image::imageops::FilterType::Triangle, ); let distinct = image::DynamicImage::ImageRgb8(image::RgbImage::from_fn(800, 600, |x, y| { image::Rgb([((x ^ y) % 256) as u8, (x % 256) as u8, (y % 256) as u8]) })); let original_fingerprint = hero_fingerprint(&original); let cropped_fingerprint = hero_fingerprint(&cropped); let distinct_fingerprint = hero_fingerprint(&distinct); let crop_distance = original_fingerprint .dhashes .iter() .flat_map(|left| { cropped_fingerprint .dhashes .iter() .map(move |right| (left ^ right).count_ones()) }) .min() .unwrap(); let color_intersection: u32 = original_fingerprint .color_histogram .iter() .zip(&cropped_fingerprint.color_histogram) .map(|(left, right)| u32::from((*left).min(*right))) .sum(); assert!( hero_fingerprints_match(&original_fingerprint, &cropped_fingerprint), "crop distance {crop_distance}, color intersection {color_intersection}" ); assert!(!hero_fingerprints_match( &original_fingerprint, &distinct_fingerprint )); // dHash alone sees all uniform fields as identical. Color evidence // prevents unrelated flat red/blue images from collapsing together. let flat_red = image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel( 800, 450, image::Rgb([220, 20, 20]), )); let flat_blue = image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel( 800, 450, image::Rgb([20, 20, 220]), )); let red_fingerprint = hero_fingerprint(&flat_red); let blue_fingerprint = hero_fingerprint(&flat_blue); assert_eq!(red_fingerprint.dhashes[0], blue_fingerprint.dhashes[0]); assert!(!hero_fingerprints_match( &red_fingerprint, &blue_fingerprint )); // With no successfully scored Strong image, retain exactly one Weak // fallback. Once a Strong image scores, Weak results are excluded. let (weak_only, _) = select_scored_hero_urls(vec![ ( "weak-a".into(), 20.0, original_fingerprint.clone(), BingIdentityConfidence::Weak, ), ( "weak-b".into(), 10.0, distinct_fingerprint.clone(), BingIdentityConfidence::Weak, ), ]); assert_eq!(weak_only, vec!["weak-a"]); assert_eq!( select_strong_hero_seed(vec![ ( "weak-a".into(), 20.0, original_fingerprint.clone(), BingIdentityConfidence::Weak, ), ( "weak-b".into(), 10.0, distinct_fingerprint.clone(), BingIdentityConfidence::Weak, ), ]), None, "a provisional Bing hero must never expose a Weak-only result", ); let (strong_only, _) = select_scored_hero_urls(vec![ ( "weak".into(), 100.0, distinct_fingerprint.clone(), BingIdentityConfidence::Weak, ), ( "strong".into(), 10.0, original_fingerprint.clone(), BingIdentityConfidence::Strong, ), ( "strong-crop".into(), 9.0, cropped_fingerprint, BingIdentityConfidence::Strong, ), ]); assert_eq!(strong_only, vec!["strong"]); assert_eq!( select_strong_hero_seed(vec![ ( "weak".into(), 100.0, distinct_fingerprint, BingIdentityConfidence::Weak, ), ( "strong".into(), 10.0, original_fingerprint, BingIdentityConfidence::Strong, ), ]), Some("strong".into()), ); } #[test] fn duplicate_bing_urls_upgrade_to_stronger_metadata() { let existing = BingImageCandidate { url: "https://s1.ticketm.net/dam/a/id/hero.jpg".into(), title: "Billy Cobb live".into(), page_url: "https://youtube.com/watch?v=x".into(), identity: BingIdentityConfidence::Weak, display_eligible: true, query_index: 0, rank_in_query: 0, }; let trusted = BingImageCandidate { url: existing.url.clone(), title: "Billy Cobb tour dates".into(), page_url: "https://www.livenation.com/artist/id/billy-cobb-events".into(), identity: BingIdentityConfidence::Strong, display_eligible: true, query_index: 4, rank_in_query: 2, }; assert!(bing_candidate_metadata_is_better(&trusted, &existing)); assert!(!bing_candidate_metadata_is_better(&existing, &trusted)); } #[test] fn hero_decode_limits_bound_remote_image_allocations() { let limits = hero_decode_limits(); assert_eq!(limits.max_alloc, Some(HERO_MAX_DECODE_ALLOC_BYTES)); assert!(limits .check_dimensions(HERO_MAX_DECODE_DIMENSION, HERO_MAX_DECODE_DIMENSION) .is_ok()); assert!(limits .check_dimensions(HERO_MAX_DECODE_DIMENSION + 1, 1) .is_err()); let image = image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel( 32, 32, image::Rgb([90, 120, 150]), )); let mut bytes = std::io::Cursor::new(Vec::new()); image.write_to(&mut bytes, image::ImageFormat::Png).unwrap(); assert!(decode_hero_image(bytes.into_inner()).is_some()); } #[test] fn video_thumbnail_sources_are_softly_deprioritized() { assert_eq!( search_thumbnail_source_penalty("https://i.ytimg.com/vi/interview/maxresdefault.jpg"), 0.58 ); assert_eq!( search_thumbnail_source_penalty("https://press.example/artist-photo.jpg"), 1.0 ); } fn album_with(titles: &[&str]) -> serde_json::Value { json!({ "tracks": { "data": titles.iter().map(|t| json!({ "title": t })).collect::>() } }) } #[test] fn live_album_from_untitled_tracks() { // Animal Collective "Ballet Slippers" — real tracklist, title gives no // hint, but every track is marked live. let ballet = album_with(&[ "In The Flowers (Live June 2, 2009 Boulder, CO)", "Who Could Win A Rabbit (Live June 9, 2009 Ft. Lauderdale, FL)", "Summertime Clothes (Live June 9, 2009 Ft. Lauderdale, FL)", "Bleed (Live May 30, 2009 Las Vegas, NV)", "My Girls (Live May 30, 2009 Las Vegas, NV)", "Brother Sport (Live June 2, 2009 Boulder, CO)", ]); assert!(detail_is_live(&ballet)); // "FEELSLive 04/05" — the "Live" in the title has no word boundary, but // the tracks settle it. let feels = album_with(&[ "Wastered (Live Bowery Ballroom 8/15/04)", "Loch Raven (Live Haverford College 4/23/05)", "Grass (Live Ottobar 11/19/04)", "Turn Into Something (Live Coltrane 4/22/05)", ]); assert!(detail_is_live(&feels)); // "(Live)" suffix form. assert!(detail_is_live(&album_with(&[ "Amanita (Live)", "My Girls (Live)", "Peacebone (Live)", ]))); } #[test] fn studio_album_is_not_live() { // Animal Collective "Time Skiffs" — clean studio tracklist. let time_skiffs = album_with(&[ "Dragon Slayer", "Car Keys", "Prester John", "Strung with Everything", "Cherokee", "We Go Back", ]); assert!(!detail_is_live(&time_skiffs)); // A studio album carrying a couple of live bonus cuts stays studio: 2 of // 8 live is well under the 60% majority. let deluxe = album_with(&[ "Track One", "Track Two", "Track Three", "Track Four", "Track Five", "Track Six", "Encore (Live)", "Closer (Live)", ]); assert!(!detail_is_live(&deluxe)); // Ordinary titles containing the word "live" are not live markers. assert!(!is_live_track_title("Long Live")); assert!(!is_live_track_title("Come Alive")); assert!(is_live_track_title("My Girls (Live)")); assert!(is_live_track_title("Encore - Live")); } #[test] fn too_few_tracks_or_no_tracklist_is_not_live() { // Fewer than three tracks: not enough to call it a live album. assert!(!detail_is_live(&album_with(&[ "A Song (Live)", "Another (Live)" ]))); // No tracklist at all (e.g. a search result) resolves false, not a panic. assert!(!detail_is_live(&json!({ "title": "Some Album" }))); } fn album_with_date(title: &str, date: &str) -> CatalogAlbum { CatalogAlbum { id: 1, title: title.to_string(), artist: "A".into(), cover: None, release_date: Some(date.to_string()), record_type: Some("album".into()), track_count: Some(10), edition_group: None, is_soundtrack: false, is_live: false, is_mixtape: false, appears_on: false, } } /// A plain studio album row, the shape the details pass leaves behind. fn catalog_album(id: u64, title: &str, date: Option<&str>) -> CatalogAlbum { CatalogAlbum { id, title: title.to_string(), artist: "A".into(), cover: None, release_date: date.map(String::from), record_type: Some("album".into()), track_count: Some(10), edition_group: None, is_soundtrack: false, is_live: false, is_mixtape: false, appears_on: false, } } /// The `/album/{id}` body shape the attribution rule reads: a sized cover /// plus `tracks.data` rows carrying id / title / duration / preview. fn album_detail(id: u64, titles: &[&str]) -> serde_json::Value { json!({ "id": id, "cover_medium": format!( "https://e-cdns-images.dzcdn.net/images/cover/{id}/250x250-000000-80-0-0.jpg" ), "tracks": { "data": titles .iter() .enumerate() .map(|(at, title)| json!({ "id": id * 1000 + at as u64, "title": title, "duration": 200, "preview": format!("https://cdns-preview.dzcdn.net/{id}-{at}.mp3"), "readable": true, })) .collect::>() } }) } /// Albums + the two maps `catalog_artist` derives from the details pass. #[allow(clippy::type_complexity)] fn attribution_fixture( rows: &[(CatalogAlbum, &[&str])], ) -> ( Vec, HashMap>, HashMap, ) { let mut albums = Vec::new(); let mut containment = HashMap::new(); let mut details = HashMap::new(); for (album, titles) in rows { containment.insert( album.id, titles .iter() .map(|title| contained_track_identity(title)) .collect(), ); details.insert(album.id, album_detail(album.id, titles)); albums.push(album.clone()); } (albums, containment, details) } fn scrobbles(plays: u64, listeners: u64) -> TrackStats { TrackStats { plays, listeners } } /// A MusicBrainz group with no secondary types — a plain studio release. fn mb_album(date: &str) -> ReleaseFacts { ReleaseFacts { first: date.to_string(), secondary: Vec::new(), ambiguous: false, } } fn mb_typed(date: &str, kinds: &[&str]) -> ReleaseFacts { ReleaseFacts { first: date.to_string(), secondary: kinds.iter().map(|k| k.to_string()).collect(), ambiguous: false, } } /// Deezer's own attribution: the licensed pressing, on some other release. fn licensed_track(id: u64, title: &str, album: &str, album_id: u64) -> CatalogTrack { let mut track = popular_track(id, title); track.album = album.to_string(); track.album_id = Some(album_id); track } fn popular_track(id: u64, title: &str) -> CatalogTrack { CatalogTrack { id, title: title.to_string(), artist: "A".into(), artist_id: None, rank: None, album: "B".into(), album_id: Some(1), cover: None, duration_secs: Some(180), preview: None, track_no: None, disk_number: None, unreleased: false, listeners: None, plays: None, origin_track_id: None, } } #[test] fn track_identity_folds_pressings_but_not_performances() { // Same recording, different pressing — these must collapse, or Last.fm's // scrobble name never finds the Deezer row. let shake = track_title_identity("Shake It Off"); assert_eq!(track_title_identity("Shake It Off (Taylor's Version)"), shake); assert_eq!(track_title_identity("Dreams - 2004 Remaster"), track_title_identity("Dreams")); assert_eq!( track_title_identity("Heartbreaker (Original Mix)"), track_title_identity("Heartbreaker") ); assert_eq!( track_title_identity("Like It (feat. Reckz'Capo)"), track_title_identity("Like It") ); assert_eq!(track_title_identity("Creep [Radio Edit]"), track_title_identity("Creep")); // Genuinely different recordings. Collapsing these would delete a real // track from the list, which is why the marker set is curated and not // "strip every parenthetical". let song = track_title_identity("Song"); assert_ne!(track_title_identity("Song (Live)"), song); assert_ne!(track_title_identity("Song (Acoustic)"), song); assert_ne!(track_title_identity("Song (Reprise)"), song); assert_ne!(track_title_identity("Song (JENNIE Remix)"), song); // A parenthetical that is part of the title survives intact. assert_eq!( track_title_identity("(I Can't Get No) Satisfaction"), normalize("(I Can't Get No) Satisfaction") ); } #[test] fn lastfm_order_ranks_by_listeners_and_keeps_unscored_below() { let pool = vec![ popular_track(1, "New Single"), // Deezer's #1, nobody has scrobbled it popular_track(2, "Deep Cut"), popular_track(3, "The Hit"), ]; let ranks = HashMap::from([ (track_title_identity("The Hit"), scrobbles(4_100_000, 900_000)), (track_title_identity("Deep Cut"), scrobbles(48_000, 12_000)), ]); let ordered = apply_lastfm_order(pool, &ranks, 10); let titles: Vec<&str> = ordered.iter().map(|t| t.title.as_str()).collect(); // Ranked tracks descend by listeners; the unscored one is kept, not // dropped, and sits below them. assert_eq!(titles, ["The Hit", "Deep Cut", "New Single"]); assert_eq!(ordered[0].listeners, Some(900_000)); // Plays ride the same table and land on the row for the album column. assert_eq!(ordered[0].plays, Some(4_100_000)); assert_eq!(ordered[2].listeners, None); assert_eq!(ordered[2].plays, None); } #[test] fn lastfm_order_never_admits_a_track_deezer_did_not_supply() { // The merged-page case: Last.fm files several bands under one name, so // its top track can belong to a different artist entirely. Membership // comes from Deezer, so the stranger simply has nowhere to appear. let pool = vec![popular_track(1, "Circus X")]; let ranks = HashMap::from([ (track_title_identity("Independent Love Song"), scrobbles(210_000, 35_745)), (track_title_identity("Circus X"), scrobbles(31_000, 7_044)), ]); let ordered = apply_lastfm_order(pool, &ranks, 10); assert_eq!(ordered.len(), 1); assert_eq!(ordered[0].title, "Circus X"); } #[test] fn lastfm_order_collapses_pressings_to_the_entry_deezer_ranks_highest() { // Both resolve to one key and would otherwise sort adjacently showing // the same count twice. Deezer's order decides which survives. let pool = vec![ popular_track(7, "Heartbreaker (Original Mix)"), popular_track(8, "Heartbreaker"), popular_track(9, "Cruel Mistress"), ]; let ranks = HashMap::from([ (track_title_identity("Heartbreaker"), scrobbles(402_000, 81_809)), (track_title_identity("Cruel Mistress"), scrobbles(160_000, 35_536)), ]); let ordered = apply_lastfm_order(pool, &ranks, 10); assert_eq!(ordered.len(), 2); assert_eq!(ordered[0].id, 7); assert_eq!(ordered[1].title, "Cruel Mistress"); } #[test] fn lastfm_order_without_ranks_is_deezers_own_order() { // Every degradation path — no MBID, no Last.fm entry, guard refused — // arrives here. Popular must fall back to what it always was. let pool = vec![ popular_track(1, "First"), popular_track(2, "Second"), popular_track(3, "Third"), ]; let ordered = apply_lastfm_order(pool, &HashMap::new(), 2); let titles: Vec<&str> = ordered.iter().map(|t| t.title.as_str()).collect(); assert_eq!(titles, ["First", "Second"]); assert!(ordered.iter().all(|t| t.listeners.is_none())); } #[test] fn album_identity_folds_pressings_onto_the_work() { // Every one of these is Deezer selling a re-press of one record. let opera = album_title_identity("A Night at the Opera"); assert_eq!(album_title_identity("A Night at the Opera (2011 Remaster)"), opera); assert_eq!(album_title_identity("A Night at the Opera - Remastered"), opera); assert_eq!(album_title_identity("A Night at the Opera (Deluxe Edition)"), opera); assert_eq!(album_title_identity("A Night at the Opera (Super Deluxe)"), opera); assert_eq!( album_title_identity("A Night at the Opera (40th Anniversary Edition)"), opera ); let idiot = album_title_identity("American Idiot"); assert_eq!(album_title_identity("American Idiot (Deluxe)"), idiot); assert_eq!(album_title_identity("Rumours (Super Deluxe)"), album_title_identity("Rumours")); // A re-recorded catalogue is its OWN work with its own first-release // date, so it must not fold onto the original -- same exemption the // frontend's EDITION_MARKER makes. assert_ne!( album_title_identity("Fearless (Taylor's Version)"), album_title_identity("Fearless") ); // Genuinely different records that merely share a word. assert_ne!( album_title_identity("Greatest Hits"), album_title_identity("Greatest Hits II") ); } #[test] fn volume_numbering_matches_across_providers() { // Measured on Queen: Deezer sells "Deep Cuts 2 (1977-1982)" while // MusicBrainz files "Deep Cuts, Volume 2 (1977–1982)" — note also the // en-dash. Keyed on raw titles these never met, so the whole Deep Cuts // series kept Deezer's wrong date and sat in the Albums tab. let deezer = album_title_identity("Deep Cuts 2 (1977-1982)"); assert_eq!(album_title_identity("Deep Cuts, Volume 2 (1977\u{2013}1982)"), deezer); assert_eq!(album_title_identity("Deep Cuts, Vol. 2 (1977-1982)"), deezer); assert_eq!(album_title_identity("Deep Cuts Vol 2 (1977-1982)"), deezer); // Different volumes stay different. assert_ne!(album_title_identity("Deep Cuts, Volume 3 (1984-1995)"), deezer); // A volume word with no number behind it is part of the title. assert_eq!(album_title_identity("Volume"), normalize("volume")); assert_eq!(album_title_identity("The Volume of Noise"), normalize("the volume of noise")); } #[test] fn original_dates_replace_deezers_reissue_dates() { // The measured case: Deezer dates the CD re-press, not the record. let mut albums = vec![ album_with_date("A Night at the Opera (2011 Remaster)", "2005-11-22"), album_with_date("The Game", "1994-10-17"), album_with_date("Some Bootleg Deezer Alone Has", "1999-01-01"), ]; let first_released = HashMap::from([ (release_date_key(Some("Album"), "A Night at the Opera"), mb_album("1975-11-21")), (release_date_key(Some("Album"), "The Game"), mb_album("1980-06-24")), ]); let corrected = apply_original_release_dates(&mut albums, &first_released); assert_eq!(corrected, 2); assert_eq!(albums[0].release_date.as_deref(), Some("1975-11-21")); assert_eq!(albums[1].release_date.as_deref(), Some("1980-06-24")); // Unmatched keeps whatever Deezer said rather than losing its date. assert_eq!(albums[2].release_date.as_deref(), Some("1999-01-01")); // And the corrected dates are what a chronological sort now uses. albums.sort_by(|a, b| b.release_date.cmp(&a.release_date)); let order: Vec<&str> = albums.iter().map(|a| a.title.as_str()).collect(); assert_eq!(order[2], "A Night at the Opera (2011 Remaster)"); } /// The measured fixture both attribution tests below run against: the /// compilation is dated EARLIER than the album it compiles, which is the /// systematic shape of real data and the reason "earliest wins" alone /// selects hits packages. #[allow(clippy::type_complexity)] fn bohemian_fixture() -> ( Vec, HashMap>, HashMap, ) { let anato = catalog_album(10, "A Night at the Opera", Some("1975-11-21")); let hits = catalog_album(11, "Greatest Hits", Some("1971-11-01")); let mut live = catalog_album(12, "Live at Wembley", Some("1986-07-12")); live.is_live = true; let mut ost = catalog_album( 13, "Bohemian Rhapsody (The Original Soundtrack)", Some("2018-10-19"), ); ost.is_soundtrack = true; let songs: &[&str] = &["Bohemian Rhapsody", "You're My Best Friend"]; attribution_fixture(&[(anato, songs), (hits, songs), (live, songs), (ost, songs)]) } #[test] fn attribution_picks_the_earliest_studio_album_that_contains_the_track() { let (albums, containment, details) = bohemian_fixture(); let mut tracks = vec![licensed_track( 999, "Bohemian Rhapsody (2011 Remaster)", "Bohemian Rhapsody (The Original Soundtrack)", 13, )]; let mb = HashMap::from([ ( release_date_key(Some("Album"), "A Night at the Opera"), mb_album("1975-11-21"), ), ( release_date_key(Some("Album"), "Greatest Hits"), mb_typed("1971-11-01", &["Compilation"]), ), ]); let moved = apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb); assert_eq!(moved, 1); let track = &tracks[0]; // The compilation is two years EARLIER and still loses: the type // exclusion outranks the date, which is the whole feature. assert_eq!(track.album, "A Night at the Opera"); assert_eq!(track.album_id, Some(10)); // Every id-bearing field moves together, or album auto-continue dies // on a queue seeded from one release and indexed by the other's id. assert_eq!(track.origin_track_id, Some(999)); assert_eq!(track.id, 10_000); assert!(track.cover.as_deref().is_some_and(|c| c.contains("/10/"))); // A SIZED cover, or the reel's 1000x1000 rewrite silently no-ops. assert!(track.cover.as_deref().is_some_and(|c| c.contains("250x250"))); assert!(track.preview.as_deref().is_some_and(|p| p.contains("10-0"))); } #[test] fn attribution_without_musicbrainz_secondary_types_would_pick_the_compilation() { // The ablation, asserted as an explicit inverse. Deleting the // secondary-type exclusion does not merely lose accuracy — it flips 16 // of 80 measured picks to WRONG, because every hits package is dated // before the records it draws from. This test exists so the guard // cannot be quietly removed and still look green. let (albums, containment, details) = bohemian_fixture(); let mut tracks = vec![licensed_track( 999, "Bohemian Rhapsody (2011 Remaster)", "Bohemian Rhapsody (The Original Soundtrack)", 13, )]; let mb = HashMap::from([ ( release_date_key(Some("Album"), "A Night at the Opera"), mb_album("1975-11-21"), ), // Same fixture, but MusicBrainz no longer says "Compilation". ( release_date_key(Some("Album"), "Greatest Hits"), mb_album("1971-11-01"), ), ]); apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb); assert_eq!(tracks[0].album, "Greatest Hits"); } #[test] fn attribution_prefers_a_plain_title_over_an_edition_title() { // Weezer, measured: `album_title_identity` strips the whole // parenthetical from "Weezer (Deluxe Edition / Blue Album)" and yields // `weezer`, which collides with seven self-titled Weezer groups and // inherits 1994-05-10. The plain "Weezer (Blue Album)" reduces to a key // MusicBrainz has never heard of and keeps no date at all — so an album // MB does not know always loses to a wrongly-dated album it does. The // edition tiebreak has to sit ABOVE the date, and doing so took the // measured pass from 74/80 to 79/80. let deluxe = catalog_album(20, "Weezer (Deluxe Edition / Blue Album)", Some("1994-05-10")); let plain = catalog_album(21, "Weezer (Blue Album)", None); let songs: &[&str] = &["Buddy Holly", "Say It Ain't So"]; let (albums, containment, details) = attribution_fixture(&[(deluxe, songs), (plain, songs)]); let mut tracks = vec![licensed_track(999, "Buddy Holly", "Some Sampler", 77)]; let mb = HashMap::from([( release_date_key(Some("Album"), "Weezer (Deluxe Edition / Blue Album)"), mb_album("1994-05-10"), )]); let moved = apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb); assert_eq!(moved, 1); assert_eq!(tracks[0].album, "Weezer (Blue Album)"); } #[test] fn attribution_prefers_the_record_a_reissue_was_built_from() { // Radiohead, measured live: Deezer carries "OK Computer" (12 tracks) // and "OK Computer OKNOTOK 1997 2017" (23) and dates BOTH 1997-05-21. // EP, edition-keyword and date all tie — "OKNOTOK 1997 2017" carries // none of the edition keywords — so the "original over a truncated // sampler" rule handed No Surprises, Karma Police and Let Down to the // 23-track reissue. let reissue = catalog_album(40, "OK Computer OKNOTOK 1997 2017", Some("1997-05-21")); let original = catalog_album(41, "OK Computer", Some("1997-05-21")); let songs: &[&str] = &["No Surprises", "Karma Police", "Let Down"]; let (albums, containment, details) = attribution_fixture(&[(reissue, songs), (original, songs)]); let mut tracks = vec![licensed_track(999, "No Surprises", "Some Compilation", 77)]; let mb = HashMap::from([ (release_date_key(Some("Album"), "OK Computer"), mb_album("1997-05-21")), ( release_date_key(Some("Album"), "OK Computer OKNOTOK 1997 2017"), mb_album("1997-05-21"), ), ]); let moved = apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb); assert_eq!(moved, 1); assert_eq!(tracks[0].album, "OK Computer"); } #[test] fn a_sequel_is_not_an_expansion_of_the_record_it_follows() { // The guard on the rule above. A sequel shares a prefix but not a // tracklist, so it must never be read as an edition of its predecessor. assert!(expands_on("OK Computer OKNOTOK 1997 2017", "OK Computer")); assert!(expands_on("Rumours (Super Deluxe)", "Rumours")); assert!(!expands_on("Greatest Hits II", "Greatest Hits")); assert!(!expands_on("Trap House 3", "Trap House")); assert!(!expands_on("Deep Cuts, Volume 2", "Deep Cuts")); // Not a prefix at all, and never itself. assert!(!expands_on("Kid A", "Amnesiac")); assert!(!expands_on("OK Computer", "OK Computer")); } #[test] fn attribution_leaves_a_track_alone_when_nothing_contains_it() { // 3 of the 80 measured tracks match nothing — a remix, an "(Original)" // and a one-off — and all three were already correct. let currents = catalog_album(30, "Currents", Some("2015-07-17")); let (albums, containment, details) = attribution_fixture(&[(currents, &["Let It Happen", "Eventually"])]); let mb = HashMap::from([( release_date_key(Some("Album"), "Currents"), mb_album("2015-07-17"), )]); let mut tracks = vec![licensed_track(999, "Dracula (JENNIE Remix)", "Dracula", 88)]; let before = format!("{:?}", tracks[0]); assert_eq!( apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb), 0 ); assert_eq!(format!("{:?}", tracks[0]), before); // And a containing album whose detail body was dropped (a Deezer quota // response never enters `details`) cannot win either — never wrong, // only less complete. let mut tracks = vec![licensed_track(999, "Let It Happen", "Some Sampler", 88)]; let before = format!("{:?}", tracks[0]); assert_eq!( apply_original_album_attribution( &mut tracks, &albums, &containment, &HashMap::new(), &mb ), 0 ); assert_eq!(format!("{:?}", tracks[0]), before); } #[test] fn attribution_prefers_an_album_over_the_ep_that_previewed_it() { // Caught in the running app on Radiohead. A hit is routinely issued as // a lead single/EP months before the record it belongs to, so on date // alone the EP always wins — "Creep" moved off *Pablo Honey*, which is // Deezer's own correct answer, onto a "Creep EP" five months earlier. let pablo = catalog_album(80, "Pablo Honey", Some("1993-02-22")); let mut ep = catalog_album(81, "Creep EP", Some("1992-09-21")); ep.record_type = Some("ep".into()); let songs: &[&str] = &["Creep"]; let (albums, containment, details) = attribution_fixture(&[(pablo, songs), (ep, songs)]); let mb = HashMap::from([( release_date_key(Some("Album"), "Pablo Honey"), mb_album("1993-02-22"), )]); let mut tracks = vec![licensed_track(999, "Creep", "Some Sampler", 88)]; assert_eq!( apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb), 1 ); assert_eq!(tracks[0].album, "Pablo Honey"); // But an EP still wins when no album carries the song, so a genuinely // EP-only track is not dragged onto an unrelated record. let other = catalog_album(82, "Amnesiac", Some("2001-06-04")); let mut ep_only = catalog_album(83, "Drill EP", Some("1992-05-05")); ep_only.record_type = Some("ep".into()); let (albums, containment, details) = attribution_fixture(&[(other, &["Pyramid Song"]), (ep_only, &["Prove Yourself"])]); let mut tracks = vec![licensed_track(999, "Prove Yourself", "Some Sampler", 88)]; assert_eq!( apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb), 1 ); assert_eq!(tracks[0].album, "Drill EP"); } #[test] fn attribution_leaves_a_row_deezer_already_got_right() { // Caught in the running app on Fleetwood Mac. "Already right" must be // an OUTPUT test, not a candidate filter: filtering the current album // out of the ranking means an already-correct row can only ever be // handed to the RUNNER-UP. Deezer had "Dreams" on plain Rumours, the // filter removed plain Rumours from consideration, and the rule // "corrected" it onto Rumours (Super Deluxe) — strictly worse, and 57 // of the 80 measured tracks sit in exactly this state. let plain = catalog_album(70, "Rumours", Some("1977-02-04")); let deluxe = catalog_album(71, "Rumours (Super Deluxe)", Some("1977-02-04")); let songs: &[&str] = &["Dreams", "Go Your Own Way"]; let (albums, containment, details) = attribution_fixture(&[(plain, songs), (deluxe, songs)]); let mb = HashMap::from([( release_date_key(Some("Album"), "Rumours"), mb_album("1977-02-04"), )]); let mut tracks = vec![licensed_track(999, "Dreams (2004 Remaster)", "Rumours", 70)]; let before = format!("{:?}", tracks[0]); assert_eq!( apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb), 0 ); assert_eq!(format!("{:?}", tracks[0]), before); } #[test] fn attribution_is_a_no_op_without_musicbrainz_evidence() { // No linked MBID, or a failed browse, leaves `first_released` empty. // The rule then returns byte-identical output to today rather than // falling back to "earliest wins", which measured WORSE than Deezer. let (albums, containment, details) = bohemian_fixture(); let mut tracks = vec![licensed_track( 999, "Bohemian Rhapsody (2011 Remaster)", "Bohemian Rhapsody (The Original Soundtrack)", 13, )]; let before = format!("{:?}", tracks[0]); assert_eq!( apply_original_album_attribution( &mut tracks, &albums, &containment, &details, &HashMap::new() ), 0 ); assert_eq!(format!("{:?}", tracks[0]), before); } #[test] fn attribution_never_moves_a_track_onto_a_guest_live_soundtrack_or_future_release() { // One candidate per rejection clause, each of which contains the song. // The Broadway cast recording is the measured failure of the earlier // date-only attempt, and it is refused three independent ways. let song: &[&str] = &["Boulevard of Broken Dreams"]; let mut guest = catalog_album(40, "A Record Someone Else Owns", Some("2005-01-01")); guest.appears_on = true; let mut live = catalog_album(41, "Bullet in a Bible", Some("2005-11-15")); live.is_live = true; let mut cast = catalog_album( 42, "American Idiot - The Original Broadway Cast Recording", Some("2010-04-12"), ); cast.is_soundtrack = true; let future = catalog_album(43, "A Record Not Out Yet", Some("2099-01-01")); let mut single = catalog_album(44, "Boulevard of Broken Dreams", Some("2004-11-29")); single.record_type = Some("single".into()); let (albums, containment, details) = attribution_fixture(&[ (guest, song), (live, song), (cast, song), (future, song), (single, song), ]); let mut tracks = vec![licensed_track( 999, "Boulevard of Broken Dreams", "Greatest Hits: God's Favorite Band", 88, )]; let mb = HashMap::from([( release_date_key(Some("Album"), "American Idiot"), mb_album("2004-09-21"), )]); assert_eq!( apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb), 0 ); assert_eq!(tracks[0].album_id, Some(88)); } #[test] fn attribution_matches_across_pressing_suffixes_and_feat_clauses() { // Deezer's /top row carries a featured-artist clause the album's own // tracklist does not, and the two sides name the same remaster // differently. `track_identity` alone misses the first, // `track_title_identity` uses a different vocabulary for the second. let hot_space = catalog_album(50, "Hot Space", Some("1982-05-04")); let (albums, containment, details) = attribution_fixture(&[( hot_space, &["Under Pressure", "Bohemian Rhapsody (Remastered 2011)"], )]); let mb = HashMap::from([( release_date_key(Some("Album"), "Hot Space"), mb_album("1982-05-04"), )]); let mut tracks = vec![ licensed_track(901, "Under Pressure (feat. David Bowie)", "Sampler", 77), licensed_track(902, "Bohemian Rhapsody (2011 Remaster)", "Sampler", 77), ]; assert_eq!( apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb), 2 ); assert!(tracks.iter().all(|track| track.album == "Hot Space")); } #[test] fn attribution_keeps_a_re_recording_on_its_first_album() { // KNOWN AND ACCEPTED — the single regression in 80 measured tracks. // "Welcome to Paradise" genuinely first appeared on Kerplunk! in 1991; // Deezer's top row is the 1994 Dookie re-recording. Containment cannot // tell a re-recording from a reissue, and a duration window was // rejected because the two cuts are seconds apart. Pinned so the // behaviour is a decision on record rather than a surprise later. let kerplunk = catalog_album(60, "Kerplunk!", Some("1991-12-17")); let dookie = catalog_album(61, "Dookie", Some("1994-02-01")); let song: &[&str] = &["Welcome to Paradise"]; let (albums, containment, details) = attribution_fixture(&[(kerplunk, song), (dookie, song)]); let mut tracks = vec![licensed_track(999, "Welcome to Paradise", "Sampler", 88)]; let mb = HashMap::from([ ( release_date_key(Some("Album"), "Kerplunk!"), mb_album("1991-12-17"), ), ( release_date_key(Some("Album"), "Dookie"), mb_album("1994-02-01"), ), ]); assert_eq!( apply_original_album_attribution(&mut tracks, &albums, &containment, &details, &mb), 1 ); assert_eq!(tracks[0].album, "Kerplunk!"); } #[test] fn release_group_facts_take_secondary_types_from_the_earliest_group() { // Fleetwood Mac, measured: MusicBrainz has three groups keyed // album/fleetwoodmac — the real 1975 album with no secondary types, a // 1979 Compilation and a 2006 Compilation+Live. Unioning the types // across the key tagged the 1975 album Compilation+Live, the // attribution rule excluded it, and "Rhiannon" landed on Greatest Hits. // The types must come from whichever group supplied the surviving date. let mut facts = HashMap::new(); for group in [ json!({"title": "Fleetwood Mac", "primary-type": "Album", "first-release-date": "2006-01-01", "secondary-types": ["Compilation", "Live"]}), json!({"title": "Fleetwood Mac", "primary-type": "Album", "first-release-date": "1975-07-11"}), json!({"title": "Fleetwood Mac", "primary-type": "Album", "first-release-date": "1979-01-01", "secondary-types": ["Compilation"]}), ] { merge_release_group_facts(&mut facts, &group); } let held = facts .get(&release_date_key(Some("Album"), "Fleetwood Mac")) .expect("the key survives"); assert_eq!(held.first, "1975-07-11"); assert!(held.secondary.is_empty()); // On an equal date the plainer group wins — a compilation that shares // both title and date with the album it compiles must not tag it. let mut tie = HashMap::new(); for group in [ json!({"title": "Tie", "primary-type": "Album", "first-release-date": "1980-01-01", "secondary-types": ["Compilation"]}), json!({"title": "Tie", "primary-type": "Album", "first-release-date": "1980-01-01"}), ] { merge_release_group_facts(&mut tie, &group); } assert!(tie .get(&release_date_key(Some("Album"), "Tie")) .expect("the key survives") .secondary .is_empty()); } #[test] fn musicbrainz_disambiguation_separates_seven_self_titled_albums() { // Measured live: MusicBrainz files SEVEN self-titled Weezer albums, // every one titled plain "Weezer", separated only by `disambiguation`. // Collapsing them to one key and keeping the earliest dated the 2026 // Gold Album — an unreleased record — 1994-05-10, and filed it beside // the debut. let mut facts = HashMap::new(); for (date, disamb) in [ ("1994-05-10", "Blue Album"), ("2001-05-07", "Green Album"), ("2008-01-01", "Red Album"), ("2016-04-01", "White Album"), ("2019-01-24", "Teal Album"), ("2019-03-01", "Black Album"), ("2026-08-21", "Gold Album"), ] { merge_release_group_facts( &mut facts, &json!({ "title": "Weezer", "primary-type": "Album", "first-release-date": date, "disambiguation": disamb, }), ); } // Deezer prints the disambiguator in its parenthetical, so each colour // now lands on its own record. for (date, title) in [ ("1994-05-10", "Weezer (Blue Album)"), ("2001-05-07", "Weezer (Green Album)"), ("2008-01-01", "Weezer (Red Album)"), ("2026-08-21", "Weezer (Gold Album)"), // Deezer's mixed pressing/disambiguator parenthetical too. ("2008-01-01", "Weezer (Deluxe Edition / Red Album)"), ] { let held = facts .get(&release_date_key(Some("Album"), title)) .unwrap_or_else(|| panic!("no entry for {title}")); assert!(!held.ambiguous, "{title} should be unambiguous"); assert_eq!(held.first, date, "{title}"); } // And the bare key, which is genuinely seven records at once, refuses // to date anything rather than guessing the earliest. let bare = facts .get(&release_date_key(Some("Album"), "Weezer")) .expect("the bare key still exists"); assert!(bare.ambiguous); let mut albums = vec![album_with_date("Weezer", "2026-08-21")]; assert_eq!(apply_original_release_dates(&mut albums, &facts), 0); assert_eq!(albums[0].release_date.as_deref(), Some("2026-08-21")); } #[test] fn a_reissue_duplicate_still_dates_from_the_earliest_group() { // The ambiguity guard must not throw away the case it was always // handling: two groups for ONE work. Same date is not ambiguous, and // the plainer of the two still carries the secondary types. let mut facts = HashMap::new(); for group in [ json!({"title": "Dookie", "primary-type": "Album", "first-release-date": "1994-02-01"}), json!({"title": "Dookie", "primary-type": "Album", "first-release-date": "1994-02-01", "secondary-types": ["Compilation"]}), ] { merge_release_group_facts(&mut facts, &group); } let held = facts .get(&release_date_key(Some("Album"), "Dookie")) .expect("the key survives"); assert!(!held.ambiguous); assert!(held.secondary.is_empty()); assert_eq!(held.first, "1994-02-01"); } #[test] fn a_mixed_edition_parenthetical_keeps_its_disambiguator() { // Deezer sells Weezer's colour records as "Weezer (Deluxe Edition / // Red Album)". Stripping that parenthetical whole threw away the only // thing saying WHICH self-titled album it is, so the 2008 Red Album // deluxe reduced to `weezer` — the same key as the 1994 debut — and // MusicBrainz, which titles all seven self-titled groups plain // "Weezer", dated it 1994-05-10. assert_eq!( album_title_identity("Weezer (Deluxe Edition / Red Album)"), album_title_identity("Weezer (Red Album)") ); assert_ne!( album_title_identity("Weezer (Deluxe Edition / Red Album)"), album_title_identity("Weezer (Deluxe Edition / Blue Album)") ); // A parenthetical that is ONLY a pressing still collapses onto the // work, which is what the whole date pass depends on. assert_eq!( album_title_identity("A Night at the Opera (2011 Remaster)"), album_title_identity("A Night at the Opera") ); assert_eq!( album_title_identity("Weezer (Deluxe Edition)"), album_title_identity("Weezer") ); // And a parenthetical with no pressing in it is untouched. assert_eq!( album_title_identity("Weezer (Blue Album)"), normalize("weezerbluealbum") ); } #[test] fn an_edition_inherits_the_date_of_the_work_it_presses() { // The tracklist-verified edition link knows which record a pressing // is; the title does not. Measured on Weezer: the Red Album deluxe // (edition_group -> the 2008 Red Album) was showing 1994-05-10. let mut red = album_with_date("Weezer (Red Album)", "2008-06-16"); red.id = 262150; red.edition_group = Some(262150); let mut red_deluxe = album_with_date("Weezer (Deluxe Edition / Red Album)", "1994-05-10"); red_deluxe.id = 320925257; red_deluxe.edition_group = Some(262150); // An unlinked album is left entirely alone. let mut green = album_with_date("Weezer (Green Album)", "2001-04-24"); green.id = 249320; let mut albums = vec![red, red_deluxe, green]; assert_eq!(apply_edition_group_dates(&mut albums), 1); assert_eq!(albums[1].release_date.as_deref(), Some("2008-06-16")); // The original is its own group and must never be rewritten. assert_eq!(albums[0].release_date.as_deref(), Some("2008-06-16")); assert_eq!(albums[2].release_date.as_deref(), Some("2001-04-24")); // Idempotent: a second pass changes nothing. assert_eq!(apply_edition_group_dates(&mut albums), 0); } #[test] fn a_single_never_dates_the_album_that_shares_its_name() { // Queen, measured: "The Miracle" exists as a 1989 album AND a 1984 // single. Keyed on title alone the single won and the album was dated // five years before it was made. let mut albums = vec![ album_with_date("The Miracle", "2023-09-29"), album_with_date("The Miracle (Collector's Edition)", "2022-11-18"), ]; let first_released = HashMap::from([ (release_date_key(Some("Album"), "The Miracle"), mb_album("1989-05-22")), (release_date_key(Some("Single"), "The Miracle"), mb_album("1984-11-27")), ]); apply_original_release_dates(&mut albums, &first_released); assert_eq!(albums[0].release_date.as_deref(), Some("1989-05-22")); // The collector's edition folds onto the same work, and onto the ALBUM. assert_eq!(albums[1].release_date.as_deref(), Some("1989-05-22")); } #[test] fn lastfm_identifier_ignores_the_two_space_encodings() { // MusicBrainz declares %20, Last.fm echoes +. A byte compare rejects 2 // of every 18 real matches, so the guard compares decoded identifiers. let declared = lastfm_artist_identifier("https://www.last.fm/music/Inland%20Knights"); let echoed = lastfm_artist_identifier("https://www.last.fm/music/Inland+Knights"); assert_eq!(declared, echoed); assert_eq!(declared.as_deref(), Some("inland knights")); // A different artist still has to fail. assert_ne!( lastfm_artist_identifier("https://www.last.fm/music/Scarlet"), lastfm_artist_identifier("https://www.last.fm/music/Scarlett") ); // Non-ASCII survives the round trip (MusicBrainz declares this one for // Taylor Swift). assert_eq!( lastfm_artist_identifier( "https://www.last.fm/music/%E3%83%86%E3%82%A4%E3%83%A9%E3%83%BC" ) .as_deref(), Some("テイラー") ); } }