diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 59607ee..c2596f3 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -257,12 +257,31 @@ fn parse_news_rss(body: &str) -> Vec { days * 86400 + h * 3600 + mi * 60 + sec } + /// The `url` attribute of an opening tag: ``. + /// `tag()` above returns element TEXT and has no attribute support. + fn attr(block: &str, name: &str, key: &str) -> Option { + let open = format!("<{name}"); + let start = block.find(&open)?; + let head_end = block[start..].find('>')? + start; + let head = &block[start..head_end]; + let at = head.find(&format!("{key}=\""))? + key.len() + 2; + let end = head[at..].find('"')? + at; + Some(decode(head[at..end].trim())) + } + /// "https://www.wwd.com/x" → "wwd.com". Bare host, no scheme, no `www.`. + fn host_of(url: &str) -> String { + let rest = url.split_once("://").map_or(url, |(_, r)| r); + let host = rest.split(['/', '?', '#']).next().unwrap_or(""); + host.trim_start_matches("www.").to_ascii_lowercase() + } + let mut out = Vec::new(); for block in body.split("").skip(1) { let block = block.split("").next().unwrap_or(""); let Some(raw_title) = tag(block, "title") else { continue }; let Some(link) = tag(block, "link") else { continue }; let source = tag(block, "source").map(|s| decode(&s)).unwrap_or_default(); + let host = attr(block, "source", "url").map(|u| host_of(&u)).unwrap_or_default(); let mut title = decode(&raw_title); // "Headline - Publication" → "Headline" when the tail names the source. if !source.is_empty() { @@ -271,15 +290,822 @@ fn parse_news_rss(body: &str) -> Vec { } } let date = tag(block, "pubDate").map(|d| rfc2822_secs(&d)).unwrap_or(0); - out.push(ArtistNewsItem { title, url: decode(&link), source, date }); - if out.len() >= 12 { - break; + out.push(ArtistNewsItem { title, url: decode(&link), source, date, host, others: 0 }); + } + // NO CAP AND NO DATE SORT HERE. + // + // This used to `break` at 12 and then sort by date. The feed returns 100 + // items in Google's RELEVANCE order, so that kept the twelve Google liked + // best and discarded ranks 13-100 — and for an ambiguously named artist + // that is exactly backwards: measured on the live feed, every genuine + // "toe" item sits at rank 15-88 and every genuine "Pond" item starts at + // rank 15. Zero of the top twelve were about either band. No amount of + // filtering downstream could recover what was thrown away up here. + // + // Relevance order is also the better tiebreaker to preserve, so the caller + // filters and dedups against the full feed and caps at the end. + out +} + +/* =========================== news relevance =============================== + * + * Deciding whether a headline is about THIS band. Researched and adversarially + * verified 2026-08-05 against seven live feeds; the plan and the measurements + * live in research/news/PLAN.md. + * + * The one idea the whole thing rests on: the signal is the name's SYNTACTIC + * ROLE, not the headline's topic. The query already contains the word "music", + * so every junk item is saturated with music vocabulary — "Every song on the + * 'Tip Toe' soundtrack" has a music-press host, a quoted work title and a + * music noun, and is still not about the band toe. What separates a real item + * is evidence AT THE NAME: a verb it governs, a role noun beside it, its + * position. Hence the two-part decision at the bottom — a total, and a + * separate floor on the occurrence-level subscore that publisher reputation + * can never satisfy on its own. + */ + +/// How much evidence a name needs before it is believed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NewsTier { + /// Multi-token distinctive name ("King Gizzard & The Lizard Wizard"). + Distinctive, + /// Any other single-token name ("Weezer", "Wand", "Interpol"). + Single, + /// A name that is also an ordinary English word ("toe", "Pond", "Girls"). + VeryCommon, +} + +impl NewsTier { + const fn threshold(self) -> i32 { + match self { + Self::Distinctive => 0, + Self::Single => 3, + Self::VeryCommon => 6, } } - out.sort_by_key(|item| -item.date); + /// The floor on occurrence-level evidence. `Distinctive` opts out. + const fn min_occurrence(self) -> i32 { + match self { + Self::Distinctive => i32::MIN, + Self::Single => 1, + Self::VeryCommon => 3, + } + } +} + +/// Names that are ordinary English words. Deliberately a short explicit list, +/// NOT a frequency table: bucketing by corpus rank was measured to send Wand +/// from 16 kept to 45 (the feed floods with "Wistoria: Wand and Sword"), and +/// toe precision from 1.00 to 0.70. +const VERY_COMMON_NAMES: &[&str] = &[ + "air", "bush", "cake", "cream", "curve", "eels", "elbow", "everything", "fear", "field", + "fire", "fuel", "garbage", "girls", "gorilla", "grass", "heart", "hole", "horse", "james", + "journey", "kansas", "keane", "low", "lush", "moon", "mountain", "oasis", "pond", "portal", + "prince", "pulp", "queen", "rain", "ride", "river", "rush", "sky", "smoke", "snow", "spirit", + "star", "steel", "stone", "sun", "texas", "toe", "train", "war", "water", "wire", "yes", +]; + +fn is_very_common(token: &str) -> bool { + VERY_COMMON_NAMES.contains(&token.to_ascii_lowercase().as_str()) +} + +/// The tier for an artist name. +pub fn news_tier(artist: &str) -> NewsTier { + let tokens: Vec<&str> = artist.split_whitespace().filter(|t| !t.is_empty()).collect(); + match tokens.len() { + 0 | 1 => { + let t = tokens.first().copied().unwrap_or(""); + if is_very_common(t) { NewsTier::VeryCommon } else { NewsTier::Single } + } + _ => { + if tokens.iter().all(|t| is_very_common(t)) { + NewsTier::VeryCommon + } else { + NewsTier::Distinctive + } + } + } +} + +/// Publisher hosts whose presence is evidence AGAINST a music story. Negative +/// classes are far more enumerable than a music-press allowlist would be — of +/// fourteen genuine `toe` items only one sits on any obvious list — so hosts +/// are only ever a nudge, never a gate. +const OFF_TOPIC_HOSTS: &[&str] = &[ + "legacy.com", "dignitymemorial.com", "echovita.com", "tributearchive.com", + "wwd.com", "footwearnews.com", "hypebeast.com", "pagesix.com", "vogue.com", + "gamesradar.com", "ign.com", "polygon.com", "kotaku.com", "pcgamer.com", "eurogamer.net", + "chowhound.com", "allrecipes.com", "tastingtable.com", "thedailymeal.com", + "espn.com", "realtor.com", "zillow.com", +]; +const MUSIC_HOSTS: &[&str] = &[ + "pitchfork.com", "stereogum.com", "brooklynvegan.com", "consequence.net", "nme.com", + "rollingstone.com", "spin.com", "billboard.com", "thelineofbestfit.com", "clashmusic.com", + "diymag.com", "loudersound.com", "alternativepress.com", "paste.com", "exclaim.ca", + "thequietus.com", "residentadvisor.net", "ra.co", "faroutmagazine.co.uk", "mojo4music.com", +]; + +/// Verbs a band GOVERNS. Kept deliberately narrow: adding "brings"/"opens" +/// admitted "bike ride brings history, music and family activities" and +/// "Frog Pond opens 16th season". +const CORE_VERBS: &[&str] = &[ + "announce", "announces", "announced", "share", "shares", "shared", "release", "releases", + "released", "drop", "drops", "dropped", "unveil", "unveils", "unveiled", "reveal", "reveals", + "tease", "teases", "teased", "detail", "details", "confirm", "confirms", "confirmed", + "debut", "debuts", "premiere", "premieres", "preview", "previews", "stream", "streams", + "perform", "performs", "headline", "headlines", "tour", "tours", "remix", "remixes", + "reissue", "reissues", "sign", "signs", "cancel", "cancels", "postpone", "postpones", + "reunite", "reunites", "reunited", "add", "adds", "plot", "plots", "map", +]; +const SOFT_VERBS: &[&str] = &[ + "return", "returns", "launch", "launches", "join", "joins", "take", "takes", "bring", + "brings", "make", "makes", "open", "opens", "set", "sets", "play", "plays", "go", "goes", +]; +const ROLE_NOUNS: &[&str] = &[ + "band", "bands", "group", "duo", "trio", "quartet", "singer", "rapper", "outfit", "rockers", + "frontman", "frontwoman", "drummer", "guitarist", "bassist", "producer", "act", "collective", +]; +const WEAK_ROLES: &[&str] = &["star", "stars", "legend", "legends", "icon", "icons"]; +const MUSIC_NOUNS: &[&str] = &[ + "album", "albums", "ep", "eps", "lp", "single", "singles", "song", "songs", "track", + "tracks", "tour", "shows", "show", "gig", "gigs", "review", "interview", "video", "festival", + "vinyl", "reissue", "record", "records", "setlist", "concert", "residency", +]; +/// Heads that turn a name into a PLACE or an EVENT rather than a band. +const ENTITY_HEADS: &[&str] = &[ + "park", "lake", "colony", "county", "hall", "studio", "project", "fest", "festival", "day", + "days", "river", "jam", "lily", "market", "club", "school", "church", "arts", "trail", + "backlot", "road", "street", "lane", "farm", "inn", "resort", "camp", "beach", "bay", + "creek", "valley", "ridge", "grove", "village", "township", "estates", "gardens", +]; +const OBIT_MARKERS: &[&str] = &[ + "obitu", "passed away", "funeral", "found dead", "in loving memory", "celebration of life", +]; + +fn words(s: &str) -> Vec { + s.split(|c: char| !c.is_alphanumeric() && c != '\'') + .filter(|w| !w.is_empty()) + .map(|w| w.to_ascii_lowercase()) + .collect() +} + +/// The name forms a headline may legitimately use. +fn name_variants(artist: &str) -> Vec { + let mut out = vec![artist.to_string()]; + if artist.contains('&') { + out.push(artist.replace('&', "and")); + } + if artist.to_ascii_lowercase().contains(" and ") { + out.push(artist.replacen(" and ", " & ", 1)); + } + let toks: Vec<&str> = artist.split_whitespace().collect(); + if toks.len() >= 4 { + // "King Gizzard & The Lizard Wizard" → "King Gizzard". Without this the + // hard gate silently drops genuine stories that use the short form. + out.push(toks[..2].join(" ")); + } + out.sort(); + out.dedup(); out } +/// Byte offsets of every whole-token occurrence of `needle` in `hay`, matched +/// case-insensitively on token boundaries. A trailing `'s` still counts — +/// without that, "King Gizzard's Stu Mackenzie" fails the hard gate. +fn token_occurrences(hay: &str, needle: &str) -> Vec<(usize, usize)> { + let h = hay.to_ascii_lowercase(); + let n = needle.to_ascii_lowercase(); + if n.is_empty() { + return Vec::new(); + } + let mut out = Vec::new(); + let mut from = 0; + while let Some(rel) = h[from..].find(&n) { + let start = from + rel; + let end = start + n.len(); + let before_ok = start == 0 + || !h[..start].chars().next_back().is_some_and(|c| c.is_alphanumeric()); + let after = h[end..].chars().next(); + let after_ok = match after { + None => true, + Some(c) if !c.is_alphanumeric() => true, + // allow the possessive "'s" which the char scan above already let + // through as an apostrophe; a bare alphanumeric is a different word + _ => false, + }; + if before_ok && after_ok { + out.push((start, end)); + } + from = end.max(start + 1); + } + out +} + +/// Is the character immediately abutting this span an ASCII hyphen? That is +/// what makes "peep-toe", "toe-tapping", "head-to-toe" and "Chorn-Pond" not +/// the band. Spaced en-dashes ("toe – 'NOW I SEE THE LIGHT'") are a different +/// character and are correctly ignored. +fn hyphen_abuts(hay: &str, start: usize, end: usize) -> bool { + let before = hay[..start].chars().next_back() == Some('-'); + let after = hay[end..].chars().next() == Some('-'); + before || after +} + +/// Title Case headlines capitalise everything, so an all-lowercase occurrence +/// carries no information there. +fn is_title_case(title: &str) -> bool { + let long: Vec<&str> = title + .split_whitespace() + .filter(|w| w.chars().filter(|c| c.is_alphabetic()).count() > 3) + .collect(); + if long.len() < 3 { + return true; // too short to tell; do not use the case rule + } + let capped = long + .iter() + .filter(|w| w.chars().next().is_some_and(char::is_uppercase)) + .count(); + capped * 10 > long.len() * 6 +} + +/// Score a headline for being about `artist`. `None` means rejected outright. +/// +/// See research/news/PLAN.md §2 for the derivation of every weight and for the +/// measurements behind the two-part decision at the end. +pub fn news_relevance(item: &ArtistNewsItem, artist: &str, tier: NewsTier) -> Option { + let title = item.title.as_str(); + let lower = title.to_ascii_lowercase(); + let variants = name_variants(artist); + + // ---- hard gate 1: the name must appear as a whole token run ---------- + let mut hits: Vec<(usize, usize)> = Vec::new(); + for v in &variants { + hits.extend(token_occurrences(title, v)); + } + if hits.is_empty() { + return None; + } + + // ---- hard gate 2: every occurrence is inside a hyphenated compound ---- + if hits.iter().all(|&(s, e)| hyphen_abuts(title, s, e)) { + return None; + } + hits.retain(|&(s, e)| !hyphen_abuts(title, s, e)); + + // ---- hard gate 3: canonical capitalisation vs an all-lowercase hit ---- + let name_is_capitalised = artist.chars().next().is_some_and(char::is_uppercase); + if name_is_capitalised && !is_title_case(title) { + let any_cased = hits + .iter() + .any(|&(s, e)| title[s..e].chars().next().is_some_and(char::is_uppercase)); + if !any_cased { + return None; + } + } + + let has = |list: &[&str], w: &str| list.contains(&w); + + // ---- document-level signals (counted once) --------------------------- + let name_tokens = artist.split_whitespace().count(); + let mut doc = match name_tokens { + 0 | 1 => 0, + 2 => 2, + _ => 4, + }; + if !item.host.is_empty() { + if OFF_TOPIC_HOSTS.iter().any(|h| item.host == *h || item.host.ends_with(&format!(".{h}"))) + { + doc -= 6; + } else if MUSIC_HOSTS.iter().any(|h| item.host == *h) { + doc += 2; + } + } + if OBIT_MARKERS.iter().any(|m| lower.contains(m)) { + doc -= 6; + } + if title.contains('"') || title.contains('\u{201c}') || title.contains('\u{2018}') { + doc += 1; + } + + // ---- occurrence-level signals — keep the MAX over occurrences -------- + // A headline may name the artist several ways; only one of them has to + // carry the evidence. + let mut best_occ = i32::MIN; + for &(s, e) in &hits { + let mut occ = 0i32; + let before_toks = words(&title[..s]); + let after_toks = words(&title[e..]); + let prev = before_toks.last().map(String::as_str); + let next = after_toks.first().map(String::as_str); + let next2 = after_toks.get(1).map(String::as_str); + // "within 2 tokens" is a DISTANCE, so it reaches index 2. Checking only + // the first two positions lost "toe Announces October Shows in Osaka", + // where the music noun is the third word after the name. + let near: Vec<&str> = after_toks.iter().take(3).map(String::as_str).collect(); + + if next.is_some_and(|w| has(CORE_VERBS, w)) { + occ += 3; + } else if next.is_some_and(|w| has(SOFT_VERBS, w)) { + occ += 1; + } + // possessive followed by a music noun within two tokens + if title[e..].starts_with('\u{2019}') || title[e..].starts_with('\'') { + if next.is_some_and(|w| has(MUSIC_NOUNS, w)) || next2.is_some_and(|w| has(MUSIC_NOUNS, w)) + { + occ += 3; + } + } + if prev.is_some_and(|w| has(ROLE_NOUNS, w)) || next.is_some_and(|w| has(ROLE_NOUNS, w)) { + occ += 3; + } else if prev.is_some_and(|w| has(WEAK_ROLES, w)) || next.is_some_and(|w| has(WEAK_ROLES, w)) + { + occ += 1; + } + if near.iter().any(|w| has(MUSIC_NOUNS, w)) { + occ += 2; + } + // Head of the headline — or head of what follows a masthead marker, + // which is the same claim: "Review: toe, 'NOW I SEE THE LIGHT'". + // char_indices, not `rfind(..) + 1`: an en/em dash is three bytes, so + // adding one lands mid-character and slicing panics. In an async Tauri + // command that panic surfaces as a promise that never resolves, which + // is why only artists whose name sits mid-headline hung. + let after_marker = title[..s] + .char_indices() + .rev() + .find(|(_, ch)| matches!(ch, ':' | '\u{2013}' | '\u{2014}')) + .is_some_and(|(c, ch)| title[c + ch.len_utf8()..s].trim().is_empty()); + if before_toks.is_empty() || after_marker { + occ += 2; + } + // " — " / ": " masthead convention + let tail = title[e..].trim_start(); + if tail.starts_with(':') || tail.starts_with('\u{2013}') || tail.starts_with('\u{2014}') { + occ += 2; + } + // "Review:" / "Listen:" prefix markers + if let Some(colon) = title.find(':') { + if colon < 40 { + let head = title[..colon].to_ascii_lowercase(); + if ["review", "interview", "premiere", "listen", "watch", "feature", "video"] + .iter() + .any(|m| head.contains(m)) + { + occ += 2; + } + } + } + // ---- negative: the name is really part of a longer proper name ---- + // Bare-whitespace adjacency ONLY. Treating any adjacent capitalised + // token as an extension wrongly kills "Review: toe, …" and + // "Pond 'Terrestrials' Review". + if s > 0 { + let gap = &title[..s]; + if gap.ends_with(' ') { + let prev_raw = gap.trim_end().rsplit(' ').next().unwrap_or(""); + let prev_cap = prev_raw.chars().next().is_some_and(char::is_uppercase); + let sentence_initial = gap.trim_end().is_empty(); + let known = prev.is_some_and(|w| { + has(CORE_VERBS, w) || has(SOFT_VERBS, w) || has(ROLE_NOUNS, w) + || has(WEAK_ROLES, w) || has(MUSIC_NOUNS, w) + }); + if prev_cap && !known && !prev_raw.contains('-') { + occ -= if sentence_initial { 2 } else { 4 }; + } + // "Amanda M. Pond" + if prev_raw.len() == 2 && prev_raw.ends_with('.') { + occ -= 5; + } + } + } + if next.is_some_and(|w| has(ENTITY_HEADS, w)) { + occ -= 4; + } else if next2.is_some_and(|w| has(ENTITY_HEADS, w)) { + occ -= 3; + } + // "across the Pond", "over the pond" + if before_toks.len() >= 2 { + let a = before_toks[before_toks.len() - 2].as_str(); + let b = before_toks[before_toks.len() - 1].as_str(); + if b == "the" && ["across", "over", "from", "in", "into", "beyond"].contains(&a) { + occ -= 6; + } + } + if prev.is_some_and(|w| ["a", "an", "the", "to"].contains(&w)) && !artist.starts_with("The ") + { + occ -= 3; + } + best_occ = best_occ.max(occ); + } + if best_occ == i32::MIN { + return None; + } + + let total = doc + best_occ; + // The occurrence floor is the whole filter. At a single combined threshold + // every false positive scored at threshold from document-level signals + // alone — "Every song on the 'Tip Toe' soundtrack" is a music-press host + // plus a quoted title plus a music noun and zero evidence at the name. + // Publisher reputation must never admit an item on its own. + if total >= tier.threshold() && best_occ >= tier.min_occurrence() { + Some(total) + } else { + None + } +} + +/* ============================ news dedup ================================== + * + * Collapsing several outlets' reports of ONE story. Runs after the filter and + * before the cap. See research/news/PLAN.md §3. + * + * Two counter-intuitive choices, both measured: + * - CONTAINMENT, not Jaccard. Headline lengths are wildly asymmetric — + * Stereogum's six-token 'King Gizzard & The Lizard Wizard – "Alien Metal"' + * against a thirty-token version of the same story — and Jaccard's union + * denominator punishes exactly that (F1 0.61 vs 0.44). + * - KEEP the artist-name tokens. Stripping them starves a six-token headline + * to two and was measured to lose real merges and invent false ones. The + * name is a uniform offset across every pair in a per-artist feed, so it + * moves the operating point rather than the ranking — which the raised + * 0.6 threshold and the non-name-overlap clause compensate for. + */ + +const NEWS_STOP: &[&str] = &[ + "a", "an", "the", "of", "to", "in", "on", "at", "for", "with", "and", "or", "from", "is", + "are", "was", "were", "be", "been", "by", "as", "it", "its", "this", "that", "new", "now", + "has", "have", "had", "will", "their", "his", "her", "they", "he", "she", "you", "your", + "our", "says", "say", "said", "more", "than", "then", "but", "not", "out", "up", "off", + "over", "into", "about", "after", "before", "some", "all", "how", "why", "what", "when", + "who", "his", "hers", "via", "amp", +]; + +/// Cap applied AFTER filtering and dedup. The frontend's band asks for up to +/// `bandCols * 3` items, so the old 12 starved a wide window. +const NEWS_KEEP: usize = 40; + +/// Score, reject, collapse, cap — the whole pipeline over one parsed feed. +/// +/// If the filter empties the list, that is the answer: an empty row is +/// strictly better than the unfiltered feed, which for an ambiguously named +/// artist is 100% junk. There is deliberately no fall back to unfiltered. +fn filter_news(raw: Vec, artist: &str, started: Instant) -> Vec { + let fetched = raw.len(); + let tier = news_tier(artist); + let mut scored: Vec<(ArtistNewsItem, i32)> = raw + .into_iter() + .filter_map(|item| news_relevance(&item, artist, tier).map(|s| (item, s))) + .collect(); + let kept = scored.len(); + // Google's relevance order is the tiebreaker worth preserving, so only + // sort by date once the set is final. + let mut out = dedup_news(std::mem::take(&mut scored), artist); + let deduped = out.len(); + // Sort BEFORE the quota so the two survivors of each story are its newest, + // and quota BEFORE the cap so the forty slots are forty stories. + out.sort_by_key(|item| -item.date); + let mut out = news_story_quota(out, artist); + let storied = out.len(); + out.truncate(NEWS_KEEP); + // "fetched 100, kept 0" and "fetched 0" must never read the same — the + // trap already recorded for MusicBrainz probing. + stage_trace( + "artist-news", + started, + &format!( + "{artist} tier={tier:?} fetched={fetched} kept={kept} deduped={deduped} storied={storied}" + ), + ); + out +} + +/// Comparable token set for one headline. The " - Publisher" suffix is already +/// stripped by `parse_news_rss`; do not re-implement it here. +fn news_signature(title: &str) -> std::collections::BTreeSet { + let folded = title + .replace('\u{2019}', "'") + .replace(['\u{201c}', '\u{201d}'], "\"") + .replace(['\u{2013}', '\u{2014}'], "-") + .replace('&', " and "); + folded + .split(|c: char| !c.is_alphanumeric()) + .filter(|w| w.len() > 2) + .map(|w| w.to_ascii_lowercase()) + .filter(|w| !NEWS_STOP.contains(&w.as_str())) + .collect() +} + +/// Collapse near-duplicate reports, keeping the best-scoring member of each +/// cluster and recording how many others it stands for. +/// +/// Leader clustering, NOT union-find: union-find at a comparable threshold +/// chained 68 of 93 King Gizzard items — a Nonagon Infinity review and a +/// Phantom Island review included — into a single "story". +fn dedup_news(items: Vec<(ArtistNewsItem, i32)>, artist: &str) -> Vec { + let name_toks: std::collections::BTreeSet = news_signature(artist); + let sigs: Vec<_> = items.iter().map(|(i, _)| news_signature(&i.title)).collect(); + + // leader index -> members + let mut clusters: Vec<(usize, Vec)> = Vec::new(); + 'item: for i in 0..items.len() { + for (leader, members) in clusters.iter_mut() { + let (a, b) = (&sigs[*leader], &sigs[i]); + let inter: Vec<&String> = a.intersection(b).collect(); + let (da, db) = (items[*leader].0.date, items[i].0.date); + + // Google truncates syndicated copies with an ellipsis, which is + // exactly the GamesRadar+/inkl.com pair. + let ta = items[*leader].0.title.trim_end_matches(['\u{2026}', '.', ' ']); + let tb = items[i].0.title.trim_end_matches(['\u{2026}', '.', ' ']); + let prefix = ta.len() >= 24 + && tb.len() >= 24 + && (ta.starts_with(tb) || tb.starts_with(ta)); + + // `rfc2822_secs` yields 0 on a parse failure, so two unparseable + // items would otherwise show |delta| = 0 and always pass the window. + let dated = da != 0 && db != 0 && (da - db).abs() <= 2 * 86_400; + let denom = a.len().min(b.len()).max(5) as f32; + let containment = inter.len() as f32 / denom; + let non_name = inter.iter().filter(|t| !name_toks.contains(**t)).count(); + + let same = prefix + || (dated && containment >= 0.6 && inter.len() >= 3 && non_name >= 2); + if same { + members.push(i); + continue 'item; + } + } + clusters.push((i, vec![i])); + } + + let mut out = Vec::new(); + for (_, members) in clusters { + // Survivor = best score; tie-break to the earliest report. + let best = *members + .iter() + .min_by(|&&x, &&y| { + items[y].1 + .cmp(&items[x].1) + .then_with(|| items[x].0.date.cmp(&items[y].0.date)) + }) + .expect("cluster is never empty"); + let mut item = items[best].0.clone(); + item.others = (members.len() - 1) as u32; + out.push(item); + } + out +} + +/* ======================= per-story display quota ========================== + * + * What dedup cannot reach. Leader clustering collapses REWORDINGS of one + * report, but it leaves stragglers: twelve outlets covering the "Alien Metal" + * announcement produced one 11-member cluster plus six loners that are similar + * to each other and not to the leader that already absorbed the rest. Measured + * on the live feed, seven of the visible twelve slots were that one record. + * + * The remedy is NOT a lower dedup threshold. Lexical similarity merges + * unrelated stories long before it catches rewrites — Cleveland's "Beloved band + * pulls its music from major streaming service" and InsideHook's "Scammers + * Share Fake King Gizzard Songs on Spotify" are the same story and share + * literally zero content words. Closing that gap needs sentence embeddings. + * See research/news/PLAN.md §3. + * + * So bound the damage instead: at most two items may name the same WORK. The + * key is the album or song title the headline is about, which syndicated + * coverage repeats verbatim even when it shares no other vocabulary. Headlines + * that name no work are never dropped — an artist's feed can shrink but can + * never be emptied by this. + */ + +/// Nouns after which a title is a work title. "toe Announces October Shows" +/// carries no work; "Detail New Album Alien Metal" does. +const WORK_CUES: &[&str] = &[ + "album", "albums", "lp", "ep", "single", "singles", "track", "song", "mixtape", "record", +]; +/// Words that are never a work title on their own. Without these, "…Album +/// Review" keys as `review` and "…New Song Next Week" as `next week`, and two +/// unrelated headlines share a story. +const GENERIC_TITLE_WORDS: &[&str] = &[ + "review", "reviews", "interview", "interviews", "stream", "listen", "watch", "premiere", + "premieres", "video", "videos", "news", "tour", "tours", "dates", "list", "setlist", "week", + "weekend", "day", "today", "tomorrow", "release", "releases", "single", "singles", "album", + "albums", "ep", "lp", "song", "songs", "track", "tracks", "next", "first", "new", "out", + "now", "more", +]; +/// A capitalised function word does not continue a title: "New Song With Cindy +/// Lee" is a credit, not a work called "With Cindy Lee". +const RUN_STOP_WORDS: &[&str] = + &["with", "from", "by", "featuring", "feat", "and", "for", "on", "in", "at", "to", "of", "via", "ft"]; + +/// Every quoted span in a headline, as (contents, byte range covering the +/// quotes themselves). +/// +/// Two rules earn their keep. A straight quote only OPENS at a word boundary — +/// otherwise a possessive ("Ware's") opens a span that closes on the next real +/// quote and misaligns every span after it. And a curly opener may close on its +/// straight twin, because real feeds mix them inside one headline +/// (`Produce ‘The Gold Album'`). +fn quoted_spans(title: &str) -> Vec<(String, usize, usize)> { + let chars: Vec<(usize, char)> = title.char_indices().collect(); + let mut out = Vec::new(); + let mut i = 0; + while i < chars.len() { + let (idx, c) = chars[i]; + let closers: &[char] = match c { + '\u{2018}' => &['\u{2019}', '\''], + '\u{201c}' => &['\u{201d}', '"'], + '"' => &['"'], + '\'' => &['\''], + _ => { + i += 1; + continue; + } + }; + let curly = matches!(c, '\u{2018}' | '\u{201c}'); + let opens = curly || i == 0 || !chars[i - 1].1.is_alphanumeric(); + let mut advance = 1; + if opens { + let closed = (i + 1..chars.len()).find(|&j| { + closers.contains(&chars[j].1) + && (curly || j + 1 >= chars.len() || !chars[j + 1].1.is_alphanumeric()) + }); + if let Some(j) = closed { + let (close_idx, close_ch) = chars[j]; + let inner = title[idx + c.len_utf8()..close_idx].to_string(); + out.push((inner, idx, close_idx + close_ch.len_utf8())); + advance = j - i + 1; + } + } + i += advance; + } + out +} + +/// Normalise a candidate work title into a comparison key, or reject it. +/// +/// Artist-name tokens and a leading article come off, which is what unifies +/// "Weezer: The Gathering" with "The Gathering" — five headlines about one +/// tour that would otherwise split into two keys and pass the quota twice. +fn work_key(raw: &str, name_toks: &HashSet) -> Option { + let mut toks: Vec = raw + .split(|c: char| !c.is_alphanumeric()) + .filter(|w| !w.is_empty()) + .map(|w| w.to_ascii_lowercase()) + .collect(); + // A span this long is speech, not a title: "It's 5 days since they released + // new music – was starting to think we'd never hear from them again". + if toks.len() > 8 { + return None; + } + toks.retain(|w| !name_toks.contains(w)); + while matches!(toks.first().map(String::as_str), Some("the" | "a" | "an")) { + toks.remove(0); + } + if toks.is_empty() || toks.len() > 6 { + return None; + } + if toks + .iter() + .all(|w| GENERIC_TITLE_WORDS.contains(&w.as_str()) || NEWS_STOP.contains(&w.as_str())) + { + return None; + } + let key = toks.join(" "); + (key.len() >= 3).then_some(key) +} + +fn bare_word(w: &str) -> String { + w.chars().filter(|c| c.is_alphanumeric()).collect::().to_ascii_lowercase() +} + +/// Whitespace-separated words with their byte offsets. `split_whitespace` +/// alone loses the offsets, and the offsets are what tell a work cue inside a +/// quoted span from one outside it. +fn offset_words(s: &str) -> Vec<(usize, &str)> { + let mut out = Vec::new(); + let mut start: Option = None; + for (i, c) in s.char_indices() { + if c.is_whitespace() { + if let Some(at) = start.take() { + out.push((at, &s[at..i])); + } + } else if start.is_none() { + start = Some(i); + } + } + if let Some(at) = start { + out.push((at, &s[at..])); + } + out +} + +/// The last whole word before a byte offset. +fn word_before(title: &str, at: usize) -> String { + title[..at] + .split(|c: char| !c.is_alphanumeric()) + .rfind(|w| !w.is_empty()) + .unwrap_or("") + .to_ascii_lowercase() +} + +/// The work a headline is about — an album or song title, normalised — or +/// `None` when it names none. `None` is common and is not a failure: a tour +/// announcement or an interview is its own story and is never quota'd. +pub fn news_story_key(title: &str, artist: &str) -> Option { + let name_toks: HashSet = artist + .split(|c: char| !c.is_alphanumeric()) + .filter(|w| !w.is_empty()) + .map(|w| w.to_ascii_lowercase()) + .collect(); + let raw_spans = quoted_spans(title); + let keyed: Vec<(String, usize)> = raw_spans + .iter() + .filter_map(|(raw, at, _)| work_key(raw, &name_toks).map(|k| (k, *at))) + .collect(); + + // 1. a quoted span right after a work cue IS the work — this outranks a + // bare quote, which is as often speech ("'We are truly doomed'"). + if let Some((key, _)) = keyed + .iter() + .find(|(_, at)| WORK_CUES.contains(&word_before(title, *at).as_str())) + { + return Some(key.clone()); + } + + // 2. an unquoted Title Case run after a work cue, closed by end of headline + // or by `:` `,` or a dash. The closing requirement is what keeps + // "Fueling Album Speculation [Watch]" from keying as a record. + let words = offset_words(title); + for i in 0..words.len() { + let (at, word) = words[i]; + if !WORK_CUES.contains(&bare_word(word).as_str()) { + continue; + } + // A cue INSIDE a quote is part of a title, not an introduction to one: + // `A “Sweater Song” Homage` was keying as the work "homage". + if raw_spans.iter().any(|&(_, s, e)| at >= s && at < e) { + continue; + } + let mut run: Vec<&str> = Vec::new(); + let mut j = i + 1; + let mut closed = false; + while j < words.len() { + let w = words[j].1; + if !w.chars().next().is_some_and(|c| c.is_uppercase() || c.is_numeric()) { + break; + } + if RUN_STOP_WORDS.contains(&bare_word(w).as_str()) { + break; + } + run.push(w); + j += 1; + if w.ends_with([':', ',', '\u{2013}', '\u{2014}']) { + closed = true; + break; + } + } + if run.is_empty() { + continue; + } + if !closed { + closed = + j >= words.len() || words[j].1.starts_with([':', ',', '\u{2013}', '\u{2014}']); + } + if !closed { + continue; + } + if let Some(key) = work_key(&run.join(" "), &name_toks) { + return Some(key); + } + } + + // 3. any other quoted span + keyed.into_iter().next().map(|(key, _)| key) +} + +/// At most this many headlines about one work. Two rather than one: a record +/// that is genuinely in the news deserves more than a single line, and the two +/// kept are usually the announcement and the release. +const NEWS_STORY_MAX: usize = 2; + +/// Keep at most `NEWS_STORY_MAX` items per work, in the order given — so +/// calling this on a date-sorted list keeps the newest of each story. +pub fn news_story_quota(items: Vec, artist: &str) -> Vec { + let mut seen: HashMap = HashMap::new(); + items + .into_iter() + .filter(|item| match news_story_key(&item.title, artist) { + None => true, + Some(key) => { + let n = seen.entry(key).or_insert(0); + *n += 1; + *n <= NEWS_STORY_MAX + } + }) + .collect() +} + /// Wikimedia hosts share one UA policy and one rate limiter, so they share /// this client's compliant UA, pacer, 429 handling and disk cache. Wikidata is /// included: it is the identity anchor that turns a MusicBrainz artist into @@ -839,6 +1665,21 @@ pub struct ArtistNewsItem { pub url: String, pub source: String, pub date: i64, + /// Publisher host from `` — "wwd.com", "pitchfork.com". + /// The element's TEXT is a display name ("GamesRadar+") and is not a host; + /// the attribute is, and it is present on every item. + /// + /// `serde(default)` is load-bearing: cached items are read back with + /// `from_value::>(..).ok()`, and `.ok()` swallows a + /// schema mismatch — a required new field would silently invalidate every + /// cached entry and fire a cold refetch for every follow at once, against + /// a rate-limited endpoint. + #[serde(default)] + pub host: String, + /// How many near-duplicate reports of this same story were collapsed into + /// it, so the UI can say "+N more outlets" instead of dropping them mutely. + #[serde(default)] + pub others: u32, } /// Lightweight artist identity + artwork response used to paint the profile @@ -1503,7 +2344,13 @@ impl MetadataClient { // whole point of using them -- an unpaid ordering must not cost a // network round on every revisit, which is exactly what TheAudioDB // suffers for NOT being here. - || url.starts_with("https://ws.audioscrobbler.com/"); + || url.starts_with("https://ws.audioscrobbler.com/") + // `artist_news` has always written and read a disk entry here, but + // news URLs never qualified, so `disk_cache_path` returned None and + // the whole branch was dead: every restart refetched every follow + // against a rate-limited endpoint. The 24h TTL this reader enforces + // is the outer bound; news carries its own 6h stamp inside the body. + || url.starts_with("https://news.google.com/rss/"); if !qualifies { return None; } @@ -7041,8 +7888,7 @@ impl MetadataClient { return Vec::new(); } }; - let items = parse_news_rss(&text); - stage_trace("artist-news", started, artist); + let items = filter_news(parse_news_rss(&text), artist, started); if let Some(path) = self.disk_cache_path(&url) { let body = serde_json::json!({ "at": now_secs(), "items": items }); write_disk_cache(&path, &url, &body); diff --git a/crates/metadata/tests/news_relevance.rs b/crates/metadata/tests/news_relevance.rs new file mode 100644 index 0000000..29975e3 --- /dev/null +++ b/crates/metadata/tests/news_relevance.rs @@ -0,0 +1,136 @@ +//! The news filter, measured against real headlines. +//! +//! Every case here was taken from a live Google News feed for one of Kyle's +//! followed artists on 2026-08-05, and hand-labelled. The four `reject_*` +//! cases are the ones he reported. Treat these numbers as tuning-set +//! performance: the weights were chosen with this corpus in view, so a new +//! bug report should arrive as a new row here rather than as a tweak to a +//! weight that is already fitted. +use metadata::{news_relevance, news_tier, ArtistNewsItem, NewsTier}; + +fn item(title: &str, host: &str) -> ArtistNewsItem { + ArtistNewsItem { + title: title.to_string(), + url: String::new(), + source: String::new(), + date: 1_785_000_000, + host: host.to_string(), + others: 0, + } +} +fn keeps(title: &str, host: &str, artist: &str) -> bool { + news_relevance(&item(title, host), artist, news_tier(artist)).is_some() +} + +#[test] +fn tiers_are_assigned_by_dictionary_not_by_length() { + // "Weezer" is six characters and perfectly safe; deciding riskiness by + // length demoted it to the strict path and cost four genuine items. + assert_eq!(news_tier("Weezer"), NewsTier::Single); + assert_eq!(news_tier("Wand"), NewsTier::Single); + assert_eq!(news_tier("toe"), NewsTier::VeryCommon); + assert_eq!(news_tier("Pond"), NewsTier::VeryCommon); + assert_eq!(news_tier("King Gizzard & The Lizard Wizard"), NewsTier::Distinctive); + assert_eq!(news_tier("Camping In Alaska"), NewsTier::Distinctive); +} + +// ---------------------------------------------------------------- rejects + +#[test] +fn rejects_hyphenated_compounds() { + // Hard gate 2 — every occurrence abuts a hyphen. + assert!(!keeps( + "Charli XCX Launches Her 'Music, Fashion, Film' Era in Patent Peep-toe Mules", + "wwd.com", + "toe", + )); + assert!(!keeps( + "Hyperpop standout Underscores makes music that can go toe-to-toe with the biggest pop acts", + "chicagoreader.com", + "toe", + )); + assert!(!keeps( + "The Toe-Tapping Music Genres You're Most Likely To Hear In Trader Joe's", + "chowhound.com", + "toe", + )); +} + +#[test] +fn rejects_the_stellar_blade_case() { + // The occurrence floor earning its keep: this has a music noun and a + // quoted title, so document-level signals alone would have passed it. + assert!(!keeps( + "Stellar Blade: Blood Rain studio CEO defends AI music video because Evie was originally \"created from head to toe\" by the devs", + "gamesradar.com", + "toe", + )); +} + +#[test] +fn rejects_the_obituary() { + assert!(!keeps( + "Obituary information for Amanda M. Pond", + "cody-whitefuneralhome.com", + "Pond", + )); +} + +#[test] +fn rejects_places_and_surnames() { + for (title, host) in [ + ("Rock the Dock 2026 bringing live music to Goose Pond Colony", "rocketcitynow.com"), + ("Jud Caswell in Round Pond", "lcnme.com"), + ("Bird Song Project sculpture trail opens at Inlet Pond County Park", "northforksun.com"), + ("On the record with Aaron Pond and the People's Music Supply", "broadstreetreview.com"), + ("Miami Music Week at Toe Jam Backlot", "timeout.com"), + ] { + let artist = if title.contains("Toe") { "toe" } else { "Pond" }; + assert!(!keeps(title, host, artist), "should have rejected: {title}"); + } +} + +// ------------------------------------------------------------------ keeps + +#[test] +fn keeps_genuine_items_for_a_very_common_name() { + assert!(keeps("toe Announces October Shows in Osaka and Tokyo", "niew.jp", "toe")); + assert!(keeps("Pond Announce New Album 'Terrestrials'", "diymag.com", "Pond")); + assert!(keeps("Ride Announce Here And Now Tour For 2027", "nme.com", "Ride")); +} + +#[test] +fn keeps_a_bare_tour_announcement_with_no_music_vocabulary() { + // The false-negative the brief demands: strict enough to kill Stellar + // Blade must not kill this. + assert!(keeps("Weezer Announces 'The Gathering' Fall Tour", "rockcellarmagazine.com", "Weezer")); +} + +#[test] +fn keeps_the_short_form_and_the_possessive() { + // Without the 4+-token short-form variant and the possessive-tolerant + // gate these silently vanish. + assert!(keeps( + "King Gizzard's Stu Mackenzie on leaving Spotify", + "pitchfork.com", + "King Gizzard & The Lizard Wizard", + )); + assert!(keeps( + "King Gizzard and The Lizard Wizard debut \"Alien Metal\"", + "northerntransmissions.com", + "King Gizzard & The Lizard Wizard", + )); +} + +#[test] +fn keeps_masthead_and_review_conventions() { + assert!(keeps("Review: toe, 'NOW I SEE THE LIGHT'", "thequietus.com", "toe")); + assert!(keeps("Pond 'Terrestrials' Review", "clashmusic.com", "Pond")); +} + +#[test] +fn a_music_press_host_alone_never_admits_an_item() { + // "Every song on the 'Tip Toe' soundtrack" is a music-press host plus a + // quoted work title plus a music noun, and zero evidence at the name. + assert!(!keeps("Every song on the 'Tip Toe' soundtrack", "nme.com", "toe")); +} diff --git a/crates/metadata/tests/news_story_quota.rs b/crates/metadata/tests/news_story_quota.rs new file mode 100644 index 0000000..0aa9d46 --- /dev/null +++ b/crates/metadata/tests/news_story_quota.rs @@ -0,0 +1,194 @@ +//! The per-story display quota, measured against real headlines. +//! +//! Dedup collapses rewordings of one report; this bounds the other kind of +//! repetition — many different reports about the same record. Every headline +//! here is verbatim from a live Google News feed on 2026-08-05, where seven of +//! the twelve visible slots were King Gizzard's "Alien Metal" announcement +//! arriving from seven outlets that shared almost no vocabulary. +use metadata::{news_story_key, news_story_quota, ArtistNewsItem}; + +const GIZZ: &str = "King Gizzard & The Lizard Wizard"; + +fn key(title: &str, artist: &str) -> Option { + news_story_key(title, artist) +} +fn item(title: &str, date: i64) -> ArtistNewsItem { + ArtistNewsItem { + title: title.to_string(), + url: title.to_string(), + source: String::new(), + date, + host: String::new(), + others: 0, + } +} + +// ------------------------------------------------------------ extraction + +#[test] +fn every_alien_metal_headline_names_the_same_work() { + // Five quote it, two only capitalise it after "Album". All seven are one + // story and between them they held seven of twelve panel slots. + for title in [ + "King Gizzard & the Lizard Wizard rave on with some \u{2018}Alien Metal\u{2019}", + "King Gizzard & the Lizard Wizard announce 28th album \u{2018}Alien Metal\u{2019}", + "King Gizzard & the Lizard Wizard Share Title Track From Forthcoming LP \u{2018}Alien Metal\u{2019}", + "King Gizzard and the Lizard Wizard announce EDM album, \u{2018}Alien Metal\u{2019}", + "King Gizzard & The Lizard Wizard Announce 28th Album 'Alien Metal,' Brooklyn Rave", + "King Gizzard & The Lizard Wizard Release Title Track from Techno Album Alien Metal: Stream", + "King Gizzard & The Lizard Wizard Detail New Album Alien Metal", + ] { + assert_eq!(key(title, GIZZ).as_deref(), Some("alien metal"), "{title}"); + } +} + +#[test] +fn a_cue_adjacent_quote_outranks_an_earlier_one() { + // The album is the story; "Two Hands" is the track it leads with. + assert_eq!( + key("Pond Announce New Album 'Terrestrials': Hear \"Two Hands\"", "Pond").as_deref(), + Some("terrestrials"), + ); + assert_eq!( + key("Pond Share New Single \"Terrestrials\": Listen", "Pond").as_deref(), + Some("terrestrials"), + ); +} + +#[test] +fn a_curly_quote_may_close_on_its_straight_twin() { + // Verbatim from the feed — one headline, two different quote characters. + assert_eq!( + key("Weezer Enlists Kenny Beats to Produce \u{2018}The Gold Album'", "Weezer").as_deref(), + Some("gold album"), + ); +} + +#[test] +fn the_artist_name_and_a_leading_article_come_off_the_key() { + // Five headlines about one tour. Without this they split into "the + // gathering" and "weezer the gathering" and pass the quota twice. + for title in [ + "Weezer Announce \u{2018}The Gathering\u{2019} Arena Tour", + "Weezer Announce \u{201c}Weezer: The Gathering\u{201d}", + "'Weezer: The Gathering' tour heading to Nationwide Arena in October", + "Weezer Play Pop-Up Rooftop Gig to Mark \u{2018}The Gathering\u{2019} Tour", + ] { + assert_eq!(key(title, "Weezer").as_deref(), Some("gathering"), "{title}"); + } +} + +#[test] +fn a_work_cue_inside_a_quote_does_not_introduce_one() { + // Three headlines about the single "C.E.O.", live on 2026-08-05. The third + // quotes an older song, and the cue inside THAT quote was keying the + // headline as a work called "homage" — a third slot for one story. + for title in [ + "Weezer\u{2019}s \u{201c}C.E.O.\u{201d} Video Has Curry Barker, Celeb Cameos, & A \u{201c}Sweater Song\u{201d} Homage", + "Weezer Unveil New Single \"C.E.O.\", Expand \"Weezer: The Gathering\" with New Tour Dates", + "Weezer Go Back in Time for \u{2018}C.E.O.\u{2019} Music Video", + ] { + assert_eq!(key(title, "Weezer").as_deref(), Some("c e o"), "{title}"); + } +} + +#[test] +fn a_possessive_apostrophe_does_not_open_a_span() { + // If it did, the span would run from "'s Rivers…" to the next real quote + // and every span after it would be misaligned. + assert_eq!( + key("Weezer's Rivers Cuomo drops new single 'Shine Again'", "Weezer").as_deref(), + Some("shine again"), + ); +} + +// ------------------------------------------------------- what is NOT a work + +#[test] +fn a_generic_run_after_a_cue_is_not_a_work() { + // Each of these keyed as a "work" before the generic list existed — + // `review`, `list` and `next week` — grouping unrelated headlines. + for (title, artist) in [ + ("Panda Bear / Sonic Boom: A ? of WHEN Album Review", "Panda Bear"), + ("Weezer Concert Setlist: Discover the Average Song List", "Weezer"), + ( + "Weezer Announce North American Tour, Tease New Album, and Are Releasing a New Song Next Week", + "Weezer", + ), + ("Track Review: Paramore \u{2013} Running Out of Time", "Panda Bear"), + ] { + assert_eq!(key(title, artist), None, "{title}"); + } +} + +#[test] +fn an_unclosed_run_is_not_a_work() { + // "Album Speculation [Watch]" — a run that neither ends the headline nor + // stops at punctuation is ordinary prose. + assert_eq!( + key("Billy Strings & King Gizzard Share Video From The Studio, Fueling Album Speculation [Watch]", GIZZ), + None, + ); + // A capitalised function word ends the run: this is a credit, not a work. + assert_eq!( + key("Panda Bear Announces Tour and New Album, Shares Video for New Song With Cindy Lee: Watch", "Panda Bear"), + None, + ); +} + +#[test] +fn a_long_quote_is_speech_not_a_title() { + assert_eq!( + key("\u{201c}It\u{2019}s 5 days since they released new music \u{2013} was starting to think we\u{2019}d never hear from them again\u{201d}: King Gizzard And The Lizard Wizard\u{2019}s 24 albums in 11 years?", GIZZ), + None, + ); +} + +#[test] +fn headlines_that_name_no_work_are_never_keyed() { + // These are the majority, and they are the reason the quota can never + // empty a feed. + assert_eq!(key("King Gizzard & The Lizard Wizard Share New Single", GIZZ), None); + assert_eq!(key("King Gizzard & The Lizard Wizard Australia 2026 Tour", GIZZ), None); + assert_eq!(key("toe Announces October Shows in Osaka and Tokyo", "toe"), None); +} + +// ----------------------------------------------------------------- quota + +#[test] +fn the_quota_keeps_the_first_two_of_a_story_and_all_the_rest() { + // The list arrives newest-first, so "first two" is "the two most recent". + let titles = [ + "King Gizzard & the Lizard Wizard rave on with some \u{2018}Alien Metal\u{2019}", + "King Gizzard & The Lizard Wizard Share New Single", + "King Gizzard & The Lizard Wizard Release Title Track from Techno Album Alien Metal: Stream", + "King Gizzard & the Lizard Wizard announce 28th album \u{2018}Alien Metal\u{2019}", + "King Gizzard & The Lizard Wizard Detail New Album Alien Metal", + "King Gizzard & The Lizard Wizard Australia 2026 Tour", + ]; + let items: Vec<_> = titles + .iter() + .enumerate() + .map(|(i, t)| item(t, 1_785_000_000 - i as i64 * 86_400)) + .collect(); + let out = news_story_quota(items, GIZZ); + let kept: Vec<&str> = out.iter().map(|i| i.title.as_str()).collect(); + assert_eq!(kept, vec![titles[0], titles[1], titles[2], titles[5]]); +} + +#[test] +fn the_quota_never_empties_a_feed() { + // Pond's whole in-feed life was five reports of one record. Two survive, + // which is the point: an empty row would be worse than a repeated one. + let items: Vec<_> = [ + "Pond Return With Psych\u{2019}d Out Track \u{2018}Terrestrials\u{2019}", + "Pond Share New Single \"Terrestrials\": Listen", + "Pond Announce North American Tour Behind New Album 'Terrestrials'", + "Pond Announce New Album 'Terrestrials': Hear \"Two Hands\"", + "Pond \u{2018}Terrestrials\u{2019} Review: Less Sometimes Is Really More", + ] + .iter() + .map(|t| item(t, 1_785_000_000)) + .collect(); + assert_eq!(news_story_quota(items, "Pond").len(), 2); +} diff --git a/crates/metadata/tests/news_utf8.rs b/crates/metadata/tests/news_utf8.rs new file mode 100644 index 0000000..c3c3440 --- /dev/null +++ b/crates/metadata/tests/news_utf8.rs @@ -0,0 +1,70 @@ +//! Byte-index safety for the news scorer. +//! +//! Google's headlines are full of curly quotes and en/em dashes. Every one of +//! these panicked at some point during development because a byte offset was +//! advanced by 1 past a multi-byte character. A panic inside an async Tauri +//! command never resolves its promise, so the symptom was a hung News band — +//! and only for artists whose name appears mid-headline, which is why it was +//! invisible on the first artist tried. +use metadata::{news_relevance, news_tier, ArtistNewsItem}; + +fn score(title: &str, artist: &str) -> Option { + let item = ArtistNewsItem { + title: title.to_string(), + url: String::new(), + source: String::new(), + date: 1_785_000_000, + host: "example.com".to_string(), + others: 0, + }; + news_relevance(&item, artist, news_tier(artist)) +} + +#[test] +fn multibyte_punctuation_before_the_name_does_not_panic() { + for (title, artist) in [ + ("Listen \u{2013} toe announce a new album", "toe"), + ("Review \u{2014} Pond announce 'Terrestrials'", "Pond"), + ("Watch: toe share \u{2018}Nothing\u{2019} video", "toe"), + ("\u{201c}Terrestrials\u{201d} \u{2013} Pond return", "Pond"), + ("Interview \u{2013} King Gizzard & The Lizard Wizard on Spotify", "King Gizzard & The Lizard Wizard"), + ("\u{2013}\u{2014}\u{2018}\u{2019}\u{201c}\u{201d} toe", "toe"), + ] { + let _ = score(title, artist); // must not panic + } +} + +#[test] +fn non_ascii_headlines_score_without_panicking() { + for (title, artist) in [ + ("Elephant Gym\u{2019}s Math Rock journey from Kaohsiung", "Elephant Gym"), + ("\u{6771}\u{4EAC}: toe announce October shows", "toe"), + ("R\u{00F6}yksopp \u{2013} toe remix premiere", "toe"), + ] { + let _ = score(title, artist); + } +} + +#[test] +fn a_multibyte_dash_marker_behaves_exactly_like_an_ascii_colon() { + // The fix must not merely avoid the panic — the masthead signal has to + // still fire. Comparing the two spellings of the same headline tests that + // directly, without depending on where a tier's threshold happens to sit. + // The lead word must NOT be one of the "Review:/Listen:" prefix markers — + // a colon would then earn a second, unrelated bonus and the two spellings + // would legitimately differ. + for (colon, dash) in [ + ("Stereogum: toe announce a new album", "Stereogum \u{2013} toe announce a new album"), + ("Mojo: Pond return with 'Terrestrials'", "Mojo \u{2014} Pond return with 'Terrestrials'"), + ] { + let artist = if colon.contains("toe") { "toe" } else { "Pond" }; + assert_eq!( + score(colon, artist), + score(dash, artist), + "marker spelling changed the score for: {dash}", + ); + } + // Whether that shape clears a tier's bar is the relevance suite's job — + // it depends on the publisher host, which this file deliberately holds + // neutral so only the byte handling is under test. +}