From 8ef730848ac8698d090968f2670da685e2c5583c Mon Sep 17 00:00:00 2001 From: KeelrLeek <156356974+KeelrLeek@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:08:35 -0500 Subject: [PATCH] news: headlines about the artists you follow, anchored to the artist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No per-artist music news API exists — every product that does this aggregates and filters, so this does too. Google News RSS is the source: free, keyless, no pagination, and no contract at all, which is why it gets the most conservative pacer in the file (4 requests per 10 seconds) and a 6-hour cache on disk as well as in memory. Nothing user-facing waits on it; the home page asks after it is already drawn. The query is the QUOTED artist name plus the word "music", and the quoting is the whole design. Measured 2026-08-04 against the real feed: a bare ambiguous name returned 0 of 10 relevant items — searching "Girls" gives Indigo Girls, The Deli Girls and a school production of Mean Girls — while the same name with a corroborating token returned 8 of 8, and an unambiguous one 10 of 10. That is the identity-anchor rule the rest of the app already lives by, applied to a provider that only speaks in names: never trust the name alone. The parser is deliberately not an XML library. One provider, four flat tags inside blocks, exactly the judgment the Bing scrape made. What it does handle is what Google actually emits: entity-encoded titles including numeric forms, a " - Publication" suffix on every headline (stripped when the tail matches the tag, so the title reads as a headline rather than a citation), and RFC 2822 pubDates converted to unix seconds by a hand-rolled civil-date conversion — nothing else in the workspace needs a date crate for one field. Freshness and relevance stay OUT of here. The 45-day window and the review filter are display policy and belong to the surface that shows them; this returns what the feed said, newest first, capped at a dozen. --- crates/metadata/src/lib.rs | 225 +++++++++++++++++++++++++++++++++++++ src-tauri/src/commands.rs | 19 ++++ src-tauri/src/lib.rs | 1 + 3 files changed, 245 insertions(+) diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 46824ba..59607ee 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -184,6 +184,102 @@ fn strip_discogs_markup(profile: &str) -> String { out.split_whitespace().collect::>().join(" ") } +/// Pull the fields the news panel needs out of a Google News RSS body. +/// +/// A real XML parser is not warranted for one provider whose format is four +/// flat tags inside `` blocks — the same judgment the Bing scrape made. +/// Google encodes item titles as plain text with entities (no CDATA observed), +/// appends " - Publication" to every title, and stamps RFC 2822 pubDates. +fn parse_news_rss(body: &str) -> Vec { + fn tag(block: &str, name: &str) -> Option { + let open = format!("<{name}"); + let start = block.find(&open)?; + let start = block[start..].find('>')? + start + 1; + let end = block[start..].find(&format!(""))? + start; + Some(block[start..end].trim().to_string()) + } + fn decode(s: &str) -> String { + // The entities Google actually emits, plus numeric forms. + let mut out = s + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'"); + while let Some(start) = out.find("&#") { + let Some(len) = out[start..].find(';') else { break }; + let code = &out[start + 2..start + len]; + let ch = code + .strip_prefix('x') + .map_or_else(|| code.parse::().ok(), |h| u32::from_str_radix(h, 16).ok()) + .and_then(char::from_u32); + match ch { + Some(c) => out.replace_range(start..start + len + 1, &c.to_string()), + None => break, + } + } + out + } + /// "Mon, 28 Jul 2026 07:00:00 GMT" → unix seconds. Hand-rolled because + /// nothing else in the workspace needs a date crate. + fn rfc2822_secs(s: &str) -> i64 { + let parts: Vec<&str> = s.split_whitespace().collect(); + // [Mon,] 28 Jul 2026 07:00:00 GMT + let off = usize::from(parts.first().is_some_and(|p| p.ends_with(','))); + let (Some(day), Some(mon), Some(year), Some(time)) = ( + parts.get(off).and_then(|v| v.parse::().ok()), + parts.get(off + 1), + parts.get(off + 2).and_then(|v| v.parse::().ok()), + parts.get(off + 3), + ) else { + return 0; + }; + const MONTHS: [&str; 12] = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + ]; + let Some(month) = MONTHS.iter().position(|m| m.eq_ignore_ascii_case(mon)) else { + return 0; + }; + let mut t = time.split(':').filter_map(|v| v.parse::().ok()); + let (h, mi, sec) = ( + t.next().unwrap_or(0), + t.next().unwrap_or(0), + t.next().unwrap_or(0), + ); + // Civil date → days since epoch (Howard Hinnant's algorithm). + let month = month as i64 + 1; + let y = if month <= 2 { year - 1 } else { year }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146097 + doe - 719468; + days * 86400 + h * 3600 + mi * 60 + sec + } + + let mut out = Vec::new(); + for block in body.split("").skip(1) { + let block = block.split("").next().unwrap_or(""); + let Some(raw_title) = tag(block, "title") else { continue }; + let Some(link) = tag(block, "link") else { continue }; + let source = tag(block, "source").map(|s| decode(&s)).unwrap_or_default(); + let mut title = decode(&raw_title); + // "Headline - Publication" → "Headline" when the tail names the source. + if !source.is_empty() { + if let Some(stripped) = title.strip_suffix(&format!(" - {source}")) { + title = stripped.to_string(); + } + } + let date = tag(block, "pubDate").map(|d| rfc2822_secs(&d)).unwrap_or(0); + out.push(ArtistNewsItem { title, url: decode(&link), source, date }); + if out.len() >= 12 { + break; + } + } + out.sort_by_key(|item| -item.date); + out +} + /// Wikimedia hosts share one UA policy and one rate limiter, so they share /// this client's compliant UA, pacer, 429 handling and disk cache. Wikidata is /// included: it is the identity anchor that turns a MusicBrainz artist into @@ -332,6 +428,16 @@ const LASTFM_KEY: &str = "69f3b3201381b56903eeee31477644a2"; /// treatment Bing got — that is the one provider that broke this way. const LASTFM_RATE_WINDOW: Duration = Duration::from_secs(1); const LASTFM_RATE_BUDGET: usize = 4; +/// Google News RSS has no published budget and no API contract at all — it is +/// scraping-adjacent, exactly the situation that broke the Bing hero tier when +/// it ran unpaced. Deliberately the most conservative pacer in this file: the +/// home page's news walk is a background task with no user waiting on any +/// single request. +const NEWS_RATE_WINDOW: Duration = Duration::from_secs(10); +const NEWS_RATE_BUDGET: usize = 4; +/// Headlines move slower than the feed refresh: within one TTL a new story is +/// at most this stale, which the home page's 45-day display window absorbs. +const NEWS_TTL: Duration = Duration::from_secs(6 * 60 * 60); /// How deep to pull the scrobble table. /// /// Raised from 100 once this table also had to serve per-track play counts on @@ -724,6 +830,17 @@ pub struct ArtistLink { pub url: String, } +/// One headline about a followed artist, for the home page's news panel. +/// `date` is unix seconds; freshness policy (the 45-day cap, the review +/// filter) is the frontend's display decision, not baked in here. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ArtistNewsItem { + pub title: String, + pub url: String, + pub source: String, + pub date: i64, +} + /// Lightweight artist identity + artwork response used to paint the profile /// while the deeper discography/edition analysis continues in parallel. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -1224,6 +1341,11 @@ pub struct MetadataClient { discogs_gate: Mutex>, /// Sliding-window pacer for ws.audioscrobbler.com (see [`LASTFM_RATE_BUDGET`]). lastfm_gate: Mutex>, + /// Sliding-window pacer for news.google.com (see [`NEWS_RATE_BUDGET`]). + news_gate: Mutex>, + /// Query URL → parsed items, so a session refresh costs nothing. Disk + /// persistence rides the generic JSON cache beside it. + news_cache: Mutex)>>, /// On-disk JSON cache for slow-moving providers (see [`DISK_CACHE_TTL`]). /// `None` (the default, and every test/example) keeps behavior purely /// in-memory. @@ -1282,6 +1404,8 @@ impl MetadataClient { wiki_cooldown: Mutex::new(None), discogs_gate: Mutex::new(VecDeque::new()), lastfm_gate: Mutex::new(VecDeque::new()), + news_gate: Mutex::new(VecDeque::new()), + news_cache: Mutex::new(HashMap::new()), disk_cache_dir: None, identity_store_dir: None, fused_portraits: std::sync::Mutex::new(HashMap::new()), @@ -6827,6 +6951,107 @@ impl MetadataClient { } } + /// Hold a Google News request until its sliding window has room. + async fn news_throttle(&self) { + loop { + let wait = { + let mut window = self.news_gate.lock().await; + let now = Instant::now(); + while window + .front() + .is_some_and(|at| now.duration_since(*at) >= NEWS_RATE_WINDOW) + { + window.pop_front(); + } + if window.len() < NEWS_RATE_BUDGET { + window.push_back(now); + return; + } + match window.front() { + Some(oldest) => NEWS_RATE_WINDOW - now.duration_since(*oldest), + None => continue, + } + }; + tokio::time::sleep(wait).await; + } + } + + /// Headlines about one artist, from Google News RSS. + /// + /// The query is the QUOTED artist name plus the word `music` — measured on + /// 2026-08-04: a bare ambiguous name scored 0/10 relevant items ("Girls"), + /// the quoted name with a corroborating token scored 8/8 and an easy name + /// 10/10. Keyless and undocumented, hence the conservative pacer and the + /// default UA (the contact UA is for providers with a policy asking for + /// it; scraped surfaces see the same UA a browser fetch would send). + /// + /// Sorting, the freshness window and the review filter are display policy + /// and live in the frontend; this returns what the feed said, newest + /// first, capped at a dozen. + pub async fn artist_news(&self, artist: &str) -> Vec { + let url = format!( + "https://news.google.com/rss/search?q={}&hl=en-US&gl=US&ceid=US:en", + urlencode(&format!("\"{artist}\" music")) + ); + let now_secs = || { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + }; + { + let cache = self.news_cache.lock().await; + if let Some((at, items)) = cache.get(&url) { + if at.elapsed() < NEWS_TTL { + return items.clone(); + } + } + } + // Disk: survives a restart. The generic reader enforces the 24h cache + // TTL; news wants 6h, so the fetch time rides inside the body and is + // checked here. + if let Some(path) = self.disk_cache_path(&url) { + if let Some(body) = read_disk_cache(&path, &url) { + let fresh = body + .get("at") + .and_then(|v| v.as_u64()) + .is_some_and(|at| now_secs().saturating_sub(at) < NEWS_TTL.as_secs()); + if fresh { + if let Some(items) = body + .get("items") + .and_then(|v| serde_json::from_value::>(v.clone()).ok()) + { + let mut cache = self.news_cache.lock().await; + cache.insert(url.clone(), (Instant::now(), items.clone())); + return items; + } + } + } + } + self.news_throttle().await; + let started = Instant::now(); + let text = match self.http.get(&url).send().await { + Ok(resp) if resp.status().is_success() => resp.text().await.unwrap_or_default(), + Ok(resp) => { + net_trace(&url, &format!("news-status {}", resp.status())); + return Vec::new(); + } + Err(e) => { + net_trace(&url, &format!("news-error {e}")); + return Vec::new(); + } + }; + let items = parse_news_rss(&text); + stage_trace("artist-news", started, artist); + if let Some(path) = self.disk_cache_path(&url) { + let body = serde_json::json!({ "at": now_secs(), "items": items }); + write_disk_cache(&path, &url, &body); + } + let mut cache = self.news_cache.lock().await; + cache.insert(url, (Instant::now(), items.clone())); + items + } + /// Hold a Discogs request until its sliding window has room. async fn discogs_throttle(&self) { loop { diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index a94332d..c25687b 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -4204,6 +4204,25 @@ pub async fn unfollow_artist( Ok(()) } +// ---- home ---- + +/// Headlines for the home page's news panel, one paced Google News query per +/// followed artist. Serial on purpose: the pacer inside `artist_news` is the +/// budget, and nothing user-blocking waits on this call — the frontend +/// requests it after the page is already up. +#[tauri::command] +pub async fn home_artist_news( + state: State<'_, LibraryState>, + artists: Vec, +) -> Result>, String> { + let mut out = std::collections::HashMap::new(); + for name in artists { + let items = state.metadata.artist_news(&name).await; + out.insert(name, items); + } + Ok(out) +} + // ---- playlists ---- fn now_secs() -> i64 { diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index a511f4f..d5b93a0 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -89,6 +89,7 @@ pub fn run() { commands::followed_artists, commands::follow_artist, commands::unfollow_artist, + commands::home_artist_news, commands::list_playlists, commands::create_playlist, commands::delete_playlist,