diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 46824ba..59607ee 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -184,6 +184,102 @@ fn strip_discogs_markup(profile: &str) -> String { out.split_whitespace().collect::>().join(" ") } +/// Pull the fields the news panel needs out of a Google News RSS body. +/// +/// A real XML parser is not warranted for one provider whose format is four +/// flat tags inside `` blocks — the same judgment the Bing scrape made. +/// Google encodes item titles as plain text with entities (no CDATA observed), +/// appends " - Publication" to every title, and stamps RFC 2822 pubDates. +fn parse_news_rss(body: &str) -> Vec { + fn tag(block: &str, name: &str) -> Option { + let open = format!("<{name}"); + let start = block.find(&open)?; + let start = block[start..].find('>')? + start + 1; + let end = block[start..].find(&format!(""))? + start; + Some(block[start..end].trim().to_string()) + } + fn decode(s: &str) -> String { + // The entities Google actually emits, plus numeric forms. + let mut out = s + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'"); + while let Some(start) = out.find("&#") { + let Some(len) = out[start..].find(';') else { break }; + let code = &out[start + 2..start + len]; + let ch = code + .strip_prefix('x') + .map_or_else(|| code.parse::().ok(), |h| u32::from_str_radix(h, 16).ok()) + .and_then(char::from_u32); + match ch { + Some(c) => out.replace_range(start..start + len + 1, &c.to_string()), + None => break, + } + } + out + } + /// "Mon, 28 Jul 2026 07:00:00 GMT" → unix seconds. Hand-rolled because + /// nothing else in the workspace needs a date crate. + fn rfc2822_secs(s: &str) -> i64 { + let parts: Vec<&str> = s.split_whitespace().collect(); + // [Mon,] 28 Jul 2026 07:00:00 GMT + let off = usize::from(parts.first().is_some_and(|p| p.ends_with(','))); + let (Some(day), Some(mon), Some(year), Some(time)) = ( + parts.get(off).and_then(|v| v.parse::().ok()), + parts.get(off + 1), + parts.get(off + 2).and_then(|v| v.parse::().ok()), + parts.get(off + 3), + ) else { + return 0; + }; + const MONTHS: [&str; 12] = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + ]; + let Some(month) = MONTHS.iter().position(|m| m.eq_ignore_ascii_case(mon)) else { + return 0; + }; + let mut t = time.split(':').filter_map(|v| v.parse::().ok()); + let (h, mi, sec) = ( + t.next().unwrap_or(0), + t.next().unwrap_or(0), + t.next().unwrap_or(0), + ); + // Civil date → days since epoch (Howard Hinnant's algorithm). + let month = month as i64 + 1; + let y = if month <= 2 { year - 1 } else { year }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146097 + doe - 719468; + days * 86400 + h * 3600 + mi * 60 + sec + } + + let mut out = Vec::new(); + for block in body.split("").skip(1) { + let block = block.split("").next().unwrap_or(""); + let Some(raw_title) = tag(block, "title") else { continue }; + let Some(link) = tag(block, "link") else { continue }; + let source = tag(block, "source").map(|s| decode(&s)).unwrap_or_default(); + let mut title = decode(&raw_title); + // "Headline - Publication" → "Headline" when the tail names the source. + if !source.is_empty() { + if let Some(stripped) = title.strip_suffix(&format!(" - {source}")) { + title = stripped.to_string(); + } + } + let date = tag(block, "pubDate").map(|d| rfc2822_secs(&d)).unwrap_or(0); + out.push(ArtistNewsItem { title, url: decode(&link), source, date }); + if out.len() >= 12 { + break; + } + } + out.sort_by_key(|item| -item.date); + out +} + /// Wikimedia hosts share one UA policy and one rate limiter, so they share /// this client's compliant UA, pacer, 429 handling and disk cache. Wikidata is /// included: it is the identity anchor that turns a MusicBrainz artist into @@ -332,6 +428,16 @@ const LASTFM_KEY: &str = "69f3b3201381b56903eeee31477644a2"; /// treatment Bing got — that is the one provider that broke this way. const LASTFM_RATE_WINDOW: Duration = Duration::from_secs(1); const LASTFM_RATE_BUDGET: usize = 4; +/// Google News RSS has no published budget and no API contract at all — it is +/// scraping-adjacent, exactly the situation that broke the Bing hero tier when +/// it ran unpaced. Deliberately the most conservative pacer in this file: the +/// home page's news walk is a background task with no user waiting on any +/// single request. +const NEWS_RATE_WINDOW: Duration = Duration::from_secs(10); +const NEWS_RATE_BUDGET: usize = 4; +/// Headlines move slower than the feed refresh: within one TTL a new story is +/// at most this stale, which the home page's 45-day display window absorbs. +const NEWS_TTL: Duration = Duration::from_secs(6 * 60 * 60); /// How deep to pull the scrobble table. /// /// Raised from 100 once this table also had to serve per-track play counts on @@ -724,6 +830,17 @@ pub struct ArtistLink { pub url: String, } +/// One headline about a followed artist, for the home page's news panel. +/// `date` is unix seconds; freshness policy (the 45-day cap, the review +/// filter) is the frontend's display decision, not baked in here. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ArtistNewsItem { + pub title: String, + pub url: String, + pub source: String, + pub date: i64, +} + /// Lightweight artist identity + artwork response used to paint the profile /// while the deeper discography/edition analysis continues in parallel. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -1224,6 +1341,11 @@ pub struct MetadataClient { discogs_gate: Mutex>, /// Sliding-window pacer for ws.audioscrobbler.com (see [`LASTFM_RATE_BUDGET`]). lastfm_gate: Mutex>, + /// Sliding-window pacer for news.google.com (see [`NEWS_RATE_BUDGET`]). + news_gate: Mutex>, + /// Query URL → parsed items, so a session refresh costs nothing. Disk + /// persistence rides the generic JSON cache beside it. + news_cache: Mutex)>>, /// On-disk JSON cache for slow-moving providers (see [`DISK_CACHE_TTL`]). /// `None` (the default, and every test/example) keeps behavior purely /// in-memory. @@ -1282,6 +1404,8 @@ impl MetadataClient { wiki_cooldown: Mutex::new(None), discogs_gate: Mutex::new(VecDeque::new()), lastfm_gate: Mutex::new(VecDeque::new()), + news_gate: Mutex::new(VecDeque::new()), + news_cache: Mutex::new(HashMap::new()), disk_cache_dir: None, identity_store_dir: None, fused_portraits: std::sync::Mutex::new(HashMap::new()), @@ -6827,6 +6951,107 @@ impl MetadataClient { } } + /// Hold a Google News request until its sliding window has room. + async fn news_throttle(&self) { + loop { + let wait = { + let mut window = self.news_gate.lock().await; + let now = Instant::now(); + while window + .front() + .is_some_and(|at| now.duration_since(*at) >= NEWS_RATE_WINDOW) + { + window.pop_front(); + } + if window.len() < NEWS_RATE_BUDGET { + window.push_back(now); + return; + } + match window.front() { + Some(oldest) => NEWS_RATE_WINDOW - now.duration_since(*oldest), + None => continue, + } + }; + tokio::time::sleep(wait).await; + } + } + + /// Headlines about one artist, from Google News RSS. + /// + /// The query is the QUOTED artist name plus the word `music` — measured on + /// 2026-08-04: a bare ambiguous name scored 0/10 relevant items ("Girls"), + /// the quoted name with a corroborating token scored 8/8 and an easy name + /// 10/10. Keyless and undocumented, hence the conservative pacer and the + /// default UA (the contact UA is for providers with a policy asking for + /// it; scraped surfaces see the same UA a browser fetch would send). + /// + /// Sorting, the freshness window and the review filter are display policy + /// and live in the frontend; this returns what the feed said, newest + /// first, capped at a dozen. + pub async fn artist_news(&self, artist: &str) -> Vec { + let url = format!( + "https://news.google.com/rss/search?q={}&hl=en-US&gl=US&ceid=US:en", + urlencode(&format!("\"{artist}\" music")) + ); + let now_secs = || { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + }; + { + let cache = self.news_cache.lock().await; + if let Some((at, items)) = cache.get(&url) { + if at.elapsed() < NEWS_TTL { + return items.clone(); + } + } + } + // Disk: survives a restart. The generic reader enforces the 24h cache + // TTL; news wants 6h, so the fetch time rides inside the body and is + // checked here. + if let Some(path) = self.disk_cache_path(&url) { + if let Some(body) = read_disk_cache(&path, &url) { + let fresh = body + .get("at") + .and_then(|v| v.as_u64()) + .is_some_and(|at| now_secs().saturating_sub(at) < NEWS_TTL.as_secs()); + if fresh { + if let Some(items) = body + .get("items") + .and_then(|v| serde_json::from_value::>(v.clone()).ok()) + { + let mut cache = self.news_cache.lock().await; + cache.insert(url.clone(), (Instant::now(), items.clone())); + return items; + } + } + } + } + self.news_throttle().await; + let started = Instant::now(); + let text = match self.http.get(&url).send().await { + Ok(resp) if resp.status().is_success() => resp.text().await.unwrap_or_default(), + Ok(resp) => { + net_trace(&url, &format!("news-status {}", resp.status())); + return Vec::new(); + } + Err(e) => { + net_trace(&url, &format!("news-error {e}")); + return Vec::new(); + } + }; + let items = parse_news_rss(&text); + stage_trace("artist-news", started, artist); + if let Some(path) = self.disk_cache_path(&url) { + let body = serde_json::json!({ "at": now_secs(), "items": items }); + write_disk_cache(&path, &url, &body); + } + let mut cache = self.news_cache.lock().await; + cache.insert(url, (Instant::now(), items.clone())); + items + } + /// Hold a Discogs request until its sliding window has room. async fn discogs_throttle(&self) { loop { diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index a94332d..c25687b 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -4204,6 +4204,25 @@ pub async fn unfollow_artist( Ok(()) } +// ---- home ---- + +/// Headlines for the home page's news panel, one paced Google News query per +/// followed artist. Serial on purpose: the pacer inside `artist_news` is the +/// budget, and nothing user-blocking waits on this call — the frontend +/// requests it after the page is already up. +#[tauri::command] +pub async fn home_artist_news( + state: State<'_, LibraryState>, + artists: Vec, +) -> Result>, String> { + let mut out = std::collections::HashMap::new(); + for name in artists { + let items = state.metadata.artist_news(&name).await; + out.insert(name, items); + } + Ok(out) +} + // ---- playlists ---- fn now_secs() -> i64 { diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index a511f4f..d5b93a0 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -89,6 +89,7 @@ pub fn run() { commands::followed_artists, commands::follow_artist, commands::unfollow_artist, + commands::home_artist_news, commands::list_playlists, commands::create_playlist, commands::delete_playlist,