forked from Keelrleek/SoulStream
news: headlines about the artists you follow, anchored to the artist
No per-artist music news API exists — every product that does this aggregates and filters, so this does too. Google News RSS is the source: free, keyless, no pagination, and no contract at all, which is why it gets the most conservative pacer in the file (4 requests per 10 seconds) and a 6-hour cache on disk as well as in memory. Nothing user-facing waits on it; the home page asks after it is already drawn. The query is the QUOTED artist name plus the word "music", and the quoting is the whole design. Measured 2026-08-04 against the real feed: a bare ambiguous name returned 0 of 10 relevant items — searching "Girls" gives Indigo Girls, The Deli Girls and a school production of Mean Girls — while the same name with a corroborating token returned 8 of 8, and an unambiguous one 10 of 10. That is the identity-anchor rule the rest of the app already lives by, applied to a provider that only speaks in names: never trust the name alone. The parser is deliberately not an XML library. One provider, four flat tags inside <item> blocks, exactly the judgment the Bing scrape made. What it does handle is what Google actually emits: entity-encoded titles including numeric forms, a " - Publication" suffix on every headline (stripped when the tail matches the <source> tag, so the title reads as a headline rather than a citation), and RFC 2822 pubDates converted to unix seconds by a hand-rolled civil-date conversion — nothing else in the workspace needs a date crate for one field. Freshness and relevance stay OUT of here. The 45-day window and the review filter are display policy and belong to the surface that shows them; this returns what the feed said, newest first, capped at a dozen.
This commit is contained in:
1 parent
50afa6a6a9
commit
8ef730848a
3 files changed
+245
No files matched your search
@@ -184,6 +184,102 @@ fn strip_discogs_markup(profile: &str) -> String {
|
||||
out.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
/// Pull the fields the news panel needs out of a Google News RSS body.
|
||||
///
|
||||
/// A real XML parser is not warranted for one provider whose format is four
|
||||
/// flat tags inside `<item>` blocks — the same judgment the Bing scrape made.
|
||||
/// Google encodes item titles as plain text with entities (no CDATA observed),
|
||||
/// appends " - Publication" to every title, and stamps RFC 2822 pubDates.
|
||||
fn parse_news_rss(body: &str) -> Vec<ArtistNewsItem> {
|
||||
fn tag(block: &str, name: &str) -> Option<String> {
|
||||
let open = format!("<{name}");
|
||||
let start = block.find(&open)?;
|
||||
let start = block[start..].find('>')? + start + 1;
|
||||
let end = block[start..].find(&format!("</{name}>"))? + start;
|
||||
Some(block[start..end].trim().to_string())
|
||||
}
|
||||
fn decode(s: &str) -> String {
|
||||
// The entities Google actually emits, plus numeric forms.
|
||||
let mut out = s
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace(""", "\"")
|
||||
.replace("'", "'");
|
||||
while let Some(start) = out.find("&#") {
|
||||
let Some(len) = out[start..].find(';') else { break };
|
||||
let code = &out[start + 2..start + len];
|
||||
let ch = code
|
||||
.strip_prefix('x')
|
||||
.map_or_else(|| code.parse::<u32>().ok(), |h| u32::from_str_radix(h, 16).ok())
|
||||
.and_then(char::from_u32);
|
||||
match ch {
|
||||
Some(c) => out.replace_range(start..start + len + 1, &c.to_string()),
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
/// "Mon, 28 Jul 2026 07:00:00 GMT" → unix seconds. Hand-rolled because
|
||||
/// nothing else in the workspace needs a date crate.
|
||||
fn rfc2822_secs(s: &str) -> i64 {
|
||||
let parts: Vec<&str> = s.split_whitespace().collect();
|
||||
// [Mon,] 28 Jul 2026 07:00:00 GMT
|
||||
let off = usize::from(parts.first().is_some_and(|p| p.ends_with(',')));
|
||||
let (Some(day), Some(mon), Some(year), Some(time)) = (
|
||||
parts.get(off).and_then(|v| v.parse::<i64>().ok()),
|
||||
parts.get(off + 1),
|
||||
parts.get(off + 2).and_then(|v| v.parse::<i64>().ok()),
|
||||
parts.get(off + 3),
|
||||
) else {
|
||||
return 0;
|
||||
};
|
||||
const MONTHS: [&str; 12] = [
|
||||
"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
|
||||
];
|
||||
let Some(month) = MONTHS.iter().position(|m| m.eq_ignore_ascii_case(mon)) else {
|
||||
return 0;
|
||||
};
|
||||
let mut t = time.split(':').filter_map(|v| v.parse::<i64>().ok());
|
||||
let (h, mi, sec) = (
|
||||
t.next().unwrap_or(0),
|
||||
t.next().unwrap_or(0),
|
||||
t.next().unwrap_or(0),
|
||||
);
|
||||
// Civil date → days since epoch (Howard Hinnant's algorithm).
|
||||
let month = month as i64 + 1;
|
||||
let y = if month <= 2 { year - 1 } else { year };
|
||||
let era = y.div_euclid(400);
|
||||
let yoe = y - era * 400;
|
||||
let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1;
|
||||
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
|
||||
let days = era * 146097 + doe - 719468;
|
||||
days * 86400 + h * 3600 + mi * 60 + sec
|
||||
}
|
||||
|
||||
let mut out = Vec::new();
|
||||
for block in body.split("<item>").skip(1) {
|
||||
let block = block.split("</item>").next().unwrap_or("");
|
||||
let Some(raw_title) = tag(block, "title") else { continue };
|
||||
let Some(link) = tag(block, "link") else { continue };
|
||||
let source = tag(block, "source").map(|s| decode(&s)).unwrap_or_default();
|
||||
let mut title = decode(&raw_title);
|
||||
// "Headline - Publication" → "Headline" when the tail names the source.
|
||||
if !source.is_empty() {
|
||||
if let Some(stripped) = title.strip_suffix(&format!(" - {source}")) {
|
||||
title = stripped.to_string();
|
||||
}
|
||||
}
|
||||
let date = tag(block, "pubDate").map(|d| rfc2822_secs(&d)).unwrap_or(0);
|
||||
out.push(ArtistNewsItem { title, url: decode(&link), source, date });
|
||||
if out.len() >= 12 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out.sort_by_key(|item| -item.date);
|
||||
out
|
||||
}
|
||||
|
||||
/// Wikimedia hosts share one UA policy and one rate limiter, so they share
|
||||
/// this client's compliant UA, pacer, 429 handling and disk cache. Wikidata is
|
||||
/// included: it is the identity anchor that turns a MusicBrainz artist into
|
||||
@@ -332,6 +428,16 @@ const LASTFM_KEY: &str = "69f3b3201381b56903eeee31477644a2";
|
||||
/// treatment Bing got — that is the one provider that broke this way.
|
||||
const LASTFM_RATE_WINDOW: Duration = Duration::from_secs(1);
|
||||
const LASTFM_RATE_BUDGET: usize = 4;
|
||||
/// Google News RSS has no published budget and no API contract at all — it is
|
||||
/// scraping-adjacent, exactly the situation that broke the Bing hero tier when
|
||||
/// it ran unpaced. Deliberately the most conservative pacer in this file: the
|
||||
/// home page's news walk is a background task with no user waiting on any
|
||||
/// single request.
|
||||
const NEWS_RATE_WINDOW: Duration = Duration::from_secs(10);
|
||||
const NEWS_RATE_BUDGET: usize = 4;
|
||||
/// Headlines move slower than the feed refresh: within one TTL a new story is
|
||||
/// at most this stale, which the home page's 45-day display window absorbs.
|
||||
const NEWS_TTL: Duration = Duration::from_secs(6 * 60 * 60);
|
||||
/// How deep to pull the scrobble table.
|
||||
///
|
||||
/// Raised from 100 once this table also had to serve per-track play counts on
|
||||
@@ -724,6 +830,17 @@ pub struct ArtistLink {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
/// One headline about a followed artist, for the home page's news panel.
|
||||
/// `date` is unix seconds; freshness policy (the 45-day cap, the review
|
||||
/// filter) is the frontend's display decision, not baked in here.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ArtistNewsItem {
|
||||
pub title: String,
|
||||
pub url: String,
|
||||
pub source: String,
|
||||
pub date: i64,
|
||||
}
|
||||
|
||||
/// Lightweight artist identity + artwork response used to paint the profile
|
||||
/// while the deeper discography/edition analysis continues in parallel.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
@@ -1224,6 +1341,11 @@ pub struct MetadataClient {
|
||||
discogs_gate: Mutex<VecDeque<Instant>>,
|
||||
/// Sliding-window pacer for ws.audioscrobbler.com (see [`LASTFM_RATE_BUDGET`]).
|
||||
lastfm_gate: Mutex<VecDeque<Instant>>,
|
||||
/// Sliding-window pacer for news.google.com (see [`NEWS_RATE_BUDGET`]).
|
||||
news_gate: Mutex<VecDeque<Instant>>,
|
||||
/// Query URL → parsed items, so a session refresh costs nothing. Disk
|
||||
/// persistence rides the generic JSON cache beside it.
|
||||
news_cache: Mutex<HashMap<String, (Instant, Vec<ArtistNewsItem>)>>,
|
||||
/// On-disk JSON cache for slow-moving providers (see [`DISK_CACHE_TTL`]).
|
||||
/// `None` (the default, and every test/example) keeps behavior purely
|
||||
/// in-memory.
|
||||
@@ -1282,6 +1404,8 @@ impl MetadataClient {
|
||||
wiki_cooldown: Mutex::new(None),
|
||||
discogs_gate: Mutex::new(VecDeque::new()),
|
||||
lastfm_gate: Mutex::new(VecDeque::new()),
|
||||
news_gate: Mutex::new(VecDeque::new()),
|
||||
news_cache: Mutex::new(HashMap::new()),
|
||||
disk_cache_dir: None,
|
||||
identity_store_dir: None,
|
||||
fused_portraits: std::sync::Mutex::new(HashMap::new()),
|
||||
@@ -6827,6 +6951,107 @@ impl MetadataClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// Hold a Google News request until its sliding window has room.
|
||||
async fn news_throttle(&self) {
|
||||
loop {
|
||||
let wait = {
|
||||
let mut window = self.news_gate.lock().await;
|
||||
let now = Instant::now();
|
||||
while window
|
||||
.front()
|
||||
.is_some_and(|at| now.duration_since(*at) >= NEWS_RATE_WINDOW)
|
||||
{
|
||||
window.pop_front();
|
||||
}
|
||||
if window.len() < NEWS_RATE_BUDGET {
|
||||
window.push_back(now);
|
||||
return;
|
||||
}
|
||||
match window.front() {
|
||||
Some(oldest) => NEWS_RATE_WINDOW - now.duration_since(*oldest),
|
||||
None => continue,
|
||||
}
|
||||
};
|
||||
tokio::time::sleep(wait).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Headlines about one artist, from Google News RSS.
|
||||
///
|
||||
/// The query is the QUOTED artist name plus the word `music` — measured on
|
||||
/// 2026-08-04: a bare ambiguous name scored 0/10 relevant items ("Girls"),
|
||||
/// the quoted name with a corroborating token scored 8/8 and an easy name
|
||||
/// 10/10. Keyless and undocumented, hence the conservative pacer and the
|
||||
/// default UA (the contact UA is for providers with a policy asking for
|
||||
/// it; scraped surfaces see the same UA a browser fetch would send).
|
||||
///
|
||||
/// Sorting, the freshness window and the review filter are display policy
|
||||
/// and live in the frontend; this returns what the feed said, newest
|
||||
/// first, capped at a dozen.
|
||||
pub async fn artist_news(&self, artist: &str) -> Vec<ArtistNewsItem> {
|
||||
let url = format!(
|
||||
"https://news.google.com/rss/search?q={}&hl=en-US&gl=US&ceid=US:en",
|
||||
urlencode(&format!("\"{artist}\" music"))
|
||||
);
|
||||
let now_secs = || {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0)
|
||||
};
|
||||
{
|
||||
let cache = self.news_cache.lock().await;
|
||||
if let Some((at, items)) = cache.get(&url) {
|
||||
if at.elapsed() < NEWS_TTL {
|
||||
return items.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
// Disk: survives a restart. The generic reader enforces the 24h cache
|
||||
// TTL; news wants 6h, so the fetch time rides inside the body and is
|
||||
// checked here.
|
||||
if let Some(path) = self.disk_cache_path(&url) {
|
||||
if let Some(body) = read_disk_cache(&path, &url) {
|
||||
let fresh = body
|
||||
.get("at")
|
||||
.and_then(|v| v.as_u64())
|
||||
.is_some_and(|at| now_secs().saturating_sub(at) < NEWS_TTL.as_secs());
|
||||
if fresh {
|
||||
if let Some(items) = body
|
||||
.get("items")
|
||||
.and_then(|v| serde_json::from_value::<Vec<ArtistNewsItem>>(v.clone()).ok())
|
||||
{
|
||||
let mut cache = self.news_cache.lock().await;
|
||||
cache.insert(url.clone(), (Instant::now(), items.clone()));
|
||||
return items;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.news_throttle().await;
|
||||
let started = Instant::now();
|
||||
let text = match self.http.get(&url).send().await {
|
||||
Ok(resp) if resp.status().is_success() => resp.text().await.unwrap_or_default(),
|
||||
Ok(resp) => {
|
||||
net_trace(&url, &format!("news-status {}", resp.status()));
|
||||
return Vec::new();
|
||||
}
|
||||
Err(e) => {
|
||||
net_trace(&url, &format!("news-error {e}"));
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
let items = parse_news_rss(&text);
|
||||
stage_trace("artist-news", started, artist);
|
||||
if let Some(path) = self.disk_cache_path(&url) {
|
||||
let body = serde_json::json!({ "at": now_secs(), "items": items });
|
||||
write_disk_cache(&path, &url, &body);
|
||||
}
|
||||
let mut cache = self.news_cache.lock().await;
|
||||
cache.insert(url, (Instant::now(), items.clone()));
|
||||
items
|
||||
}
|
||||
|
||||
/// Hold a Discogs request until its sliding window has room.
|
||||
async fn discogs_throttle(&self) {
|
||||
loop {
|
||||
|
||||
@@ -4204,6 +4204,25 @@ pub async fn unfollow_artist(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- home ----
|
||||
|
||||
/// Headlines for the home page's news panel, one paced Google News query per
|
||||
/// followed artist. Serial on purpose: the pacer inside `artist_news` is the
|
||||
/// budget, and nothing user-blocking waits on this call — the frontend
|
||||
/// requests it after the page is already up.
|
||||
#[tauri::command]
|
||||
pub async fn home_artist_news(
|
||||
state: State<'_, LibraryState>,
|
||||
artists: Vec<String>,
|
||||
) -> Result<std::collections::HashMap<String, Vec<metadata::ArtistNewsItem>>, String> {
|
||||
let mut out = std::collections::HashMap::new();
|
||||
for name in artists {
|
||||
let items = state.metadata.artist_news(&name).await;
|
||||
out.insert(name, items);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// ---- playlists ----
|
||||
|
||||
fn now_secs() -> i64 {
|
||||
|
||||
@@ -89,6 +89,7 @@ pub fn run() {
|
||||
commands::followed_artists,
|
||||
commands::follow_artist,
|
||||
commands::unfollow_artist,
|
||||
commands::home_artist_news,
|
||||
commands::list_playlists,
|
||||
commands::create_playlist,
|
||||
commands::delete_playlist,
|
||||
|
||||
Reference in new issue
Block a user