diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 5000d04..c9ba319 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -570,6 +570,10 @@ const HERO_SEARCH_WAVE_CONCURRENCY: usize = 3; /// small result set. Bing's genuine pages run 12-100 cards; the degraded shape /// is one card of something unrelated. const HERO_DEGRADED_PAGE_CARDS: usize = 2; +/// Bing's own default adult-content preference, sent as a cookie. The +/// `&safesearch=moderate` query parameter measures identical to sending no +/// preference at all; this cookie actually selects the bucket. +const BING_MODERATE_COOKIE: &str = "SRCHHPGUSR=ADLT=DEMOTE"; const HERO_SCORE_CACHE_MAX_ENTRIES: usize = 128; const HERO_SCORE_RECIPE: &str = "bing-identity-visual-v1"; @@ -1139,8 +1143,9 @@ impl MetadataClient { /// Hero artwork for the artist — wide background banner + band ClearLOGO — /// best quality first: /// - /// 1. **Fanart.tv** `artistbackground` (1920×1080) + `hdmusiclogo` (HD - /// transparent PNG), keyed by MusicBrainz id — one call yields both. + /// 1. **Fanart.tv** `artist4kbackground` (3840×2160) then `artistbackground` + /// (1920×1080), plus `hdmusiclogo` (HD transparent PNG), keyed by + /// MusicBrainz id — one call yields both. /// 2. **TheAudioDB** 1280×720 artist fanart, restored as the broad-coverage /// fallback and upscaled by the app's dedicated hero proxy. /// 3. **Bing image search**, with Extra-Large + Wide filters, exact artist @@ -1794,6 +1799,33 @@ impl MetadataClient { .await } + /// One Bing image read, with the degraded-answer retry. + /// + /// Bing answers the SAME url with wildly different pages: measured back to + /// back, one card on one read and 25-80 on the next. A near-empty page is a + /// degraded answer, not an empty result set, so drop it and ask once more + /// rather than caching it as this artist's verdict for the next quarter + /// hour. The test used to be `is_empty()`, which let the commonest degraded + /// shape straight through: a ONE-card page carrying something unrelated + /// (15 of 30 waves for MSPAINT and Billy Cobb in one audit). Keep whichever + /// read is richer so a thin retry can never lose a good first answer. + async fn bing_wave_cards(&self, url: &str, cookie: Option<&str>) -> Vec { + let mut parsed = match self.browser_html_cookie(url, cookie).await { + Some(page) => parse_bing_image_results(&page), + None => Vec::new(), + }; + if parsed.len() <= HERO_DEGRADED_PAGE_CARDS { + self.forget_html(url).await; + if let Some(page) = self.browser_html_cookie(url, cookie).await { + let retried = parse_bing_image_results(&page); + if retried.len() > parsed.len() { + parsed = retried; + } + } + } + parsed + } + async fn bing_image_candidates_limited( &self, artist: &str, @@ -1871,56 +1903,49 @@ impl MetadataClient { // together, but consume it in the original wave order so candidate // membership, metadata upgrades, ranks and early-stop behavior stay // identical to the sequential implementation. - // `safesearch=strict` is load-bearing for RELEVANCE, not just for - // decency. Our client keeps no cookie jar, so every read reaches - // Bing with its adult-content preference undefined -- and in that - // state Bing answers with wholly unrelated cached result sets: four - // identical back-to-back reads for "Fishmans band photo" returned - // porn, an Indonesian chord page, porn again, and a printable 2026 - // calendar, each internally coherent with honest titles and purls. - // Measured 27/27 reads with zero relevant cards without the flag - // and 27/27 correct with it. Only `strict` fixes it; `moderate` and - // `off` behave exactly like sending nothing at all. This was - // misdiagnosed twice (as a poisoned IP, then as Bing rate-limiting) - // because the failure looks external and clears on its own. + // READ BOTH ADULT-CONTENT BUCKETS AND POOL THEM. Bing's safesearch + // preference selects which cached result set answers a query, and + // any single bucket can be serving a DIFFERENT query's results -- + // measured in both directions on the same machine minutes apart: + // Fishmans strict 32/35 relevant | unsignalled 0/35 (porn, an + // Indonesian chord page, a printable 2026 calendar) + // Nikki Nair strict 0/35 | unsignalled 35/35 (every card + // a real press photo -- RA, Rinse, Boiler Room, DJ Mag) + // So there is no correct global setting: strict alone silently loses + // artists Bing can see perfectly well, and no signal at all loses + // different ones. Ask both and let the identity gate choose; it + // rejected ~3,000 cards in the audit behind this with no observed + // false rejects, and it is what keeps the unfiltered bucket's + // explicit results out of the pool. let wave_urls = waves .iter() .take(wave_limit) .map(|wave| { - format!( - "https://www.bing.com/images/search?q={q}&qft={}&form=HDRSC3&first=1&cc=us&setlang=en-us&tsc=ImageHoverTitle&safesearch=strict", + let base = format!( + "https://www.bing.com/images/search?q={q}&qft={}&form=HDRSC3&first=1&cc=us&setlang=en-us&tsc=ImageHoverTitle", urlencode(wave) - ) + ); + // Distinct URLs so the html cache keeps the two buckets + // apart; the moderate read carries Bing's own default + // preference as a cookie instead of the query parameter, + // because `&safesearch=moderate` measures identical to + // sending nothing while the cookie actually takes effect. + (format!("{base}&safesearch=strict"), base) }) .collect::>(); let mut wave_pages = stream::iter(wave_urls) - .map(|url| async move { - // Bing answers the SAME url with wildly different pages: - // measured back to back, one card on one read and 25-80 on - // the next. A near-empty page is a degraded answer, not an - // empty result set, so drop it and ask once more rather than - // caching it as this artist's verdict for the next quarter - // hour. - // - // The emptiness test used to be `is_empty()`, which let the - // commonest degraded shape straight through: a ONE-card page - // carrying something unrelated. Measured across one audit, - // 15 of 30 waves for MSPAINT and Billy Cobb, 6 of 15 for - // DJ Sabrina and 9 of 15 for TAGABOW came back with exactly - // one junk card -- each one enough to blank that artist for - // a full CATALOG_TTL. Keep whichever read is richer so a - // thin retry can never lose a good first answer. - let mut parsed = match self.browser_html(&url).await { - Some(page) => parse_bing_image_results(&page), - None => Vec::new(), - }; - if parsed.len() <= HERO_DEGRADED_PAGE_CARDS { - self.forget_html(&url).await; - if let Some(page) = self.browser_html(&url).await { - let retried = parse_bing_image_results(&page); - if retried.len() > parsed.len() { - parsed = retried; - } + .map(|(strict_url, moderate_url)| async move { + let (strict, moderate) = tokio::join!( + self.bing_wave_cards(&strict_url, None), + self.bing_wave_cards(&moderate_url, Some(BING_MODERATE_COOKIE)), + ); + // Pool, preferring the strict bucket's ordering. Bing repeats + // one image under several cards, so dedupe on the media url. + let mut seen: HashSet = HashSet::new(); + let mut parsed = Vec::with_capacity(strict.len() + moderate.len()); + for card in strict.into_iter().chain(moderate) { + if seen.insert(card.url.clone()) { + parsed.push(card); } } parsed @@ -2092,7 +2117,13 @@ impl MetadataClient { self.html_cache.lock().await.remove(url); } - async fn browser_html(&self, url: &str) -> Option { + /// A page's HTML fetched with a browser-ish identity, with an optional + /// Cookie header. Bing's adult-content + /// preference selects which cached result bucket answers a query, and any + /// one bucket can be serving another query's results, so the hero search + /// reads more than one. Callers must keep the cookie deterministic per URL: + /// the cache is still keyed by URL alone. + async fn browser_html_cookie(&self, url: &str, cookie: Option<&str>) -> Option { { let cache = self.html_cache.lock().await; if let Some((at, html)) = cache.get(url) { @@ -2111,22 +2142,23 @@ impl MetadataClient { }; let html = fetch .get_or_init(|| async { - let response = tokio::time::timeout( - Duration::from_secs(6), - self.http - .get(url) - .header( - "user-agent", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0", - ) - .header("accept-language", "en-US,en;q=0.9") - .header( - "accept", - "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - ) - .header("cache-control", "no-cache") - .send(), - ) + let mut request = self + .http + .get(url) + .header( + "user-agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0", + ) + .header("accept-language", "en-US,en;q=0.9") + .header( + "accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + ) + .header("cache-control", "no-cache"); + if let Some(cookie) = cookie { + request = request.header("cookie", cookie); + } + let response = tokio::time::timeout(Duration::from_secs(6), request.send()) .await .ok()? .ok()?; @@ -2566,10 +2598,24 @@ impl MetadataClient { if logos.is_empty() { logos = top_liked_urls(&body, "musiclogo", 6); } + // fanart.tv publishes newer uploads under `artist4kbackground` and only + // older ones under `artistbackground`; a record can carry one, the + // other, or both. Reading only the legacy key silently discarded the + // ONLY background some artists have -- MSPAINT's record is literally + // just {artist4kbackground: [3840x2160 band photo], hdmusiclogo: [...]}, + // fetched and thrown away on every page load while the artist showed no + // hero at all. 4K first: it outranks the 1920x1080 legacy tier. + let mut backgrounds = top_liked_urls(&body, "artist4kbackground", 8); + for url in top_liked_urls(&body, "artistbackground", 8) { + if !backgrounds.contains(&url) { + backgrounds.push(url); + } + } // Keep every useful candidate from this best-quality tier. Eight is // enough variety without making a long-lived artist page accumulate an // unbounded set of processed 4K images in the frontend cache. - (top_liked_urls(&body, "artistbackground", 8), logos) + backgrounds.truncate(8); + (backgrounds, logos) } /// Catalog search: artists, albums and tracks for the query, in parallel.