From 3718dad667cd2df3292f6e5d5dbe8e13798f5f7b Mon Sep 17 00:00:00 2001 From: KeelrLeek <156356974+KeelrLeek@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:38:01 -0500 Subject: [PATCH] metadata: say why the Bing hero tier found nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Bing hero tier currently returns no banner for artists it used to cover. Every gate in the candidate loop is a silent `continue`, so a tier that throws away all 525 parsed results looks identical to Bing returning nothing at all — which is why this has now read as a fresh regression four times without anything being able to confirm or deny it. Emit HERO_GATES alongside HERO_WAVE: what was parsed, what was accepted, and the count discarded by each gate, plus the first few identity rejections in full. That immediately separates "our filter broke" from "Bing sent us junk". It says the second one. For Camping in Alaska: parsed=525, accepted=0, identity=510 — and the rejected cards are campground photos from recreation.gov and thedyrt.com. Partiboi69 returns Bruce Lee wallpapers, Fishmans returns Tommy Fury boxing coverage, Hermeth returns aesthetic quote wallpapers. The results are unrelated to the query and to each other, so the identity gate is doing exactly its job: refusing to hang another person's face on the artist. Confirmed outside the app, straight from curl. Zero query-relevant result cards on any variant — with and without qft/form/first/cc/setlang/tsc, with full browser client-hint and sec-fetch headers, and against the /images/async grid endpoint the page itself uses. The async endpoint answers "Fishmans band photo" with AI-course ads and "Partiboi69 band photo" with celebrity photoshoots. Our url is well formed and the parser is fine: `q` is properly encoded, and cards still parse (252-525 of them). Bing is simply no longer honouring the query for this client. Also stop one bad answer from sticking. Bing serves the SAME url a card-less page on one read and 25-80 cards on the next, and browser_html caches whatever came back for CATALOG_TTL — so a single degraded response left an artist heroless for fifteen minutes. A wave that parses to nothing now drops the cached page and asks once more. This does not address the wrong-results failure above; it removes a compounding one found while chasing it. No hero behaviour is changed by this commit. --- crates/metadata/src/lib.rs | 73 ++++++++++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 6 deletions(-) diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index b91b5fe..0769580 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -1833,6 +1833,18 @@ impl MetadataClient { ), ]; let mut out: Vec = Vec::new(); + // Why results died, per gate. Every gate below is a silent `continue`, + // so a gate that starts rejecting EVERYTHING looks exactly like Bing + // returning nothing — which is how this tier has now broken four times + // without anything saying so. HERO_WAVE proves the scrape parsed, this + // proves what threw the results away. + let mut rejected_scheme = 0usize; + let mut rejected_identity = 0usize; + let mut rejected_surname = 0usize; + let mut rejected_multi_artist = 0usize; + let mut rejected_unsuitable = 0usize; + let mut rejected_uncorroborated = 0usize; + let mut parsed_total = 0usize; 'queries: for (query_index, query) in queries.into_iter().take(query_limit).enumerate() { let waves = if query_index == 0 { &MANUAL_WAVES @@ -1856,16 +1868,30 @@ impl MetadataClient { }) .collect::>(); let mut wave_pages = stream::iter(wave_urls) - .map(|url| async move { self.browser_html(&url).await }) + .map(|url| async move { + // Bing answers the SAME url with wildly different pages: + // measured back to back, one card on one read and 25-80 on + // the next. A card-less page is a degraded answer, not an + // empty result set, so drop it and ask once more rather than + // caching it as this artist's verdict for the next quarter + // hour. + let mut parsed = match self.browser_html(&url).await { + Some(page) => parse_bing_image_results(&page), + None => Vec::new(), + }; + if parsed.is_empty() { + self.forget_html(&url).await; + if let Some(page) = self.browser_html(&url).await { + parsed = parse_bing_image_results(&page); + } + } + parsed + }) .buffered(HERO_SEARCH_WAVE_CONCURRENCY.min(wave_limit.max(1))); - while let Some(html) = wave_pages.next().await { + while let Some(parsed) = wave_pages.next().await { if out.len() >= total_limit { break 'queries; } - let Some(html) = html else { - continue; - }; - let parsed = parse_bing_image_results(&html); if self.hero_diagnostics { eprintln!( "HERO_WAVE\t{}\tq{}\tparsed={}\t{}", @@ -1875,6 +1901,7 @@ impl MetadataClient { query, ); } + parsed_total += parsed.len(); for result in parsed { // Bing occasionally publishes a legacy http image URL even // when the same asset is available over TLS. Upgrade it so @@ -1890,12 +1917,20 @@ impl MetadataClient { || lower.ends_with(".gif") || lower.ends_with(".svg") { + rejected_scheme += 1; continue; } let title = result.title; let purl = result.page_url; let Some(identity) = bing_identity_confidence(&title, &purl, &murl, artist) else { + rejected_identity += 1; + if self.hero_diagnostics && rejected_identity <= 6 { + eprintln!( + "HERO_REJECT_ID\t{}\ttitle={}\tpurl={}\tmurl={}", + artist, title, purl, murl, + ); + } continue; }; // Result metadata can repeat the requested name while the @@ -1903,6 +1938,7 @@ impl MetadataClient { // same surname. Bing did this for Billy Cobb with a // `jimmy-cobb` photo under a misleading Billy Cobb title. if image_path_conflicts_with_two_part_artist(&murl, artist) { + rejected_surname += 1; continue; } // Roundups can name the requested artist while displaying @@ -1910,6 +1946,7 @@ impl MetadataClient { // multi-artist headline is paired with a filename naming // somebody else, wait for the artist-specific image. if multi_artist_result_mismatches_image(&title, &murl, artist) { + rejected_multi_artist += 1; continue; } // A winner-release card on an artist-named host may prove @@ -1919,6 +1956,7 @@ impl MetadataClient { // every other unsuitable result is discarded immediately. let product_result = is_product_image_result(&title, &purl, &murl); if !product_result && is_unsuitable_bing_result(&title, &purl, &murl, artist) { + rejected_unsuitable += 1; continue; } let rank_in_query = out.len() - query_start; @@ -1934,6 +1972,7 @@ impl MetadataClient { if product_result && !bing_candidate_can_corroborate_artist_host(&candidate, policy, artist) { + rejected_uncorroborated += 1; continue; } // Bing can repeat one murl under weaker and stronger cards. @@ -1986,6 +2025,20 @@ impl MetadataClient { } } } + if self.hero_diagnostics { + eprintln!( + "HERO_GATES\t{}\tparsed={}\taccepted={}\tscheme={}\tidentity={}\tsurname={}\tmulti={}\tunsuitable={}\tuncorroborated={}", + artist, + parsed_total, + out.len(), + rejected_scheme, + rejected_identity, + rejected_surname, + rejected_multi_artist, + rejected_unsuitable, + rejected_uncorroborated, + ); + } // Strong-vs-Weak selection happens after download/decode/scoring. A // broken metadata-Strong URL must not erase the only usable fallback. out @@ -1993,6 +2046,14 @@ impl MetadataClient { /// A page's HTML fetched with a browser-ish identity — several providers /// serve a stub (or a wall) to unknown user agents. + /// Drop a cached page so the next read re-fetches it. Bing can answer a + /// perfectly valid image search with a page carrying no result cards, and + /// `browser_html` would otherwise hold that answer for CATALOG_TTL — one bad + /// response leaving an artist heroless for fifteen minutes. + async fn forget_html(&self, url: &str) { + self.html_cache.lock().await.remove(url); + } + async fn browser_html(&self, url: &str) -> Option { { let cache = self.html_cache.lock().await;