diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index b91b5fe..0769580 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -1833,6 +1833,18 @@ impl MetadataClient { ), ]; let mut out: Vec = Vec::new(); + // Why results died, per gate. Every gate below is a silent `continue`, + // so a gate that starts rejecting EVERYTHING looks exactly like Bing + // returning nothing — which is how this tier has now broken four times + // without anything saying so. HERO_WAVE proves the scrape parsed, this + // proves what threw the results away. + let mut rejected_scheme = 0usize; + let mut rejected_identity = 0usize; + let mut rejected_surname = 0usize; + let mut rejected_multi_artist = 0usize; + let mut rejected_unsuitable = 0usize; + let mut rejected_uncorroborated = 0usize; + let mut parsed_total = 0usize; 'queries: for (query_index, query) in queries.into_iter().take(query_limit).enumerate() { let waves = if query_index == 0 { &MANUAL_WAVES @@ -1856,16 +1868,30 @@ impl MetadataClient { }) .collect::>(); let mut wave_pages = stream::iter(wave_urls) - .map(|url| async move { self.browser_html(&url).await }) + .map(|url| async move { + // Bing answers the SAME url with wildly different pages: + // measured back to back, one card on one read and 25-80 on + // the next. A card-less page is a degraded answer, not an + // empty result set, so drop it and ask once more rather than + // caching it as this artist's verdict for the next quarter + // hour. + let mut parsed = match self.browser_html(&url).await { + Some(page) => parse_bing_image_results(&page), + None => Vec::new(), + }; + if parsed.is_empty() { + self.forget_html(&url).await; + if let Some(page) = self.browser_html(&url).await { + parsed = parse_bing_image_results(&page); + } + } + parsed + }) .buffered(HERO_SEARCH_WAVE_CONCURRENCY.min(wave_limit.max(1))); - while let Some(html) = wave_pages.next().await { + while let Some(parsed) = wave_pages.next().await { if out.len() >= total_limit { break 'queries; } - let Some(html) = html else { - continue; - }; - let parsed = parse_bing_image_results(&html); if self.hero_diagnostics { eprintln!( "HERO_WAVE\t{}\tq{}\tparsed={}\t{}", @@ -1875,6 +1901,7 @@ impl MetadataClient { query, ); } + parsed_total += parsed.len(); for result in parsed { // Bing occasionally publishes a legacy http image URL even // when the same asset is available over TLS. Upgrade it so @@ -1890,12 +1917,20 @@ impl MetadataClient { || lower.ends_with(".gif") || lower.ends_with(".svg") { + rejected_scheme += 1; continue; } let title = result.title; let purl = result.page_url; let Some(identity) = bing_identity_confidence(&title, &purl, &murl, artist) else { + rejected_identity += 1; + if self.hero_diagnostics && rejected_identity <= 6 { + eprintln!( + "HERO_REJECT_ID\t{}\ttitle={}\tpurl={}\tmurl={}", + artist, title, purl, murl, + ); + } continue; }; // Result metadata can repeat the requested name while the @@ -1903,6 +1938,7 @@ impl MetadataClient { // same surname. Bing did this for Billy Cobb with a // `jimmy-cobb` photo under a misleading Billy Cobb title. if image_path_conflicts_with_two_part_artist(&murl, artist) { + rejected_surname += 1; continue; } // Roundups can name the requested artist while displaying @@ -1910,6 +1946,7 @@ impl MetadataClient { // multi-artist headline is paired with a filename naming // somebody else, wait for the artist-specific image. if multi_artist_result_mismatches_image(&title, &murl, artist) { + rejected_multi_artist += 1; continue; } // A winner-release card on an artist-named host may prove @@ -1919,6 +1956,7 @@ impl MetadataClient { // every other unsuitable result is discarded immediately. let product_result = is_product_image_result(&title, &purl, &murl); if !product_result && is_unsuitable_bing_result(&title, &purl, &murl, artist) { + rejected_unsuitable += 1; continue; } let rank_in_query = out.len() - query_start; @@ -1934,6 +1972,7 @@ impl MetadataClient { if product_result && !bing_candidate_can_corroborate_artist_host(&candidate, policy, artist) { + rejected_uncorroborated += 1; continue; } // Bing can repeat one murl under weaker and stronger cards. @@ -1986,6 +2025,20 @@ impl MetadataClient { } } } + if self.hero_diagnostics { + eprintln!( + "HERO_GATES\t{}\tparsed={}\taccepted={}\tscheme={}\tidentity={}\tsurname={}\tmulti={}\tunsuitable={}\tuncorroborated={}", + artist, + parsed_total, + out.len(), + rejected_scheme, + rejected_identity, + rejected_surname, + rejected_multi_artist, + rejected_unsuitable, + rejected_uncorroborated, + ); + } // Strong-vs-Weak selection happens after download/decode/scoring. A // broken metadata-Strong URL must not erase the only usable fallback. out @@ -1993,6 +2046,14 @@ impl MetadataClient { /// A page's HTML fetched with a browser-ish identity — several providers /// serve a stub (or a wall) to unknown user agents. + /// Drop a cached page so the next read re-fetches it. Bing can answer a + /// perfectly valid image search with a page carrying no result cards, and + /// `browser_html` would otherwise hold that answer for CATALOG_TTL — one bad + /// response leaving an artist heroless for fifteen minutes. + async fn forget_html(&self, url: &str) { + self.html_cache.lock().await.remove(url); + } + async fn browser_html(&self, url: &str) -> Option { { let cache = self.html_cache.lock().await;