sidebar and artist pipeline: one list in the rail, recents anchored to ids, and popular takes its own lane
This commit is contained in:
1 parent
606250c361
commit
eba29824c9
12 files changed
+1595
-281
No files matched your search
@@ -0,0 +1,171 @@
|
||||
//! Cold-start bug audit for the artist-page pipeline. Companion to
|
||||
//! `artist_page_bench` (latency) — this one checks INVARIANTS across a spread
|
||||
//! of artist shapes and prints a statistics row per artist:
|
||||
//!
|
||||
//! - overview / page / identity wall-clock, run the way the frontend runs
|
||||
//! them (all three overlapped, one fresh client per artist, no disk cache)
|
||||
//! - album and popular-row counts at each stage
|
||||
//! - listeners coverage (the Last.fm anchor working end-to-end)
|
||||
//! - release-date coverage (the MusicBrainz dates pass reaching the page)
|
||||
//! - BUG lines for anything that violates an invariant
|
||||
//!
|
||||
//! Artists are audited SEQUENTIALLY — each fresh client has its own
|
||||
//! MusicBrainz pacer, and overlapping artists would multiply request rate
|
||||
//! against the 1 req/s policy (and contaminate the numbers, see the
|
||||
//! paced-probe rule).
|
||||
//!
|
||||
//! Usage:
|
||||
//! cargo run -p metadata --release --example cold_audit -- ["Artist" ...]
|
||||
//! (no args = the built-in spread)
|
||||
|
||||
use metadata::MetadataClient;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let mut artists: Vec<String> = std::env::args().skip(1).collect();
|
||||
if artists.is_empty() {
|
||||
artists = [
|
||||
"Alvvays", // small, clean MB identity
|
||||
"Weezer", // medium, deep catalog
|
||||
"Queen", // huge, 1119 MB pressings
|
||||
"Death Grips", // mixtape/street edge
|
||||
"King Gizzard & The Lizard Wizard", // Last.fm mbid-index miss + '&'
|
||||
"Boards of Canada", // electronic, sparse Deezer
|
||||
"Duster", // namesake-collision shape
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
}
|
||||
|
||||
let mut bugs: Vec<String> = Vec::new();
|
||||
println!(
|
||||
"{:<36} {:>8} {:>8} {:>8} {:>6} {:>6} {:>7} {:>9} {:>8}",
|
||||
"artist", "overview", "page", "identity", "ov-alb", "pg-alb", "popular", "listeners", "dates"
|
||||
);
|
||||
|
||||
for artist in &artists {
|
||||
let client = Arc::new(MetadataClient::new());
|
||||
let search = client.catalog_search(artist).await;
|
||||
let Some(found) = search
|
||||
.artists
|
||||
.iter()
|
||||
.find(|a| normalize(&a.name) == normalize(artist))
|
||||
else {
|
||||
bugs.push(format!("{artist}: no exact-name artist in catalog search"));
|
||||
continue;
|
||||
};
|
||||
let id = found.id;
|
||||
|
||||
// The frontend shape: overview, page and the identity warm-up all in
|
||||
// flight together on one client.
|
||||
let (overview, page, identity) = tokio::join!(
|
||||
async {
|
||||
let t = Instant::now();
|
||||
let overview = client.catalog_artist_overview(id).await;
|
||||
(t.elapsed().as_millis(), overview)
|
||||
},
|
||||
async {
|
||||
let t = Instant::now();
|
||||
let page = client.catalog_artist(id).await;
|
||||
(t.elapsed().as_millis(), page)
|
||||
},
|
||||
async {
|
||||
let t = Instant::now();
|
||||
let identity = client.catalog_artist_identity(id, artist).await;
|
||||
(t.elapsed().as_millis(), identity)
|
||||
},
|
||||
);
|
||||
|
||||
let (ov_ms, overview) = overview;
|
||||
let (pg_ms, page) = page;
|
||||
let (id_ms, identity) = identity;
|
||||
|
||||
let mut flag = |what: &str| bugs.push(format!("{artist}: {what}"));
|
||||
|
||||
let (ov_albums, _ov_popular) = match &overview {
|
||||
Some(ov) => {
|
||||
if ov.artist.id != id {
|
||||
flag("overview echoed a different artist id");
|
||||
}
|
||||
if ov.artist.name.is_empty() {
|
||||
flag("overview artist name empty");
|
||||
}
|
||||
if ov.albums.is_empty() {
|
||||
flag("overview albums empty");
|
||||
}
|
||||
if ov.popular_tracks.is_empty() {
|
||||
flag("overview popular empty");
|
||||
}
|
||||
(ov.albums.len(), ov.popular_tracks.len())
|
||||
}
|
||||
None => {
|
||||
flag("overview returned None");
|
||||
(0, 0)
|
||||
}
|
||||
};
|
||||
|
||||
let (pg_albums, pg_popular, with_listeners, with_dates) = match &page {
|
||||
Some(pg) => {
|
||||
if pg.artist.id != id {
|
||||
flag("page echoed a different artist id");
|
||||
}
|
||||
if pg.albums.is_empty() {
|
||||
flag("page albums empty");
|
||||
}
|
||||
if pg.popular_tracks.is_empty() {
|
||||
flag("page popular empty");
|
||||
}
|
||||
let listeners = pg
|
||||
.popular_tracks
|
||||
.iter()
|
||||
.filter(|t| t.listeners.is_some())
|
||||
.count();
|
||||
let dates = pg
|
||||
.albums
|
||||
.iter()
|
||||
.filter(|a| a.release_date.is_some())
|
||||
.count();
|
||||
// The page waits for real supplements; zero listener counts on
|
||||
// a popular row set means the Last.fm anchor died end-to-end.
|
||||
if !pg.popular_tracks.is_empty() && listeners == 0 {
|
||||
flag("page popular has zero Last.fm listener counts");
|
||||
}
|
||||
(pg.albums.len(), pg.popular_tracks.len(), listeners, dates)
|
||||
}
|
||||
None => {
|
||||
flag("page returned None");
|
||||
(0, 0, 0, 0)
|
||||
}
|
||||
};
|
||||
|
||||
if identity.artist_id != id {
|
||||
flag("identity echoed a different artist id");
|
||||
}
|
||||
|
||||
println!(
|
||||
"{:<36} {:>6}ms {:>6}ms {:>6}ms {:>6} {:>6} {:>7} {:>6}/{:<2} {:>4}/{:<3}",
|
||||
artist, ov_ms, pg_ms, id_ms, ov_albums, pg_albums, pg_popular,
|
||||
with_listeners, pg_popular, with_dates, pg_albums
|
||||
);
|
||||
}
|
||||
|
||||
println!();
|
||||
if bugs.is_empty() {
|
||||
println!("NO INVARIANT FAILURES");
|
||||
} else {
|
||||
println!("FAILURES ({}):", bugs.len());
|
||||
for bug in &bugs {
|
||||
println!(" BUG {bug}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize(s: &str) -> String {
|
||||
s.to_lowercase()
|
||||
.chars()
|
||||
.filter(|c| c.is_alphanumeric())
|
||||
.collect()
|
||||
}
|
||||
+294
-61
@@ -361,6 +361,15 @@ const MB_RELEASE_GROUP_PAGES: usize = 3;
|
||||
/// The failures this exists for are transient by nature — a 503 out of the
|
||||
/// 1 req/s limiter, or this machine having throttled itself with probes — and
|
||||
/// the cost of giving up is the whole date + attribution pass for that artist.
|
||||
/// How long the overview holds first paint for the supplements chain (linked
|
||||
/// MBID → Last.fm ranks + original release dates). Warm the chain answers in
|
||||
/// single-digit milliseconds and the budget never bites; cold it is 2.2s+ of
|
||||
/// MusicBrainz behind the 1 req/s gate, and the overview paints without it
|
||||
/// rather than becoming the slow page it exists to preempt. Wide enough for
|
||||
/// a disk-cache read on a busy runtime, an order of magnitude under a cold
|
||||
/// network chain.
|
||||
const OVERVIEW_SUPPLEMENTS_BUDGET: Duration = Duration::from_millis(300);
|
||||
|
||||
/// Two retries behind a widening backoff is enough to ride out a blip without
|
||||
/// turning a genuine outage into a slow page: the request rides the same
|
||||
/// 1 req/s gate as everything else, and the identity pass is deferred, so
|
||||
@@ -1110,6 +1119,13 @@ pub struct MetadataClient {
|
||||
/// The fast artwork response and full page intentionally overlap, so this
|
||||
/// prevents their common profile/provider lookups from being duplicated.
|
||||
catalog_inflight: Mutex<HashMap<String, CatalogFetch>>,
|
||||
/// URLs whose most recent failure was a 4xx — an answer about the
|
||||
/// RESOURCE (it does not exist), not about the request. `cached_json`
|
||||
/// deliberately caches no failures, so its `None` conflates "MusicBrainz
|
||||
/// has no such entity" with "MusicBrainz hiccuped"; this marker is what
|
||||
/// lets a caller retry the hiccup without punishing every legitimately
|
||||
/// absent resource with the same ladder. Entries clear on later success.
|
||||
definitive_miss: Mutex<HashSet<String>>,
|
||||
/// Artist name → (fetched-at, resolved art). Art resolution hits two
|
||||
/// providers (TheAudioDB + Fanart.tv); this keeps repeat artist visits
|
||||
/// instant and keeps us polite to both services.
|
||||
@@ -1209,6 +1225,7 @@ impl MetadataClient {
|
||||
deezer_gate: Mutex::new(VecDeque::new()),
|
||||
catalog_cache: Mutex::new(HashMap::new()),
|
||||
catalog_inflight: Mutex::new(HashMap::new()),
|
||||
definitive_miss: Mutex::new(HashSet::new()),
|
||||
art_cache: Mutex::new(HashMap::new()),
|
||||
art_inflight: Mutex::new(HashMap::new()),
|
||||
provider_art_cache: Mutex::new(HashMap::new()),
|
||||
@@ -1600,6 +1617,13 @@ impl MetadataClient {
|
||||
Some(Instant::now() + retry_after);
|
||||
return Err(());
|
||||
}
|
||||
// A 4xx is definitive — the resource is absent,
|
||||
// not the server unwell. Mark it so retry ladders
|
||||
// upstream can tell the two apart (see
|
||||
// `definitive_miss`).
|
||||
if status.is_client_error() {
|
||||
self.definitive_miss.lock().await.insert(url.to_string());
|
||||
}
|
||||
net_trace(url, &format!("http-{status}"));
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -1625,6 +1649,7 @@ impl MetadataClient {
|
||||
let Some(body) = body else {
|
||||
return None;
|
||||
};
|
||||
self.definitive_miss.lock().await.remove(url);
|
||||
if body.get("error").is_none() {
|
||||
if let Some(path) = disk.as_deref() {
|
||||
write_disk_cache(path, url, &body);
|
||||
@@ -3453,24 +3478,72 @@ impl MetadataClient {
|
||||
/// artist id is known, so a cold page pays one network round rather than
|
||||
/// waiting for provider artwork or the exhaustive detail/classification
|
||||
/// pass used by `catalog_artist`.
|
||||
pub async fn catalog_artist_overview(&self, artist_id: u64) -> Option<CatalogArtistOverview> {
|
||||
pub async fn catalog_artist_overview(
|
||||
self: &Arc<Self>,
|
||||
artist_id: u64,
|
||||
) -> Option<CatalogArtistOverview> {
|
||||
let overview_started = Instant::now();
|
||||
let profile_url = format!("https://api.deezer.com/artist/{artist_id}");
|
||||
let albums_url =
|
||||
format!("https://api.deezer.com/artist/{artist_id}/albums?limit=50&index=0");
|
||||
let popular_url =
|
||||
format!("https://api.deezer.com/artist/{artist_id}/top?limit={POPULAR_POOL_LIMIT}");
|
||||
// The Last.fm chain needs only the artist id, so it rides in this join
|
||||
// rather than extending the critical path behind it. Both of its hops
|
||||
// are disk cached for a day; a cold page overlaps them with the Apple
|
||||
// identity work below, which takes longer.
|
||||
let (profile, album_page, popular, supplements) = tokio::join!(
|
||||
// The supplements chain (linked MBID → Last.fm ranks + original
|
||||
// release dates) rides this join under a BUDGET. Warm — memory or
|
||||
// same-day disk cache, every visit after the first — it answers in
|
||||
// single-digit milliseconds and rides for free, and the page is the
|
||||
// page it always was. Cold it is a MusicBrainz chain serialized
|
||||
// behind the 1 req/s gate, measured 2.2–2.6s on a THREE-album artist
|
||||
// against ~300ms for everything else in this join — it had quietly
|
||||
// become the whole critical path of the stage that exists to paint
|
||||
// fast. A missed budget cancels nothing: the task is spawned, keeps
|
||||
// filling the exact caches the full page is about to read, and that
|
||||
// page — which genuinely needs the dates for classification and
|
||||
// replaces this stage's content wholesale — collects the answer
|
||||
// moments later. The discography FLIP already glides that correction.
|
||||
// Overview: no recordings browse (the `false`). It paints fast and
|
||||
// the full page replaces its Popular list wholesale.
|
||||
let supplements_task = tokio::spawn({
|
||||
let client = Arc::clone(self);
|
||||
async move { client.artist_supplements(artist_id, false).await }
|
||||
});
|
||||
// Candidate discovery needs only the name; the identity SELECTION
|
||||
// needs this join's album evidence and is pure CPU. Split exactly the
|
||||
// way the full page splits it, so the Apple round trip rides the join
|
||||
// instead of queueing serially behind it.
|
||||
let apple_fut = async {
|
||||
let name = self
|
||||
.deezer(&profile_url)
|
||||
.await
|
||||
.and_then(|profile| Some(profile.get("name")?.as_str()?.to_string()));
|
||||
match name {
|
||||
Some(name) => self.apple_artist_candidates(&name).await,
|
||||
None => Arc::new(Vec::new()),
|
||||
}
|
||||
};
|
||||
let (profile, album_page, popular, apple_candidates, supplements) = tokio::join!(
|
||||
self.deezer(&profile_url),
|
||||
self.deezer(&albums_url),
|
||||
self.deezer(&popular_url),
|
||||
// Overview: no recordings browse. It paints fast and the full page
|
||||
// replaces its Popular list wholesale.
|
||||
self.artist_supplements(artist_id, false),
|
||||
apple_fut,
|
||||
async {
|
||||
let supplements_started = Instant::now();
|
||||
match tokio::time::timeout(OVERVIEW_SUPPLEMENTS_BUDGET, supplements_task).await {
|
||||
Ok(Ok(supplements)) => {
|
||||
stage_trace("overview.supplements", supplements_started, "within-budget");
|
||||
supplements
|
||||
}
|
||||
Ok(Err(_)) => ArtistSupplements::default(),
|
||||
Err(_) => {
|
||||
stage_trace(
|
||||
"overview.supplements",
|
||||
supplements_started,
|
||||
"budget-missed, detached",
|
||||
);
|
||||
ArtistSupplements::default()
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
stage_trace("overview.deezer_join", overview_started, "");
|
||||
let mut artist = parse_artist(&profile?)?;
|
||||
@@ -3481,13 +3554,13 @@ impl MetadataClient {
|
||||
.iter()
|
||||
.filter_map(deezer_release_evidence)
|
||||
.collect();
|
||||
// Resolve every exact-name Apple performer, then use the requested
|
||||
// Deezer id's weighted release evidence to identify the right one.
|
||||
// This adds no MusicBrainz traffic and shares its Apple I/O with the
|
||||
// full page when both frontend stages overlap.
|
||||
// Select among the exact-name Apple performers fetched in the join,
|
||||
// using the requested Deezer id's weighted release evidence to
|
||||
// identify the right one. This adds no MusicBrainz traffic and shares
|
||||
// its Apple I/O with the full page when both frontend stages overlap.
|
||||
let apple_started = Instant::now();
|
||||
let apple = self
|
||||
.apple_catalog_for_deezer(artist_id, &artist.name, &evidence)
|
||||
.apple_catalog_for_deezer_candidates(artist_id, &artist.name, apple_candidates, &evidence)
|
||||
.await;
|
||||
stage_trace("overview.apple", apple_started, "");
|
||||
stage_trace("overview.total", overview_started, "");
|
||||
@@ -3527,14 +3600,91 @@ impl MetadataClient {
|
||||
})
|
||||
}
|
||||
|
||||
/// Popular's fast lane. The overview paints the section immediately under
|
||||
/// its supplements BUDGET — cold, that means Deezer's order with no
|
||||
/// listener counts — and the REAL list (Last.fm order + counts) used to
|
||||
/// arrive only with the full page, queued behind the dates and recordings
|
||||
/// walks it never reads. This waits on exactly what Popular consumes: the
|
||||
/// linked MBID and one Last.fm call. Every Deezer/Apple input rides the
|
||||
/// caches the overview just filled, and the ranks chain singleflights
|
||||
/// with the overview's detached supplements task, so this usually JOINS
|
||||
/// in-flight work rather than adding any. Cold it answers in ~2s against
|
||||
/// the full page's 6–9s; warm it is the overview's own list.
|
||||
///
|
||||
/// The full page still owns the final word — re-attribution and the
|
||||
/// Last.fm-ranked tracks Deezer does not sell arrive with it.
|
||||
pub async fn catalog_artist_popular(
|
||||
self: &Arc<Self>,
|
||||
artist_id: u64,
|
||||
) -> Option<Vec<CatalogTrack>> {
|
||||
let popular_started = Instant::now();
|
||||
let profile_url = format!("https://api.deezer.com/artist/{artist_id}");
|
||||
let albums_url =
|
||||
format!("https://api.deezer.com/artist/{artist_id}/albums?limit=50&index=0");
|
||||
let popular_url =
|
||||
format!("https://api.deezer.com/artist/{artist_id}/top?limit={POPULAR_POOL_LIMIT}");
|
||||
let ranks_fut = async {
|
||||
let mbid = self.deezer_linked_mbid(artist_id).await?;
|
||||
self.lastfm_listener_ranks(&mbid).await
|
||||
};
|
||||
let apple_fut = async {
|
||||
let name = self
|
||||
.deezer(&profile_url)
|
||||
.await
|
||||
.and_then(|profile| Some(profile.get("name")?.as_str()?.to_string()));
|
||||
match name {
|
||||
Some(name) => self.apple_artist_candidates(&name).await,
|
||||
None => Arc::new(Vec::new()),
|
||||
}
|
||||
};
|
||||
let (profile, album_page, popular, apple_candidates, ranks) = tokio::join!(
|
||||
self.deezer(&profile_url),
|
||||
self.deezer(&albums_url),
|
||||
self.deezer(&popular_url),
|
||||
apple_fut,
|
||||
ranks_fut,
|
||||
);
|
||||
let artist = parse_artist(&profile?)?;
|
||||
let album_page = album_page?;
|
||||
let album_items = album_page.get("data")?.as_array()?;
|
||||
let evidence: Vec<DeezerReleaseEvidence> = album_items
|
||||
.iter()
|
||||
.filter_map(deezer_release_evidence)
|
||||
.collect();
|
||||
let apple = self
|
||||
.apple_catalog_for_deezer_candidates(
|
||||
artist_id,
|
||||
&artist.name,
|
||||
apple_candidates,
|
||||
&evidence,
|
||||
)
|
||||
.await;
|
||||
if apple.identity_unavailable {
|
||||
return None;
|
||||
}
|
||||
// The same foreign filter the overview applies to its pool.
|
||||
let mut foreign_album_ids = HashSet::new();
|
||||
for item in album_items {
|
||||
let Some(album) = parse_album(item) else { continue };
|
||||
let genre_id = item.get("genre_id").and_then(|id| id.as_u64());
|
||||
if apple_release_is_foreign(&apple, &album.title, genre_id) {
|
||||
foreign_album_ids.insert(album.id);
|
||||
}
|
||||
}
|
||||
let pool = deezer_popular_pool(popular.as_ref(), artist_id, &foreign_album_ids, &apple);
|
||||
let ranks = ranks.unwrap_or_default();
|
||||
let tracks = apply_lastfm_order(pool, &ranks.by_key, POPULAR_KEEP);
|
||||
stage_trace(
|
||||
"popular.total",
|
||||
popular_started,
|
||||
&format!("{} rows, ranked={}", tracks.len(), !ranks.by_key.is_empty()),
|
||||
);
|
||||
(!tracks.is_empty()).then_some(tracks)
|
||||
}
|
||||
|
||||
pub async fn catalog_artist(self: &Arc<Self>, artist_id: u64) -> Option<CatalogArtistPage> {
|
||||
let page_started = Instant::now();
|
||||
let profile = self
|
||||
.deezer(&format!("https://api.deezer.com/artist/{artist_id}"))
|
||||
.await?;
|
||||
stage_trace("page.profile", page_started, "");
|
||||
let mut artist = parse_artist(&profile)?;
|
||||
self.apply_known_portrait(&mut artist);
|
||||
let profile_url = format!("https://api.deezer.com/artist/{artist_id}");
|
||||
|
||||
// Provider artwork is an independent frontend stage. Do not make the
|
||||
// exhaustive release list wait for its Bing hero fallback: that used to
|
||||
@@ -3601,8 +3751,20 @@ impl MetadataClient {
|
||||
albums, evidence,
|
||||
))
|
||||
};
|
||||
let q = urlencode(&format!("artist:\"{}\"", artist.name));
|
||||
// Only the search and Apple legs need the NAME. They ride the same
|
||||
// singleflighted profile fetch inside the join instead of queueing the
|
||||
// whole gather behind a serial profile round trip — the id-keyed legs
|
||||
// (curated, popular, and above all the cold supplements chain, which
|
||||
// is the join's long pole) start one Deezer round trip sooner.
|
||||
let search_fut = async {
|
||||
let Some(name) = self
|
||||
.deezer(&profile_url)
|
||||
.await
|
||||
.and_then(|profile| Some(profile.get("name")?.as_str()?.to_string()))
|
||||
else {
|
||||
return Vec::new(); // no profile, no page — the join's own bail
|
||||
};
|
||||
let q = urlencode(&format!("artist:\"{name}\""));
|
||||
let mut candidates = Vec::new();
|
||||
let mut index = 0;
|
||||
for _ in 0..2 {
|
||||
@@ -3637,20 +3799,48 @@ impl MetadataClient {
|
||||
}
|
||||
candidates
|
||||
};
|
||||
let apple_fut = async {
|
||||
let name = self
|
||||
.deezer(&profile_url)
|
||||
.await
|
||||
.and_then(|profile| Some(profile.get("name")?.as_str()?.to_string()));
|
||||
match name {
|
||||
Some(name) => self.apple_artist_candidates(&name).await,
|
||||
None => Arc::new(Vec::new()),
|
||||
}
|
||||
};
|
||||
let popular_url =
|
||||
format!("https://api.deezer.com/artist/{artist_id}/top?limit={POPULAR_POOL_LIMIT}");
|
||||
let gather_started = Instant::now();
|
||||
// Rides the gather join for the same reason it does in the overview:
|
||||
// it needs only the artist id, and both its hops are disk cached, so
|
||||
// overlapping costs nothing the exhaustive album walk was not paying.
|
||||
let (curated, search_candidates, popular, apple_candidates, supplements) = tokio::join!(
|
||||
// The supplements chain (cold: several MusicBrainz requests behind
|
||||
// the mandatory 1 req/s gate — the longest thing this function does)
|
||||
// is SPAWNED here and collected beside the album-detail wave below,
|
||||
// the only other multi-second leg. Its output is first consumed after
|
||||
// the details land (dates, attribution, popular), so holding the
|
||||
// gather for it bought nothing — audited on Queen, the details wave
|
||||
// cost 5.3s serialized after a gather that was itself pinned to the
|
||||
// supplements chain. Spawned, not merely deferred: an
|
||||
// identity-unavailable bail below must leave it warming the caches
|
||||
// the identity pass reads next, exactly like the overview's budget
|
||||
// miss does.
|
||||
let supplements_task = tokio::spawn({
|
||||
let client = Arc::clone(self);
|
||||
async move { client.artist_supplements(artist_id, true).await }
|
||||
});
|
||||
let (profile, curated, search_candidates, popular, apple_candidates) = tokio::join!(
|
||||
async {
|
||||
let profile = self.deezer(&profile_url).await;
|
||||
stage_trace("page.profile", page_started, "");
|
||||
profile
|
||||
},
|
||||
curated_fut,
|
||||
search_fut,
|
||||
self.deezer(&popular_url),
|
||||
self.apple_artist_candidates(&artist.name),
|
||||
self.artist_supplements(artist_id, true),
|
||||
apple_fut,
|
||||
);
|
||||
stage_trace("page.gather_join", gather_started, "");
|
||||
let mut artist = parse_artist(&profile?)?;
|
||||
self.apply_known_portrait(&mut artist);
|
||||
let (curated, evidence) = curated?;
|
||||
let apple_started = Instant::now();
|
||||
let apple = self
|
||||
@@ -3838,7 +4028,7 @@ impl MetadataClient {
|
||||
}
|
||||
let details_started = Instant::now();
|
||||
let detail_count = need.len();
|
||||
let fetched = stream::iter(need.iter().copied())
|
||||
let details_fut = stream::iter(need.iter().copied())
|
||||
.map(|id| {
|
||||
let url = format!("https://api.deezer.com/album/{id}");
|
||||
async move { self.deezer(&url).await }
|
||||
@@ -3847,12 +4037,26 @@ impl MetadataClient {
|
||||
// request completes. Fixed 25-item waves made every later album
|
||||
// wait for the single slowest response in the preceding wave.
|
||||
.buffer_unordered(25)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
stage_trace(
|
||||
"page.details",
|
||||
details_started,
|
||||
&format!("count={detail_count}"),
|
||||
.collect::<Vec<_>>();
|
||||
// Collected together: the Deezer detail wave and the MusicBrainz
|
||||
// supplements tail are the two remaining multi-second legs, and
|
||||
// neither reads the other. Each leg stamps its own trace, so an audit
|
||||
// still sees which of the two the join was actually waiting on.
|
||||
let (fetched, supplements) = tokio::join!(
|
||||
async {
|
||||
let fetched = details_fut.await;
|
||||
stage_trace(
|
||||
"page.details",
|
||||
details_started,
|
||||
&format!("count={detail_count}"),
|
||||
);
|
||||
fetched
|
||||
},
|
||||
async {
|
||||
let supplements = supplements_task.await.unwrap_or_default();
|
||||
stage_trace("page.supplements_wait", details_started, "");
|
||||
supplements
|
||||
}
|
||||
);
|
||||
let mut details: HashMap<u64, serde_json::Value> = HashMap::new();
|
||||
for d in fetched.into_iter().flatten() {
|
||||
@@ -4412,27 +4616,35 @@ impl MetadataClient {
|
||||
return empty_catalog_artist_identity(artist_id);
|
||||
};
|
||||
stage_trace("identity.linked_mbid", identity_started, "");
|
||||
let ownership_started = Instant::now();
|
||||
let ownership = self
|
||||
.artist_release_ownership(artist_id, artist, &mbid)
|
||||
.await;
|
||||
stage_trace(
|
||||
"identity.ownership",
|
||||
ownership_started,
|
||||
&format!(
|
||||
"owned={} foreign={} fused={}",
|
||||
ownership.owned.len(),
|
||||
ownership.foreign.len(),
|
||||
ownership.cotenancy_proven
|
||||
),
|
||||
);
|
||||
let join_started = Instant::now();
|
||||
// Socials are footer content, not sorting evidence, yet every request
|
||||
// in this join serializes behind MusicBrainz's 1 req/s gate — so the
|
||||
// socials lookup was ~1 s of extra time-to-correctly-sorted on every
|
||||
// cold artist. They now ride [`Self::catalog_artist_socials`], which
|
||||
// the frontend calls only after this merge-critical response lands.
|
||||
let (albums, mixtapes, type_hints) = tokio::join!(
|
||||
// Ownership rides the join too: its MusicBrainz legs share the gate
|
||||
// with the three walks either way (the request count is what the
|
||||
// policy prices), but serialized ahead of them its non-MusicBrainz
|
||||
// leg — the Apple candidate fetch — added a tail the gate never
|
||||
// required.
|
||||
let (ownership, albums, mixtapes, type_hints) = tokio::join!(
|
||||
async {
|
||||
let ownership_started = Instant::now();
|
||||
let ownership = self
|
||||
.artist_release_ownership(artist_id, artist, &mbid)
|
||||
.await;
|
||||
stage_trace(
|
||||
"identity.ownership",
|
||||
ownership_started,
|
||||
&format!(
|
||||
"owned={} foreign={} fused={}",
|
||||
ownership.owned.len(),
|
||||
ownership.foreign.len(),
|
||||
ownership.cotenancy_proven
|
||||
),
|
||||
);
|
||||
ownership
|
||||
},
|
||||
self.musicbrainz_studio_albums(artist, &mbid),
|
||||
self.musicbrainz_mixtapes(artist, &mbid),
|
||||
self.musicbrainz_type_hints(&mbid)
|
||||
@@ -5045,7 +5257,31 @@ impl MetadataClient {
|
||||
let url = format!(
|
||||
"https://musicbrainz.org/ws/2/url?resource={resource}&inc=artist-rels&fmt=json"
|
||||
);
|
||||
let body = self.musicbrainz_json(&url).await?;
|
||||
let mut body = self.musicbrainz_json(&url).await;
|
||||
// THE anchor request: dates, attribution, listeners and the identity
|
||||
// supplement all hang off this one lookup, so a single transient
|
||||
// failure here used to orphan the whole visit from MusicBrainz with
|
||||
// no second ask — observed live: a 242ms failure at open left a page
|
||||
// with 0/24 dates and an unranked Popular, while the identical
|
||||
// lookup answered normally two seconds later. Retried on the same
|
||||
// ladder the dates fetch runs — but ONLY when the miss was not a
|
||||
// 4xx: "MusicBrainz has no Deezer link for this artist" is a real
|
||||
// answer, and an artist without one must not pay two gated requests
|
||||
// and 3.6s of backoff on every cold visit to hear it again.
|
||||
if body.is_none() && !self.definitive_miss.lock().await.contains(&url) {
|
||||
for attempt in 0..MB_RETRY_ATTEMPTS {
|
||||
tokio::time::sleep(MB_RETRY_BACKOFF * (attempt + 1)).await;
|
||||
net_trace(&url, &format!("mb-retry {}", attempt + 1));
|
||||
body = self.musicbrainz_json(&url).await;
|
||||
if body.is_some() {
|
||||
break;
|
||||
}
|
||||
if self.definitive_miss.lock().await.contains(&url) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let body = body?;
|
||||
body.get("relations")?
|
||||
.as_array()?
|
||||
.iter()
|
||||
@@ -5948,9 +6184,17 @@ impl MetadataClient {
|
||||
artist_id: u64,
|
||||
with_recordings: bool,
|
||||
) -> ArtistSupplements {
|
||||
// Traced separately from the joined legs below: this hop is serial —
|
||||
// nothing downstream can start without the MBID — and it was invisible
|
||||
// in the stage audit that found the overview's critical path, because
|
||||
// every leg it gates IS traced and the ~1.5s cold gap between them and
|
||||
// the join total belonged to no stage.
|
||||
let mbid_started = Instant::now();
|
||||
let Some(mbid) = self.deezer_linked_mbid(artist_id).await else {
|
||||
stage_trace("supplements.linked_mbid", mbid_started, "no-mbid");
|
||||
return ArtistSupplements::default();
|
||||
};
|
||||
stage_trace("supplements.linked_mbid", mbid_started, "");
|
||||
let (ranks, (first_released, musicbrainz_unavailable), recordings) = tokio::join!(
|
||||
self.lastfm_listener_ranks(&mbid),
|
||||
self.musicbrainz_first_release_dates(&mbid),
|
||||
@@ -6815,17 +7059,6 @@ impl MetadataClient {
|
||||
catalog
|
||||
}
|
||||
|
||||
async fn apple_catalog_for_deezer(
|
||||
&self,
|
||||
deezer_id: u64,
|
||||
artist: &str,
|
||||
evidence: &[DeezerReleaseEvidence],
|
||||
) -> Arc<AppleArtistCatalog> {
|
||||
let candidates = self.apple_artist_candidates(artist).await;
|
||||
self.apple_catalog_for_deezer_candidates(deezer_id, artist, candidates, evidence)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn apple_catalog_for_deezer_candidates(
|
||||
&self,
|
||||
deezer_id: u64,
|
||||
|
||||
Reference in new issue
Block a user