diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 84d1b81..0d47164 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -346,9 +346,9 @@ const LASTFM_TRACK_LIMIT: usize = 1000; /// the curve flattens (79% at 100) and every extra track is one more row that /// can never be displayed. const POPULAR_POOL_LIMIT: usize = 50; -/// How many reordered tracks reach the frontend. Unchanged: the UI shows five, -/// expands to ten, and `openPreviews` snapshots ten. -const POPULAR_KEEP: usize = 10; +/// How many ranked tracks reach the frontend. The UI shows five and expands to +/// the rest, so this is the size of the expanded list. +const POPULAR_KEEP: usize = 20; /// Release-group paging for the first-release-date pass. 100 is MusicBrainz's /// maximum page; three pages covers a very prolific catalogue (Queen has ~100 /// album groups alone) without ever walking a discography forever behind the @@ -367,6 +367,17 @@ const MB_RELEASE_GROUP_PAGES: usize = 3; /// nothing on screen is waiting on it. const MB_RETRY_ATTEMPTS: u32 = 2; const MB_RETRY_BACKOFF: Duration = Duration::from_millis(1200); +/// Release paging for [`MetadataClient::musicbrainz_recordings`]. 100 is +/// MusicBrainz's maximum page and each one carries every tracklist on it, so +/// two pages reach 200 pressings — comfortably the whole discography for the +/// artists this exists for, and bounded because each page costs a second behind +/// the 1 req/s gate and roughly 200 KB of body. +const MB_RECORDING_PAGE: usize = 100; +const MB_RECORDING_PAGES: usize = 2; +/// How deep Last.fm's rank order is kept for building Popular. Only +/// [`POPULAR_KEEP`] rows are shown, but a track can fail to resolve to anything +/// playable, so the list needs headroom to fill from underneath. +const LASTFM_RANKED_KEEP: usize = 40; /// A resolved artist banner rarely changes and costs two external calls, so it /// is cached far longer than ordinary catalog pages. @@ -501,6 +512,61 @@ pub struct CatalogTrack { pub origin_track_id: Option, } +/// One entry of Last.fm's top-tracks table, in rank order. +#[derive(Debug, Clone)] +struct LastfmTrack { + /// [`track_title_identity`] of `title`. + key: String, + /// Last.fm's own spelling, used when no Deezer row supplies one. + title: String, + stats: TrackStats, +} + +/// Last.fm's top-tracks table, both ways round: by identity for the lookups +/// that annotate an existing row, and in rank order for the ones that BUILD the +/// Popular list. +#[derive(Debug, Clone, Default)] +struct LastfmRanks { + by_key: HashMap, + ranked: Vec, +} + +/// One recording of one song, and the record it is best attributed to. +/// Built by [`MetadataClient::musicbrainz_recordings`]. +#[derive(Debug, Clone)] +struct MbRecording { + release_group_id: String, + album_title: String, + /// MusicBrainz's spelling of the track, used when Deezer has no row to take + /// a display title from. + title: String, + release_date: Option, + length_ms: Option, + /// Primary type Album with NO secondary type — i.e. a studio record rather + /// than a compilation, live album, soundtrack or remix set. + plain_album: bool, +} + +impl MbRecording { + /// Is this a better home for the song than `other`? + /// + /// A plain album always beats a typed one, however early the typed one is + /// dated — the point of the test is that compilations are dated by their + /// own release and would otherwise win on date alone. Between two of the + /// same kind, the earlier record wins; a missing date loses to any date. + fn outranks(&self, other: &Self) -> bool { + match (self.plain_album, other.plain_album) { + (true, false) => true, + (false, true) => false, + _ => match (self.release_date.as_deref(), other.release_date.as_deref()) { + (Some(mine), Some(theirs)) => mine < theirs, + (Some(_), None) => true, + _ => false, + }, + } + } +} + /// What one MusicBrainz release GROUP says about a work: when it first came /// out, and what kind of release it is. /// @@ -546,6 +612,11 @@ struct ArtistSupplements { listeners: HashMap, /// MusicBrainz first-release facts, keyed by [`release_date_key`]. first_released: HashMap, + /// Last.fm's top tracks IN RANK ORDER, which is what Popular is built from. + ranked: Vec, + /// Which record each recording belongs to, for Last.fm entries Deezer has + /// no pressing of. Empty on the overview stage, which does not pay for it. + recordings: HashMap, /// MusicBrainz never answered, as distinct from having nothing to say. /// /// The two used to be the same value — a failed page `break`s and leaves an @@ -3344,7 +3415,9 @@ impl MetadataClient { self.deezer(&profile_url), self.deezer(&albums_url), self.deezer(&popular_url), - self.artist_supplements(artist_id), + // Overview: no recordings browse. It paints fast and the full page + // replaces its Popular list wholesale. + self.artist_supplements(artist_id, false), ); stage_trace("overview.deezer_join", overview_started, ""); let mut artist = parse_artist(&profile?)?; @@ -3522,7 +3595,7 @@ impl MetadataClient { search_fut, self.deezer(&popular_url), self.apple_artist_candidates(&artist.name), - self.artist_supplements(artist_id), + self.artist_supplements(artist_id, true), ); stage_trace("page.gather_join", gather_started, ""); let (curated, evidence) = curated?; @@ -4197,7 +4270,13 @@ impl MetadataClient { artist.album_count = Some(albums.len() as u32); albums.sort_by(|a, b| b.release_date.cmp(&a.release_date)); let pool = deezer_popular_pool(popular.as_ref(), artist_id, &foreign_album_ids, &apple); - let mut popular_tracks = apply_lastfm_order(pool, &supplements.listeners, POPULAR_KEEP); + let mut popular_tracks = build_popular( + &supplements.ranked, + pool, + &supplements.recordings, + &artist.name, + POPULAR_KEEP, + ); // Runs LAST, on the ten rows that survived: after the album pipeline // has foreign-filtered, ownership-checked, classified and folded, and // after `apply_original_release_dates` so the date tiebreak compares @@ -4212,11 +4291,19 @@ impl MetadataClient { &details, &supplements.first_released, ); + // Last, and only for songs whose original record Deezer has no copy of + // at all — the case attribution structurally cannot reach. + let relabelled = apply_musicbrainz_track_albums( + &mut popular_tracks, + &supplements.recordings, + &albums, + &supplements.first_released, + ); stage_trace( "page.attribution", attribution_started, &format!( - "{reattributed}/{} re-attributed{}", + "{reattributed}/{} re-attributed, {relabelled} relabelled{}", popular_tracks.len(), // Attribution reads the same facts, so it fails the same way: // with no secondary types the rule is a no-op and every popular @@ -5451,6 +5538,7 @@ impl MetadataClient { let Some(ranks) = self.lastfm_listener_ranks(&mbid).await else { return; }; + let ranks = ranks.by_key; let started = Instant::now(); let mut matched = 0; let mut summed_plays = 0_u64; @@ -5697,16 +5785,34 @@ impl MetadataClient { /// No MusicBrainz entry for this Deezer artist means no anchor for either /// provider, and the caller falls back to Deezer's own ordering and dates — /// which is exactly what the page did before any of this existed. - async fn artist_supplements(&self, artist_id: u64) -> ArtistSupplements { + async fn artist_supplements( + &self, + artist_id: u64, + with_recordings: bool, + ) -> ArtistSupplements { let Some(mbid) = self.deezer_linked_mbid(artist_id).await else { return ArtistSupplements::default(); }; - let (listeners, (first_released, musicbrainz_unavailable)) = tokio::join!( + let (ranks, (first_released, musicbrainz_unavailable), recordings) = tokio::join!( self.lastfm_listener_ranks(&mbid), self.musicbrainz_first_release_dates(&mbid), + // Only the full page pays for this: it is one or two more requests + // behind the 1 req/s gate, and the overview exists to paint fast. + // The overview keeps ordering Deezer's pool, and the full page + // replaces that list wholesale a moment later. + async { + if with_recordings { + self.musicbrainz_recordings(&mbid).await + } else { + HashMap::new() + } + }, ); + let ranks = ranks.unwrap_or_default(); ArtistSupplements { - listeners: listeners.unwrap_or_default(), + listeners: ranks.by_key, + ranked: ranks.ranked, + recordings, first_released, musicbrainz_unavailable, } @@ -5804,6 +5910,117 @@ impl MetadataClient { (dates, false) } + /// Which record each of this artist's recordings comes from, keyed by + /// [`track_title_identity`]. + /// + /// This is what lets Popular show a song Deezer does not sell. Deezer has + /// none of Loveless — 8 of My Bloody Valentine's Last.fm top ten are absent + /// from its catalogue outright — so a row for "When You Sleep" has no + /// Deezer album, cover or duration to inherit. MusicBrainz has all three. + /// + /// One browse carries the whole discography: `inc=recordings` returns every + /// release WITH its tracklist, and `release-groups` names the work each + /// pressing belongs to. Measured on My Bloody Valentine, one 208 KB request + /// covered 64 releases folding to 27 works — versus one request per album + /// behind the 1 req/s gate, which is the reason this is a browse and not a + /// lookup per supplemented row. + /// + /// A recording usually appears on several pressings, so the winner is + /// chosen the way [`apply_original_album_attribution`] chooses one: a plain + /// album outranks anything carrying a secondary type, and the earliest date + /// breaks the tie. Without the type test the compilations win, because a + /// hits package is dated earlier than nothing and later than everything. + async fn musicbrainz_recordings(&self, mbid: &str) -> HashMap { + let started = Instant::now(); + let mut found: HashMap = HashMap::new(); + for page in 0..MB_RECORDING_PAGES { + let offset = page * MB_RECORDING_PAGE; + let url = format!( + "https://musicbrainz.org/ws/2/release?artist={mbid}\ + &inc=recordings+release-groups&limit={MB_RECORDING_PAGE}&offset={offset}&fmt=json" + ); + let Some(body) = self.musicbrainz_json(&url).await else { + break; + }; + let Some(releases) = body.get("releases").and_then(|v| v.as_array()) else { + break; + }; + let returned = releases.len(); + for release in releases { + let Some(group) = release.get("release-group") else { + continue; + }; + let Some(album_title) = group.get("title").and_then(|v| v.as_str()) else { + continue; + }; + let Some(release_group_id) = group.get("id").and_then(|v| v.as_str()) else { + continue; + }; + // Mixtape/Street counts as a studio record here for the same + // reason it gets its own tab rather than being thrown away: + // Exmilitary is Death Grips' debut and Acid Rap is Chance the + // Rapper's best-scrobbled record. Excluding it left "Guillotine" + // attributed to the SINGLE named after it. Every other secondary + // type — Compilation, Live, Soundtrack, Remix — still disqualifies, + // which is the whole point of the test. + let secondary: Vec<&str> = group + .get("secondary-types") + .and_then(|v| v.as_array()) + .map(|types| types.iter().filter_map(|t| t.as_str()).collect()) + .unwrap_or_default(); + let plain_album = group.get("primary-type").and_then(|v| v.as_str()) + == Some("Album") + && secondary + .iter() + .all(|kind| kind.eq_ignore_ascii_case("Mixtape/Street")); + let release_date = group + .get("first-release-date") + .and_then(|v| v.as_str()) + .filter(|v| !v.is_empty()) + .map(String::from); + let tracks = release + .get("media") + .and_then(|v| v.as_array()) + .into_iter() + .flatten() + .filter_map(|medium| medium.get("tracks").and_then(|v| v.as_array())) + .flatten(); + for track in tracks { + let Some(title) = track.get("title").and_then(|v| v.as_str()) else { + continue; + }; + let key = track_title_identity(title); + if key.is_empty() { + continue; + } + let candidate = MbRecording { + release_group_id: release_group_id.to_string(), + album_title: album_title.to_string(), + title: title.to_string(), + release_date: release_date.clone(), + length_ms: track.get("length").and_then(|v| v.as_u64()), + plain_album, + }; + match found.get(&key) { + Some(held) if !candidate.outranks(held) => {} + _ => { + found.insert(key, candidate); + } + } + } + } + if returned < MB_RECORDING_PAGE { + break; + } + } + stage_trace( + "musicbrainz.recordings", + started, + &format!("{} recordings", found.len()), + ); + found + } + /// All-time Last.fm listener counts for one artist's tracks, keyed by /// [`track_title_identity`]. /// @@ -5829,7 +6046,7 @@ impl MetadataClient { /// Step 2 costs no extra MusicBrainz traffic — it reads the same /// `artist_detail_url` body [`Self::artist_socials`] already fetches, which /// is memory and disk cached. - async fn lastfm_listener_ranks(&self, mbid: &str) -> Option> { + async fn lastfm_listener_ranks(&self, mbid: &str) -> Option { let started = Instant::now(); let by_mbid = format!( "https://ws.audioscrobbler.com/2.0/?method=artist.gettoptracks\ @@ -5867,7 +6084,15 @@ impl MetadataClient { body = Some(candidate); } - let tracks = body? + // The ORDER is kept as well as the counts. Popular used to take + // Deezer's fifty and merely re-sort them, so a song Last.fm ranks first + // could not appear at all if Deezer did not sell it — which is the + // normal case for an artist whose catalogue Deezer carries badly. Eight + // of My Bloody Valentine's Last.fm top ten are absent from Deezer + // entirely, so its Popular showed b-sides and soundtrack cuts. Keeping + // this list means Last.fm decides WHAT is popular and Deezer is only + // consulted for a playable pressing. + let entries: Vec<(String, String, TrackStats)> = body? .pointer("/toptracks/track")? .as_array()? .iter() @@ -5883,16 +6108,28 @@ impl MetadataClient { .and_then(|value| value.parse().ok()) .unwrap_or(0); let key = track_title_identity(title); - (!key.is_empty()).then_some((key, TrackStats { plays, listeners })) + (!key.is_empty()).then_some(( + key, + title.to_string(), + TrackStats { plays, listeners }, + )) }) + .collect(); + + let mut by_key: HashMap = HashMap::new(); + let mut ranked: Vec = Vec::new(); + for (key, title, stats) in entries { // Last.fm's list is already listeners-descending, so an earlier // entry wins a key collision (e.g. a remaster listed twice). - .fold(HashMap::new(), |mut acc, (key, stats)| { - acc.entry(key).or_insert(stats); - acc - }); - stage_trace("lastfm.ranks", started, &format!("{} tracks", tracks.len())); - (!tracks.is_empty()).then_some(tracks) + if by_key.insert(key.clone(), stats).is_some() { + continue; + } + if ranked.len() < LASTFM_RANKED_KEEP { + ranked.push(LastfmTrack { key, title, stats }); + } + } + stage_trace("lastfm.ranks", started, &format!("{} tracks", by_key.len())); + (!by_key.is_empty()).then_some(LastfmRanks { by_key, ranked }) } /// The `last.fm` URL MusicBrainz declares for this artist, if any. Shares @@ -7737,6 +7974,199 @@ fn deezer_popular_pool( /// is still a real track. Same-recording pressings are collapsed to the entry /// Deezer ranks highest, because they would otherwise sort adjacently showing /// an identical count twice. +/// Build Popular from Last.fm's ranking, using Deezer only for playable rows. +/// +/// The old rule took Deezer's `/top` fifty and re-sorted them by scrobbles, +/// which silently capped the list at what Deezer happens to sell. For most +/// artists that is invisible — measured across Queen, Radiohead, Weezer, Green +/// Day, Fleetwood Mac and Tame Impala the two agree on 56 of 60 rows. For an +/// artist Deezer carries badly it is the whole list: Deezer has no Loveless at +/// all, so 8 of My Bloody Valentine's Last.fm top ten were unreachable and +/// Popular filled up with b-sides, a compilation cut and a soundtrack cut. +/// +/// So Last.fm decides what is popular, in its order, and each entry is resolved +/// in preference order: +/// +/// 1. A Deezer track of the same [`track_title_identity`] — the ideal case, +/// since it brings a real album, cover, preview and duration, and stays +/// eligible for [`apply_original_album_attribution`] afterwards. +/// 2. Otherwise a MusicBrainz recording, which supplies the record it belongs +/// to and the track length. There is no preview, and that is the accepted +/// cost: playback resolves over Soulseek by artist and title, so the row +/// plays exactly like any other. +/// 3. Otherwise the entry is dropped — a title with no album behind it from +/// either provider is not something the page can meaningfully show. +/// +/// Deezer rows that Last.fm has never seen still backfill from underneath, so a +/// thin Last.fm artist keeps a full list rather than an empty one. +/// Put a popular row on the record its song came from, when that record is one +/// Deezer does not sell. +/// +/// [`apply_original_album_attribution`] can only ever move a track onto ANOTHER +/// DEEZER ALBUM, because it swaps the row onto that album's real track. When +/// the original is missing from Deezer entirely there is nothing to swap to, so +/// the row keeps whatever pressing Deezer licensed — and the pressing it +/// licensed is exactly the kind of record the rule exists to get rid of. My +/// Bloody Valentine is the clean example: Deezer carries just two Loveless +/// songs, "Only Shallow" on the compilation *Amateur* and "Sometimes" on the +/// *Lost In Translation* soundtrack, so those two sat under a compilation and a +/// soundtrack while the eighteen tracks Deezer does NOT carry were already +/// showing Loveless. +/// +/// This is a RELABEL, not a swap: `album`, `album_id` and `cover` follow +/// MusicBrainz while `id`, `preview` and `duration_secs` stay on the Deezer +/// pressing, because that pressing is still the playable one. Nothing is +/// misrepresented — it is the same recording, and the album named is the album +/// the recording is from. +/// +/// Only a PLAIN album may claim a row, and only away from a record that is not +/// itself one. Without the second half a legitimately-attributed row churns: +/// MusicBrainz would happily relabel Queen's "Bohemian Rhapsody" from Deezer's +/// *A Night At The Opera* pressing to its own, which is the same record under a +/// different spelling. +fn apply_musicbrainz_track_albums( + tracks: &mut [CatalogTrack], + recordings: &HashMap, + albums: &[CatalogAlbum], + mb: &HashMap, +) -> usize { + let mut relabelled = 0; + for track in tracks.iter_mut() { + let Some(recording) = recordings.get(&track_title_identity(&track.title)) else { + continue; + }; + if !recording.plain_album { + continue; + } + // Already on that record, however the two providers spell it. + if album_title_identity(&track.album) == album_title_identity(&recording.album_title) { + continue; + } + // Only take a row AWAY from a record that is not a studio album. A row + // already sitting on one is either correctly attributed or was just + // placed there by `apply_original_album_attribution`, and overruling it + // would trade a real Deezer album card for a synthetic id with no card + // in the discography behind it. + let current = track + .album_id + .and_then(|id| albums.iter().find(|album| album.id == id)); + let displaceable = match current { + Some(album) => { + album.is_soundtrack + || album.is_live + || album.record_type.as_deref() != Some("album") + || musicbrainz_excludes_original(mb, album) + } + // No album row on this page at all — nothing to protect. + None => true, + }; + if !displaceable { + continue; + } + track.album = recording.album_title.clone(); + track.album_id = Some(synthetic_catalog_id(&format!( + "release-group:{}", + recording.release_group_id + ))); + track.cover = Some(format!( + "https://coverartarchive.org/release-group/{}/front-500", + recording.release_group_id + )); + relabelled += 1; + } + relabelled +} + +fn build_popular( + ranked: &[LastfmTrack], + pool: Vec, + recordings: &HashMap, + artist: &str, + keep: usize, +) -> Vec { + let mut by_key: HashMap = HashMap::new(); + let mut pool_order: Vec = Vec::new(); + for track in pool { + let key = track_title_identity(&track.title); + if key.is_empty() { + continue; + } + if !by_key.contains_key(&key) { + pool_order.push(key.clone()); + by_key.insert(key, track); + } + } + + let mut out: Vec = Vec::new(); + let mut used: HashSet = HashSet::new(); + for entry in ranked { + if out.len() >= keep { + break; + } + if let Some(mut track) = by_key.remove(&entry.key) { + track.listeners = Some(entry.stats.listeners); + track.plays = (entry.stats.plays > 0).then_some(entry.stats.plays); + used.insert(entry.key.clone()); + out.push(track); + continue; + } + let Some(recording) = recordings.get(&entry.key) else { + continue; + }; + used.insert(entry.key.clone()); + out.push(CatalogTrack { + // Stable across loads and distinct from every Deezer id, the same + // way supplemented ALBUM rows are keyed. + id: synthetic_catalog_id(&format!( + "lastfm-track:{}:{}", + recording.release_group_id, entry.key + )), + // Deezer's spelling is preferred wherever it exists; here there is + // none, so MusicBrainz's is the canonical one rather than Last.fm's + // scrobbled string, which carries whatever users typed. + title: recording.title.clone(), + artist: artist.to_string(), + album: recording.album_title.clone(), + // The SAME synthetic id the album supplement mints for this release + // group, so the row points at the card the discography already + // shows and the now-playing marker lines up. + album_id: Some(synthetic_catalog_id(&format!( + "release-group:{}", + recording.release_group_id + ))), + track_no: None, + disk_number: None, + unreleased: false, + cover: Some(format!( + "https://coverartarchive.org/release-group/{}/front-500", + recording.release_group_id + )), + // No Deezer pressing means no 30-second preview. The row still + // plays: streaming resolves by artist and title over Soulseek. + preview: None, + duration_secs: recording.length_ms.map(|ms| (ms / 1000) as u32), + listeners: Some(entry.stats.listeners), + plays: (entry.stats.plays > 0).then_some(entry.stats.plays), + origin_track_id: None, + }); + } + + // Backfill in Deezer's own order for anything Last.fm never ranked, so a + // sparsely-scrobbled artist still gets a full list. + for key in pool_order { + if out.len() >= keep { + break; + } + if used.contains(&key) { + continue; + } + if let Some(track) = by_key.remove(&key) { + out.push(track); + } + } + out +} + fn apply_lastfm_order( tracks: Vec, ranks: &HashMap, diff --git a/src/views/CatalogArtist.svelte b/src/views/CatalogArtist.svelte index fbe5629..0fdbcb9 100644 --- a/src/views/CatalogArtist.svelte +++ b/src/views/CatalogArtist.svelte @@ -710,11 +710,15 @@ const banner = $derived(displayedBanner.kind === "gpu" ? null : displayedBanner.url); const heroPainted = $derived(Boolean(displayedBanner.url)); - // Popular songs from Deezer's play-ranked /top. Show five, Spotify-style, - // with a "Show more" that reveals the rest (up to ten). Collapse again when - // the artist changes so a new page never opens pre-expanded. + // Popular songs, ranked by Last.fm listeners. Show five, Spotify-style, + // with a "Show more" that reveals the rest. Collapse again when the artist + // changes so a new page never opens pre-expanded. + // + // The length is the BACKEND's to decide (`POPULAR_KEEP`) rather than a + // second cap here: the two were both 10 and silently disagreed the moment + // one moved. let showAllPopular = $state(false); - const allPopular = $derived(page?.popular_tracks?.slice(0, 10) ?? []); + const allPopular = $derived(page?.popular_tracks ?? []); const popularTracks = $derived(showAllPopular ? allPopular : allPopular.slice(0, 5)); $effect(() => { void artistIdentity; @@ -750,7 +754,7 @@ * pipeline via the same path the Popular rows use. */ function openPreviews() { if (!page?.popular_tracks?.length) return; - const snapshotTracks = page.popular_tracks.slice(0, 10); + const snapshotTracks = page.popular_tracks; const byProviderId = new Map(snapshotTracks.map((t) => [String(t.id), t])); const manifest = buildArtistManifest({ artist: {