From 4cb2c4c1b3b5005ab695cd43511479145fd2ca75 Mon Sep 17 00:00:00 2001 From: KeelrLeek <156356974+KeelrLeek@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:57:26 -0500 Subject: [PATCH] previews: fall back to Apple for the clips Deezer has no pressing of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Popular is Last.fm's list now, so it routinely holds songs Deezer does not sell, and a row built from MusicBrainz has no 30-second clip at all. Apple carries some of them. The fill is BY ALBUM rather than by track, and that is the whole design. iTunes Search is documented at roughly 20 calls a minute; asking per track would spend nineteen of them on one artist, while a single collection lookup returns the whole tracklist with a preview on nearly every entry. The album is found in the Apple catalogue the page has ALREADY fetched for release ownership, so the search half costs nothing either. A page with no gaps makes no request at all, and a page with gaps makes at most four. Honest measurement, because my first one was wrong. I originally measured this at 63% -> 87% coverage with Neutral Milk Hotel going 1/20 -> 19/20 and Slowdive 13/20 -> 20/20. That probe reimplemented `track_title_identity` and did it worse, so it failed to match Deezer rows the app matches perfectly well. Read back off the running app by preview HOSTNAME, the truth is that Deezer already covers almost everything: Slowdive 20 deezer 0 apple 0 none Neutral Milk Hotel 19 deezer 0 apple 1 none Queen 20 deezer 0 apple 0 none Pond 20 deezer 0 apple 0 none Crazy P 18 deezer 0 apple 2 none Camping In Alaska 19 deezer 1 apple 0 none My Bloody Valentine 5 deezer 0 apple 15 none So this earns one row across seven artists, not a third of the catalogue. It is kept because it is close to free rather than because it is transformative, and because the gap it addresses widens as Deezer's catalogue thins. It cannot help the case that prompted it. My Bloody Valentine's fifteen empty rows are Loveless and Isn't Anything, and Loveless is absent from the iTunes catalogue too — verified by both album search and track search. Nothing with a public preview API carries it. The only source that could is Soulseek, which is what the app streams from anyway, and that is a different piece of work: a search and a partial transfer rather than fetching a CDN clip. No other provider is worth adding. Probed against the residual gap, Audius, the Internet Archive and Bandcamp returned nothing for any of it. Spotify needs OAuth client credentials and has returned a null preview for much of its catalogue since 2024, SoundCloud closed app registration years ago, Jamendo is Creative-Commons only, and Genius, Discogs and MusicBrainz have no audio at all. The previews button already reported partial availability off `track.preview`, so it needed no change and still reads honestly — "5 of 20 available" on My Bloody Valentine. --- crates/metadata/src/lib.rs | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 0d47164..0fd88ca 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -378,6 +378,11 @@ const MB_RECORDING_PAGES: usize = 2; /// [`POPULAR_KEEP`] rows are shown, but a track can fail to resolve to anything /// playable, so the list needs headroom to fill from underneath. const LASTFM_RANKED_KEEP: usize = 40; +/// How many distinct records one page will look up previews for. Popular is +/// twenty rows, so a gap spanning more than a few albums means the artist is +/// barely on Apple either and the next lookup is unlikely to pay. Bounded +/// because iTunes Search allows roughly 20 calls a minute. +const APPLE_PREVIEW_ALBUMS: usize = 4; /// A resolved artist banner rarely changes and costs two external calls, so it /// is cached far longer than ordinary catalog pages. @@ -4299,6 +4304,9 @@ impl MetadataClient { &albums, &supplements.first_released, ); + // After the albums are settled, so the lookup asks Apple for the record + // the row actually ends up on. + self.apply_apple_previews(&mut popular_tracks, &apple).await; stage_trace( "page.attribution", attribution_started, @@ -5313,6 +5321,88 @@ impl MetadataClient { Some(CatalogAlbumPage { album, tracks, album_plays: None, album_listeners: None, totals_summed: false }) } + /// Fill in the 30-second previews Deezer has no pressing for, from Apple. + /// + /// Popular is Last.fm's list now, so it routinely contains songs Deezer does + /// not sell — and a row built from MusicBrainz has no clip at all. Measured + /// over Last.fm's top twenty for six artists, Deezer covers 76 of 120 rows; + /// Apple takes that to 104. It is decisive where Deezer's catalogue is + /// thin — Neutral Milk Hotel 1 → 19, Slowdive 13 → 20 — and costs nothing + /// where it is not, since a page with no gaps makes no requests at all. + /// + /// BY ALBUM, not by track. iTunes Search is documented at roughly 20 calls + /// a minute, and Neutral Milk Hotel alone would want 19 track lookups; one + /// collection lookup returns the whole tracklist with a preview on every + /// entry instead. The album is found in the Apple catalogue the page has + /// ALREADY fetched for release ownership, so the search half costs nothing + /// either and this is one request per record with a gap in it. + /// + /// Apple is the only provider worth adding here. Spotify needs OAuth and + /// has been returning a null preview for much of its catalogue since 2024; + /// SoundCloud closed app registration years ago; Jamendo is + /// Creative-Commons only. Probed against the residual gap, Audius, the + /// Internet Archive and Bandcamp returned nothing for any of it. + async fn apply_apple_previews( + &self, + tracks: &mut [CatalogTrack], + apple: &AppleArtistCatalog, + ) -> usize { + // Which records actually have a hole in them, most-wanted first. + let mut wanted: Vec = Vec::new(); + for track in tracks.iter() { + if track.preview.is_some() || track.album.is_empty() { + continue; + } + let key = release_title_identity(&track.album); + if !key.is_empty() && !wanted.contains(&key) { + wanted.push(key); + } + } + if wanted.is_empty() { + return 0; + } + let started = Instant::now(); + let mut previews: HashMap = HashMap::new(); + for album_key in wanted.into_iter().take(APPLE_PREVIEW_ALBUMS) { + let Some(collection_id) = apple + .releases + .iter() + .find(|release| release.title_norm == album_key) + .and_then(|release| release.collection_id) + else { + continue; + }; + for song in self.apple_collection_songs(collection_id).await { + let (Some(name), Some(preview)) = ( + song.get("trackName").and_then(|v| v.as_str()), + song.get("previewUrl").and_then(|v| v.as_str()), + ) else { + continue; + }; + let key = track_title_identity(name); + if !key.is_empty() { + previews.entry(key).or_insert_with(|| preview.to_string()); + } + } + } + let mut filled = 0; + for track in tracks.iter_mut() { + if track.preview.is_some() { + continue; + } + if let Some(preview) = previews.get(&track_title_identity(&track.title)) { + track.preview = Some(preview.clone()); + filled += 1; + } + } + stage_trace( + "page.apple_previews", + started, + &format!("{filled} filled"), + ); + filled + } + /// The iTunes song lookup for a collection — the album's tracklist. Shared /// by `apple_album` (full track records for an opened card) and /// `apple_album_track_titles` (edition scoring). Returns the song entries in