Files
SoulStream/src-tauri/src/provider_playlist.rs
T

1349 lines
50 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Playlist import from the streaming services — Spotify, Apple Music,
//! TIDAL, Deezer — off each service's own PUBLIC surface, the same honest
//! metadata posture as the YouTube doors: nothing is downloaded, nothing
//! needs an account, and what a service refuses to show anonymously is
//! reported instead of worked around.
//!
//! Where each list comes from (all live-probed 2026-08-24):
//! - **Deezer**: the open JSON API (`api.deezer.com/playlist/{id}`), no
//! auth at all, paginated by the `next` links it hands back.
//! - **Spotify**: the embed player page's `__NEXT_DATA__` JSON — the only
//! anonymous surface left. Both token endpoints are closed to us on
//! purpose: `/get_access_token` is blocked at their edge (Fastly, 403
//! "URL Blocked"), and `/api/token` answers 400 with "Usage of this
//! endpoint is not permitted under the Spotify Developer Terms" — so
//! this module does not go near them. The embed lists AT MOST 100 tracks
//! and accepts no offset/limit/page parameter (all four probed), which is
//! why pasted TRACK links exist below: the per-track pages carry the same
//! data with no ceiling.
//! - **Apple Music**: the playlist page's `serialized-server-data` JSON —
//! full tracklists inline (`trackCount` is checked against what was
//! actually present).
//! - **TIDAL**: `api.tidal.com/v1` with the public anonymous token their
//! own embed widget ships to every browser; if that token rotates, the
//! current one is re-mined from the embed bundle before giving up.
//! Paginated to the full list.
use crate::youtube_setlist::walk;
use serde_json::Value;
pub struct Row {
pub artist: String,
pub title: String,
pub album: Option<String>,
pub duration_secs: Option<u32>,
}
pub struct ProviderFetch {
/// Draft-format slug: "spotify" | "apple-music" | "tidal" | "deezer".
pub provider: &'static str,
pub title: String,
pub author: String,
pub rows: Vec<Row>,
pub cover_url: Option<String>,
pub warnings: Vec<String>,
}
/// Rows a single import will take, across every provider.
const MAX_TRACKS: usize = 1000;
const UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \
(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
/// The TIDAL embed widget's anonymous token, as shipped 2026-08.
const TIDAL_PUBLIC_TOKEN: &str = "TPIsV0A9lyiqKl9u";
fn client(secs: u64) -> Result<reqwest::Client, String> {
reqwest::Client::builder()
.user_agent(UA)
.timeout(std::time::Duration::from_secs(secs))
.build()
.map_err(|e| e.to_string())
}
// ---- URL detection ----------------------------------------------------------
#[derive(Debug, PartialEq)]
pub enum ProviderUrl {
Spotify(String),
Apple { url: String },
Tidal(String),
Deezer(String),
}
/// Which service a pasted link belongs to, and the playlist id it names.
/// None for anything else — including YouTube, which has its own module.
pub fn detect(url: &str) -> Option<ProviderUrl> {
let s = url.trim();
let after_scheme = s.split("://").nth(1).unwrap_or(s);
let (host, rest) = after_scheme.split_once('/')?;
let host = host.split('@').next_back().unwrap_or(host).to_lowercase();
let host = host.split(':').next().unwrap_or(&host).to_string();
let path = rest.split(['?', '#']).next().unwrap_or(rest);
let segs: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect();
if host == "open.spotify.com" {
// open.spotify.com/playlist/{id} — possibly behind /intl-xx/.
let at = segs.iter().position(|s| *s == "playlist")?;
let id = segs.get(at + 1)?;
let ok = (16..=32).contains(&id.len()) && id.bytes().all(|b| b.is_ascii_alphanumeric());
return ok.then(|| ProviderUrl::Spotify((*id).to_string()));
}
if host == "music.apple.com" {
// music.apple.com/{storefront}/playlist/{slug}/pl.{id} — the whole
// URL is the identity (storefront matters), so keep it as given.
let is_pl = segs.iter().any(|s| *s == "playlist") && segs.iter().any(|s| s.starts_with("pl."));
return is_pl.then(|| ProviderUrl::Apple {
url: format!("https://music.apple.com/{}", segs.join("/")),
});
}
if host == "tidal.com" || host.ends_with(".tidal.com") {
// tidal.com/browse/playlist/{uuid} · listen.tidal.com/playlist/{uuid}
let at = segs.iter().position(|s| *s == "playlist")?;
let id = segs.get(at + 1)?;
let ok = id.len() == 36
&& id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-')
&& id.bytes().filter(|b| *b == b'-').count() == 4;
return ok.then(|| ProviderUrl::Tidal((*id).to_string()));
}
if host == "www.deezer.com" || host == "deezer.com" || host == "deezer.page.link" {
// deezer.com/{lang}/playlist/{digits}
let at = segs.iter().position(|s| *s == "playlist")?;
let id = segs.get(at + 1)?;
let ok = !id.is_empty() && id.bytes().all(|b| b.is_ascii_digit());
return ok.then(|| ProviderUrl::Deezer((*id).to_string()));
}
None
}
// ---- shared JSON slicing ----------------------------------------------------
/// The JSON body of a `<script … id="{id}" …>` tag. Both Spotify and Apple
/// serve their page state this way — pure JSON between the tags, no `var =`.
fn script_json<'a>(html: &'a str, id_attr: &str) -> Option<&'a str> {
let at = html.find(id_attr)?;
let rest = &html[at..];
let open = rest.find('>')? + 1;
let close = rest[open..].find("</script>")?;
Some(rest[open..open + close].trim())
}
fn og_image(html: &str) -> Option<String> {
let at = html.find("property=\"og:image\" content=\"")?;
let rest = &html[at + "property=\"og:image\" content=\"".len()..];
rest.split('"').next().map(str::to_string)
}
// ---- Spotify ----------------------------------------------------------------
/// Tracks out of the embed page's entity. Artists arrive pre-joined
/// ("Shakira, Burna Boy"); durations in milliseconds.
pub(crate) fn spotify_rows(entity: &Value) -> Vec<Row> {
let mut rows = Vec::new();
if let Some(list) = entity.get("trackList").and_then(Value::as_array) {
for t in list {
let title = t.get("title").and_then(Value::as_str).unwrap_or("").trim();
let artist = t.get("subtitle").and_then(Value::as_str).unwrap_or("").trim();
if title.is_empty() {
continue;
}
rows.push(Row {
artist: artist.to_string(),
title: title.to_string(),
album: None,
duration_secs: t
.get("duration")
.and_then(Value::as_i64)
.filter(|d| *d > 0)
.map(|ms| (ms / 1000) as u32),
});
}
}
rows
}
async fn fetch_spotify(id: &str) -> Result<ProviderFetch, String> {
let client = client(20)?;
let html = client
.get(format!("https://open.spotify.com/embed/playlist/{id}"))
.send()
.await
.map_err(|e| format!("Spotify did not answer: {e}"))?
.error_for_status()
.map_err(|e| format!("Spotify refused the page: {e}"))?
.text()
.await
.map_err(|e| e.to_string())?;
let data: Value = script_json(&html, "id=\"__NEXT_DATA__\"")
.and_then(|s| serde_json::from_str(s).ok())
.ok_or("Could not read the playlist's data — is the link public?")?;
let entity = &data["props"]["pageProps"]["state"]["data"]["entity"];
if entity.get("type").and_then(Value::as_str) != Some("playlist") {
return Err("That Spotify link is not a playlist.".into());
}
let rows = spotify_rows(entity);
if rows.is_empty() {
return Err("Spotify listed no tracks for this playlist — it may be private.".into());
}
let mut warnings = Vec::new();
if rows.len() == 100 {
warnings.push(
"Spotify shows only the first 100 tracks of a playlist to anyone not signed \
in, so a longer playlist stops here."
.into(),
);
}
Ok(ProviderFetch {
provider: "spotify",
title: entity
.get("name")
.and_then(Value::as_str)
.unwrap_or("Spotify playlist")
.to_string(),
author: entity
.get("subtitle")
.and_then(Value::as_str)
.unwrap_or("")
.to_string(),
cover_url: entity["coverArt"]["sources"]
.as_array()
.and_then(|s| s.last())
.and_then(|s| s.get("url"))
.and_then(Value::as_str)
.map(str::to_string),
rows,
warnings,
})
}
// ---- Apple Music ------------------------------------------------------------
/// Track rows and the header out of `serialized-server-data`. The tree is
/// deep and renderer-happy, so sections are FOUND (key walk), not pathed.
pub(crate) fn apple_parse(data: &Value) -> (Vec<Row>, Option<String>, Option<u64>) {
let mut rows = Vec::new();
let mut header_title = None;
let mut track_count = None;
walk(data, "sections", &mut |secs| {
let Some(list) = secs.as_array() else { return };
for sec in list {
let kind = sec.get("itemKind").and_then(Value::as_str).unwrap_or("");
let items = sec.get("items").and_then(Value::as_array);
match (kind, items) {
("containerDetailHeaderLockup", Some(items)) => {
if let Some(h) = items.first() {
if header_title.is_none() {
header_title = h
.get("title")
.and_then(Value::as_str)
.map(str::to_string);
}
if track_count.is_none() {
track_count = h.get("trackCount").and_then(Value::as_u64);
}
}
}
("trackLockup", Some(items)) => {
for it in items {
let title =
it.get("title").and_then(Value::as_str).unwrap_or("").trim();
let artist = it
.get("artistName")
.and_then(Value::as_str)
.unwrap_or("")
.trim();
if title.is_empty() {
continue;
}
rows.push(Row {
artist: artist.to_string(),
title: title.to_string(),
album: it["tertiaryLinks"]
.as_array()
.and_then(|l| l.first())
.and_then(|l| l.get("title"))
.and_then(Value::as_str)
.map(str::to_string),
duration_secs: it
.get("duration")
.and_then(Value::as_i64)
.filter(|d| *d > 0)
.map(|ms| (ms / 1000) as u32),
});
}
}
_ => {}
}
}
});
(rows, header_title, track_count)
}
async fn fetch_apple(url: &str) -> Result<ProviderFetch, String> {
let client = client(25)?;
let html = client
.get(url)
.send()
.await
.map_err(|e| format!("Apple Music did not answer: {e}"))?
.error_for_status()
.map_err(|e| format!("Apple Music refused the page: {e}"))?
.text()
.await
.map_err(|e| e.to_string())?;
let data: Value = script_json(&html, "id=\"serialized-server-data\"")
.and_then(|s| serde_json::from_str(s).ok())
.ok_or("Could not read the playlist's data — is the link public?")?;
let (rows, header_title, track_count) = apple_parse(&data);
if rows.is_empty() {
return Err("Apple Music listed no tracks for this playlist — it may be private or region-locked.".into());
}
let mut warnings = Vec::new();
if let Some(total) = track_count {
if (rows.len() as u64) < total {
warnings.push(format!(
"The page lists the first {} of {total} tracks.",
rows.len()
));
}
}
let author = {
let mut curator = None;
walk(&data, "curatorName", &mut |c| {
if curator.is_none() {
curator = c.as_str().map(str::to_string);
}
});
curator.unwrap_or_default()
};
Ok(ProviderFetch {
provider: "apple-music",
title: header_title.unwrap_or_else(|| "Apple Music playlist".into()),
author,
cover_url: og_image(&html),
rows,
warnings,
})
}
// ---- TIDAL ------------------------------------------------------------------
/// One TIDAL API GET with the anonymous token.
async fn tidal_get(client: &reqwest::Client, token: &str, url: &str) -> Result<reqwest::Response, String> {
client
.get(url)
.header("X-Tidal-Token", token)
.send()
.await
.map_err(|e| format!("TIDAL did not answer: {e}"))
}
/// The current anonymous token out of the embed widget's app bundle — the
/// fallback for the day the baked constant rotates.
async fn tidal_mine_token(client: &reqwest::Client) -> Option<String> {
let shell = client
.get("https://embed.tidal.com/playlists/00000000-0000-0000-0000-000000000000")
.send()
.await
.ok()?
.text()
.await
.ok()?;
let src = shell
.split("src=\"")
.find_map(|part| part.split('"').next().filter(|p| p.contains("/js/app-")))?;
let js = client
.get(format!("https://embed.tidal.com{src}"))
.send()
.await
.ok()?
.text()
.await
.ok()?;
let at = js.find("X-Tidal-Token\",\"")?;
js[at + "X-Tidal-Token\",\"".len()..]
.split('"')
.next()
.filter(|t| (8..=64).contains(&t.len()))
.map(str::to_string)
}
pub(crate) fn tidal_rows(items: &[Value]) -> Vec<Row> {
let mut rows = Vec::new();
for t in items {
let base = t.get("title").and_then(Value::as_str).unwrap_or("").trim();
if base.is_empty() {
continue;
}
// TIDAL keeps the version qualifier ("Remix", "Extended Mix") in its
// own field; fold it back into the display title the way every other
// source states it.
let version = t.get("version").and_then(Value::as_str).unwrap_or("").trim();
let title = if !version.is_empty()
&& !base.to_lowercase().contains(&version.to_lowercase())
{
format!("{base} ({version})")
} else {
base.to_string()
};
rows.push(Row {
artist: t["artist"]["name"].as_str().unwrap_or("").trim().to_string(),
title,
album: t["album"]["title"].as_str().map(str::to_string),
duration_secs: t
.get("duration")
.and_then(Value::as_i64)
.filter(|d| *d > 0)
.map(|s| s as u32),
});
}
rows
}
async fn fetch_tidal(id: &str) -> Result<ProviderFetch, String> {
let client = client(20)?;
let mut token = TIDAL_PUBLIC_TOKEN.to_string();
let meta_url = format!("https://api.tidal.com/v1/playlists/{id}?countryCode=US");
let mut resp = tidal_get(&client, &token, &meta_url).await?;
if matches!(resp.status().as_u16(), 401 | 403) {
token = tidal_mine_token(&client)
.await
.ok_or("TIDAL rejected the public token and a fresh one could not be found.")?;
resp = tidal_get(&client, &token, &meta_url).await?;
}
if resp.status().as_u16() == 404 {
return Err("TIDAL knows no playlist at that link — it may be private or deleted.".into());
}
let meta: Value = resp
.error_for_status()
.map_err(|e| format!("TIDAL refused: {e}"))?
.json()
.await
.map_err(|e| e.to_string())?;
let mut rows = Vec::new();
let mut warnings = Vec::new();
let mut offset = 0usize;
loop {
let page: Value = tidal_get(
&client,
&token,
&format!(
"https://api.tidal.com/v1/playlists/{id}/tracks?countryCode=US&limit=100&offset={offset}"
),
)
.await?
.error_for_status()
.map_err(|e| format!("TIDAL refused the tracks: {e}"))?
.json()
.await
.map_err(|e| e.to_string())?;
let items = page.get("items").and_then(Value::as_array).cloned().unwrap_or_default();
if items.is_empty() {
break;
}
rows.extend(tidal_rows(&items));
offset += items.len();
let total = page
.get("totalNumberOfItems")
.and_then(Value::as_u64)
.unwrap_or(0) as usize;
if offset >= total {
break;
}
if rows.len() >= MAX_TRACKS {
warnings.push(format!(
"Only the first {MAX_TRACKS} tracks were taken — the playlist goes on."
));
break;
}
}
if rows.is_empty() {
return Err("TIDAL listed no tracks for this playlist.".into());
}
let cover_url = meta
.get("squareImage")
.or_else(|| meta.get("image"))
.and_then(Value::as_str)
.map(|u| format!("https://resources.tidal.com/images/{}/640x640.jpg", u.replace('-', "/")));
Ok(ProviderFetch {
provider: "tidal",
title: meta
.get("title")
.and_then(Value::as_str)
.unwrap_or("TIDAL playlist")
.to_string(),
author: meta["creator"]["name"].as_str().unwrap_or("").to_string(),
cover_url,
rows,
warnings,
})
}
// ---- Deezer -----------------------------------------------------------------
pub(crate) fn deezer_rows(items: &[Value]) -> Vec<Row> {
let mut rows = Vec::new();
for t in items {
let title = t.get("title").and_then(Value::as_str).unwrap_or("").trim();
if title.is_empty() {
continue;
}
rows.push(Row {
artist: t["artist"]["name"].as_str().unwrap_or("").trim().to_string(),
title: title.to_string(),
album: t["album"]["title"].as_str().map(str::to_string),
duration_secs: t
.get("duration")
.and_then(Value::as_i64)
.filter(|d| *d > 0)
.map(|s| s as u32),
});
}
rows
}
async fn fetch_deezer(id: &str) -> Result<ProviderFetch, String> {
let client = client(20)?;
let root: Value = client
.get(format!("https://api.deezer.com/playlist/{id}"))
.send()
.await
.map_err(|e| format!("Deezer did not answer: {e}"))?
.json()
.await
.map_err(|e| e.to_string())?;
if let Some(err) = root.get("error") {
let msg = err["message"].as_str().unwrap_or("unknown error");
return Err(format!("Deezer refused the playlist — {msg}."));
}
let mut rows = deezer_rows(
root["tracks"]["data"].as_array().map(Vec::as_slice).unwrap_or(&[]),
);
let mut warnings = Vec::new();
// Long playlists page through `next` links the API itself hands out.
let mut next = root["tracks"]["next"].as_str().map(str::to_string);
while let Some(url) = next.take() {
if rows.len() >= MAX_TRACKS {
warnings.push(format!(
"Only the first {MAX_TRACKS} tracks were taken — the playlist goes on."
));
break;
}
if !url.starts_with("https://api.deezer.com/") {
break;
}
let page: Value = match client.get(&url).send().await {
Err(e) => {
warnings.push(format!("Only the first {} tracks could be fetched: {e}", rows.len()));
break;
}
Ok(r) => match r.json().await {
Err(e) => {
warnings.push(format!("Only the first {} tracks could be fetched: {e}", rows.len()));
break;
}
Ok(v) => v,
},
};
rows.extend(deezer_rows(
page.get("data").and_then(Value::as_array).map(Vec::as_slice).unwrap_or(&[]),
));
next = page.get("next").and_then(Value::as_str).map(str::to_string);
}
if rows.is_empty() {
return Err("Deezer listed no tracks for this playlist — it may be private.".into());
}
Ok(ProviderFetch {
provider: "deezer",
title: root
.get("title")
.and_then(Value::as_str)
.unwrap_or("Deezer playlist")
.to_string(),
author: root["creator"]["name"].as_str().unwrap_or("").to_string(),
cover_url: root
.get("picture_xl")
.or_else(|| root.get("picture_big"))
.and_then(Value::as_str)
.map(str::to_string),
rows,
warnings,
})
}
// ---- single track links -----------------------------------------------------
// The escape hatch for Spotify's 100-track ceiling (and a courtesy for the
// rest): a service's PLAYLIST page may cap what it shows, but every single
// TRACK has its own public page. Paste the track links — Spotify's own app
// hands them over with select-all + copy — and each is read from its own
// page, so the playlist's length stops mattering.
#[derive(Debug, Clone, PartialEq)]
pub enum TrackLink {
Spotify(String),
Deezer(String),
Tidal(String),
/// Apple names a song by URL, not a bare id — the storefront is part of
/// the identity.
Apple(String),
}
fn is_spotify_id(id: &str) -> bool {
id.len() == 22 && id.bytes().all(|b| b.is_ascii_alphanumeric())
}
fn all_digits(s: &str) -> bool {
!s.is_empty() && s.bytes().all(|b| b.is_ascii_digit())
}
/// The first track link inside one pasted line, if it holds one. A line can
/// carry other text around the link ("1. ", tab-separated columns, quotes),
/// so every separated token is tried.
pub fn detect_track(line: &str) -> Option<TrackLink> {
for tok in line.split(|c: char| c.is_whitespace() || c == ',' || c == '"') {
let tok = tok.trim_matches(|c: char| "()[]<>'\u{201c}\u{201d}".contains(c));
if tok.is_empty() {
continue;
}
// Spotify's own clipboard form, straight out of the desktop app.
if let Some(id) = tok.strip_prefix("spotify:track:") {
if is_spotify_id(id) {
return Some(TrackLink::Spotify(id.to_string()));
}
continue;
}
let after_scheme = match tok.split_once("://") {
Some((_, rest)) => rest,
None => tok,
};
let Some((host, rest)) = after_scheme.split_once('/') else {
continue;
};
let host = host.split('@').next_back().unwrap_or(host).to_lowercase();
let host = host.split(':').next().unwrap_or(&host).to_string();
let path = rest.split(['?', '#']).next().unwrap_or(rest);
let query = rest
.split('#')
.next()
.unwrap_or(rest)
.split_once('?')
.map(|(_, q)| q);
let segs: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect();
let after = |name: &str| -> Option<&str> {
segs.iter()
.position(|s| *s == name)
.and_then(|at| segs.get(at + 1))
.copied()
};
if host == "open.spotify.com" {
if let Some(id) = after("track").filter(|id| is_spotify_id(id)) {
return Some(TrackLink::Spotify(id.to_string()));
}
} else if host == "www.deezer.com" || host == "deezer.com" {
if let Some(id) = after("track").filter(|id| all_digits(id)) {
return Some(TrackLink::Deezer(id.to_string()));
}
} else if host == "tidal.com" || host.ends_with(".tidal.com") {
if let Some(id) = after("track").filter(|id| all_digits(id)) {
return Some(TrackLink::Tidal(id.to_string()));
}
} else if host == "music.apple.com" {
// Two spellings: /song/{slug}/{id}, and an album link whose `?i=`
// names one track on it.
let is_song = segs.iter().any(|s| *s == "song");
let has_i = query
.map(|q| q.split('&').any(|kv| kv.starts_with("i=")))
.unwrap_or(false);
if is_song || has_i {
return Some(TrackLink::Apple(format!(
"https://music.apple.com/{}{}",
segs.join("/"),
query.map(|q| format!("?{q}")).unwrap_or_default()
)));
}
}
}
None
}
// ---- pacing ------------------------------------------------------------
// Spotify rate-limits these pages, and its 429 is expensive: measured
// 2026-08-24, a burst of ~350 lookups tripped it and the block held for
// **211 seconds**, with `Retry-After: 0` on every refusal — the header is
// worthless, so the backoff here is our own. 150 lookups in one run were
// fine; the ceiling is a few hundred per window across ALL of a session's
// requests, so the run paces itself from the start rather than sprinting
// into a wall it then has to wait out.
/// Track pages in flight at once.
const TRACK_FETCH_CONCURRENCY: usize = 3;
/// Minimum spacing between request STARTS, before any throttling is seen.
/// ~6/s: 350 tracks in about a minute, comfortably under what tripped it.
const TRACK_PACE_MS: u64 = 160;
/// Spacing never grows past this, however many times we are refused.
const TRACK_PACE_MAX_MS: u64 = 1_200;
/// How long to stand down when refused. The measured block was ~3.5 min,
/// but it was being poked every 10s throughout; a quiet minute first is the
/// polite read, and the run resumes at half speed either way.
const TRACK_COOLDOWN_SECS: u64 = 60;
/// Stand-downs before the run stops trying and reports what it got.
const MAX_COOLDOWNS: u32 = 3;
/// Attempts per track, across all stand-downs.
const TRACK_ATTEMPTS: u32 = 3;
/// Shared pacing state for one paste run: a slot reservation clock, the
/// current spacing, and a stand-down deadline every worker honours — so one
/// worker meeting a 429 parks the whole run instead of the other two
/// carrying on and deepening the penalty.
#[derive(Default)]
struct Throttle {
next_slot: std::sync::Mutex<Option<std::time::Instant>>,
interval_ms: std::sync::Mutex<u64>,
cooldown_until: std::sync::Mutex<Option<std::time::Instant>>,
cooldowns: std::sync::Mutex<u32>,
/// Set while standing down, so the UI can say so instead of looking hung.
waiting: std::sync::atomic::AtomicBool,
}
impl Throttle {
fn new() -> Self {
Throttle {
interval_ms: std::sync::Mutex::new(TRACK_PACE_MS),
..Default::default()
}
}
/// Reserve the next send slot and sleep until it comes up.
async fn slot(&self) {
let wait = {
let now = std::time::Instant::now();
let interval =
std::time::Duration::from_millis(*self.interval_ms.lock().unwrap());
let cooldown = *self.cooldown_until.lock().unwrap();
let mut next = self.next_slot.lock().unwrap();
let earliest = [Some(now), *next, cooldown]
.into_iter()
.flatten()
.max()
.unwrap_or(now);
*next = Some(earliest + interval);
earliest.saturating_duration_since(now)
};
if !wait.is_zero() {
tokio::time::sleep(wait).await;
}
}
/// Record a refusal. Returns false once the run has stood down as often
/// as it is going to.
fn back_off(&self) -> bool {
let mut count = self.cooldowns.lock().unwrap();
*count += 1;
if *count > MAX_COOLDOWNS {
return false;
}
let mut interval = self.interval_ms.lock().unwrap();
*interval = (*interval * 2).min(TRACK_PACE_MAX_MS);
*self.cooldown_until.lock().unwrap() =
Some(std::time::Instant::now() + std::time::Duration::from_secs(TRACK_COOLDOWN_SECS));
self.waiting
.store(true, std::sync::atomic::Ordering::Relaxed);
true
}
fn is_waiting(&self) -> bool {
match *self.cooldown_until.lock().unwrap() {
Some(until) if until > std::time::Instant::now() => true,
_ => {
self.waiting
.store(false, std::sync::atomic::Ordering::Relaxed);
false
}
}
}
}
/// What one lookup came back as.
enum Fetched {
Body(String),
/// The service refused us for going too fast — worth retrying.
RateLimited,
/// No such track, or the page could not be read — retrying will not help.
Missing,
}
/// One paced GET, with 429 recognized as its own answer.
async fn paced_get(
client: &reqwest::Client,
throttle: &Throttle,
url: &str,
token: Option<&str>,
) -> Fetched {
throttle.slot().await;
let mut request = client.get(url);
if let Some(t) = token {
request = request.header("X-Tidal-Token", t);
}
match request.send().await {
Err(_) => Fetched::Missing,
Ok(response) => {
if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS {
return Fetched::RateLimited;
}
if !response.status().is_success() {
return Fetched::Missing;
}
match response.text().await {
Ok(body) => Fetched::Body(body),
Err(_) => Fetched::Missing,
}
}
}
}
async fn resolve_spotify_track(
client: &reqwest::Client,
throttle: &Throttle,
id: &str,
) -> Result<Option<Row>, ()> {
let html = match paced_get(
client,
throttle,
&format!("https://open.spotify.com/embed/track/{id}"),
None,
)
.await
{
Fetched::Body(b) => b,
Fetched::RateLimited => return Err(()),
Fetched::Missing => return Ok(None),
};
let Some(json) = script_json(&html, "id=\"__NEXT_DATA__\"") else {
return Ok(None);
};
let Ok(data) = serde_json::from_str::<Value>(json) else {
return Ok(None);
};
let e = &data["props"]["pageProps"]["state"]["data"]["entity"];
let title = match e.get("title").and_then(Value::as_str) {
Some(t) if !t.trim().is_empty() => t.trim().to_string(),
_ => return Ok(None),
};
// A TRACK embed lists its artists; a PLAYLIST embed pre-joins them into
// `subtitle`. Take whichever this page carries.
let artist = e
.get("artists")
.and_then(Value::as_array)
.map(|list| {
list.iter()
.filter_map(|a| a.get("name").and_then(Value::as_str))
.collect::<Vec<_>>()
.join(", ")
})
.filter(|a| !a.is_empty())
.or_else(|| e.get("subtitle").and_then(Value::as_str).map(str::to_string))
.unwrap_or_default();
Ok(Some(Row {
artist,
title,
album: None,
duration_secs: e
.get("duration")
.and_then(Value::as_i64)
.filter(|d| *d > 0)
.map(|ms| (ms / 1000) as u32),
}))
}
async fn resolve_deezer_track(
client: &reqwest::Client,
throttle: &Throttle,
id: &str,
) -> Result<Option<Row>, ()> {
let body = match paced_get(
client,
throttle,
&format!("https://api.deezer.com/track/{id}"),
None,
)
.await
{
Fetched::Body(b) => b,
Fetched::RateLimited => return Err(()),
Fetched::Missing => return Ok(None),
};
let Ok(v) = serde_json::from_str::<Value>(&body) else {
return Ok(None);
};
// Deezer answers 200 with an error body; code 4 is its quota refusal,
// which is a 429 by another name.
if let Some(code) = v["error"]["code"].as_i64() {
return if code == 4 { Err(()) } else { Ok(None) };
}
Ok(deezer_rows(std::slice::from_ref(&v)).into_iter().next())
}
async fn resolve_tidal_track(
client: &reqwest::Client,
throttle: &Throttle,
id: &str,
) -> Result<Option<Row>, ()> {
let body = match paced_get(
client,
throttle,
&format!("https://api.tidal.com/v1/tracks/{id}?countryCode=US"),
Some(TIDAL_PUBLIC_TOKEN),
)
.await
{
Fetched::Body(b) => b,
Fetched::RateLimited => return Err(()),
Fetched::Missing => return Ok(None),
};
let Ok(v) = serde_json::from_str::<Value>(&body) else {
return Ok(None);
};
Ok(tidal_rows(std::slice::from_ref(&v)).into_iter().next())
}
/// A song page states itself twice: in a `songDetailHeader` section of its
/// page JSON (title plus artist links — structured and the same in every
/// storefront), and in OpenGraph tags. The JSON is read first; the tags are
/// the fallback, and they carry the only duration either surface offers.
///
/// The tags are also where Apple writes "on Apple\u{a0}Music" with a NO-BREAK
/// SPACE — a plain " on Apple Music" suffix strip silently misses, and the
/// artist comes out as "Drake on Apple Music". Text off this page is
/// normalized before anything is matched against it.
async fn resolve_apple_track(
client: &reqwest::Client,
throttle: &Throttle,
url: &str,
) -> Result<Option<Row>, ()> {
let html = match paced_get(client, throttle, url, None).await {
Fetched::Body(b) => b,
Fetched::RateLimited => return Err(()),
Fetched::Missing => return Ok(None),
};
let nbsp = |s: &str| s.replace('\u{a0}', " ").trim().to_string();
let mut title = String::new();
let mut artist = String::new();
if let Some(data) = script_json(&html, "id=\"serialized-server-data\"")
.and_then(|j| serde_json::from_str::<Value>(j).ok())
{
walk(&data, "sections", &mut |secs| {
let Some(list) = secs.as_array() else { return };
for sec in list {
if sec.get("itemKind").and_then(Value::as_str) != Some("songDetailHeader") {
continue;
}
let Some(item) = sec.get("items").and_then(Value::as_array).and_then(|i| i.first())
else {
continue;
};
if title.is_empty() {
if let Some(t) = item.get("title").and_then(Value::as_str) {
title = nbsp(t);
}
}
if artist.is_empty() {
if let Some(links) = item.get("artistLinks").and_then(Value::as_array) {
artist = links
.iter()
.filter_map(|l| l.get("title").and_then(Value::as_str))
.map(|a| nbsp(a))
.collect::<Vec<_>>()
.join(", ");
}
}
}
});
}
let og = |prop: &str| -> Option<String> {
let needle = format!("property=\"{prop}\" content=\"");
let at = html.find(&needle)?;
html[at + needle.len()..].split('"').next().map(nbsp)
};
if title.is_empty() {
// "TITLE by ARTIST on Apple Music" — the wording is localized, so the
// suffix is only trimmed when it is actually there.
let Some(raw) = og("og:title") else {
return Ok(None);
};
let raw = raw.strip_suffix(" on Apple Music").unwrap_or(&raw).trim();
match raw.rsplit_once(" by ") {
Some((t, a)) => {
title = t.trim().to_string();
if artist.is_empty() {
artist = a.trim().to_string();
}
}
None => title = raw.to_string(),
}
}
if title.is_empty() {
return Ok(None);
}
let duration_secs = og("og:description").and_then(|d| {
let at = d.find("Duration ")? + "Duration ".len();
let tok = d[at..].split(|c: char| c.is_whitespace()).next()?;
crate::playlist_formats::parse_timestamp(tok).map(|ms| (ms / 1000) as u32)
});
Ok(Some(Row {
artist,
title,
album: None,
duration_secs,
}))
}
/// One pasted line: a track link to look up, or plain text to read as-is.
#[derive(Debug, Clone)]
pub enum PastedLine {
Link(TrackLink),
Text(String),
}
/// Read a pasted block into lines the resolver understands. Blank lines and
/// section headers drop out; everything else is either a link or a name.
pub fn pasted_lines(text: &str) -> Vec<PastedLine> {
let mut out = Vec::new();
for line in text.lines() {
let raw = line.trim();
if raw.is_empty() || raw.len() > 4096 {
continue;
}
let bare = raw.trim_matches(['-', '=', '*', '#', ' ']);
if bare.is_empty()
|| bare.eq_ignore_ascii_case("tracklist")
|| bare.eq_ignore_ascii_case("tracklist:")
{
continue;
}
match detect_track(raw) {
Some(link) => out.push(PastedLine::Link(link)),
None => out.push(PastedLine::Text(raw.to_string())),
}
}
out
}
/// What a whole paste run came to.
pub struct PastedOutcome {
pub rows: Vec<Option<Row>>,
/// Links attempted.
pub links: usize,
/// Links given up on because the service kept refusing us — the number
/// that would have arrived on a quieter day, reported separately from
/// links that simply name nothing.
pub throttled_out: usize,
/// Whether the run ever had to stand down.
pub was_throttled: bool,
}
/// Resolve every line, in order, a few pages at a time and paced. `progress`
/// is called with (done, total, waiting) as lookups land — `waiting` is true
/// while standing down from a refusal, so a minute of silence reads as what
/// it is. A link that cannot be read comes back as None, never a guess.
pub async fn resolve_pasted<F>(lines: Vec<PastedLine>, progress: F) -> Result<PastedOutcome, String>
where
F: Fn(usize, usize, bool) + Send + Sync + 'static,
{
let client = client(20)?;
let links = lines
.iter()
.filter(|l| matches!(l, PastedLine::Link(_)))
.count();
let throttle = std::sync::Arc::new(Throttle::new());
let mut rows: Vec<Option<Row>> = Vec::with_capacity(lines.len());
let mut done = 0usize;
let mut throttled_out = 0usize;
for chunk in lines.chunks(TRACK_FETCH_CONCURRENCY) {
let mut handles = Vec::with_capacity(chunk.len());
for line in chunk {
let client = client.clone();
let line = line.clone();
let throttle = throttle.clone();
let is_link = matches!(line, PastedLine::Link(_));
handles.push((
is_link,
tauri::async_runtime::spawn(async move {
// A plain line is read the way every pasted tracklist
// already is — no lookup, no network.
let PastedLine::Link(link) = line else {
let PastedLine::Text(t) = line else {
unreachable!()
};
let (artist, title) = crate::playlist_formats::split_artist_title(&t);
return (
title.map(|title| Row {
artist: artist.unwrap_or_default(),
title,
album: None,
duration_secs: None,
}),
false,
);
};
for attempt in 0..TRACK_ATTEMPTS {
let got = match &link {
TrackLink::Spotify(id) => {
resolve_spotify_track(&client, &throttle, id).await
}
TrackLink::Deezer(id) => {
resolve_deezer_track(&client, &throttle, id).await
}
TrackLink::Tidal(id) => {
resolve_tidal_track(&client, &throttle, id).await
}
TrackLink::Apple(u) => {
resolve_apple_track(&client, &throttle, u).await
}
};
match got {
Ok(row) => return (row, false),
Err(()) => {
// Refused. Park the whole run, then try again
// — unless we have already stood down as often
// as we are going to.
if attempt + 1 == TRACK_ATTEMPTS || !throttle.back_off() {
return (None, true);
}
}
}
}
(None, true)
}),
));
}
for (is_link, handle) in handles {
let (row, gave_up_throttled) = handle.await.unwrap_or((None, false));
if is_link {
done += 1;
if gave_up_throttled {
throttled_out += 1;
}
progress(done, links, throttle.is_waiting());
}
rows.push(row);
}
}
let was_throttled = *throttle.cooldowns.lock().unwrap() > 0;
Ok(PastedOutcome {
rows,
links,
throttled_out,
was_throttled,
})
}
// ---- entry ------------------------------------------------------------------
pub async fn fetch(url: &str) -> Result<ProviderFetch, String> {
match detect(url) {
Some(ProviderUrl::Spotify(id)) => fetch_spotify(&id).await,
Some(ProviderUrl::Apple { url }) => fetch_apple(&url).await,
Some(ProviderUrl::Tidal(id)) => fetch_tidal(&id).await,
Some(ProviderUrl::Deezer(id)) => fetch_deezer(&id).await,
None => Err(
"That link isn't a playlist from a service this door knows — it takes \
YouTube, Spotify, Apple Music, TIDAL, and Deezer playlist links."
.into(),
),
}
}
/// Draft rows from a provider fetch. The services state artist and title as
/// separate fields, so no display-string splitting or title cleaning happens
/// — what the service said is what the review shows.
pub fn to_parsed(fetch: &ProviderFetch) -> crate::playlist_formats::ParsedPlaylist {
let entries = fetch
.rows
.iter()
.enumerate()
.map(|(i, r)| crate::playlist_formats::ParsedEntry {
ordinal: i as u32,
title: Some(r.title.clone()),
artist: (!r.artist.is_empty()).then(|| r.artist.clone()),
album: r.album.clone(),
duration_secs: r.duration_secs,
location: None,
start_ms: None,
raw: format!("{} - {}", r.artist, r.title),
})
.collect();
crate::playlist_formats::ParsedPlaylist {
format: fetch.provider.into(),
title: Some(fetch.title.clone()),
description: None,
entries,
warnings: Vec::new(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_each_provider_and_rejects_the_rest() {
assert_eq!(
detect("https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M?si=abc"),
Some(ProviderUrl::Spotify("37i9dQZF1DXcBWIGoYBM5M".into()))
);
assert_eq!(
detect("https://open.spotify.com/intl-de/playlist/37i9dQZF1DXcBWIGoYBM5M"),
Some(ProviderUrl::Spotify("37i9dQZF1DXcBWIGoYBM5M".into()))
);
assert_eq!(
detect("https://music.apple.com/us/playlist/todays-hits/pl.f4d106fed2bd41149aaacabb233eb5eb"),
Some(ProviderUrl::Apple {
url: "https://music.apple.com/us/playlist/todays-hits/pl.f4d106fed2bd41149aaacabb233eb5eb".into()
})
);
assert_eq!(
detect("https://tidal.com/browse/playlist/edf3b7d2-cb42-41d7-93c0-afa2a395521b"),
Some(ProviderUrl::Tidal("edf3b7d2-cb42-41d7-93c0-afa2a395521b".into()))
);
assert_eq!(
detect("https://listen.tidal.com/playlist/edf3b7d2-cb42-41d7-93c0-afa2a395521b"),
Some(ProviderUrl::Tidal("edf3b7d2-cb42-41d7-93c0-afa2a395521b".into()))
);
assert_eq!(
detect("https://www.deezer.com/en/playlist/908622995"),
Some(ProviderUrl::Deezer("908622995".into()))
);
assert_eq!(detect("https://www.youtube.com/playlist?list=PLabc"), None);
assert_eq!(detect("https://open.spotify.com/track/4uLU6hMCjMI75M1A2tKUQC"), None);
assert_eq!(detect("https://example.com/playlist/123"), None);
}
#[test]
fn spotify_rows_from_entity_fixture() {
let entity: Value = serde_json::json!({
"type": "playlist",
"trackList": [
{ "title": "Dai Dai", "subtitle": "Shakira, Burna Boy", "duration": 223448 },
{ "title": "", "subtitle": "nobody", "duration": 1000 },
{ "title": "Loser", "subtitle": "Tame Impala", "duration": 0 }
]
});
let rows = spotify_rows(&entity);
assert_eq!(rows.len(), 2);
assert_eq!(rows[0].artist, "Shakira, Burna Boy");
assert_eq!(rows[0].duration_secs, Some(223));
assert_eq!(rows[1].duration_secs, None);
}
#[test]
fn apple_sections_walked_for_header_and_tracks() {
let data: Value = serde_json::json!({ "data": [ { "data": { "sections": [
{ "itemKind": "containerDetailHeaderLockup",
"items": [ { "title": "Today’s Hits", "trackCount": 3 } ] },
{ "itemKind": "trackLockup", "items": [
{ "title": "Janice STFU", "artistName": "Drake", "duration": 237344,
"tertiaryLinks": [ { "title": "ICEMAN" } ] },
{ "title": "Golden", "artistName": "HUNTR/X", "duration": 200000 }
]}
]}}]});
let (rows, title, count) = apple_parse(&data);
assert_eq!(title.as_deref(), Some("Today’s Hits"));
assert_eq!(count, Some(3));
assert_eq!(rows.len(), 2);
assert_eq!(rows[0].album.as_deref(), Some("ICEMAN"));
assert_eq!(rows[0].duration_secs, Some(237));
assert_eq!(rows[1].album, None);
}
#[test]
fn tidal_version_folds_into_the_title() {
let items = vec![
serde_json::json!({ "title": "Song", "version": "VIP Mix",
"artist": { "name": "A" }, "album": { "title": "B" }, "duration": 200 }),
serde_json::json!({ "title": "Song (VIP Mix)", "version": "VIP Mix",
"artist": { "name": "A" }, "duration": 100 }),
];
let rows = tidal_rows(&items);
assert_eq!(rows[0].title, "Song (VIP Mix)");
assert_eq!(rows[1].title, "Song (VIP Mix)");
}
#[test]
fn track_links_detected_in_every_spelling() {
let sp = "7bxaFZ1O3cHkgLKMsdC3xR";
assert_eq!(
detect_track(&format!("https://open.spotify.com/track/{sp}?si=abc123")),
Some(TrackLink::Spotify(sp.into()))
);
// The desktop app's clipboard form, and a link with row noise around it.
assert_eq!(
detect_track(&format!("spotify:track:{sp}")),
Some(TrackLink::Spotify(sp.into()))
);
assert_eq!(
detect_track(&format!("12. Tame Impala - Loser\thttps://open.spotify.com/track/{sp}")),
Some(TrackLink::Spotify(sp.into()))
);
assert_eq!(
detect_track("https://www.deezer.com/en/track/3135556"),
Some(TrackLink::Deezer("3135556".into()))
);
assert_eq!(
detect_track("https://listen.tidal.com/track/77646168"),
Some(TrackLink::Tidal("77646168".into()))
);
assert_eq!(
detect_track("https://music.apple.com/us/song/janice-stfu/6769568596"),
Some(TrackLink::Apple(
"https://music.apple.com/us/song/janice-stfu/6769568596".into()
))
);
// An Apple ALBUM link naming one track on it.
assert_eq!(
detect_track("https://music.apple.com/us/album/janice-stfu/6769568449?i=6769568596"),
Some(TrackLink::Apple(
"https://music.apple.com/us/album/janice-stfu/6769568449?i=6769568596".into()
))
);
// Not tracks: a playlist link, an album link with no track, plain text.
assert_eq!(detect_track("https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M"), None);
assert_eq!(detect_track("https://music.apple.com/us/album/discovery/697194953"), None);
assert_eq!(detect_track("Daft Punk - Around the World"), None);
// A malformed Spotify id must not pass as one.
assert_eq!(detect_track("https://open.spotify.com/track/tooshort"), None);
}
#[test]
fn pasted_lines_split_links_from_names_and_drop_headers() {
let text = "Tracklist:\n\
https://open.spotify.com/track/7bxaFZ1O3cHkgLKMsdC3xR\n\
\n\
Daft Punk - Around the World\n\
---\n\
spotify:track:4LfCY65LvojKjWEnU7fNN4\n";
let lines = pasted_lines(text);
assert_eq!(lines.len(), 3);
assert!(matches!(lines[0], PastedLine::Link(TrackLink::Spotify(_))));
assert!(matches!(lines[1], PastedLine::Text(_)));
assert!(matches!(lines[2], PastedLine::Link(TrackLink::Spotify(_))));
}
#[test]
fn deezer_rows_from_api_fixture() {
let items = vec![serde_json::json!({
"title": "Hey Jude (Remastered 2015)",
"artist": { "name": "The Beatles" },
"album": { "title": "1 (Remastered)" },
"duration": 429
})];
let rows = deezer_rows(&items);
assert_eq!(rows[0].artist, "The Beatles");
assert_eq!(rows[0].duration_secs, Some(429));
}
}