playlists + player: every service's list through one door with its names matched for real, and shuffle that readies exactly what it promises

src-tauri/src/youtube_playlist.rs

@@ -0,0 +1,613 @@
+//! YouTube PLAYLIST import — one row per video, straight off the playlist's
+//! public page. This is the metadata-import posture of `youtube_setlist`
+//! applied to the other kind of link: nothing is downloaded and nothing is
+//! recognized — the page already states every video's title, channel and
+//! length, and those become ordinary draft rows for the existing
+//! resolve → review → commit pipeline.
+//!
+//! Everything runs in-process over reqwest: one GET of the playlist page,
+//! plus one innertube `browse` POST per additional hundred videos (the same
+//! dialect the setlist module's comments call speaks). Since 2024 the
+//! desktop page renders videos as `lockupViewModel` nodes; the old
+//! `playlistVideoRenderer` shape is still walked as insurance, since YouTube
+//! has served both in the wild.
+
+use crate::youtube_setlist::{collect_runs, slice_json_after, walk};
+use serde_json::Value;
+
+/// One video as the playlist page stated it.
+pub struct VideoRow {
+    pub video_id: String,
+    /// The raw video title, junk suffixes and all — cleaning happens at
+    /// draft-building so the review's provenance line can show the original.
+    pub title: String,
+    /// The uploading channel's display name ("Skrillex", "Artist - Topic").
+    pub byline: String,
+    pub duration_secs: Option<u32>,
+}
+
+pub struct PlaylistFetch {
+    pub title: String,
+    /// The playlist's owner ("Tommy"), not any track's artist.
+    pub author: String,
+    pub rows: Vec<VideoRow>,
+    pub warnings: Vec<String>,
+}
+
+/// Videos a single import will take. A playlist page holds 100; each
+/// continuation another 100. Ten requests is a generous ceiling — past that
+/// the review modal stops being a review.
+const MAX_VIDEOS: usize = 1000;
+
+// ---- URL handling -----------------------------------------------------------
+
+/// The `list=` id out of any YouTube-shaped URL (playlist pages, watch links
+/// carrying a list, music.youtube.com), or a bare id pasted alone.
+pub fn playlist_id_from_url(url: &str) -> Option<String> {
+    let s = url.trim();
+    let is_id = |c: &str| {
+        (10..=64).contains(&c.len())
+            && c.bytes()
+                .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
+    };
+    // A bare id: playlist ids start with a recognizable prefix; a raw video
+    // id must NOT be mistaken for one (11 chars, no such prefix).
+    if is_id(s)
+        && ["PL", "OLAK5uy_", "FL", "UU", "LL", "RD"]
+            .iter()
+            .any(|p| s.starts_with(p))
+    {
+        return Some(s.to_string());
+    }
+    let after_scheme = s.split("://").nth(1).unwrap_or(s);
+    let (host, rest) = after_scheme.split_once('/')?;
+    let host = host.split('@').next_back().unwrap_or(host).to_lowercase();
+    let host = host.split(':').next().unwrap_or(&host);
+    if !(host.ends_with("youtube.com") || host == "youtu.be" || host.ends_with("youtube-nocookie.com")) {
+        return None;
+    }
+    let query = rest.split('#').next().unwrap_or(rest).split_once('?')?.1;
+    query
+        .split('&')
+        .find_map(|kv| kv.strip_prefix("list=").map(str::to_string))
+        .filter(|c| is_id(c))
+}
+
+// ---- page shapes ------------------------------------------------------------
+
+/// Video rows out of any subtree, both page generations:
+/// `lockupViewModel` (2024+ desktop) and `playlistVideoRenderer` (classic).
+/// Order follows the document, which is the playlist's own order.
+pub(crate) fn video_rows(root: &Value, skipped_unavailable: &mut usize) -> Vec<VideoRow> {
+    let mut rows: Vec<VideoRow> = Vec::new();
+
+    walk(root, "lockupViewModel", &mut |lk| {
+        if lk.get("contentType").and_then(Value::as_str) != Some("LOCKUP_CONTENT_TYPE_VIDEO") {
+            return;
+        }
+        let Some(id) = lk.get("contentId").and_then(Value::as_str) else {
+            return;
+        };
+        let md = &lk["metadata"]["lockupMetadataViewModel"];
+        let title = md
+            .get("title")
+            .and_then(|t| t.get("content"))
+            .and_then(Value::as_str)
+            .unwrap_or("")
+            .trim()
+            .to_string();
+        if title.is_empty() || is_unavailable_stub(&title) {
+            *skipped_unavailable += usize::from(!title.is_empty());
+            return;
+        }
+        // First metadata row's first part is the channel line.
+        let mut byline = String::new();
+        walk(md, "metadataParts", &mut |parts| {
+            if byline.is_empty() {
+                if let Some(t) = parts
+                    .get(0)
+                    .and_then(|p| p.get("text"))
+                    .and_then(|t| t.get("content"))
+                    .and_then(Value::as_str)
+                {
+                    byline = t.trim().to_string();
+                }
+            }
+        });
+        // Duration rides a thumbnail badge as "3:20"; other badges say
+        // "LIVE"/"SHORTS" and simply fail the timestamp parse.
+        let mut secs = None;
+        walk(lk, "thumbnailBadgeViewModel", &mut |b| {
+            if secs.is_none() {
+                secs = b
+                    .get("text")
+                    .and_then(Value::as_str)
+                    .and_then(|t| crate::playlist_formats::parse_timestamp(t.trim()))
+                    .map(|ms| (ms / 1000) as u32);
+            }
+        });
+        rows.push(VideoRow {
+            video_id: id.to_string(),
+            title,
+            byline,
+            duration_secs: secs,
+        });
+    });
+
+    walk(root, "playlistVideoRenderer", &mut |v| {
+        let Some(id) = v.get("videoId").and_then(Value::as_str) else {
+            return;
+        };
+        let title = v.get("title").map(collect_runs).unwrap_or_default().trim().to_string();
+        if title.is_empty() || is_unavailable_stub(&title) {
+            *skipped_unavailable += usize::from(!title.is_empty());
+            return;
+        }
+        let byline = v
+            .get("shortBylineText")
+            .map(collect_runs)
+            .unwrap_or_default()
+            .trim()
+            .to_string();
+        let secs = v
+            .get("lengthSeconds")
+            .and_then(Value::as_str)
+            .and_then(|s| s.parse::<u32>().ok());
+        rows.push(VideoRow {
+            video_id: id.to_string(),
+            title,
+            byline,
+            duration_secs: secs,
+        });
+    });
+
+    rows
+}
+
+/// The placeholder rows YouTube leaves for videos it can no longer show.
+fn is_unavailable_stub(title: &str) -> bool {
+    matches!(title, "[Private video]" | "[Deleted video]" | "[Unavailable]")
+}
+
+/// The next page's continuation token, wherever the shape put it.
+pub(crate) fn continuation_token(root: &Value) -> Option<String> {
+    let mut token = None;
+    for key in ["continuationItemViewModel", "continuationItemRenderer"] {
+        walk(root, key, &mut |c| {
+            if token.is_none() {
+                walk(c, "token", &mut |t| {
+                    if token.is_none() {
+                        if let Some(s) = t.as_str() {
+                            token = Some(s.to_string());
+                        }
+                    }
+                });
+            }
+        });
+        if token.is_some() {
+            break;
+        }
+    }
+    token
+}
+
+/// The playlist's own name, across header generations.
+fn playlist_title(initial_data: &Value) -> Option<String> {
+    let mut title = None;
+    for key in ["playlistMetadataRenderer", "microformatDataRenderer", "pageHeaderRenderer"] {
+        walk(initial_data, key, &mut |n| {
+            if title.is_none() {
+                title = n
+                    .get("title")
+                    .or_else(|| n.get("pageTitle"))
+                    .and_then(Value::as_str)
+                    .map(str::to_string);
+            }
+        });
+        if title.is_some() {
+            break;
+        }
+    }
+    title
+}
+
+/// The playlist owner's display name. The new header states it as an avatar
+/// stack captioned "by Tommy"; the classic one as `ownerText` runs.
+fn playlist_author(initial_data: &Value) -> String {
+    let mut author = String::new();
+    walk(initial_data, "avatarStackViewModel", &mut |a| {
+        if author.is_empty() {
+            if let Some(t) = a
+                .get("text")
+                .and_then(|t| t.get("content"))
+                .and_then(Value::as_str)
+            {
+                author = t.trim().strip_prefix("by ").unwrap_or(t.trim()).to_string();
+            }
+        }
+    });
+    if author.is_empty() {
+        walk(initial_data, "ownerText", &mut |o| {
+            if author.is_empty() {
+                let t = collect_runs(o);
+                if !t.trim().is_empty() {
+                    author = t.trim().to_string();
+                }
+            }
+        });
+    }
+    author
+}
+
+/// INFO banners worth repeating ("Unavailable videos are hidden") and the
+/// fatal alert text for dead lists ("The playlist does not exist.").
+fn alert_texts(initial_data: &Value) -> Vec<String> {
+    let mut out = Vec::new();
+    for key in ["alertRenderer", "alertWithButtonRenderer"] {
+        walk(initial_data, key, &mut |a| {
+            if let Some(t) = a
+                .get("text")
+                .and_then(|t| crate::youtube_setlist::first_string(t, &["simpleText", "text"]))
+            {
+                if !t.trim().is_empty() {
+                    out.push(t.trim().to_string());
+                }
+            }
+        });
+    }
+    out
+}
+
+// ---- title cleaning and the artist policy -----------------------------------
+
+/// Strip the upload-side junk a video title carries that a TRACK title does
+/// not: "(Official Video)", "[Official Audio]", "(Lyric Video)", "[4K]"…
+/// Version qualifiers are load-bearing (the DJ-set work proved remix naming
+/// matters), so any group naming a remix/edit/mix/version survives even when
+/// it also says "official" — "(Official Remix)" stays.
+pub fn clean_video_title(title: &str) -> String {
+    const KEEP: &[&str] = &[
+        "remix", "edit", "bootleg", "mix", "vip", "flip", "mashup", "rework",
+        "cover", "version", "feat", "ft.", "ft ", "live", "acoustic", "demo",
+        "instrumental", "extended", "radio", "dub", "remaster",
+    ];
+    const JUNK: &[&str] = &[
+        "official video", "official music video", "official audio",
+        "official lyric", "official visualizer", "official visualiser",
+        "music video", "lyric video", "lyrics video", "lyric", "lyrics",
+        "visualizer", "visualiser", "official", "out now", "free download",
+        "free dl", "premiere", "monstercat release", "ncs release",
+        "video oficial", "audio oficial", "clip officiel",
+    ];
+    let mut s = title.trim().to_string();
+    loop {
+        let t = s.trim_end();
+        let Some((open, close)) = (match t.as_bytes().last() {
+            Some(b')') => Some(('(', ')')),
+            Some(b']') => Some(('[', ']')),
+            _ => None,
+        }) else {
+            break;
+        };
+        let Some(at) = t.rfind(open) else { break };
+        let inner = t[at + 1..t.len() - 1].trim().to_lowercase();
+        let _ = close;
+        let keeps = KEEP.iter().any(|k| inner.contains(k));
+        let junk = !keeps
+            && (JUNK.iter().any(|j| inner.contains(j))
+                || matches!(inner.as_str(), "hd" | "hq" | "4k" | "8k" | "mv" | "m/v" | "video" | "audio" | "full version"));
+        if !junk || at == 0 {
+            break;
+        }
+        s = t[..at].trim_end().to_string();
+    }
+    if s.is_empty() {
+        title.trim().to_string()
+    } else {
+        s
+    }
+}
+
+/// The artist a channel name implies, when it does. Auto-generated music
+/// channels ("Artist - Topic") and VEVO channels ("ArtistVEVO") are the
+/// artist by construction; any other channel is offered as-is — for artist
+/// channels it is exactly right, and the review shows every row's artist
+/// before anything commits.
+pub fn byline_artist(byline: &str) -> Option<String> {
+    let b = byline.trim();
+    if b.is_empty() {
+        return None;
+    }
+    if let Some(a) = b.strip_suffix(" - Topic") {
+        let a = a.trim();
+        return (!a.is_empty()).then(|| a.to_string());
+    }
+    if let Some(a) = b.strip_suffix("VEVO").filter(|a| !a.is_empty()) {
+        // "TaylorSwiftVEVO" → "TaylorSwift"; normalized matching downstream
+        // strips spacing anyway.
+        return Some(a.trim().to_string());
+    }
+    Some(b.to_string())
+}
+
+/// Draft rows from the fetched videos: the title's own "Artist - Title"
+/// split wins (a curator's upload states the artist there); a bare title
+/// takes the channel as artist.
+pub fn to_parsed(fetch: &PlaylistFetch) -> crate::playlist_formats::ParsedPlaylist {
+    let mut entries = Vec::new();
+    for (i, row) in fetch.rows.iter().enumerate() {
+        let cleaned = clean_video_title(&row.title);
+        let (artist, title) = match crate::playlist_formats::split_artist_title(&cleaned) {
+            (Some(a), Some(t)) => (Some(a), Some(t)),
+            (_, t) => (byline_artist(&row.byline), t),
+        };
+        entries.push(crate::playlist_formats::ParsedEntry {
+            ordinal: i as u32,
+            title,
+            artist,
+            album: None,
+            duration_secs: row.duration_secs,
+            location: None,
+            start_ms: None,
+            raw: row.title.clone(),
+        });
+    }
+    crate::playlist_formats::ParsedPlaylist {
+        format: "youtube-playlist".into(),
+        title: Some(fetch.title.clone()),
+        description: None,
+        entries,
+        warnings: Vec::new(),
+    }
+}
+
+// ---- the fetch ----------------------------------------------------------------
+
+/// Fetch the playlist's full video list: the page, then one `browse` POST
+/// per continuation. Continuation failures degrade to "first N videos" with
+/// a warning instead of failing the import.
+pub async fn fetch(url: &str) -> Result<PlaylistFetch, String> {
+    let id = playlist_id_from_url(url)
+        .ok_or("That doesn't look like a YouTube playlist link — it needs a list= id")?;
+    let client = reqwest::Client::builder()
+        .user_agent(
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
+             Chrome/126.0.0.0 Safari/537.36",
+        )
+        .timeout(std::time::Duration::from_secs(25))
+        .build()
+        .map_err(|e| e.to_string())?;
+    let html = client
+        .get(format!("https://www.youtube.com/playlist?list={id}&hl=en"))
+        .header("Cookie", "CONSENT=YES+1; SOCS=CAI")
+        .header("Accept-Language", "en")
+        .send()
+        .await
+        .map_err(|e| format!("YouTube did not answer: {e}"))?
+        .error_for_status()
+        .map_err(|e| format!("YouTube refused the page: {e}"))?
+        .text()
+        .await
+        .map_err(|e| e.to_string())?;
+
+    let initial_data: Value = slice_json_after(&html, "ytInitialData")
+        .and_then(|s| serde_json::from_str(s).ok())
+        .ok_or("Could not read the playlist's data — is the link public?")?;
+
+    let mut warnings = Vec::new();
+    let mut skipped = 0usize;
+    let mut rows = video_rows(&initial_data, &mut skipped);
+    let alerts = alert_texts(&initial_data);
+    if rows.is_empty() {
+        let why = alerts
+            .first()
+            .cloned()
+            .unwrap_or_else(|| "it may be private or deleted".into());
+        return Err(format!("YouTube listed no videos for this playlist — {why}"));
+    }
+    // "Unavailable videos are hidden" and its kin — YouTube already counted
+    // what it dropped, so repeat it rather than pretending completeness.
+    warnings.extend(alerts);
+
+    let title = playlist_title(&initial_data).unwrap_or_else(|| "YouTube playlist".into());
+    let author = playlist_author(&initial_data);
+    let version = crate::youtube_setlist::client_version(&html);
+
+    let mut token = continuation_token(&initial_data);
+    while let Some(t) = token.take() {
+        if rows.len() >= MAX_VIDEOS {
+            warnings.push(format!(
+                "Only the first {MAX_VIDEOS} videos were taken — the playlist goes on."
+            ));
+            break;
+        }
+        let body = serde_json::json!({
+            "context": { "client": {
+                "clientName": "WEB", "clientVersion": version,
+                "hl": "en", "gl": "US",
+            }},
+            "continuation": t,
+        });
+        let resp = client
+            .post("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false")
+            .json(&body)
+            .send()
+            .await;
+        let next: Value = match resp {
+            Err(e) => {
+                warnings.push(format!(
+                    "Only the first {} videos could be fetched: {e}",
+                    rows.len()
+                ));
+                break;
+            }
+            Ok(r) => match r.json().await {
+                Err(e) => {
+                    warnings.push(format!(
+                        "Only the first {} videos could be fetched: {e}",
+                        rows.len()
+                    ));
+                    break;
+                }
+                Ok(v) => v,
+            },
+        };
+        let mut more = video_rows(&next, &mut skipped);
+        if more.is_empty() {
+            break;
+        }
+        rows.append(&mut more);
+        token = continuation_token(&next);
+    }
+
+    if skipped > 0 {
+        warnings.push(format!(
+            "{skipped} unavailable video{} skipped (private or deleted).",
+            if skipped == 1 { "" } else { "s" }
+        ));
+    }
+
+    Ok(PlaylistFetch {
+        title,
+        author,
+        rows,
+        warnings,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn playlist_ids_from_all_the_spellings() {
+        let id = "PLRJiC-nOkffKTsUPqj4NfkOYtBX490wI1";
+        for url in [
+            &format!("https://www.youtube.com/playlist?list={id}"),
+            &format!("https://music.youtube.com/playlist?list={id}"),
+            &format!("https://www.youtube.com/watch?v=abc123def45&list={id}&index=2"),
+            &format!("https://youtu.be/abc123def45?list={id}"),
+            &format!("youtube.com/playlist?list={id}#top"),
+            id,
+        ] {
+            assert_eq!(playlist_id_from_url(url).as_deref(), Some(id), "{url}");
+        }
+        // A bare VIDEO id must not read as a playlist.
+        assert_eq!(playlist_id_from_url("dQw4w9WgXcQ"), None);
+        assert_eq!(playlist_id_from_url("https://example.com/playlist?list=PLabc123456"), None);
+        assert_eq!(playlist_id_from_url("https://www.youtube.com/watch?v=abc123def45"), None);
+    }
+
+    #[test]
+    fn junk_suffixes_stripped_qualifiers_kept() {
+        assert_eq!(clean_video_title("Song (Official Video)"), "Song");
+        assert_eq!(clean_video_title("Song [Official Music Video] (4K)"), "Song");
+        assert_eq!(clean_video_title("Song (Ben Böhmer Remix)"), "Song (Ben Böhmer Remix)");
+        assert_eq!(
+            clean_video_title("Song (Unofficial Remix) [HD]"),
+            "Song (Unofficial Remix)"
+        );
+        assert_eq!(clean_video_title("Only You (feat. Haley)"), "Only You (feat. Haley)");
+        assert_eq!(clean_video_title("Song (Official Audio)"), "Song");
+        // A title that IS one junk-looking group survives whole.
+        assert_eq!(clean_video_title("(Official Video)"), "(Official Video)");
+    }
+
+    #[test]
+    fn byline_artist_policy() {
+        assert_eq!(byline_artist("Ben Böhmer - Topic").as_deref(), Some("Ben Böhmer"));
+        assert_eq!(byline_artist("TaylorSwiftVEVO").as_deref(), Some("TaylorSwift"));
+        assert_eq!(byline_artist("Skrillex").as_deref(), Some("Skrillex"));
+        assert_eq!(byline_artist(""), None);
+    }
+
+    #[test]
+    fn lockup_rows_and_continuation_from_fixture() {
+        let page: Value = serde_json::json!({
+            "contents": [
+                { "lockupViewModel": {
+                    "contentId": "1FSqIJTiovQ",
+                    "contentType": "LOCKUP_CONTENT_TYPE_VIDEO",
+                    "contentImage": { "overlays": [ { "thumbnailBadgeViewModel": { "text": "3:20" } } ] },
+                    "metadata": { "lockupMetadataViewModel": {
+                        "title": { "content": "When We Were Young" },
+                        "metadata": { "contentMetadataViewModel": { "metadataRows": [
+                            { "metadataParts": [ { "text": { "content": "Dillon Francis" } } ] }
+                        ]}}
+                    }}
+                }},
+                { "lockupViewModel": {
+                    "contentId": "zzzzzzzzzzz",
+                    "contentType": "LOCKUP_CONTENT_TYPE_VIDEO",
+                    "metadata": { "lockupMetadataViewModel": { "title": { "content": "[Private video]" } } }
+                }},
+                { "lockupViewModel": {
+                    "contentId": "PLnotavideo",
+                    "contentType": "LOCKUP_CONTENT_TYPE_PLAYLIST",
+                    "metadata": { "lockupMetadataViewModel": { "title": { "content": "A shelf" } } }
+                }},
+                { "continuationItemViewModel": { "continuationCommand": {
+                    "innertubeCommand": { "continuationCommand": { "token": "NEXT_PAGE" } }
+                }}}
+            ]
+        });
+        let mut skipped = 0;
+        let rows = video_rows(&page, &mut skipped);
+        assert_eq!(rows.len(), 1);
+        assert_eq!(rows[0].video_id, "1FSqIJTiovQ");
+        assert_eq!(rows[0].title, "When We Were Young");
+        assert_eq!(rows[0].byline, "Dillon Francis");
+        assert_eq!(rows[0].duration_secs, Some(200));
+        assert_eq!(skipped, 1);
+        assert_eq!(continuation_token(&page).as_deref(), Some("NEXT_PAGE"));
+    }
+
+    #[test]
+    fn classic_renderer_still_read() {
+        let page: Value = serde_json::json!({
+            "contents": [ { "playlistVideoRenderer": {
+                "videoId": "abc123def45",
+                "title": { "runs": [ { "text": "Artist - Song" } ] },
+                "shortBylineText": { "runs": [ { "text": "Some Channel" } ] },
+                "lengthSeconds": "253"
+            }}]
+        });
+        let mut skipped = 0;
+        let rows = video_rows(&page, &mut skipped);
+        assert_eq!(rows.len(), 1);
+        assert_eq!(rows[0].duration_secs, Some(253));
+        assert_eq!(rows[0].byline, "Some Channel");
+    }
+
+    #[test]
+    fn split_wins_over_byline_and_byline_fills_bare_titles() {
+        let fetch = PlaylistFetch {
+            title: "party playlist".into(),
+            author: "Tommy".into(),
+            warnings: Vec::new(),
+            rows: vec![
+                VideoRow {
+                    video_id: "a".into(),
+                    title: "Jessi Malay - Summer Love (Funk LeBlanc Remix)".into(),
+                    byline: "Funk LeBlanc".into(),
+                    duration_secs: Some(278),
+                },
+                VideoRow {
+                    video_id: "b".into(),
+                    title: "RATATA (Official Video)".into(),
+                    byline: "Skrillex".into(),
+                    duration_secs: Some(127),
+                },
+            ],
+        };
+        let parsed = to_parsed(&fetch);
+        assert_eq!(parsed.entries[0].artist.as_deref(), Some("Jessi Malay"));
+        assert_eq!(
+            parsed.entries[0].title.as_deref(),
+            Some("Summer Love (Funk LeBlanc Remix)")
+        );
+        assert_eq!(parsed.entries[1].artist.as_deref(), Some("Skrillex"));
+        assert_eq!(parsed.entries[1].title.as_deref(), Some("RATATA"));
+        assert!(parsed.entries.iter().all(|e| e.start_ms.is_none()));
+    }
+}