Files
SoulStream/src-tauri/src/youtube_setlist.rs
T

653 lines
25 KiB
Rust

//! YouTube DJ-set tracklist extraction — the evidence rungs the feasibility
//! research ranks FIRST (creator- and viewer-declared data): the video's
//! chapters, its description, and timestamped tracklists in top comments.
//! (TODO-Feature-Docs/playlist-management-and-setlist-import, research
//! §3/§15: "the high-value, reliable subset is metadata import … broader
//! automatic recognition must remain optional".) No audio is fetched and
//! nothing is recognized — this is metadata import, honest about it, and a
//! set with no posted tracklist simply reports that instead of guessing.
//!
//! Everything runs in-process over reqwest (the no-child-processes posture):
//! one GET of the watch page, plus at most one innertube POST for comments
//! when the page itself carried no tracklist. The extracted lines feed the
//! EXISTING pasted-tracklist pipeline (playlist_formats::parse_text →
//! resolve_draft → review modal → commit), so every downstream behavior —
//! setlist stamps, library matching, stream fallbacks, unresolved rows —
//! is the one already shipped and tested.
use serde_json::Value;
/// What one video yielded: identity for the draft header plus the chosen
/// tracklist text and where it came from.
pub struct SetlistFetch {
pub title: String,
pub author: String,
pub tracklist: String,
/// "chapters" | "description" | "a comment by …" — shown in the modal's
/// warnings so the evidence rung is visible (research §3: "show which
/// rung produced a result").
pub provenance: String,
pub warnings: Vec<String>,
}
/// Minimum timestamped lines before a block counts as a tracklist — one or
/// two stamps are a lyric quote or a "best drop" comment, not a setlist.
const MIN_TRACKLIST_LINES: usize = 3;
/// Top-level comments scanned for a tracklist before giving up.
const MAX_COMMENTS_SCANNED: usize = 40;
// ---- URL handling -----------------------------------------------------------
/// Accepts watch/short/live/embed/youtu.be URLs (and a bare 11-char id) and
/// returns the canonical video id.
pub fn video_id_from_url(url: &str) -> Option<String> {
let s = url.trim();
let is_id = |c: &str| {
c.len() == 11
&& c.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
};
if is_id(s) {
return Some(s.to_string());
}
let after_scheme = s.split("://").nth(1).unwrap_or(s);
let (host, rest) = after_scheme.split_once('/')?;
let host = host.split('@').next_back().unwrap_or(host).to_lowercase();
let host = host.split(':').next().unwrap_or(&host);
let path_q = rest.split('#').next().unwrap_or(rest);
let (path, query) = match path_q.split_once('?') {
Some((p, q)) => (p, Some(q)),
None => (path_q, None),
};
let candidate = if host == "youtu.be" {
path.split('/').next().map(str::to_string)
} else if host.ends_with("youtube.com") || host.ends_with("youtube-nocookie.com") {
let segs: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
match segs.first().copied() {
Some("watch") => query.and_then(|q| {
q.split('&')
.find_map(|kv| kv.strip_prefix("v=").map(str::to_string))
}),
Some("shorts") | Some("live") | Some("embed") | Some("v") => {
segs.get(1).map(|s| s.to_string())
}
_ => None,
}
} else {
None
};
candidate.filter(|c| is_id(c))
}
// ---- watch-page JSON slicing --------------------------------------------------
/// Slice the JSON object that begins at the first `{` after `needle`,
/// brace-counting with string/escape awareness — the watch page embeds its
/// data as `var ytInitialData = {...};` inside script tags, megabytes long.
pub(crate) fn slice_json_after<'a>(html: &'a str, needle: &str) -> Option<&'a str> {
let at = html.find(needle)?;
let rest = &html[at + needle.len()..];
let open = rest.find('{')?;
let bytes = rest.as_bytes();
let mut depth = 0usize;
let mut in_str = false;
let mut escaped = false;
for (i, &b) in bytes.iter().enumerate().skip(open) {
if in_str {
if escaped {
escaped = false;
} else if b == b'\\' {
escaped = true;
} else if b == b'"' {
in_str = false;
}
continue;
}
match b {
b'"' => in_str = true,
b'{' => depth += 1,
b'}' => {
depth -= 1;
if depth == 0 {
return Some(&rest[open..=i]);
}
}
_ => {}
}
}
None
}
/// Depth-first search for every object under `root` that contains `key`,
/// applying `f` to the value. Youtube's trees are deep, volatile, and
/// renderer-name-happy; searching for leaf keys survives their reshuffles
/// better than hardcoded paths.
pub(crate) fn walk<'a>(root: &'a Value, key: &str, f: &mut dyn FnMut(&'a Value)) {
match root {
Value::Object(map) => {
if let Some(v) = map.get(key) {
f(v);
}
for v in map.values() {
walk(v, key, f);
}
}
Value::Array(items) => {
for v in items {
walk(v, key, f);
}
}
_ => {}
}
}
/// First string under any of `keys` in reading order — for label-ish leaves
/// ("simpleText", nested runs) where youtube varies the wrapper.
pub(crate) fn first_string(v: &Value, keys: &[&str]) -> Option<String> {
match v {
Value::String(s) => Some(s.clone()),
_ => {
for k in keys {
let mut found = None;
walk(v, k, &mut |x| {
if found.is_none() {
if let Some(s) = x.as_str() {
found = Some(s.to_string());
}
}
});
if found.is_some() {
return found;
}
}
None
}
}
}
// ---- the three evidence rungs -------------------------------------------------
/// A line "looks timestamped" when it carries a `m:ss`-or-longer stamp plus
/// some letters — the same acceptance the pasted-tracklist parser applies,
/// reused for candidate SCORING before anything is parsed for real.
pub(crate) fn is_timestamped_line(line: &str) -> bool {
let has_letters = line.chars().filter(|c| c.is_alphabetic()).count() >= 2;
if !has_letters {
return false;
}
line.split(|c: char| c.is_whitespace() || c == '[' || c == ']' || c == '(' || c == ')')
.any(|tok| {
let t = tok.trim_matches(|c: char| !c.is_ascii_digit() && c != ':');
crate::playlist_formats::parse_timestamp(t).is_some()
})
}
/// Chapters out of ytInitialData: every `chapterRenderer` carries a title
/// and `timeRangeStartMillis`. Ordered, creator-authored — the top rung.
pub(crate) fn chapters_tracklist(initial_data: &Value) -> Option<String> {
let mut rows: Vec<(i64, String)> = Vec::new();
walk(initial_data, "chapterRenderer", &mut |ch| {
let ms = ch
.get("timeRangeStartMillis")
.and_then(Value::as_i64)
.unwrap_or(-1);
let title = ch
.get("title")
.and_then(|t| first_string(t, &["simpleText", "text"]));
if ms >= 0 {
if let Some(t) = title {
rows.push((ms, t));
}
}
});
if rows.len() < MIN_TRACKLIST_LINES {
return None;
}
rows.sort_by_key(|(ms, _)| *ms);
rows.dedup();
Some(
rows.into_iter()
.map(|(ms, t)| format!("{} {}", fmt_ts(ms), t))
.collect::<Vec<_>>()
.join("\n"),
)
}
/// The description's best block: the longest run of timestamped lines with
/// at most one untimed line between them (headers like "Tracklist:" break
/// nothing; prose paragraphs do).
pub(crate) fn description_tracklist(description: &str) -> Option<String> {
fn flush<'a>(
current: &mut Vec<&'a str>,
stamped: &mut usize,
best: &mut Option<(usize, Vec<&'a str>)>,
) {
if *stamped >= MIN_TRACKLIST_LINES
&& best.as_ref().map(|(n, _)| *stamped > *n).unwrap_or(true)
{
*best = Some((*stamped, current.clone()));
}
current.clear();
*stamped = 0;
}
let mut best: Option<(usize, Vec<&str>)> = None;
let mut current: Vec<&str> = Vec::new();
let mut gap = 0usize;
let mut stamped = 0usize;
for line in description.lines() {
let trimmed = line.trim();
if is_timestamped_line(trimmed) {
current.push(trimmed);
stamped += 1;
gap = 0;
} else if !current.is_empty() && !trimmed.is_empty() && gap == 0 {
// One stray untimed line rides along (a wrapped title, an "ID"),
// a second breaks the block.
gap = 1;
} else if !current.is_empty() {
flush(&mut current, &mut stamped, &mut best);
gap = 0;
}
}
flush(&mut current, &mut stamped, &mut best);
best.map(|(_, lines)| lines.join("\n"))
}
/// Comment texts out of an innertube `next` response. Youtube has shipped
/// two shapes in the wild — legacy `contentText` runs and the newer
/// `commentEntityPayload` — so both are walked; each yields (author, text).
pub(crate) fn comment_texts(next_response: &Value) -> Vec<(String, String)> {
let mut out: Vec<(String, String)> = Vec::new();
walk(next_response, "commentEntityPayload", &mut |p| {
let text = p
.get("properties")
.and_then(|pr| pr.get("content"))
.and_then(|c| c.get("content"))
.and_then(Value::as_str);
let author = p
.get("author")
.and_then(|a| a.get("displayName"))
.and_then(Value::as_str)
.unwrap_or("a commenter");
if let Some(t) = text {
out.push((author.to_string(), t.to_string()));
}
});
walk(next_response, "commentRenderer", &mut |c| {
let text = c
.get("contentText")
.map(|ct| collect_runs(ct))
.filter(|t| !t.is_empty());
let author = c
.get("authorText")
.and_then(|a| first_string(a, &["simpleText"]))
.unwrap_or_else(|| "a commenter".into());
if let Some(t) = text {
out.push((author, t));
}
});
out.truncate(MAX_COMMENTS_SCANNED);
out
}
/// Flatten a `runs`/`simpleText` rich-text node into plain text, preserving
/// the run boundaries' own newlines (timestamps arrive as separate runs).
pub(crate) fn collect_runs(v: &Value) -> String {
if let Some(s) = v.get("simpleText").and_then(Value::as_str) {
return s.to_string();
}
let mut out = String::new();
walk(v, "text", &mut |t| {
if let Some(s) = t.as_str() {
out.push_str(s);
}
});
out
}
/// The comments continuation token for the initial `next` call, discovered
/// generically: the token whose enclosing area names the comment section.
pub(crate) fn comments_token(initial_data: &Value) -> Option<String> {
// The comment section's continuation is the one inside an
// itemSectionRenderer whose identifier mentions comments.
let mut token: Option<String> = None;
walk(initial_data, "itemSectionRenderer", &mut |sec| {
if token.is_some() {
return;
}
let ident = sec
.get("sectionIdentifier")
.and_then(Value::as_str)
.unwrap_or("");
if !ident.contains("comment") {
return;
}
walk(sec, "continuationCommand", &mut |cc| {
if token.is_none() {
if let Some(t) = cc.get("token").and_then(Value::as_str) {
token = Some(t.to_string());
}
}
});
});
token
}
fn fmt_ts(ms: i64) -> String {
let total = ms / 1000;
let h = total / 3600;
let m = (total % 3600) / 60;
let s = total % 60;
if h > 0 {
format!("{h}:{m:02}:{s:02}")
} else {
format!("{m}:{s:02}")
}
}
/// Innertube client version, read from the page so the comments call speaks
/// the same dialect the page was served with; a dated fallback otherwise.
pub(crate) fn client_version(html: &str) -> String {
slice_after_quote(html, "\"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"")
.unwrap_or("2.20260101.00.00")
.to_string()
}
fn slice_after_quote<'a>(html: &'a str, needle: &str) -> Option<&'a str> {
let at = html.find(needle)?;
let rest = &html[at + needle.len()..];
rest.split('"').next()
}
// ---- the fetch ----------------------------------------------------------------
/// Just the video's identity (title, author) — for the LISTENER's draft
/// header, whose sets typically have NO posted tracklist (that's why they
/// get listened to): the full `fetch` returns Err for those and took the
/// title down with it (Kyle's first own-set run was named by raw video id).
pub async fn fetch_title(url: &str) -> Option<(String, String)> {
let id = video_id_from_url(url)?;
let client = reqwest::Client::builder()
.user_agent(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/126.0.0.0 Safari/537.36",
)
.timeout(std::time::Duration::from_secs(15))
.build()
.ok()?;
let html = client
.get(format!("https://www.youtube.com/watch?v={id}&hl=en"))
.header("Cookie", "CONSENT=YES+1; SOCS=CAI")
.send()
.await
.ok()?
.text()
.await
.ok()?;
let player: Value = slice_json_after(&html, "ytInitialPlayerResponse")
.and_then(|s| serde_json::from_str(s).ok())?;
let details = player.get("videoDetails")?;
let title = details.get("title").and_then(Value::as_str)?.to_string();
let author = details
.get("author")
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
Some((title, author))
}
/// Fetch one video's metadata and pick its best tracklist. At most two
/// requests; every rung failure downgrades to the next with a warning
/// instead of failing the whole import (research §12).
pub async fn fetch(url: &str) -> Result<SetlistFetch, String> {
let id = video_id_from_url(url).ok_or("That doesn't look like a YouTube video link")?;
let client = reqwest::Client::builder()
.user_agent(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/126.0.0.0 Safari/537.36",
)
.timeout(std::time::Duration::from_secs(25))
.build()
.map_err(|e| e.to_string())?;
let html = client
.get(format!("https://www.youtube.com/watch?v={id}&hl=en"))
.header("Cookie", "CONSENT=YES+1; SOCS=CAI")
.header("Accept-Language", "en")
.send()
.await
.map_err(|e| format!("YouTube did not answer: {e}"))?
.error_for_status()
.map_err(|e| format!("YouTube refused the page: {e}"))?
.text()
.await
.map_err(|e| e.to_string())?;
let player: Value = slice_json_after(&html, "ytInitialPlayerResponse")
.and_then(|s| serde_json::from_str(s).ok())
.ok_or("Could not read the video's data — is the link public?")?;
// Unavailable/private/region-locked videos still serve a page — with a
// playability error instead of details. Say that plainly (research §8).
let playability = player
.get("playabilityStatus")
.and_then(|p| p.get("status"))
.and_then(Value::as_str)
.unwrap_or("OK");
if playability != "OK" && player.get("videoDetails").is_none() {
let reason = player
.get("playabilityStatus")
.and_then(|p| p.get("reason"))
.and_then(Value::as_str)
.unwrap_or("it may be private, removed, or region-locked");
return Err(format!("YouTube says this video is unavailable — {reason}"));
}
let details = player.get("videoDetails").cloned().unwrap_or(Value::Null);
let title = details
.get("title")
.and_then(Value::as_str)
.unwrap_or("YouTube set")
.to_string();
let author = details
.get("author")
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
let description = details
.get("shortDescription")
.and_then(Value::as_str)
.unwrap_or("")
.to_string();
let initial_data: Value = slice_json_after(&html, "ytInitialData")
.and_then(|s| serde_json::from_str(s).ok())
.unwrap_or(Value::Null);
let mut warnings = Vec::new();
// Rung 1: chapters. Rung 2: the description's timestamped block.
if let Some(list) = chapters_tracklist(&initial_data) {
return Ok(SetlistFetch {
title,
author,
tracklist: list,
provenance: "the video's chapters".into(),
warnings,
});
}
if let Some(list) = description_tracklist(&description) {
return Ok(SetlistFetch {
title,
author,
tracklist: list,
provenance: "the video description".into(),
warnings,
});
}
// Rung 3: top comments — one innertube call, failure-soft.
match comments_token(&initial_data) {
None => warnings.push("The video page exposed no comments to scan.".into()),
Some(token) => {
let version = client_version(&html);
let body = serde_json::json!({
"context": { "client": {
"clientName": "WEB", "clientVersion": version,
"hl": "en", "gl": "US",
}},
"continuation": token,
});
let resp = client
.post("https://www.youtube.com/youtubei/v1/next?prettyPrint=false")
.json(&body)
.send()
.await;
match resp {
Err(e) => warnings.push(format!("Comments could not be scanned: {e}")),
Ok(r) => match r.json::<Value>().await {
Err(e) => warnings.push(format!("Comments could not be read: {e}")),
Ok(next) => {
let mut best: Option<(usize, String, String)> = None;
for (who, text) in comment_texts(&next) {
let stamped =
text.lines().filter(|l| is_timestamped_line(l.trim())).count();
if stamped >= MIN_TRACKLIST_LINES
&& best.as_ref().map(|(n, _, _)| stamped > *n).unwrap_or(true)
{
let block: Vec<&str> = text
.lines()
.map(str::trim)
.filter(|l| is_timestamped_line(l))
.collect();
best = Some((stamped, who.clone(), block.join("\n")));
}
}
if let Some((n, who, list)) = best {
warnings.push(format!(
"Tracklist found in a comment by {who} ({n} timestamped lines) — viewer-supplied, worth a skim."
));
return Ok(SetlistFetch {
title,
author,
tracklist: list,
provenance: format!("a comment by {who}"),
warnings,
});
}
warnings.push("No comment carried a timestamped tracklist.".into());
}
},
}
}
}
Err(format!(
"No tracklist found on \"{title}\" — no chapters, no timestamped description, and {} \
Paste one into the tracklist door if you have it.",
if warnings.iter().any(|w| w.contains("comment")) {
"none of the top comments carry one."
} else {
"the comments could not be scanned."
}
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn video_ids_from_every_url_shape() {
for (url, want) in [
("https://www.youtube.com/watch?v=dQw4w9WgXcQ", Some("dQw4w9WgXcQ")),
("https://youtu.be/dQw4w9WgXcQ?t=42", Some("dQw4w9WgXcQ")),
("https://www.youtube.com/live/abc-DEF_123?feature=share", Some("abc-DEF_123")),
("https://music.youtube.com/watch?v=dQw4w9WgXcQ&list=RD", Some("dQw4w9WgXcQ")),
("www.youtube.com/shorts/abc-DEF_123", Some("abc-DEF_123")),
("dQw4w9WgXcQ", Some("dQw4w9WgXcQ")),
("https://example.com/watch?v=dQw4w9WgXcQ", None),
("https://www.youtube.com/watch?v=tooShort", None),
] {
assert_eq!(
video_id_from_url(url).as_deref(),
want,
"{url} must yield {want:?}"
);
}
}
#[test]
fn json_slicer_handles_braces_in_strings() {
let html = r#"<script>var ytInitialData = {"a":"br{ace \" quote","b":{"c":1}};</script>"#;
let json = slice_json_after(html, "ytInitialData").unwrap();
let v: Value = serde_json::from_str(json).unwrap();
assert_eq!(v["b"]["c"], 1);
}
#[test]
fn chapters_walk_and_format() {
let data: Value = serde_json::from_str(
r#"{"markersMap":[{"value":{"chapters":[
{"chapterRenderer":{"title":{"simpleText":"Bicep - Glue"},"timeRangeStartMillis":0}},
{"chapterRenderer":{"title":{"simpleText":"Overmono - So U Kno"},"timeRangeStartMillis":330000}},
{"chapterRenderer":{"title":{"simpleText":"Joy O - Sin Palta"},"timeRangeStartMillis":3900000}}
]}}]}"#,
)
.unwrap();
let list = chapters_tracklist(&data).unwrap();
assert_eq!(
list,
"0:00 Bicep - Glue\n5:30 Overmono - So U Kno\n1:05:00 Joy O - Sin Palta"
);
}
#[test]
fn description_block_beats_prose_and_survives_headers() {
let desc = "Recorded live at the warehouse.\nBig thanks to everyone!\n\nTracklist:\n00:00 Artist One - Opener\n04:12 Artist Two - Middle (Remix)\nID - ID\n12:44 Artist Three - Closer\n\nFollow me on socials at 10:00 tonight";
let block = description_tracklist(desc).unwrap();
assert_eq!(
block,
"00:00 Artist One - Opener\n04:12 Artist Two - Middle (Remix)\n12:44 Artist Three - Closer"
);
}
#[test]
fn too_few_stamps_is_no_tracklist() {
assert!(description_tracklist("00:01 hi\nsome words\n03:30 drop!").is_none());
assert!(!is_timestamped_line("10000 subscribers!"));
assert!(is_timestamped_line("1:23:45 Artist - Title"));
assert!(is_timestamped_line("Artist - Title [04:20]"));
}
#[test]
fn comment_payloads_both_shapes() {
let next: Value = serde_json::from_str(
r#"{"frameworkUpdates":{"entityBatchUpdate":{"mutations":[
{"payload":{"commentEntityPayload":{
"properties":{"content":{"content":"00:00 A - B\n03:00 C - D\n06:00 E - F"}},
"author":{"displayName":"@digger"}}}}
]}},
"old":[{"commentRenderer":{
"authorText":{"simpleText":"@legacy"},
"contentText":{"runs":[{"text":"great set"}]}}}]}"#,
)
.unwrap();
let texts = comment_texts(&next);
assert_eq!(texts.len(), 2);
assert_eq!(texts[0].0, "@digger");
assert!(texts[0].1.contains("03:00 C - D"));
assert_eq!(texts[1], ("@legacy".into(), "great set".into()));
}
#[test]
fn comments_token_found_by_section_identifier() {
let data: Value = serde_json::from_str(
r#"{"contents":[
{"itemSectionRenderer":{"sectionIdentifier":"related-items",
"contents":[{"continuationItemRenderer":{"continuationEndpoint":{"continuationCommand":{"token":"WRONG"}}}}]}},
{"itemSectionRenderer":{"sectionIdentifier":"comment-item-section",
"contents":[{"continuationItemRenderer":{"continuationEndpoint":{"continuationCommand":{"token":"RIGHT"}}}}]}}
]}"#,
)
.unwrap();
assert_eq!(comments_token(&data).as_deref(), Some("RIGHT"));
}
}