playlists + player: every service's list through one door with its names matched for real, and shuffle that readies exactly what it promises

This commit is contained in:
KeelrLeek committed 2026-08-24 18:10:34 -05:00
1 parent 30eb6fe025
commit 50414dae5c
16 files changed
+4157 -154

No files matched your search

+201 -12
View File
@@ -1557,6 +1557,32 @@ pub async fn catalog_search(
Ok(state.metadata.catalog_search(&query).await)
}
#[tauri::command]
pub async fn catalog_search_tracks(
state: State<'_, LibraryState>,
query: String,
limit: Option<u32>,
) -> Result<Vec<metadata::CatalogTrack>, String> {
state.touch_catalog_activity();
Ok(state
.metadata
.catalog_search_tracks(&query, limit.unwrap_or(10))
.await)
}
#[tauri::command]
pub async fn catalog_artist_track_candidates(
state: State<'_, LibraryState>,
artist: String,
title: String,
) -> Result<Vec<metadata::CatalogTrack>, String> {
state.touch_catalog_activity();
Ok(state
.metadata
.catalog_artist_track_candidates(&artist, &title)
.await)
}
#[tauri::command]
pub async fn catalog_track_by_isrc(
state: State<'_, LibraryState>,
@@ -3592,7 +3618,16 @@ pub async fn save_to_library(
let mut organized = 0u32;
let mut queued = 0u32;
for f in files {
let cached = crate::playback::cache_path(&app, &f.username, &f.path).filter(|c| c.exists());
// Only a BYTE-COMPLETE cache saves directly: a mid-stream (or
// abandoned) cache file is a prefix of the track, and the organizer
// would file the truncation as the real thing. Anything shorter
// falls through — to the in-flight transfer if one is running, else
// to a fresh download of the known file.
let cached = crate::playback::cache_path(&app, &f.username, &f.path)
.filter(|c| c.exists())
.filter(|c| {
f.size == 0 || c.metadata().map(|m| m.len() == f.size).unwrap_or(false)
});
if let Some(cache) = cached {
if organize_cached_save(&app, &downloads_dir, &cache, &f.path).await? {
organized += 1;
@@ -3639,6 +3674,17 @@ pub async fn save_to_library(
Ok((organized, queued))
}
/// Byte length of the stream-cache copy of one peer file, if it exists.
/// The caller compares against the peer-stated size to tell a finished
/// cache from a mid-stream prefix. Read-only — never contacts the peer.
#[tauri::command]
pub fn stream_cache_len(app: tauri::AppHandle, username: String, path: String) -> Option<u64> {
crate::playback::cache_path(&app, &username, &path)
.filter(|c| c.exists())
.and_then(|c| c.metadata().ok())
.map(|m| m.len())
}
/// Copy a completed stream cache into staging, then run the normal tagging and
/// library organizer. Returns false only when the copy itself did not happen.
async fn organize_cached_save(
@@ -4172,19 +4218,100 @@ pub async fn refresh_all_album_art(app: tauri::AppHandle) -> Result<usize, Strin
/// A source already within bounds is passed through rather than re-encoded, so
/// this never costs quality it was meant to protect.
#[tauri::command]
pub async fn album_art(path: String, max_edge: Option<u32>) -> Result<String, String> {
let bytes = tauri::async_runtime::spawn_blocking(move || -> Result<Vec<u8>, String> {
let raw = std::fs::read(&path).map_err(|e| e.to_string())?;
let Some(edge) = max_edge else { return Ok(raw) };
// An undecodable or already-small source falls through to the original
// bytes: a thumbnail is an optimisation, never a reason to show nothing.
match pipeline::thumbnail_jpeg(&raw, edge) {
Ok(Some(thumb)) => Ok(thumb),
_ => Ok(raw),
}
})
pub async fn album_art(
app: tauri::AppHandle,
path: String,
max_edge: Option<u32>,
) -> Result<String, String> {
// Thumbnails used to cost a FULL master decode + Lanczos resize per
// request, on the unbounded blocking pool, with nothing remembered —
// so a Library page scrolling past a few hundred tiles pinned every
// core in image decodes and the AUDIO thread started to starve (the
// player decodes on its own std::thread, which only chops when the
// whole machine is saturated). Two changes, and either alone would
// have hidden the other's need:
// 1. a decode SEMAPHORE — at most 2 covers decode at once, ever;
// 2. a thumbnail disk cache keyed by (path, mtime, len, edge) — each
// cover is decoded once per size PER LIFETIME, then it's a small
// file read. Covers change by being replaced (cover_pick writes a
// new file), so the mtime+len key self-invalidates; orphaned
// entries are tens of KB and left to lie.
// Master requests (no edge: the player-bar hero) stay a plain read —
// no decode, no permit, no cache.
let thumb_dir = app
.path()
.app_cache_dir()
.ok()
.map(|d| d.join("art-thumbs-v1"));
// Phase 1 — cheap and permit-free: stat, cache probe, raw read.
let probe_path = path.clone();
let probe_dir = thumb_dir.clone();
let (bytes, write_to) = tauri::async_runtime::spawn_blocking(
move || -> Result<(Vec<u8>, Option<std::path::PathBuf>), String> {
let Some(edge) = max_edge else {
return Ok((std::fs::read(&probe_path).map_err(|e| e.to_string())?, None));
};
let target = probe_dir.and_then(|dir| {
let meta = std::fs::metadata(&probe_path).ok()?;
let mtime = meta
.modified()
.ok()?
.duration_since(std::time::UNIX_EPOCH)
.ok()?
.as_nanos();
let mut hash: u64 = 0xcbf29ce484222325;
for b in probe_path
.as_bytes()
.iter()
.chain(mtime.to_le_bytes().iter())
.chain(meta.len().to_le_bytes().iter())
{
hash ^= u64::from(*b);
hash = hash.wrapping_mul(0x100000001b3);
}
Some(dir.join(format!("{hash:016x}-{edge}.jpg")))
});
if let Some(t) = &target {
if let Ok(cached) = std::fs::read(t) {
return Ok((cached, None));
}
}
Ok((std::fs::read(&probe_path).map_err(|e| e.to_string())?, target))
},
)
.await
.map_err(|e| e.to_string())??;
// Phase 2 — the decode, gated. Runs only on a cache miss.
let bytes = match write_to {
None => bytes,
Some(target) => {
let _permit = ART_DECODE_SLOTS
.acquire()
.await
.map_err(|e| e.to_string())?;
let edge = max_edge.unwrap_or(0);
tauri::async_runtime::spawn_blocking(move || {
// An undecodable or already-small source falls through to the
// original bytes: a thumbnail is an optimisation, never a
// reason to show nothing. Either result is CACHED, so the
// "already small" discovery (itself a full decode) is also
// paid once.
let out = match pipeline::thumbnail_jpeg(&bytes, edge) {
Ok(Some(thumb)) => thumb,
_ => bytes,
};
if let Some(dir) = target.parent() {
let _ = std::fs::create_dir_all(dir);
}
let _ = std::fs::write(&target, &out); // failure-soft
out
})
.await
.map_err(|e| e.to_string())?
}
};
use base64::Engine as _;
Ok(format!(
"data:image/jpeg;base64,{}",
@@ -4192,6 +4319,68 @@ pub async fn album_art(path: String, max_edge: Option<u32>) -> Result<String, St
))
}
/// At most this many album-art decodes run at once, machine-wide. Art is
/// cosmetic; audio and the UI are not — two permits keep a burst of tile
/// requests from ever owning more than two cores.
static ART_DECODE_SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(2);
/// The cover a streamed track carries INSIDE its own bytes, extracted to a
/// file the ordinary art path can read.
///
/// A Soulseek transfer is a tagged audio file, and in the reference library
/// 449 of 515 records keep their best cover embedded — so a stream that
/// arrived without any catalog artwork usually still has the real thing a
/// tag read away, and that cover needs no name matching to be certain it
/// belongs to this recording. Extracted once, keyed by the bytes' own hash
/// so two tracks off one record share the file.
#[tauri::command]
pub async fn stream_embedded_art(
app: tauri::AppHandle,
username: String,
path: String,
) -> Result<Option<String>, String> {
let Some(cache) = playback::cache_path(&app, &username, &path) else {
return Ok(None);
};
// Beside the library, not in AppData. The path is persisted onto the
// playlist row, and `playlist_items.art_path` is one of the columns the
// portable-library rebase rewrites — a cover kept in AppData would opt
// out of that and dangle the first time the drive is opened elsewhere.
let dir = config::ensure_library_data_dir(&app, &config::load(&app))
.map_err(|e| e.to_string())?
.join("stream-art");
tauri::async_runtime::spawn_blocking(move || -> Result<Option<String>, String> {
// A transfer still in flight has no cache file yet (the bytes land in
// `<name>.partial`), and a tag read over a half-written one simply
// fails — either way the answer is "nothing yet", and the caller asks
// again when the transfer completes.
if !cache.exists() {
return Ok(None);
}
let Ok(Some(bytes)) = tagger::extract_art(&cache) else {
return Ok(None);
};
if bytes.is_empty() {
return Ok(None);
}
let mut hash: u64 = 0xcbf29ce484222325;
for b in &bytes {
hash ^= *b as u64;
hash = hash.wrapping_mul(0x100000001b3);
}
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
// No extension the loader trusts: JPEG and PNG both appear in tags and
// the art pipeline sniffs the bytes anyway.
let out = dir.join(format!("{hash:016x}.img"));
if !matches!(std::fs::metadata(&out), Ok(md) if md.len() == bytes.len() as u64) {
std::fs::write(&out, &bytes).map_err(|e| e.to_string())?;
}
Ok(Some(out.to_string_lossy().to_string()))
})
.await
.map_err(|e| e.to_string())?
}
/// Open an http(s) URL in the system's default browser. The WebView blocks
/// `target="_blank"` navigation (no new-window handler is installed), so
/// external links such as artist socials route through
+9
View File
@@ -13,6 +13,7 @@ mod pipeline;
mod playback;
mod playlist_formats;
mod playlists;
mod provider_playlist;
mod preview;
mod setlist_scan;
mod shazam;
@@ -20,6 +21,7 @@ mod smtc;
mod state;
mod thumbbar;
mod vocal_sep;
mod youtube_playlist;
mod youtube_setlist;
mod ytdlp;
@@ -72,6 +74,7 @@ pub fn run() {
commands::download_file,
commands::copy_stream_to_folder,
commands::save_to_library,
commands::stream_cache_len,
commands::cancel_download,
commands::show_in_folder,
commands::refresh_artist_art,
@@ -85,6 +88,7 @@ pub fn run() {
commands::library_album_stats,
commands::album_tracks,
commands::album_art,
commands::stream_embedded_art,
commands::refresh_album_art,
commands::refresh_all_album_art,
commands::get_settings,
@@ -122,6 +126,9 @@ pub fn run() {
playlists::playlist_import_folder,
playlists::playlist_import_text,
playlists::playlist_import_youtube,
playlists::playlist_import_youtube_playlist,
playlists::playlist_import_link,
playlists::playlist_import_track_links,
playlists::playlist_import_commit,
setlist_scan::playlist_scan_youtube,
setlist_scan::playlist_pick_set_file,
@@ -143,6 +150,8 @@ pub fn run() {
lyrics::lyrics_save_text,
lyrics::lyrics_clear,
commands::catalog_search,
commands::catalog_search_tracks,
commands::catalog_artist_track_candidates,
commands::catalog_track_by_isrc,
import::import_scan,
import::import_portable_root,
+1 -1
View File
@@ -233,7 +233,7 @@ fn parse_m3u(bytes: &[u8]) -> Result<ParsedPlaylist, String> {
/// "Artist - Title" (or the em/en-dash forms) split. Returns (artist, title);
/// no separator means the whole display string is the title.
fn split_artist_title(display: &str) -> (Option<String>, Option<String>) {
pub(crate) fn split_artist_title(display: &str) -> (Option<String>, Option<String>) {
let display = display.trim();
if display.is_empty() {
return (None, None);
+190
View File
@@ -946,6 +946,196 @@ pub async fn playlist_import_youtube(
Ok(draft)
}
/// Import a YouTube PLAYLIST — one row per video, off the playlist page's
/// own metadata (title, channel, length). No tools, no audio, no listening:
/// this is the same honest metadata-import posture as the DJ-set door, for
/// links where each video IS one track.
#[tauri::command]
pub async fn playlist_import_youtube_playlist(
lib: State<'_, LibraryState>,
url: String,
) -> Result<ImportDraft, String> {
let fetched = crate::youtube_playlist::fetch(&url).await?;
let parsed = crate::youtube_playlist::to_parsed(&fetched);
let source = if fetched.author.is_empty() {
fetched.title.clone()
} else {
format!("{} — {}", fetched.author, fetched.title)
};
let mut draft = resolve_draft(&lib, parsed, source, None).await?;
draft.format = "youtube-playlist".into();
draft.name = Some(fetched.title.clone());
// The first video's thumbnail is the playlist's default face — the same
// choice YouTube itself makes.
if let Some(first) = fetched.rows.first() {
draft.cover_url = Some(format!(
"https://i.ytimg.com/vi/{}/maxresdefault.jpg",
first.video_id
));
}
draft.warnings.insert(
0,
format!(
"{} video{} from the playlist.",
fetched.rows.len(),
if fetched.rows.len() == 1 { "" } else { "s" }
),
);
draft.warnings.extend(fetched.warnings);
Ok(draft)
}
/// Import a playlist from any supported LINK — YouTube, Spotify, Apple
/// Music, TIDAL, Deezer — one door, dispatched by host. Each source is
/// metadata-only and anonymous; rows ride the same resolve → review →
/// commit pipeline as every other import.
#[tauri::command]
pub async fn playlist_import_link(
lib: State<'_, LibraryState>,
url: String,
) -> Result<ImportDraft, String> {
// YouTube keeps its own module (lockup pages, continuations, title
// cleaning); everything else is a provider fetch.
if crate::youtube_playlist::playlist_id_from_url(&url).is_some() {
return playlist_import_youtube_playlist(lib, url).await;
}
let fetched = crate::provider_playlist::fetch(&url).await?;
let parsed = crate::provider_playlist::to_parsed(&fetched);
let source = if fetched.author.is_empty() {
fetched.title.clone()
} else {
format!("{} — {}", fetched.author, fetched.title)
};
let mut draft = resolve_draft(&lib, parsed, source, None).await?;
draft.format = fetched.provider.into();
draft.name = Some(fetched.title.clone());
draft.cover_url = fetched.cover_url.clone();
draft.warnings.insert(
0,
format!(
"{} track{} from the playlist.",
fetched.rows.len(),
if fetched.rows.len() == 1 { "" } else { "s" }
),
);
draft.warnings.extend(fetched.warnings);
Ok(draft)
}
/// Track links one paste will look up. Each costs a page fetch, so this is a
/// politeness ceiling as much as a sanity one.
const MAX_PASTED_LINKS: usize = 2000;
/// Import a pasted block of TRACK links — the way past a service's own
/// playlist ceiling. Spotify's public playlist page stops at 100 tracks, but
/// every track has its own public page, and Spotify's app hands over all the
/// links at once (select all, copy). Each line is looked up on its own; a
/// line that is not a link is read as ordinary "Artist - Title" text, so a
/// mixed paste works too.
#[tauri::command]
pub async fn playlist_import_track_links(
app: tauri::AppHandle,
lib: State<'_, LibraryState>,
text: String,
name: Option<String>,
) -> Result<ImportDraft, String> {
let lines = crate::provider_playlist::pasted_lines(&text);
if lines.is_empty() {
return Err("Nothing to import in that paste.".into());
}
let link_count = lines
.iter()
.filter(|l| matches!(l, crate::provider_playlist::PastedLine::Link(_)))
.count();
if link_count > MAX_PASTED_LINKS {
return Err(format!(
"That paste holds {link_count} track links — more than the {MAX_PASTED_LINKS} one import takes."
));
}
// The lookups take a page each, so the modal gets a live count instead of
// a frozen button.
let emitter = app.clone();
let outcome =
crate::provider_playlist::resolve_pasted(lines, move |done, total, waiting| {
let _ = emitter.emit(
"playlist:link-progress",
serde_json::json!({ "done": done, "total": total, "waiting": waiting }),
);
})
.await?;
let total = outcome.links;
let unread = outcome.rows.iter().filter(|r| r.is_none()).count();
let entries: Vec<_> = outcome
.rows
.into_iter()
.flatten()
.enumerate()
.map(|(i, r)| playlist_formats::ParsedEntry {
ordinal: i as u32,
title: Some(r.title.clone()),
artist: (!r.artist.is_empty()).then(|| r.artist.clone()),
album: r.album.clone(),
duration_secs: r.duration_secs,
location: None,
start_ms: None,
raw: format!("{} - {}", r.artist, r.title),
})
.collect();
if entries.is_empty() {
return Err("None of those links could be read.".into());
}
let parsed = playlist_formats::ParsedPlaylist {
format: "links".into(),
title: name.clone(),
description: None,
entries,
warnings: Vec::new(),
};
let source = name.clone().unwrap_or_else(|| "Pasted track links".into());
let mut draft = resolve_draft(&lib, parsed, source, None).await?;
draft.format = "links".into();
draft.name = name;
draft.warnings.insert(
0,
format!(
"{} track{} read from {total} pasted link{}.",
draft.entries.len(),
if draft.entries.len() == 1 { "" } else { "s" },
if total == 1 { "" } else { "s" }
),
);
// A link the service refused us on is a DIFFERENT failure from a link
// that names nothing, and only one of them is worth retrying.
let missing = unread.saturating_sub(outcome.throttled_out);
if outcome.throttled_out > 0 {
draft.warnings.insert(
1,
format!(
"{} link{} went unread because the service throttled the lookups. \
Paste those again in a few minutes and they will come through.",
outcome.throttled_out,
if outcome.throttled_out == 1 { "" } else { "s" }
),
);
} else if outcome.was_throttled {
draft.warnings.insert(
1,
"The service throttled the lookups partway; the import slowed down and \
finished anyway."
.into(),
);
}
if missing > 0 {
draft.warnings.push(format!(
"{missing} line{} named nothing readable and {} left out.",
if missing == 1 { "" } else { "s" },
if missing == 1 { "was" } else { "were" }
));
}
Ok(draft)
}
/// Commit a reviewed draft as a new playlist, atomically.
#[tauri::command]
pub async fn playlist_import_commit(
File diff suppressed because it is too large. Load diff
+613
View File
@@ -0,0 +1,613 @@
//! YouTube PLAYLIST import — one row per video, straight off the playlist's
//! public page. This is the metadata-import posture of `youtube_setlist`
//! applied to the other kind of link: nothing is downloaded and nothing is
//! recognized — the page already states every video's title, channel and
//! length, and those become ordinary draft rows for the existing
//! resolve → review → commit pipeline.
//!
//! Everything runs in-process over reqwest: one GET of the playlist page,
//! plus one innertube `browse` POST per additional hundred videos (the same
//! dialect the setlist module's comments call speaks). Since 2024 the
//! desktop page renders videos as `lockupViewModel` nodes; the old
//! `playlistVideoRenderer` shape is still walked as insurance, since YouTube
//! has served both in the wild.
use crate::youtube_setlist::{collect_runs, slice_json_after, walk};
use serde_json::Value;
/// One video as the playlist page stated it.
pub struct VideoRow {
pub video_id: String,
/// The raw video title, junk suffixes and all — cleaning happens at
/// draft-building so the review's provenance line can show the original.
pub title: String,
/// The uploading channel's display name ("Skrillex", "Artist - Topic").
pub byline: String,
pub duration_secs: Option<u32>,
}
pub struct PlaylistFetch {
pub title: String,
/// The playlist's owner ("Tommy"), not any track's artist.
pub author: String,
pub rows: Vec<VideoRow>,
pub warnings: Vec<String>,
}
/// Videos a single import will take. A playlist page holds 100; each
/// continuation another 100. Ten requests is a generous ceiling — past that
/// the review modal stops being a review.
const MAX_VIDEOS: usize = 1000;
// ---- URL handling -----------------------------------------------------------
/// The `list=` id out of any YouTube-shaped URL (playlist pages, watch links
/// carrying a list, music.youtube.com), or a bare id pasted alone.
pub fn playlist_id_from_url(url: &str) -> Option<String> {
let s = url.trim();
let is_id = |c: &str| {
(10..=64).contains(&c.len())
&& c.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
};
// A bare id: playlist ids start with a recognizable prefix; a raw video
// id must NOT be mistaken for one (11 chars, no such prefix).
if is_id(s)
&& ["PL", "OLAK5uy_", "FL", "UU", "LL", "RD"]
.iter()
.any(|p| s.starts_with(p))
{
return Some(s.to_string());
}
let after_scheme = s.split("://").nth(1).unwrap_or(s);
let (host, rest) = after_scheme.split_once('/')?;
let host = host.split('@').next_back().unwrap_or(host).to_lowercase();
let host = host.split(':').next().unwrap_or(&host);
if !(host.ends_with("youtube.com") || host == "youtu.be" || host.ends_with("youtube-nocookie.com")) {
return None;
}
let query = rest.split('#').next().unwrap_or(rest).split_once('?')?.1;
query
.split('&')
.find_map(|kv| kv.strip_prefix("list=").map(str::to_string))
.filter(|c| is_id(c))
}
// ---- page shapes ------------------------------------------------------------
/// Video rows out of any subtree, both page generations:
/// `lockupViewModel` (2024+ desktop) and `playlistVideoRenderer` (classic).
/// Order follows the document, which is the playlist's own order.
pub(crate) fn video_rows(root: &Value, skipped_unavailable: &mut usize) -> Vec<VideoRow> {
let mut rows: Vec<VideoRow> = Vec::new();
walk(root, "lockupViewModel", &mut |lk| {
if lk.get("contentType").and_then(Value::as_str) != Some("LOCKUP_CONTENT_TYPE_VIDEO") {
return;
}
let Some(id) = lk.get("contentId").and_then(Value::as_str) else {
return;
};
let md = &lk["metadata"]["lockupMetadataViewModel"];
let title = md
.get("title")
.and_then(|t| t.get("content"))
.and_then(Value::as_str)
.unwrap_or("")
.trim()
.to_string();
if title.is_empty() || is_unavailable_stub(&title) {
*skipped_unavailable += usize::from(!title.is_empty());
return;
}
// First metadata row's first part is the channel line.
let mut byline = String::new();
walk(md, "metadataParts", &mut |parts| {
if byline.is_empty() {
if let Some(t) = parts
.get(0)
.and_then(|p| p.get("text"))
.and_then(|t| t.get("content"))
.and_then(Value::as_str)
{
byline = t.trim().to_string();
}
}
});
// Duration rides a thumbnail badge as "3:20"; other badges say
// "LIVE"/"SHORTS" and simply fail the timestamp parse.
let mut secs = None;
walk(lk, "thumbnailBadgeViewModel", &mut |b| {
if secs.is_none() {
secs = b
.get("text")
.and_then(Value::as_str)
.and_then(|t| crate::playlist_formats::parse_timestamp(t.trim()))
.map(|ms| (ms / 1000) as u32);
}
});
rows.push(VideoRow {
video_id: id.to_string(),
title,
byline,
duration_secs: secs,
});
});
walk(root, "playlistVideoRenderer", &mut |v| {
let Some(id) = v.get("videoId").and_then(Value::as_str) else {
return;
};
let title = v.get("title").map(collect_runs).unwrap_or_default().trim().to_string();
if title.is_empty() || is_unavailable_stub(&title) {
*skipped_unavailable += usize::from(!title.is_empty());
return;
}
let byline = v
.get("shortBylineText")
.map(collect_runs)
.unwrap_or_default()
.trim()
.to_string();
let secs = v
.get("lengthSeconds")
.and_then(Value::as_str)
.and_then(|s| s.parse::<u32>().ok());
rows.push(VideoRow {
video_id: id.to_string(),
title,
byline,
duration_secs: secs,
});
});
rows
}
/// The placeholder rows YouTube leaves for videos it can no longer show.
fn is_unavailable_stub(title: &str) -> bool {
matches!(title, "[Private video]" | "[Deleted video]" | "[Unavailable]")
}
/// The next page's continuation token, wherever the shape put it.
pub(crate) fn continuation_token(root: &Value) -> Option<String> {
let mut token = None;
for key in ["continuationItemViewModel", "continuationItemRenderer"] {
walk(root, key, &mut |c| {
if token.is_none() {
walk(c, "token", &mut |t| {
if token.is_none() {
if let Some(s) = t.as_str() {
token = Some(s.to_string());
}
}
});
}
});
if token.is_some() {
break;
}
}
token
}
/// The playlist's own name, across header generations.
fn playlist_title(initial_data: &Value) -> Option<String> {
let mut title = None;
for key in ["playlistMetadataRenderer", "microformatDataRenderer", "pageHeaderRenderer"] {
walk(initial_data, key, &mut |n| {
if title.is_none() {
title = n
.get("title")
.or_else(|| n.get("pageTitle"))
.and_then(Value::as_str)
.map(str::to_string);
}
});
if title.is_some() {
break;
}
}
title
}
/// The playlist owner's display name. The new header states it as an avatar
/// stack captioned "by Tommy"; the classic one as `ownerText` runs.
fn playlist_author(initial_data: &Value) -> String {
let mut author = String::new();
walk(initial_data, "avatarStackViewModel", &mut |a| {
if author.is_empty() {
if let Some(t) = a
.get("text")
.and_then(|t| t.get("content"))
.and_then(Value::as_str)
{
author = t.trim().strip_prefix("by ").unwrap_or(t.trim()).to_string();
}
}
});
if author.is_empty() {
walk(initial_data, "ownerText", &mut |o| {
if author.is_empty() {
let t = collect_runs(o);
if !t.trim().is_empty() {
author = t.trim().to_string();
}
}
});
}
author
}
/// INFO banners worth repeating ("Unavailable videos are hidden") and the
/// fatal alert text for dead lists ("The playlist does not exist.").
fn alert_texts(initial_data: &Value) -> Vec<String> {
let mut out = Vec::new();
for key in ["alertRenderer", "alertWithButtonRenderer"] {
walk(initial_data, key, &mut |a| {
if let Some(t) = a
.get("text")
.and_then(|t| crate::youtube_setlist::first_string(t, &["simpleText", "text"]))
{
if !t.trim().is_empty() {
out.push(t.trim().to_string());
}
}
});
}
out
}
// ---- title cleaning and the artist policy -----------------------------------
/// Strip the upload-side junk a video title carries that a TRACK title does
/// not: "(Official Video)", "[Official Audio]", "(Lyric Video)", "[4K]"…
/// Version qualifiers are load-bearing (the DJ-set work proved remix naming
/// matters), so any group naming a remix/edit/mix/version survives even when
/// it also says "official" — "(Official Remix)" stays.
pub fn clean_video_title(title: &str) -> String {
const KEEP: &[&str] = &[
"remix", "edit", "bootleg", "mix", "vip", "flip", "mashup", "rework",
"cover", "version", "feat", "ft.", "ft ", "live", "acoustic", "demo",
"instrumental", "extended", "radio", "dub", "remaster",
];
const JUNK: &[&str] = &[
"official video", "official music video", "official audio",
"official lyric", "official visualizer", "official visualiser",
"music video", "lyric video", "lyrics video", "lyric", "lyrics",
"visualizer", "visualiser", "official", "out now", "free download",
"free dl", "premiere", "monstercat release", "ncs release",
"video oficial", "audio oficial", "clip officiel",
];
let mut s = title.trim().to_string();
loop {
let t = s.trim_end();
let Some((open, close)) = (match t.as_bytes().last() {
Some(b')') => Some(('(', ')')),
Some(b']') => Some(('[', ']')),
_ => None,
}) else {
break;
};
let Some(at) = t.rfind(open) else { break };
let inner = t[at + 1..t.len() - 1].trim().to_lowercase();
let _ = close;
let keeps = KEEP.iter().any(|k| inner.contains(k));
let junk = !keeps
&& (JUNK.iter().any(|j| inner.contains(j))
|| matches!(inner.as_str(), "hd" | "hq" | "4k" | "8k" | "mv" | "m/v" | "video" | "audio" | "full version"));
if !junk || at == 0 {
break;
}
s = t[..at].trim_end().to_string();
}
if s.is_empty() {
title.trim().to_string()
} else {
s
}
}
/// The artist a channel name implies, when it does. Auto-generated music
/// channels ("Artist - Topic") and VEVO channels ("ArtistVEVO") are the
/// artist by construction; any other channel is offered as-is — for artist
/// channels it is exactly right, and the review shows every row's artist
/// before anything commits.
pub fn byline_artist(byline: &str) -> Option<String> {
let b = byline.trim();
if b.is_empty() {
return None;
}
if let Some(a) = b.strip_suffix(" - Topic") {
let a = a.trim();
return (!a.is_empty()).then(|| a.to_string());
}
if let Some(a) = b.strip_suffix("VEVO").filter(|a| !a.is_empty()) {
// "TaylorSwiftVEVO" → "TaylorSwift"; normalized matching downstream
// strips spacing anyway.
return Some(a.trim().to_string());
}
Some(b.to_string())
}
/// Draft rows from the fetched videos: the title's own "Artist - Title"
/// split wins (a curator's upload states the artist there); a bare title
/// takes the channel as artist.
pub fn to_parsed(fetch: &PlaylistFetch) -> crate::playlist_formats::ParsedPlaylist {
let mut entries = Vec::new();
for (i, row) in fetch.rows.iter().enumerate() {
let cleaned = clean_video_title(&row.title);
let (artist, title) = match crate::playlist_formats::split_artist_title(&cleaned) {
(Some(a), Some(t)) => (Some(a), Some(t)),
(_, t) => (byline_artist(&row.byline), t),
};
entries.push(crate::playlist_formats::ParsedEntry {
ordinal: i as u32,
title,
artist,
album: None,
duration_secs: row.duration_secs,
location: None,
start_ms: None,
raw: row.title.clone(),
});
}
crate::playlist_formats::ParsedPlaylist {
format: "youtube-playlist".into(),
title: Some(fetch.title.clone()),
description: None,
entries,
warnings: Vec::new(),
}
}
// ---- the fetch ----------------------------------------------------------------
/// Fetch the playlist's full video list: the page, then one `browse` POST
/// per continuation. Continuation failures degrade to "first N videos" with
/// a warning instead of failing the import.
pub async fn fetch(url: &str) -> Result<PlaylistFetch, String> {
let id = playlist_id_from_url(url)
.ok_or("That doesn't look like a YouTube playlist link — it needs a list= id")?;
let client = reqwest::Client::builder()
.user_agent(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/126.0.0.0 Safari/537.36",
)
.timeout(std::time::Duration::from_secs(25))
.build()
.map_err(|e| e.to_string())?;
let html = client
.get(format!("https://www.youtube.com/playlist?list={id}&hl=en"))
.header("Cookie", "CONSENT=YES+1; SOCS=CAI")
.header("Accept-Language", "en")
.send()
.await
.map_err(|e| format!("YouTube did not answer: {e}"))?
.error_for_status()
.map_err(|e| format!("YouTube refused the page: {e}"))?
.text()
.await
.map_err(|e| e.to_string())?;
let initial_data: Value = slice_json_after(&html, "ytInitialData")
.and_then(|s| serde_json::from_str(s).ok())
.ok_or("Could not read the playlist's data — is the link public?")?;
let mut warnings = Vec::new();
let mut skipped = 0usize;
let mut rows = video_rows(&initial_data, &mut skipped);
let alerts = alert_texts(&initial_data);
if rows.is_empty() {
let why = alerts
.first()
.cloned()
.unwrap_or_else(|| "it may be private or deleted".into());
return Err(format!("YouTube listed no videos for this playlist — {why}"));
}
// "Unavailable videos are hidden" and its kin — YouTube already counted
// what it dropped, so repeat it rather than pretending completeness.
warnings.extend(alerts);
let title = playlist_title(&initial_data).unwrap_or_else(|| "YouTube playlist".into());
let author = playlist_author(&initial_data);
let version = crate::youtube_setlist::client_version(&html);
let mut token = continuation_token(&initial_data);
while let Some(t) = token.take() {
if rows.len() >= MAX_VIDEOS {
warnings.push(format!(
"Only the first {MAX_VIDEOS} videos were taken — the playlist goes on."
));
break;
}
let body = serde_json::json!({
"context": { "client": {
"clientName": "WEB", "clientVersion": version,
"hl": "en", "gl": "US",
}},
"continuation": t,
});
let resp = client
.post("https://www.youtube.com/youtubei/v1/browse?prettyPrint=false")
.json(&body)
.send()
.await;
let next: Value = match resp {
Err(e) => {
warnings.push(format!(
"Only the first {} videos could be fetched: {e}",
rows.len()
));
break;
}
Ok(r) => match r.json().await {
Err(e) => {
warnings.push(format!(
"Only the first {} videos could be fetched: {e}",
rows.len()
));
break;
}
Ok(v) => v,
},
};
let mut more = video_rows(&next, &mut skipped);
if more.is_empty() {
break;
}
rows.append(&mut more);
token = continuation_token(&next);
}
if skipped > 0 {
warnings.push(format!(
"{skipped} unavailable video{} skipped (private or deleted).",
if skipped == 1 { "" } else { "s" }
));
}
Ok(PlaylistFetch {
title,
author,
rows,
warnings,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn playlist_ids_from_all_the_spellings() {
let id = "PLRJiC-nOkffKTsUPqj4NfkOYtBX490wI1";
for url in [
&format!("https://www.youtube.com/playlist?list={id}"),
&format!("https://music.youtube.com/playlist?list={id}"),
&format!("https://www.youtube.com/watch?v=abc123def45&list={id}&index=2"),
&format!("https://youtu.be/abc123def45?list={id}"),
&format!("youtube.com/playlist?list={id}#top"),
id,
] {
assert_eq!(playlist_id_from_url(url).as_deref(), Some(id), "{url}");
}
// A bare VIDEO id must not read as a playlist.
assert_eq!(playlist_id_from_url("dQw4w9WgXcQ"), None);
assert_eq!(playlist_id_from_url("https://example.com/playlist?list=PLabc123456"), None);
assert_eq!(playlist_id_from_url("https://www.youtube.com/watch?v=abc123def45"), None);
}
#[test]
fn junk_suffixes_stripped_qualifiers_kept() {
assert_eq!(clean_video_title("Song (Official Video)"), "Song");
assert_eq!(clean_video_title("Song [Official Music Video] (4K)"), "Song");
assert_eq!(clean_video_title("Song (Ben Böhmer Remix)"), "Song (Ben Böhmer Remix)");
assert_eq!(
clean_video_title("Song (Unofficial Remix) [HD]"),
"Song (Unofficial Remix)"
);
assert_eq!(clean_video_title("Only You (feat. Haley)"), "Only You (feat. Haley)");
assert_eq!(clean_video_title("Song (Official Audio)"), "Song");
// A title that IS one junk-looking group survives whole.
assert_eq!(clean_video_title("(Official Video)"), "(Official Video)");
}
#[test]
fn byline_artist_policy() {
assert_eq!(byline_artist("Ben Böhmer - Topic").as_deref(), Some("Ben Böhmer"));
assert_eq!(byline_artist("TaylorSwiftVEVO").as_deref(), Some("TaylorSwift"));
assert_eq!(byline_artist("Skrillex").as_deref(), Some("Skrillex"));
assert_eq!(byline_artist(""), None);
}
#[test]
fn lockup_rows_and_continuation_from_fixture() {
let page: Value = serde_json::json!({
"contents": [
{ "lockupViewModel": {
"contentId": "1FSqIJTiovQ",
"contentType": "LOCKUP_CONTENT_TYPE_VIDEO",
"contentImage": { "overlays": [ { "thumbnailBadgeViewModel": { "text": "3:20" } } ] },
"metadata": { "lockupMetadataViewModel": {
"title": { "content": "When We Were Young" },
"metadata": { "contentMetadataViewModel": { "metadataRows": [
{ "metadataParts": [ { "text": { "content": "Dillon Francis" } } ] }
]}}
}}
}},
{ "lockupViewModel": {
"contentId": "zzzzzzzzzzz",
"contentType": "LOCKUP_CONTENT_TYPE_VIDEO",
"metadata": { "lockupMetadataViewModel": { "title": { "content": "[Private video]" } } }
}},
{ "lockupViewModel": {
"contentId": "PLnotavideo",
"contentType": "LOCKUP_CONTENT_TYPE_PLAYLIST",
"metadata": { "lockupMetadataViewModel": { "title": { "content": "A shelf" } } }
}},
{ "continuationItemViewModel": { "continuationCommand": {
"innertubeCommand": { "continuationCommand": { "token": "NEXT_PAGE" } }
}}}
]
});
let mut skipped = 0;
let rows = video_rows(&page, &mut skipped);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].video_id, "1FSqIJTiovQ");
assert_eq!(rows[0].title, "When We Were Young");
assert_eq!(rows[0].byline, "Dillon Francis");
assert_eq!(rows[0].duration_secs, Some(200));
assert_eq!(skipped, 1);
assert_eq!(continuation_token(&page).as_deref(), Some("NEXT_PAGE"));
}
#[test]
fn classic_renderer_still_read() {
let page: Value = serde_json::json!({
"contents": [ { "playlistVideoRenderer": {
"videoId": "abc123def45",
"title": { "runs": [ { "text": "Artist - Song" } ] },
"shortBylineText": { "runs": [ { "text": "Some Channel" } ] },
"lengthSeconds": "253"
}}]
});
let mut skipped = 0;
let rows = video_rows(&page, &mut skipped);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].duration_secs, Some(253));
assert_eq!(rows[0].byline, "Some Channel");
}
#[test]
fn split_wins_over_byline_and_byline_fills_bare_titles() {
let fetch = PlaylistFetch {
title: "party playlist".into(),
author: "Tommy".into(),
warnings: Vec::new(),
rows: vec![
VideoRow {
video_id: "a".into(),
title: "Jessi Malay - Summer Love (Funk LeBlanc Remix)".into(),
byline: "Funk LeBlanc".into(),
duration_secs: Some(278),
},
VideoRow {
video_id: "b".into(),
title: "RATATA (Official Video)".into(),
byline: "Skrillex".into(),
duration_secs: Some(127),
},
],
};
let parsed = to_parsed(&fetch);
assert_eq!(parsed.entries[0].artist.as_deref(), Some("Jessi Malay"));
assert_eq!(
parsed.entries[0].title.as_deref(),
Some("Summer Love (Funk LeBlanc Remix)")
);
assert_eq!(parsed.entries[1].artist.as_deref(), Some("Skrillex"));
assert_eq!(parsed.entries[1].title.as_deref(), Some("RATATA"));
assert!(parsed.entries.iter().all(|e| e.start_ms.is_none()));
}
}
+3 -3
View File
@@ -122,7 +122,7 @@ pub(crate) fn slice_json_after<'a>(html: &'a str, needle: &str) -> Option<&'a st
/// applying `f` to the value. Youtube's trees are deep, volatile, and
/// renderer-name-happy; searching for leaf keys survives their reshuffles
/// better than hardcoded paths.
fn walk<'a>(root: &'a Value, key: &str, f: &mut dyn FnMut(&'a Value)) {
pub(crate) fn walk<'a>(root: &'a Value, key: &str, f: &mut dyn FnMut(&'a Value)) {
match root {
Value::Object(map) => {
if let Some(v) = map.get(key) {
@@ -143,7 +143,7 @@ fn walk<'a>(root: &'a Value, key: &str, f: &mut dyn FnMut(&'a Value)) {
/// First string under any of `keys` in reading order — for label-ish leaves
/// ("simpleText", nested runs) where youtube varies the wrapper.
fn first_string(v: &Value, keys: &[&str]) -> Option<String> {
pub(crate) fn first_string(v: &Value, keys: &[&str]) -> Option<String> {
match v {
Value::String(s) => Some(s.clone()),
_ => {
@@ -292,7 +292,7 @@ pub(crate) fn comment_texts(next_response: &Value) -> Vec<(String, String)> {
/// Flatten a `runs`/`simpleText` rich-text node into plain text, preserving
/// the run boundaries' own newlines (timestamps arrive as separate runs).
fn collect_runs(v: &Value) -> String {
pub(crate) fn collect_runs(v: &Value) -> String {
if let Some(s) = v.get("simpleText").and_then(Value::as_str) {
return s.to_string();
}