inputnoise/youclonedl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: use iOS player client and increase metadata gap to reduce throttling

Browse Source 
YouTube has been aggressively throttling the default web client.
The iOS client bypasses PO-token requirements and gets less aggressive
rate limiting from Google's side.

- Add --extractor-args youtube:player_client=ios to all yt-dlp calls:
  fetch_video_metadata, fetch_channel_metadata, search_youtube,
  fetch_available_subs, download_subs_only, fetch_video_comments,
  popular fetch flat-playlist crawl, and start_download
- Increase _META_MIN_GAP from 5s to 12s between metadata calls
- Widen jitter from ±2.5s to ±5s for less predictable request timing

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Mattias Thall
2026-05-27 03:46:54 +02:00
parent 8a5108425b
commit 8dcbad6e70
2 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/routers/channels.py
View File

@@ -743,6 +743,7 @@ def _fetch_popular_task(channel_id: int, youtube_channel_id: str, channel_name:
"yt-dlp", url, "yt-dlp", url,
"--dump-json", "--flat-playlist", "--dump-json", "--flat-playlist",
"--quiet", "--quiet",
"--extractor-args", "youtube:player_client=ios",
*ytdlp._cookie_args(), *ytdlp._cookie_args(),
], timeout=120) ], timeout=120)

14
backend/services/ytdlp.py
View File

@@ -53,7 +53,7 @@ def _run(args: list[str], timeout: int = 60) -> tuple[str, str, int]:
# hammering YouTube and invalidating cookies. # hammering YouTube and invalidating cookies.
_meta_lock = threading.Lock() _meta_lock = threading.Lock()
_meta_last_call: float = 0.0 _meta_last_call: float = 0.0
_META_MIN_GAP = 5.0 # seconds between any two metadata requests _META_MIN_GAP = 12.0 # seconds between any two metadata requests
# Active download counter — _meta_run pauses while any download is running so # Active download counter — _meta_run pauses while any download is running so
# background discovery and a concurrent download never share the YouTube session # background discovery and a concurrent download never share the YouTube session
@@ -77,7 +77,7 @@ def _meta_run(args: list[str], timeout: int = 60) -> tuple[str, str, int]:
now = time.monotonic() now = time.monotonic()
wait = _META_MIN_GAP - (now - _meta_last_call) wait = _META_MIN_GAP - (now - _meta_last_call)
if wait > 0: if wait > 0:
time.sleep(wait + random.uniform(0.5, 2.5)) time.sleep(wait + random.uniform(1.0, 5.0))
try: try:
return _run(args, timeout=timeout) return _run(args, timeout=timeout)
finally: finally:
@@ -211,6 +211,7 @@ def search_youtube(query: str, max_results: int = 40, polite: bool = False) -> l
"--dump-json", "--dump-json",
"--flat-playlist", "--flat-playlist",
"--quiet", "--quiet",
"--extractor-args", "youtube:player_client=ios",
*_cookie_args(), *_cookie_args(),
], timeout=60) ], timeout=60)
@@ -309,6 +310,7 @@ def fetch_video_metadata(video_id: str, polite: bool = False) -> dict | None:
base_cmd = [ base_cmd = [
"yt-dlp", url, "yt-dlp", url,
"--dump-json", "--no-download", "--no-playlist", "--dump-json", "--no-download", "--no-playlist",
"--extractor-args", "youtube:player_client=ios",
] ]
runner = _meta_run if polite else _run runner = _meta_run if polite else _run
stdout, stderr, code = runner([*base_cmd, *cookie_args], timeout=30) stdout, stderr, code = runner([*base_cmd, *cookie_args], timeout=30)
@@ -379,6 +381,7 @@ def fetch_channel_metadata(channel_id: str, max_videos: int = 30, start_video: i
"--dump-single-json", "--dump-single-json",
"--flat-playlist", "--flat-playlist",
"--quiet", "--quiet",
"--extractor-args", "youtube:player_client=ios",
*_cookie_args(), *_cookie_args(),
] ]
if start_video > 1: if start_video > 1:
@@ -613,6 +616,7 @@ def download_subs_only(video_id: str, subtitle_langs: str) -> bool:
"--write-subs", "--write-auto-subs", "--write-subs", "--write-auto-subs",
"--sub-langs", subtitle_langs, "--sub-langs", subtitle_langs,
"--convert-subs", "vtt", "--convert-subs", "vtt",
"--extractor-args", "youtube:player_client=ios",
"-o", output_template, "-o", output_template,
*_cookie_args(), *_cookie_args(),
], timeout=60) ], timeout=60)
@@ -628,7 +632,8 @@ def fetch_available_subs(video_id: str) -> dict:
BCP-47 lang codes. Manual = human-made; auto = auto-generated captions. BCP-47 lang codes. Manual = human-made; auto = auto-generated captions.
""" """
url = f"https://www.youtube.com/watch?v={video_id}" url = f"https://www.youtube.com/watch?v={video_id}"
base_cmd = ["yt-dlp", url, "--dump-json", "--no-download", "--no-playlist"] base_cmd = ["yt-dlp", url, "--dump-json", "--no-download", "--no-playlist",
"--extractor-args", "youtube:player_client=ios"]
cookie_args = _cookie_args() cookie_args = _cookie_args()
stdout, _, code = _meta_run([*base_cmd, *cookie_args], timeout=30) stdout, _, code = _meta_run([*base_cmd, *cookie_args], timeout=30)
if code != 0 and cookie_args: if code != 0 and cookie_args:
@@ -670,7 +675,7 @@ def fetch_video_comments(youtube_video_id: str, max_comments: int = 20) -> list[
"--write-info-json", "--write-info-json",
"--write-comments", "--write-comments",
# Format: thread_count,total_count,replies_per_thread,reply_pages # Format: thread_count,total_count,replies_per_thread,reply_pages
"--extractor-args", f"youtube:max_comments={max_comments},{max_comments},0,0;comment_sort=top", "--extractor-args", f"youtube:player_client=ios;max_comments={max_comments},{max_comments},0,0;comment_sort=top",
"--skip-download", "--skip-download",
"--no-playlist", "--no-playlist",
"--output", out_tmpl, "--output", out_tmpl,
@@ -927,6 +932,7 @@ def start_download(
"--no-part", "--no-mtime", "--no-part", "--no-mtime",
"-o", output_template, "-o", output_template,
"--newline", "--progress", "--no-colors", "--newline", "--progress", "--no-colors",
"--extractor-args", "youtube:player_client=ios",
*subtitle_args, *subtitle_args,
*cookie_args, *cookie_args,
] ]