Fix cookie invalidation: give each yt-dlp process a private cookie file copy

Downloads run for minutes via Popen while metadata calls continue in parallel.
Both processes read from AND write back to the same --cookies file, causing
concurrent writes that corrupt the session cookie state.

Fix: _make_private_cookie_copy() intercepts --cookies <file> in any arg list
and swaps it for a NamedTemporaryFile copy. Each yt-dlp process gets its own
snapshot; write-backs go to the throwaway copy and are discarded on cleanup.

- _run() uses this for all subprocess.run calls (metadata, subtitles, comments)
- start_download() uses it for the long-lived Popen download process
- _meta_run() benefits automatically since it calls _run()

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-05-27 01:48:29 +02:00
parent 592194f2ca
commit 4d255647a1

View File

@@ -1,8 +1,11 @@
"""Subprocess wrapper for yt-dlp."""
import json
import os
import random
import re
import shutil
import subprocess
import tempfile
import threading
import time
import urllib.request
@@ -14,9 +17,36 @@ from typing import Any
from ..config import settings
def _make_private_cookie_copy(args: list[str]) -> tuple[list[str], str | None]:
"""Replace --cookies <file> with a private temp copy so concurrent yt-dlp
processes never write to the same cookie jar simultaneously."""
for i, arg in enumerate(args):
if arg == "--cookies" and i + 1 < len(args):
source = args[i + 1]
if Path(source).exists():
try:
tmp = tempfile.NamedTemporaryFile(suffix=".txt", delete=False)
tmp.close()
shutil.copy2(source, tmp.name)
modified = list(args)
modified[i + 1] = tmp.name
return modified, tmp.name
except Exception:
break
return list(args), None
def _run(args: list[str], timeout: int = 60) -> tuple[str, str, int]:
args, tmp_path = _make_private_cookie_copy(args)
try:
result = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
return result.stdout, result.stderr, result.returncode
finally:
if tmp_path:
try:
os.unlink(tmp_path)
except Exception:
pass
# Global rate limiter for all metadata fetches — prevents concurrent tasks from
@@ -857,8 +887,7 @@ def start_download(
with _SEMAPHORE:
cookie_args = _cookie_args()
print(f"[ytdlp] cookie_args={cookie_args!r}", flush=True)
process = subprocess.Popen(
[
cmd = [
"yt-dlp", url,
"-f", fmt,
"--merge-output-format", "mp4",
@@ -867,12 +896,20 @@ def start_download(
"--newline", "--progress", "--no-colors",
*subtitle_args,
*cookie_args,
],
]
# Private cookie copy — download runs for minutes; without this,
# concurrent metadata calls would write-back to the same cookie file
# and corrupt the session.
cmd, tmp_cookie_path = _make_private_cookie_copy(cmd)
try:
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
try:
file_path = None
stream_index = 0
output_lines: list[str] = []
@@ -900,6 +937,12 @@ def start_download(
import logging
logging.getLogger(__name__).error("yt-dlp failed (code %d):\n%s", process.returncode, tail)
on_error(download_id, f"yt-dlp exited with code {process.returncode}:\n{tail}")
finally:
if tmp_cookie_path:
try:
os.unlink(tmp_cookie_path)
except Exception:
pass
thread = threading.Thread(target=_run_download, daemon=True)
thread.start()