Reduce parallel YouTube request workers to avoid cookie invalidation
8 simultaneous yt-dlp processes hitting video pages looks like a bot attack and causes YouTube to nuke the session cookies. Drop to: - Popular fetch view_count enrichment: 8→3 workers - Discovery search: 8→4 workers - Graph signal (featured channels): 8→3 workers Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -742,7 +742,7 @@ def _fetch_popular_task(channel_id: int, youtube_channel_id: str):
|
||||
if not video_ids:
|
||||
return
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
with ThreadPoolExecutor(max_workers=3) as pool:
|
||||
futures = {pool.submit(ytdlp.fetch_video_metadata, vid): vid for vid in video_ids}
|
||||
results = {}
|
||||
for future in as_completed(futures):
|
||||
|
||||
@@ -112,7 +112,7 @@ def _search_and_store(
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
futures = {pool.submit(_do_search, q): q for q in queries}
|
||||
for fut in as_completed(futures):
|
||||
for video in fut.result():
|
||||
@@ -620,7 +620,7 @@ def update_graph_signal(db: Session, user_id: int):
|
||||
return []
|
||||
|
||||
featured_map: dict[str, list[str]] = {}
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
with ThreadPoolExecutor(max_workers=3) as pool:
|
||||
futures = {pool.submit(_fetch, row["youtube_channel_id"]): row for row in sample}
|
||||
for fut in as_completed(futures):
|
||||
row = futures[fut]
|
||||
|
||||
Reference in New Issue
Block a user