From 1cd86459574f49b72ea6953f536e864190e9d38c Mon Sep 17 00:00:00 2001
From: Mattias Tall <mattias.tall@ucr.uu.se>
Date: Tue, 26 May 2026 16:00:37 +0200
Subject: [PATCH] Fix YouTube hammering, sync rate limiting, and Following load
 time

Sync throttling:
- sync-all now skips channels crawled within the last 6 hours (prevents
  re-scraping 1266 channels on every button press)
- Channels are queued into a single _index_channels_batch task that runs
  with 1.5s delay between each yt-dlp call instead of firing 1266
  background tasks simultaneously
- Startup enrich task reduced from 10 to 3 videos (3 yt-dlp calls on
  each container restart)
- Enrich task adds 2s sleep between metadata fetches

SQLite stability:
- busy_timeout=5000 prevents SQLITE_BUSY errors under concurrent load
- synchronous=NORMAL speeds up writes without data loss risk (safe with WAL)

Following page:
- staleTime: 60s on channels query so cached data is reused immediately
  on revisit; gcTime keeps it in memory for 5 min

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 backend/database.py              |  2 ++
 backend/main.py                  |  2 +-
 backend/routers/channels.py      | 23 ++++++++++++++++++-----
 frontend/src/pages/Following.jsx |  2 ++
 4 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/backend/database.py b/backend/database.py
index f7e3e5f..51c132a 100644
--- a/backend/database.py
+++ b/backend/database.py
@@ -15,6 +15,8 @@ def set_sqlite_pragma(dbapi_conn, _):
     cursor = dbapi_conn.cursor()
     cursor.execute("PRAGMA journal_mode=WAL")
     cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.execute("PRAGMA busy_timeout=5000")
+    cursor.execute("PRAGMA synchronous=NORMAL")
     cursor.close()
 
 
diff --git a/backend/main.py b/backend/main.py
index d8e9c8e..8870c93 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -152,7 +152,7 @@ def on_startup():
     # Backfill descriptions for videos that don't have them yet (runs in background)
     import threading
     from .routers.channels import _enrich_missing_task
-    threading.Thread(target=_enrich_missing_task, args=(10,), daemon=True).start()
+    threading.Thread(target=_enrich_missing_task, args=(3,), daemon=True).start()
 
 
 @app.get("/api/health")
diff --git a/backend/routers/channels.py b/backend/routers/channels.py
index 43784fc..4364814 100644
--- a/backend/routers/channels.py
+++ b/backend/routers/channels.py
@@ -105,6 +105,15 @@ def _get_channel_or_404(db: Session, channel_id: int) -> Channel:
     return c
 
 
+def _index_channels_batch(channel_ids: list[int], user_id: int, delay: float = 1.5):
+    """Run channel syncs sequentially with a polite delay between requests."""
+    import time
+    for i, cid in enumerate(channel_ids):
+        if i > 0:
+            time.sleep(delay)
+        _index_channel_task(cid, user_id)
+
+
 def _index_channel_task(channel_id: int, user_id: int):
     from ..database import SessionLocal
     db = SessionLocal()
@@ -224,7 +233,9 @@ def _enrich_missing_task(limit: int = 20):
             """),
             {"limit": limit},
         ).mappings().all()
-        for row in rows:
+        for i, row in enumerate(rows):
+            if i > 0:
+                import time; time.sleep(2)
             try:
                 meta = ytdlp.fetch_video_metadata(row["youtube_video_id"])
                 if meta:
@@ -278,22 +289,24 @@ def sync_all_channels(
     db: Session = Depends(get_db),
     current_user: User = Depends(get_current_user),
 ):
+    # Only sync channels not touched in the last 6 hours to avoid hammering YouTube
     channels = db.execute(
         text("""
             SELECT c.id FROM channels c
             JOIN user_channels uc ON c.id = uc.channel_id
             WHERE uc.user_id = :uid AND uc.status = 'followed'
+              AND (c.crawled_at IS NULL OR c.crawled_at < datetime('now', '-6 hours'))
+            ORDER BY COALESCE(c.crawled_at, '1970-01-01') ASC
         """),
         {"uid": current_user.id},
     ).mappings().all()
 
-    for row in channels:
-        background_tasks.add_task(_index_channel_task, row["id"], current_user.id)
-
     if channels:
+        ids = [row["id"] for row in channels]
+        background_tasks.add_task(_index_channels_batch, ids, current_user.id)
         background_tasks.add_task(_discovery_task, current_user.id)
 
-    background_tasks.add_task(_enrich_missing_task, 20)
+    background_tasks.add_task(_enrich_missing_task, 5)
 
     return {"indexing": len(channels)}
 
diff --git a/frontend/src/pages/Following.jsx b/frontend/src/pages/Following.jsx
index d0d00be..634728b 100644
--- a/frontend/src/pages/Following.jsx
+++ b/frontend/src/pages/Following.jsx
@@ -579,6 +579,8 @@ export default function Following() {
   const { data: channels = [], isLoading: loadingChannels } = useQuery({
     queryKey: ["channels"],
     queryFn: () => getChannels().then((r) => r.data),
+    staleTime: 60_000,
+    gcTime: 5 * 60_000,
   });
 
   const { data: appSettings } = useQuery({