diff --git a/backend/database.py b/backend/database.py index f7e3e5f..51c132a 100644 --- a/backend/database.py +++ b/backend/database.py @@ -15,6 +15,8 @@ def set_sqlite_pragma(dbapi_conn, _): cursor = dbapi_conn.cursor() cursor.execute("PRAGMA journal_mode=WAL") cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute("PRAGMA busy_timeout=5000") + cursor.execute("PRAGMA synchronous=NORMAL") cursor.close() diff --git a/backend/main.py b/backend/main.py index d8e9c8e..8870c93 100644 --- a/backend/main.py +++ b/backend/main.py @@ -152,7 +152,7 @@ def on_startup(): # Backfill descriptions for videos that don't have them yet (runs in background) import threading from .routers.channels import _enrich_missing_task - threading.Thread(target=_enrich_missing_task, args=(10,), daemon=True).start() + threading.Thread(target=_enrich_missing_task, args=(3,), daemon=True).start() @app.get("/api/health") diff --git a/backend/routers/channels.py b/backend/routers/channels.py index 43784fc..4364814 100644 --- a/backend/routers/channels.py +++ b/backend/routers/channels.py @@ -105,6 +105,15 @@ def _get_channel_or_404(db: Session, channel_id: int) -> Channel: return c +def _index_channels_batch(channel_ids: list[int], user_id: int, delay: float = 1.5): + """Run channel syncs sequentially with a polite delay between requests.""" + import time + for i, cid in enumerate(channel_ids): + if i > 0: + time.sleep(delay) + _index_channel_task(cid, user_id) + + def _index_channel_task(channel_id: int, user_id: int): from ..database import SessionLocal db = SessionLocal() @@ -224,7 +233,9 @@ def _enrich_missing_task(limit: int = 20): """), {"limit": limit}, ).mappings().all() - for row in rows: + for i, row in enumerate(rows): + if i > 0: + import time; time.sleep(2) try: meta = ytdlp.fetch_video_metadata(row["youtube_video_id"]) if meta: @@ -278,22 +289,24 @@ def sync_all_channels( db: Session = Depends(get_db), current_user: User = Depends(get_current_user), ): + # Only sync channels not touched in the last 6 hours to avoid hammering YouTube channels = db.execute( text(""" SELECT c.id FROM channels c JOIN user_channels uc ON c.id = uc.channel_id WHERE uc.user_id = :uid AND uc.status = 'followed' + AND (c.crawled_at IS NULL OR c.crawled_at < datetime('now', '-6 hours')) + ORDER BY COALESCE(c.crawled_at, '1970-01-01') ASC """), {"uid": current_user.id}, ).mappings().all() - for row in channels: - background_tasks.add_task(_index_channel_task, row["id"], current_user.id) - if channels: + ids = [row["id"] for row in channels] + background_tasks.add_task(_index_channels_batch, ids, current_user.id) background_tasks.add_task(_discovery_task, current_user.id) - background_tasks.add_task(_enrich_missing_task, 20) + background_tasks.add_task(_enrich_missing_task, 5) return {"indexing": len(channels)} diff --git a/frontend/src/pages/Following.jsx b/frontend/src/pages/Following.jsx index d0d00be..634728b 100644 --- a/frontend/src/pages/Following.jsx +++ b/frontend/src/pages/Following.jsx @@ -579,6 +579,8 @@ export default function Following() { const { data: channels = [], isLoading: loadingChannels } = useQuery({ queryKey: ["channels"], queryFn: () => getChannels().then((r) => r.data), + staleTime: 60_000, + gcTime: 5 * 60_000, }); const { data: appSettings } = useQuery({