inputnoise/youclonedl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Following page: replace 9-subquery-per-channel stats with 2 CTEs + indexes

Browse Source 
The old _CHANNEL_STATS_SELECT ran 9 correlated subqueries for each
channel row. With 1266 channels that was ~11000 sub-executions per
GET /channels request, causing multi-second (or timeout) delays.

New approach: 2 CTEs (vinfo for counts/sums, nc for new_count) each do
a single aggregated pass over all followed-channel videos, joined back
to channels. Only 2 correlated LIMIT-1 subqueries remain for
latest_video_id/title (fast with the new index).

Also adds 4 indexes on startup (IF NOT EXISTS — safe to deploy):
- videos(channel_id, published_at DESC)  — latest video lookups
- videos(channel_id, indexed_at)         — new_count filter
- user_videos(video_id, user_id)         — watch/download aggregation
- user_channels(user_id, status)         — followed channel filter

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Mattias Tall
2026-05-26 16:04:41 +02:00
parent 1cd8645957
commit 74e9a52096
2 changed files with 54 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
backend/database.py
View File

@@ -89,6 +89,18 @@ def init_db():
_add_column_if_missing(raw_conn, "videos", "like_count", "INTEGER") _add_column_if_missing(raw_conn, "videos", "like_count", "INTEGER")
_add_column_if_missing(raw_conn, "videos", "dislike_count", "INTEGER") _add_column_if_missing(raw_conn, "videos", "dislike_count", "INTEGER")
raw_conn.commit() raw_conn.commit()
# Indexes that make the channel-stats CTE query fast with many channels
for idx_sql in [
"CREATE INDEX IF NOT EXISTS idx_videos_channel_published ON videos(channel_id, published_at DESC)",
"CREATE INDEX IF NOT EXISTS idx_videos_channel_indexed ON videos(channel_id, indexed_at)",
"CREATE INDEX IF NOT EXISTS idx_user_videos_video_user ON user_videos(video_id, user_id)",
"CREATE INDEX IF NOT EXISTS idx_user_channels_user_status ON user_channels(user_id, status)",
]:
try:
raw_conn.execute(idx_sql)
except Exception:
pass
raw_conn.commit()
# executescript handles multi-statement SQL including trigger BEGIN...END blocks # executescript handles multi-statement SQL including trigger BEGIN...END blocks
raw_conn.executescript(FTS_SETUP_SQL) raw_conn.executescript(FTS_SETUP_SQL)
finally: finally:

67
backend/routers/channels.py
View File

@@ -67,31 +67,48 @@ class VideoOut(BaseModel):
_CHANNEL_STATS_SELECT = """ _CHANNEL_STATS_SELECT = """
SELECT c.*, uc.status, uc.auto_download, uc.muted_until, uc.notes, WITH followed AS (
(SELECT COUNT(*) FROM videos WHERE channel_id = c.id) AS video_count, SELECT channel_id, last_seen_at
(SELECT MAX(v.published_at) FROM videos v WHERE v.channel_id = c.id) AS last_published_at, FROM user_channels
(SELECT COUNT(*) FROM videos v WHERE user_id = :user_id AND status = 'followed'
LEFT JOIN user_videos uv ON v.id = uv.video_id AND uv.user_id = :user_id ),
WHERE v.channel_id = c.id AND COALESCE(uv.watched, 0) = 0) AS unwatched_count, vinfo AS (
(SELECT COUNT(*) FROM videos v SELECT
JOIN user_videos uv ON v.id = uv.video_id AND uv.user_id = :user_id v.channel_id,
WHERE v.channel_id = c.id AND uv.watched = 1) AS watched_count, COUNT(*) AS video_count,
(SELECT COUNT(*) FROM videos v MIN(v.published_at) AS oldest_published,
JOIN user_videos uv ON v.id = uv.video_id AND uv.user_id = :user_id MAX(v.published_at) AS last_published_at,
WHERE v.channel_id = c.id AND uv.downloaded = 1) AS downloaded_count, SUM(CASE WHEN COALESCE(uv.watched, 0) = 0 THEN 1 ELSE 0 END) AS unwatched_count,
(SELECT COUNT(*) FROM videos v SUM(CASE WHEN uv.watched = 1 THEN 1 ELSE 0 END) AS watched_count,
WHERE v.channel_id = c.id SUM(CASE WHEN uv.downloaded = 1 THEN 1 ELSE 0 END) AS downloaded_count
AND (uc.last_seen_at IS NULL OR v.indexed_at > uc.last_seen_at)) AS new_count, FROM videos v
(SELECT v.youtube_video_id FROM videos v JOIN followed f ON f.channel_id = v.channel_id
WHERE v.channel_id = c.id ORDER BY v.published_at DESC LIMIT 1) AS latest_video_id, LEFT JOIN user_videos uv ON uv.video_id = v.id AND uv.user_id = :user_id
(SELECT v.title FROM videos v GROUP BY v.channel_id
WHERE v.channel_id = c.id ORDER BY v.published_at DESC LIMIT 1) AS latest_video_title, ),
(SELECT nc AS (
CASE WHEN COUNT(*) < 2 THEN NULL SELECT v.channel_id, COUNT(*) AS new_count
ELSE CAST((julianday(MAX(sub.published_at)) - julianday(MIN(sub.published_at))) AS REAL) / (COUNT(*) - 1) FROM videos v
END JOIN followed f ON f.channel_id = v.channel_id
FROM (SELECT published_at FROM videos WHERE channel_id = c.id AND published_at IS NOT NULL ORDER BY published_at DESC LIMIT 15) sub WHERE f.last_seen_at IS NULL OR v.indexed_at > f.last_seen_at
) AS upload_frequency_days GROUP BY v.channel_id
)
SELECT
c.*, uc.status, uc.auto_download, uc.muted_until, uc.notes,
COALESCE(vi.video_count, 0) AS video_count,
vi.last_published_at,
COALESCE(vi.unwatched_count, 0) AS unwatched_count,
COALESCE(vi.watched_count, 0) AS watched_count,
COALESCE(vi.downloaded_count, 0) AS downloaded_count,
COALESCE(nc.new_count, 0) AS new_count,
CASE WHEN COALESCE(vi.video_count, 0) < 2 THEN NULL
ELSE (julianday(vi.last_published_at) - julianday(vi.oldest_published))
/ (vi.video_count - 1.0)
END AS upload_frequency_days,
(SELECT v2.youtube_video_id FROM videos v2
WHERE v2.channel_id = c.id ORDER BY v2.published_at DESC LIMIT 1) AS latest_video_id,
(SELECT v2.title FROM videos v2
WHERE v2.channel_id = c.id ORDER BY v2.published_at DESC LIMIT 1) AS latest_video_title
FROM channels c FROM channels c
JOIN user_channels uc ON c.id = uc.channel_id JOIN user_channels uc ON c.id = uc.channel_id
WHERE uc.user_id = :user_id AND uc.status = 'followed' WHERE uc.user_id = :user_id AND uc.status = 'followed'