Optimise Following page: 4 aggregated queries, no correlated subqueries
- Rewrite list_channels to run exactly 4 SQL queries regardless of channel count: channel rows, aggregated video stats (GROUP BY), new-video counts, and latest video (derived-table JOIN replaces per-row correlated subquery) - Remove dead _CHANNEL_STATS_SELECT (orphaned after the rewrite) - Fix upload_frequency_days: use pre-computed date_span_days from vstats instead of a broken per-channel db.execute() call - Restrict new_counts query to id_csv so it uses idx_videos_channel_indexed - markChannelsSeen: optimistic setQueryData instead of invalidateQueries, eliminating a full channel-list re-fetch on every Following page visit - DownloadIndicator idle poll: 10s → 30s (no need to hit DB when idle) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Mattias Tall
@@ -66,37 +66,6 @@ class VideoOut(BaseModel):
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
_CHANNEL_STATS_SELECT = """
|
||||
SELECT c.*, uc.status, uc.auto_download, uc.muted_until, uc.notes,
|
||||
(SELECT COUNT(*) FROM videos WHERE channel_id = c.id) AS video_count,
|
||||
(SELECT MAX(v.published_at) FROM videos v WHERE v.channel_id = c.id) AS last_published_at,
|
||||
(SELECT COUNT(*) FROM videos v
|
||||
LEFT JOIN user_videos uv ON v.id = uv.video_id AND uv.user_id = :user_id
|
||||
WHERE v.channel_id = c.id AND COALESCE(uv.watched, 0) = 0) AS unwatched_count,
|
||||
(SELECT COUNT(*) FROM videos v
|
||||
JOIN user_videos uv ON v.id = uv.video_id AND uv.user_id = :user_id
|
||||
WHERE v.channel_id = c.id AND uv.watched = 1) AS watched_count,
|
||||
(SELECT COUNT(*) FROM videos v
|
||||
JOIN user_videos uv ON v.id = uv.video_id AND uv.user_id = :user_id
|
||||
WHERE v.channel_id = c.id AND uv.downloaded = 1) AS downloaded_count,
|
||||
(SELECT COUNT(*) FROM videos v
|
||||
WHERE v.channel_id = c.id
|
||||
AND (uc.last_seen_at IS NULL OR v.indexed_at > uc.last_seen_at)) AS new_count,
|
||||
(SELECT v.youtube_video_id FROM videos v
|
||||
WHERE v.channel_id = c.id ORDER BY v.published_at DESC LIMIT 1) AS latest_video_id,
|
||||
(SELECT v.title FROM videos v
|
||||
WHERE v.channel_id = c.id ORDER BY v.published_at DESC LIMIT 1) AS latest_video_title,
|
||||
(SELECT
|
||||
CASE WHEN COUNT(*) < 2 THEN NULL
|
||||
ELSE CAST((julianday(MAX(sub.published_at)) - julianday(MIN(sub.published_at))) AS REAL) / (COUNT(*) - 1)
|
||||
END
|
||||
FROM (SELECT published_at FROM videos WHERE channel_id = c.id AND published_at IS NOT NULL ORDER BY published_at DESC LIMIT 15) sub
|
||||
) AS upload_frequency_days
|
||||
FROM channels c
|
||||
JOIN user_channels uc ON c.id = uc.channel_id
|
||||
WHERE uc.user_id = :user_id AND uc.status = 'followed'
|
||||
"""
|
||||
|
||||
|
||||
def _get_channel_or_404(db: Session, channel_id: int) -> Channel:
|
||||
c = db.query(Channel).filter(Channel.id == channel_id).first()
|
||||
@@ -328,11 +297,122 @@ def list_channels(
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
rows = db.execute(
|
||||
text(_CHANNEL_STATS_SELECT + "ORDER BY last_published_at DESC"),
|
||||
{"user_id": current_user.id},
|
||||
uid = current_user.id
|
||||
|
||||
# Step 1 — channel rows + user_channel metadata (fast, no video stats)
|
||||
ch_rows = db.execute(
|
||||
text("""
|
||||
SELECT c.id, c.youtube_channel_id, c.name, c.description,
|
||||
c.thumbnail_url, c.banner_url, c.subscriber_count, c.crawled_at,
|
||||
uc.status, uc.auto_download, uc.muted_until, uc.notes, uc.last_seen_at
|
||||
FROM channels c
|
||||
JOIN user_channels uc ON c.id = uc.channel_id
|
||||
WHERE uc.user_id = :uid AND uc.status = 'followed'
|
||||
"""),
|
||||
{"uid": uid},
|
||||
).mappings().all()
|
||||
return [ChannelOut(**dict(r)) for r in rows]
|
||||
|
||||
if not ch_rows:
|
||||
return []
|
||||
|
||||
id_csv = ",".join(str(r["id"]) for r in ch_rows)
|
||||
last_seen = {r["id"]: r["last_seen_at"] for r in ch_rows}
|
||||
|
||||
# Step 2 — aggregated video stats for all channels in one query
|
||||
vstats = {
|
||||
r["channel_id"]: r
|
||||
for r in db.execute(
|
||||
text(f"""
|
||||
SELECT v.channel_id,
|
||||
COUNT(*) AS video_count,
|
||||
MAX(v.published_at) AS last_published_at,
|
||||
julianday(MAX(v.published_at)) - julianday(MIN(v.published_at)) AS date_span_days,
|
||||
SUM(CASE WHEN COALESCE(uv.watched, 0) = 0 THEN 1 ELSE 0 END) AS unwatched_count,
|
||||
SUM(CASE WHEN uv.watched = 1 THEN 1 ELSE 0 END) AS watched_count,
|
||||
SUM(CASE WHEN uv.downloaded = 1 THEN 1 ELSE 0 END) AS downloaded_count
|
||||
FROM videos v
|
||||
LEFT JOIN user_videos uv ON uv.video_id = v.id AND uv.user_id = :uid
|
||||
WHERE v.channel_id IN ({id_csv})
|
||||
GROUP BY v.channel_id
|
||||
"""),
|
||||
{"uid": uid},
|
||||
).mappings().all()
|
||||
}
|
||||
|
||||
# Step 3 — new-video count per channel (videos indexed after last_seen_at)
|
||||
new_counts = {
|
||||
r["channel_id"]: r["new_count"]
|
||||
for r in db.execute(
|
||||
text(f"""
|
||||
SELECT v.channel_id, COUNT(*) AS new_count
|
||||
FROM videos v
|
||||
JOIN user_channels uc
|
||||
ON uc.channel_id = v.channel_id
|
||||
AND uc.user_id = :uid
|
||||
WHERE v.channel_id IN ({id_csv})
|
||||
AND (uc.last_seen_at IS NULL OR v.indexed_at > uc.last_seen_at)
|
||||
GROUP BY v.channel_id
|
||||
"""),
|
||||
{"uid": uid},
|
||||
).mappings().all()
|
||||
}
|
||||
|
||||
# Step 4 — latest video id + title per channel (derived-table join, no correlated subquery)
|
||||
latest = {
|
||||
r["channel_id"]: r
|
||||
for r in db.execute(
|
||||
text(f"""
|
||||
SELECT v.channel_id,
|
||||
v.youtube_video_id AS latest_video_id,
|
||||
v.title AS latest_video_title
|
||||
FROM videos v
|
||||
JOIN (
|
||||
SELECT channel_id, MAX(published_at) AS max_pub
|
||||
FROM videos
|
||||
WHERE channel_id IN ({id_csv})
|
||||
GROUP BY channel_id
|
||||
) m ON v.channel_id = m.channel_id AND v.published_at = m.max_pub
|
||||
GROUP BY v.channel_id
|
||||
"""),
|
||||
).mappings().all()
|
||||
}
|
||||
|
||||
# Merge and build response
|
||||
result = []
|
||||
for r in ch_rows:
|
||||
cid = r["id"]
|
||||
vs = vstats.get(cid) or {}
|
||||
vc = vs.get("video_count") or 0
|
||||
newest = vs.get("last_published_at")
|
||||
span = vs.get("date_span_days")
|
||||
freq = (span / (vc - 1.0)) if (vc >= 2 and span is not None) else None
|
||||
|
||||
result.append(ChannelOut(
|
||||
id=cid,
|
||||
youtube_channel_id=r["youtube_channel_id"],
|
||||
name=r["name"],
|
||||
description=r["description"],
|
||||
thumbnail_url=r["thumbnail_url"],
|
||||
banner_url=r.get("banner_url"),
|
||||
subscriber_count=r.get("subscriber_count"),
|
||||
crawled_at=r.get("crawled_at"),
|
||||
status=r["status"],
|
||||
auto_download=r.get("auto_download"),
|
||||
muted_until=r.get("muted_until"),
|
||||
notes=r.get("notes") or "",
|
||||
video_count=vc,
|
||||
last_published_at=newest,
|
||||
unwatched_count=vs.get("unwatched_count") or 0,
|
||||
watched_count=vs.get("watched_count") or 0,
|
||||
downloaded_count=vs.get("downloaded_count") or 0,
|
||||
new_count=new_counts.get(cid, 0),
|
||||
latest_video_id=latest.get(cid, {}).get("latest_video_id"),
|
||||
latest_video_title=latest.get(cid, {}).get("latest_video_title"),
|
||||
upload_frequency_days=freq,
|
||||
))
|
||||
|
||||
result.sort(key=lambda c: c.last_published_at or datetime.min, reverse=True)
|
||||
return result
|
||||
|
||||
|
||||
# ── Channel Groups (must be before /{channel_id} to avoid route shadowing) ───
|
||||
|
||||
@@ -80,7 +80,7 @@ function DownloadIndicator() {
|
||||
const active = (query.state.data ?? []).some(
|
||||
(d) => d.status === "pending" || d.status === "downloading"
|
||||
);
|
||||
return active ? 1500 : 10_000;
|
||||
return active ? 1500 : 30_000;
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -612,7 +612,10 @@ export default function Following() {
|
||||
useEffect(() => {
|
||||
if (channels.length > 0) {
|
||||
markChannelsSeen().then(() => {
|
||||
qc.invalidateQueries({ queryKey: ["channels"] });
|
||||
// Zero out new_count optimistically — avoids a full re-fetch just to clear badges
|
||||
qc.setQueryData(["channels"], (old) =>
|
||||
old ? old.map((c) => ({ ...c, new_count: 0 })) : old
|
||||
);
|
||||
});
|
||||
}
|
||||
}, []); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
Reference in New Issue
Block a user