Add older content exploration: channel page + home feed Rediscover mode
Channel page:
- "Explore older videos" button fetches 100 videos at a time further back
in the channel history using yt-dlp --playlist-start/--playlist-end
- "Fetch entire history" still available for full crawl
- Backend: /channels/{id}/explore?page=N endpoint + playlist offset support
in fetch_channel_metadata(start_video=N)
Home feed:
- New "Rediscover" mode: older unwatched videos (90+ days old) from
followed channels, randomly sampled then re-ranked by tag affinity
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -758,6 +758,54 @@ def index_channel_full(
|
|||||||
return {"detail": "Full index started"}
|
return {"detail": "Full index started"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{channel_id}/explore", status_code=status.HTTP_202_ACCEPTED)
|
||||||
|
def explore_channel_older(
|
||||||
|
channel_id: int,
|
||||||
|
page: int = 2,
|
||||||
|
background_tasks: BackgroundTasks = None,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current_user: User = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
"""Fetch a page of older videos from this channel (page 1 = newest 30, page 2 = next 100, etc.)."""
|
||||||
|
_get_channel_or_404(db, channel_id)
|
||||||
|
start = 1 if page <= 1 else (30 + (page - 2) * 100 + 1)
|
||||||
|
background_tasks.add_task(_index_channel_explore_task, channel_id, current_user.id, start, 100)
|
||||||
|
return {"detail": f"Fetching older videos (page {page})", "start": start}
|
||||||
|
|
||||||
|
|
||||||
|
def _index_channel_explore_task(channel_id: int, user_id: int, start_video: int, count: int):
|
||||||
|
from ..database import SessionLocal
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
channel = db.query(Channel).filter_by(id=channel_id).first()
|
||||||
|
if not channel:
|
||||||
|
return
|
||||||
|
result = ytdlp.fetch_channel_metadata(channel.youtube_channel_id, max_videos=count, start_video=start_video)
|
||||||
|
if not result:
|
||||||
|
return
|
||||||
|
for vdata in result.get("videos", []):
|
||||||
|
yt_id = vdata.get("youtube_video_id")
|
||||||
|
if not yt_id or not vdata.get("published_at"):
|
||||||
|
continue
|
||||||
|
if not db.query(Video).filter_by(youtube_video_id=yt_id).first():
|
||||||
|
db.add(Video(
|
||||||
|
youtube_video_id=yt_id,
|
||||||
|
channel_id=channel.id,
|
||||||
|
title=vdata.get("title", ""),
|
||||||
|
description=vdata.get("description"),
|
||||||
|
thumbnail_url=vdata.get("thumbnail_url"),
|
||||||
|
duration_seconds=vdata.get("duration_seconds"),
|
||||||
|
published_at=vdata.get("published_at"),
|
||||||
|
tags=vdata.get("tags"),
|
||||||
|
category=vdata.get("category"),
|
||||||
|
))
|
||||||
|
db.commit()
|
||||||
|
except Exception:
|
||||||
|
db.rollback()
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
@router.post("/follow-bulk", status_code=200)
|
@router.post("/follow-bulk", status_code=200)
|
||||||
def follow_bulk(
|
def follow_bulk(
|
||||||
body: dict,
|
body: dict,
|
||||||
|
|||||||
@@ -208,6 +208,58 @@ def home_feed(
|
|||||||
for r in rows
|
for r in rows
|
||||||
]
|
]
|
||||||
|
|
||||||
|
if mode == "rediscover":
|
||||||
|
# Older unwatched videos from followed channels, ranked by tag affinity
|
||||||
|
affinity_rows = db.execute(
|
||||||
|
text("SELECT tag, score FROM user_tag_affinity WHERE user_id = :user_id AND score > 0"),
|
||||||
|
{"user_id": current_user.id},
|
||||||
|
).mappings().all()
|
||||||
|
affinity = {r["tag"]: r["score"] for r in affinity_rows}
|
||||||
|
|
||||||
|
rows = db.execute(
|
||||||
|
text(f"""
|
||||||
|
SELECT v.id, v.youtube_video_id, v.title, v.description, v.thumbnail_url,
|
||||||
|
v.duration_seconds, v.published_at, v.tags, v.category,
|
||||||
|
c.id AS channel_id, c.name AS channel_name,
|
||||||
|
c.youtube_channel_id AS channel_youtube_id,
|
||||||
|
c.thumbnail_url AS channel_thumbnail_url,
|
||||||
|
COALESCE(uv.watched, 0) AS watched,
|
||||||
|
COALESCE(uv.watch_progress_seconds, 0) AS watch_progress_seconds,
|
||||||
|
COALESCE(uv.downloaded, 0) AS is_downloaded,
|
||||||
|
COALESCE(uv.queued, 0) AS queued,
|
||||||
|
NULL AS file_path
|
||||||
|
FROM videos v
|
||||||
|
JOIN channels c ON v.channel_id = c.id
|
||||||
|
JOIN user_channels uc
|
||||||
|
ON c.id = uc.channel_id AND uc.user_id = :user_id AND uc.status = 'followed'
|
||||||
|
LEFT JOIN user_videos uv ON v.id = uv.video_id AND uv.user_id = :user_id
|
||||||
|
WHERE COALESCE(uv.watched, 0) = 0
|
||||||
|
AND v.published_at < datetime('now', '-90 days')
|
||||||
|
AND (uc.muted_until IS NULL OR datetime(uc.muted_until) < datetime('now'))
|
||||||
|
{duration_clause}
|
||||||
|
ORDER BY RANDOM()
|
||||||
|
LIMIT :limit OFFSET :offset
|
||||||
|
"""),
|
||||||
|
{"user_id": current_user.id, "limit": min(limit * 4, 200), "offset": offset},
|
||||||
|
).mappings().all()
|
||||||
|
|
||||||
|
if affinity:
|
||||||
|
import json as _json
|
||||||
|
def _affinity_score(row):
|
||||||
|
try:
|
||||||
|
tags = _json.loads(row["tags"] or "[]")
|
||||||
|
return sum(affinity.get(t.lower().strip(), 0) for t in tags if isinstance(t, str))
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
rows = sorted(rows, key=_affinity_score, reverse=True)
|
||||||
|
|
||||||
|
rows = rows[:limit]
|
||||||
|
return [
|
||||||
|
VideoDetail(**{k: v for k, v in dict(r).items() if k not in ("watched",)},
|
||||||
|
is_watched=bool(r["watched"]))
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
|
||||||
if mode == "inbox":
|
if mode == "inbox":
|
||||||
rows = db.execute(
|
rows = db.execute(
|
||||||
text(f"""
|
text(f"""
|
||||||
|
|||||||
@@ -287,7 +287,7 @@ def _rss_dates(uc_channel_id: str) -> dict[str, datetime]:
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def fetch_channel_metadata(channel_id: str, max_videos: int = 30) -> dict | None:
|
def fetch_channel_metadata(channel_id: str, max_videos: int = 30, start_video: int = 1) -> dict | None:
|
||||||
"""Fetch channel info + recent videos.
|
"""Fetch channel info + recent videos.
|
||||||
|
|
||||||
Uses --dump-single-json --flat-playlist for speed, then enriches video dates
|
Uses --dump-single-json --flat-playlist for speed, then enriches video dates
|
||||||
@@ -304,8 +304,11 @@ def fetch_channel_metadata(channel_id: str, max_videos: int = 30) -> dict | None
|
|||||||
"--quiet",
|
"--quiet",
|
||||||
*_cookie_args(),
|
*_cookie_args(),
|
||||||
]
|
]
|
||||||
|
if start_video > 1:
|
||||||
|
args += ["--playlist-start", str(start_video)]
|
||||||
if max_videos > 0:
|
if max_videos > 0:
|
||||||
args += ["--playlist-end", str(max_videos)]
|
end = (start_video - 1 + max_videos) if start_video > 1 else max_videos
|
||||||
|
args += ["--playlist-end", str(end)]
|
||||||
|
|
||||||
stdout, _, code = _run(args, timeout=60)
|
stdout, _, code = _run(args, timeout=60)
|
||||||
if not stdout.strip():
|
if not stdout.strip():
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ export const followChannel = (id) => api.post(`/channels/${id}/follow`);
|
|||||||
export const unfollowChannel = (id) => api.delete(`/channels/${id}/follow`);
|
export const unfollowChannel = (id) => api.delete(`/channels/${id}/follow`);
|
||||||
export const indexChannel = (id) => api.post(`/channels/${id}/index`);
|
export const indexChannel = (id) => api.post(`/channels/${id}/index`);
|
||||||
export const indexChannelFull = (id) => api.post(`/channels/${id}/index-full`);
|
export const indexChannelFull = (id) => api.post(`/channels/${id}/index-full`);
|
||||||
|
export const exploreChannelOlder = (id, page) => api.post(`/channels/${id}/explore`, null, { params: { page } });
|
||||||
export const followChannelByUrl = (data) => api.post("/channels/follow-by-url", data);
|
export const followChannelByUrl = (data) => api.post("/channels/follow-by-url", data);
|
||||||
export const setChannelAutoDownload = (id, value) => api.patch(`/channels/${id}/auto-download`, { auto_download: value });
|
export const setChannelAutoDownload = (id, value) => api.patch(`/channels/${id}/auto-download`, { auto_download: value });
|
||||||
export const markChannelsSeen = () => api.post("/channels/mark-seen");
|
export const markChannelsSeen = () => api.post("/channels/mark-seen");
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { useParams } from "react-router-dom";
|
|||||||
import { useQuery, useMutation, useQueryClient, useInfiniteQuery } from "@tanstack/react-query";
|
import { useQuery, useMutation, useQueryClient, useInfiniteQuery } from "@tanstack/react-query";
|
||||||
import {
|
import {
|
||||||
getChannel, getChannelVideos, searchChannelYoutube,
|
getChannel, getChannelVideos, searchChannelYoutube,
|
||||||
followChannel, unfollowChannel, indexChannel, indexChannelFull, downloadChannel,
|
followChannel, unfollowChannel, indexChannel, indexChannelFull, exploreChannelOlder, downloadChannel,
|
||||||
} from "../api";
|
} from "../api";
|
||||||
import VideoCard from "../components/VideoCard";
|
import VideoCard from "../components/VideoCard";
|
||||||
|
|
||||||
@@ -30,6 +30,7 @@ export default function ChannelPage() {
|
|||||||
const [search, setSearch] = useState("");
|
const [search, setSearch] = useState("");
|
||||||
const [activeQ, setActiveQ] = useState("");
|
const [activeQ, setActiveQ] = useState("");
|
||||||
const [indexing, setIndexing] = useState(false);
|
const [indexing, setIndexing] = useState(false);
|
||||||
|
const [explorePage, setExplorePage] = useState(2);
|
||||||
const searchInputRef = useRef(null);
|
const searchInputRef = useRef(null);
|
||||||
|
|
||||||
const { data: channel, isLoading: loadingChannel } = useQuery({
|
const { data: channel, isLoading: loadingChannel } = useQuery({
|
||||||
@@ -94,6 +95,14 @@ export default function ChannelPage() {
|
|||||||
onSuccess: () => scheduleRefetch(45000),
|
onSuccess: () => scheduleRefetch(45000),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const exploreMut = useMutation({
|
||||||
|
mutationFn: () => exploreChannelOlder(id, explorePage),
|
||||||
|
onSuccess: () => {
|
||||||
|
setExplorePage(p => p + 1);
|
||||||
|
scheduleRefetch(20000);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
const deepSearchMut = useMutation({
|
const deepSearchMut = useMutation({
|
||||||
mutationFn: () => searchChannelYoutube(id, activeQ || search),
|
mutationFn: () => searchChannelYoutube(id, activeQ || search),
|
||||||
onSuccess: () => scheduleRefetch(20000),
|
onSuccess: () => scheduleRefetch(20000),
|
||||||
@@ -130,7 +139,7 @@ export default function ChannelPage() {
|
|||||||
if (!channel) return <p className="text-zinc-500">Channel not found.</p>;
|
if (!channel) return <p className="text-zinc-500">Channel not found.</p>;
|
||||||
|
|
||||||
const isFollowed = channel.status === "followed";
|
const isFollowed = channel.status === "followed";
|
||||||
const isPending = indexMut.isPending || fullIndexMut.isPending || deepSearchMut.isPending || indexing;
|
const isPending = indexMut.isPending || fullIndexMut.isPending || deepSearchMut.isPending || exploreMut.isPending || indexing;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="flex flex-col gap-5">
|
<div className="flex flex-col gap-5">
|
||||||
@@ -282,12 +291,19 @@ export default function ChannelPage() {
|
|||||||
{isFetchingNextPage ? "Loading…" : "Load more"}
|
{isFetchingNextPage ? "Loading…" : "Load more"}
|
||||||
</button>
|
</button>
|
||||||
) : !activeQ && (
|
) : !activeQ && (
|
||||||
<div className="mt-4 flex flex-col items-center gap-2 py-4 border-t border-zinc-800/50">
|
<div className="mt-4 flex flex-col items-center gap-3 py-4 border-t border-zinc-800/50">
|
||||||
<p className="text-xs text-zinc-600">All {videos.length} indexed videos shown</p>
|
<p className="text-xs text-zinc-600">{videos.length} videos indexed</p>
|
||||||
<button onClick={() => fullIndexMut.mutate()} disabled={isPending}
|
<div className="flex items-center gap-4">
|
||||||
className="text-xs text-zinc-500 hover:text-zinc-300 transition-colors disabled:opacity-40">
|
<button onClick={() => exploreMut.mutate()} disabled={isPending}
|
||||||
Fetch full history from YouTube
|
className="text-xs text-zinc-400 hover:text-zinc-100 font-medium transition-colors disabled:opacity-40">
|
||||||
</button>
|
Explore older videos
|
||||||
|
</button>
|
||||||
|
<span className="text-zinc-800 text-xs">·</span>
|
||||||
|
<button onClick={() => fullIndexMut.mutate()} disabled={isPending}
|
||||||
|
className="text-xs text-zinc-500 hover:text-zinc-300 transition-colors disabled:opacity-40">
|
||||||
|
Fetch entire history
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ const PAGE_SIZE = 25;
|
|||||||
const FEED_MODES = [
|
const FEED_MODES = [
|
||||||
{ value: "ranked", label: "For you", hint: "Ranked by your taste" },
|
{ value: "ranked", label: "For you", hint: "Ranked by your taste" },
|
||||||
{ value: "chronological", label: "New", hint: "Everything in date order" },
|
{ value: "chronological", label: "New", hint: "Everything in date order" },
|
||||||
|
{ value: "rediscover", label: "Rediscover", hint: "Older unwatched videos ranked by your taste" },
|
||||||
{ value: "random", label: "Explore", hint: "Random from discovery pool" },
|
{ value: "random", label: "Explore", hint: "Random from discovery pool" },
|
||||||
{ value: "inbox", label: "Inbox", hint: "New from followed channels since last visit" },
|
{ value: "inbox", label: "Inbox", hint: "New from followed channels since last visit" },
|
||||||
];
|
];
|
||||||
|
|||||||
Reference in New Issue
Block a user