diff --git a/backend/main.py b/backend/main.py index 7fc27d0..f414bd1 100644 --- a/backend/main.py +++ b/backend/main.py @@ -6,7 +6,7 @@ from fastapi.staticfiles import StaticFiles from .config import settings from .database import init_db, SessionLocal from .services import ytdlp as ytdlp_service -from .routers import auth, channels, videos, search, downloads, discovery, settings as settings_router, stats as stats_router, export as export_router, collections as collections_router, admin as admin_router +from .routers import auth, channels, videos, search, downloads, discovery, settings as settings_router, stats as stats_router, export as export_router, collections as collections_router, admin as admin_router, playlists as playlists_router app = FastAPI(title="YouTube Hub", version="0.1.0") @@ -29,6 +29,7 @@ app.include_router(stats_router.router, prefix="/api/stats", tags=["stats"]) app.include_router(export_router.router, prefix="/api/export", tags=["export"]) app.include_router(collections_router.router, prefix="/api/collections", tags=["collections"]) app.include_router(admin_router.router, prefix="/api/admin", tags=["admin"]) +app.include_router(playlists_router.router, prefix="/api/playlists", tags=["playlists"]) os.makedirs(settings.download_path, exist_ok=True) @@ -73,6 +74,19 @@ def on_startup(): "ALTER TABLE user_settings ADD COLUMN use_oauth2 INTEGER DEFAULT 0", "ALTER TABLE user_settings ADD COLUMN sync_interval_hours INTEGER DEFAULT 0", "ALTER TABLE user_settings ADD COLUMN subtitle_langs TEXT DEFAULT ''", + """CREATE TABLE IF NOT EXISTS playlists ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + youtube_playlist_id TEXT NOT NULL UNIQUE, + channel_id INTEGER REFERENCES channels(id) ON DELETE CASCADE, + title TEXT NOT NULL, + description TEXT, + thumbnail_url TEXT, + video_count INTEGER DEFAULT 0, + video_ids TEXT, + indexed_at DATETIME, + crawled_at DATETIME DEFAULT CURRENT_TIMESTAMP + )""", + "ALTER TABLE playlists ADD COLUMN video_ids TEXT", """CREATE TABLE IF NOT EXISTS search_history ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, diff --git a/backend/models.py b/backend/models.py index 0813059..c2110c7 100644 --- a/backend/models.py +++ b/backend/models.py @@ -203,6 +203,21 @@ class CollectionItem(Base): added_at = Column(DateTime, default=datetime.utcnow) +class Playlist(Base): + __tablename__ = "playlists" + + id = Column(Integer, primary_key=True, index=True) + youtube_playlist_id = Column(String, unique=True, nullable=False, index=True) + channel_id = Column(Integer, ForeignKey("channels.id", ondelete="CASCADE"), nullable=True) + title = Column(String, nullable=False) + description = Column(Text) + thumbnail_url = Column(String) + video_count = Column(Integer, default=0) + video_ids = Column(Text) # JSON array of youtube_video_id strings + indexed_at = Column(DateTime) + crawled_at = Column(DateTime, default=datetime.utcnow) + + class GraphEdge(Base): __tablename__ = "graph_edges" __table_args__ = (UniqueConstraint("from_channel_id", "to_channel_id"),) diff --git a/backend/routers/channels.py b/backend/routers/channels.py index 5c1f6bc..129285d 100644 --- a/backend/routers/channels.py +++ b/backend/routers/channels.py @@ -858,7 +858,7 @@ def _index_channel_explore_task(channel_id: int, user_id: int, start_video: int, return for vdata in result.get("videos", []): yt_id = vdata.get("youtube_video_id") - if not yt_id or not vdata.get("published_at"): + if not yt_id: continue if not db.query(Video).filter_by(youtube_video_id=yt_id).first(): db.add(Video( diff --git a/backend/routers/playlists.py b/backend/routers/playlists.py new file mode 100644 index 0000000..19ca392 --- /dev/null +++ b/backend/routers/playlists.py @@ -0,0 +1,196 @@ +import json +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status +from pydantic import BaseModel +from sqlalchemy.orm import Session +from sqlalchemy import text + +from ..auth_utils import get_current_user +from ..database import get_db +from ..models import Channel, Playlist, Video, User +from ..services import ytdlp + +router = APIRouter() + + +class PlaylistOut(BaseModel): + id: int + youtube_playlist_id: str + channel_id: Optional[int] + title: str + description: Optional[str] + thumbnail_url: Optional[str] + video_count: int + indexed_at: Optional[datetime] + + model_config = {"from_attributes": True} + + +class PlaylistVideoOut(BaseModel): + id: int + youtube_video_id: str + title: str + thumbnail_url: Optional[str] + duration_seconds: Optional[int] + published_at: Optional[datetime] + view_count: Optional[int] + is_downloaded: bool = False + is_watched: bool = False + channel_id: Optional[int] = None + channel_name: Optional[str] = None + + model_config = {"from_attributes": True} + + +@router.get("/channel/{channel_id}", response_model=list[PlaylistOut]) +def get_channel_playlists( + channel_id: int, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + rows = db.query(Playlist).filter_by(channel_id=channel_id).order_by(Playlist.video_count.desc()).all() + return rows + + +@router.post("/channel/{channel_id}/fetch", status_code=status.HTTP_202_ACCEPTED) +def fetch_channel_playlists( + channel_id: int, + background_tasks: BackgroundTasks, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + channel = db.query(Channel).filter_by(id=channel_id).first() + if not channel: + raise HTTPException(status_code=404, detail="Channel not found") + background_tasks.add_task(_fetch_playlists_task, channel_id, channel.youtube_channel_id) + return {"detail": "Fetching playlists"} + + +def _fetch_playlists_task(channel_id: int, youtube_channel_id: str): + from ..database import SessionLocal + db = SessionLocal() + try: + playlists = ytdlp.fetch_channel_playlists(youtube_channel_id) + for pl in playlists: + pl_id = pl["youtube_playlist_id"] + existing = db.query(Playlist).filter_by(youtube_playlist_id=pl_id).first() + if existing: + existing.title = pl["title"] or existing.title + if pl.get("video_count"): + existing.video_count = pl["video_count"] + if pl.get("thumbnail_url") and not existing.thumbnail_url: + existing.thumbnail_url = pl["thumbnail_url"] + else: + db.add(Playlist( + youtube_playlist_id=pl_id, + channel_id=channel_id, + title=pl["title"], + description=pl.get("description"), + thumbnail_url=pl.get("thumbnail_url"), + video_count=pl.get("video_count") or 0, + )) + db.commit() + except Exception: + db.rollback() + finally: + db.close() + + +@router.get("/{playlist_id}/videos", response_model=list[PlaylistVideoOut]) +def get_playlist_videos( + playlist_id: int, + offset: int = 0, + limit: int = 60, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + playlist = db.query(Playlist).filter_by(id=playlist_id).first() + if not playlist: + raise HTTPException(status_code=404, detail="Playlist not found") + if not playlist.indexed_at: + return [] + rows = db.execute( + text(""" + SELECT v.id, v.youtube_video_id, v.title, v.thumbnail_url, + v.duration_seconds, v.published_at, v.view_count, + c.id AS channel_id, c.name AS channel_name, + COALESCE(uv.downloaded, 0) AS is_downloaded, + COALESCE(uv.watched, 0) AS is_watched + FROM videos v + LEFT JOIN channels c ON v.channel_id = c.id + LEFT JOIN user_videos uv ON v.id = uv.video_id AND uv.user_id = :user_id + WHERE v.youtube_video_id IN ( + SELECT value FROM json_each(( + SELECT video_ids FROM playlists WHERE id = :playlist_id + )) + ) + ORDER BY v.published_at DESC NULLS LAST + LIMIT :limit OFFSET :offset + """), + {"user_id": current_user.id, "playlist_id": playlist_id, "limit": limit, "offset": offset}, + ).mappings().all() + return [PlaylistVideoOut(**dict(r)) for r in rows] + + +@router.post("/{playlist_id}/index", status_code=status.HTTP_202_ACCEPTED) +def index_playlist( + playlist_id: int, + background_tasks: BackgroundTasks, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + playlist = db.query(Playlist).filter_by(id=playlist_id).first() + if not playlist: + raise HTTPException(status_code=404, detail="Playlist not found") + background_tasks.add_task(_index_playlist_task, playlist_id, playlist.youtube_playlist_id, playlist.channel_id) + return {"detail": "Indexing playlist"} + + +def _index_playlist_task(playlist_id: int, youtube_playlist_id: str, channel_id: Optional[int]): + from ..database import SessionLocal + db = SessionLocal() + try: + videos = ytdlp.fetch_playlist_videos(youtube_playlist_id) + playlist = db.query(Playlist).filter_by(id=playlist_id).first() + if not playlist: + return + + video_yt_ids = [] + for vdata in videos: + yt_id = vdata.get("youtube_video_id") + if not yt_id: + continue + video_yt_ids.append(yt_id) + existing = db.query(Video).filter_by(youtube_video_id=yt_id).first() + if existing: + if vdata.get("view_count") is not None: + existing.view_count = vdata["view_count"] + else: + ch_id = channel_id + if not ch_id and vdata.get("channel", {}).get("youtube_channel_id"): + ch = db.query(Channel).filter_by( + youtube_channel_id=vdata["channel"]["youtube_channel_id"] + ).first() + if ch: + ch_id = ch.id + db.add(Video( + youtube_video_id=yt_id, + channel_id=ch_id, + title=vdata.get("title", ""), + thumbnail_url=vdata.get("thumbnail_url"), + duration_seconds=vdata.get("duration_seconds"), + published_at=vdata.get("published_at"), + view_count=vdata.get("view_count"), + tags="[]", + )) + + playlist.video_count = len(video_yt_ids) + playlist.indexed_at = datetime.utcnow() + playlist.video_ids = json.dumps(video_yt_ids) + db.commit() + except Exception: + db.rollback() + finally: + db.close() diff --git a/backend/services/ytdlp.py b/backend/services/ytdlp.py index 679aaa5..940e218 100644 --- a/backend/services/ytdlp.py +++ b/backend/services/ytdlp.py @@ -353,6 +353,83 @@ def fetch_channel_metadata(channel_id: str, max_videos: int = 30, start_video: i return {"channel": channel_info, "videos": videos} +def fetch_channel_playlists(channel_id: str, max_results: int = 100) -> list[dict]: + """Fetch the playlists listed on a channel's /playlists tab.""" + if channel_id.startswith("@"): + url = f"https://www.youtube.com/{channel_id}/playlists" + else: + url = f"https://www.youtube.com/channel/{channel_id}/playlists" + stdout, _, code = _run([ + "yt-dlp", url, + "--dump-json", "--flat-playlist", + "--playlist-end", str(max_results), + "--quiet", + *_cookie_args(), + ], timeout=60) + + playlists = [] + for line in stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + info = json.loads(line) + pl_id = info.get("id") or info.get("playlist_id") + title = info.get("title") or info.get("playlist_title") or "" + if not pl_id or not title or pl_id == channel_id: + continue + playlists.append({ + "youtube_playlist_id": pl_id, + "title": title, + "description": info.get("description"), + "thumbnail_url": _stable_thumbnail(info.get("id")) if info.get("_type") == "url" else None, + "video_count": info.get("playlist_count") or info.get("n_entries") or 0, + }) + except json.JSONDecodeError: + continue + return playlists + + +def fetch_playlist_videos(playlist_id: str, max_videos: int = 200) -> list[dict]: + """Fetch videos from a YouTube playlist by playlist ID.""" + url = f"https://www.youtube.com/playlist?list={playlist_id}" + args = [ + "yt-dlp", url, + "--dump-json", "--flat-playlist", + "--quiet", + *_cookie_args(), + ] + if max_videos > 0: + args += ["--playlist-end", str(max_videos)] + stdout, _, code = _run(args, timeout=120) + + videos = [] + for line in stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + info = json.loads(line) + vid_id = info.get("id") + if not vid_id: + continue + videos.append({ + "youtube_video_id": vid_id, + "title": info.get("title", ""), + "thumbnail_url": _stable_thumbnail(vid_id), + "duration_seconds": info.get("duration"), + "published_at": _parse_published(info), + "view_count": info.get("view_count"), + "channel": { + "youtube_channel_id": info.get("channel_id"), + "name": info.get("channel") or info.get("uploader") or "", + }, + }) + except json.JSONDecodeError: + continue + return videos + + def fetch_featured_channels(channel_id: str) -> list[str]: """Fetch channel IDs from the /channels tab of a YouTube channel. diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 720221f..dc126c3 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -155,3 +155,10 @@ export const deleteCollection = (id) => api.delete(`/collections/${id}`); export const getCollectionVideos = (id) => api.get(`/collections/${id}/videos`); export const addToCollection = (collectionId, videoId) => api.post(`/collections/${collectionId}/videos`, { video_id: videoId }); export const removeFromCollection = (collectionId, videoId) => api.delete(`/collections/${collectionId}/videos/${videoId}`); + +// Playlists +export const getChannelPlaylists = (channelId) => api.get(`/playlists/channel/${channelId}`); +export const fetchChannelPlaylists = (channelId) => api.post(`/playlists/channel/${channelId}/fetch`); +export const getPlaylistVideos = (playlistId, offset = 0, limit = 60) => + api.get(`/playlists/${playlistId}/videos`, { params: { offset, limit } }); +export const indexPlaylist = (playlistId) => api.post(`/playlists/${playlistId}/index`); diff --git a/frontend/src/pages/Channel.jsx b/frontend/src/pages/Channel.jsx index 418dce5..599d907 100644 --- a/frontend/src/pages/Channel.jsx +++ b/frontend/src/pages/Channel.jsx @@ -4,14 +4,16 @@ import { useQuery, useMutation, useQueryClient, useInfiniteQuery } from "@tansta import { getChannel, getChannelVideos, searchChannelYoutube, followChannel, unfollowChannel, indexChannel, indexChannelFull, exploreChannelOlder, fetchPopularVideos, downloadChannel, + getChannelPlaylists, fetchChannelPlaylists, getPlaylistVideos, indexPlaylist, } from "../api"; import VideoCard from "../components/VideoCard"; const LIMIT = 60; const TABS = [ - { value: "videos", label: "Videos" }, - { value: "popular", label: "Popular" }, + { value: "videos", label: "Videos" }, + { value: "popular", label: "Popular" }, + { value: "playlists", label: "Playlists" }, ]; const SORTS = [ @@ -38,6 +40,8 @@ export default function ChannelPage() { const [indexing, setIndexing] = useState(false); const [explorePage, setExplorePage] = useState(2); const searchInputRef = useRef(null); + const [openPlaylistId, setOpenPlaylistId] = useState(null); + const [playlistOffset, setPlaylistOffset] = useState(0); const { data: channel, isLoading: loadingChannel } = useQuery({ queryKey: ["channel", id], @@ -121,6 +125,28 @@ export default function ChannelPage() { onSuccess: () => scheduleRefetch(20000), }); + const { data: playlists = [], refetch: refetchPlaylists } = useQuery({ + queryKey: ["channel-playlists", id], + queryFn: () => getChannelPlaylists(id).then((r) => r.data), + enabled: !!id && tab === "playlists", + }); + + const fetchPlaylistsMut = useMutation({ + mutationFn: () => fetchChannelPlaylists(id), + onSuccess: () => setTimeout(() => refetchPlaylists(), 8000), + }); + + const { data: playlistVideos, isLoading: loadingPlaylistVideos, refetch: refetchPlaylistVideos } = useQuery({ + queryKey: ["playlist-videos", openPlaylistId, playlistOffset], + queryFn: () => getPlaylistVideos(openPlaylistId, playlistOffset, 60).then((r) => r.data), + enabled: !!openPlaylistId, + }); + + const indexPlaylistMut = useMutation({ + mutationFn: (plId) => indexPlaylist(plId), + onSuccess: () => setTimeout(() => refetchPlaylistVideos(), 15000), + }); + const [dlResult, setDlResult] = useState(null); const dlMut = useMutation({ mutationFn: () => downloadChannel(id), @@ -310,8 +336,115 @@ export default function ChannelPage() { )} + {/* Playlist browser */} + {tab === "playlists" && ( + openPlaylistId ? ( +
+
+ + {(() => { + const pl = playlists.find(p => p.id === openPlaylistId); + return pl ? {pl.title} : null; + })()} + +
+ {loadingPlaylistVideos ? ( +
+
+
+ ) : playlistVideos?.length ? ( + <> +
+ {playlistVideos.map((v) => ( + + ))} +
+
+ {playlistOffset > 0 && ( + + )} + {playlistVideos.length === 60 && ( + + )} +
+ + ) : ( +
+

No videos indexed for this playlist yet.

+ +
+ )} +
+ ) : ( +
+
+ {playlists.length} playlists + +
+ {playlists.length === 0 ? ( +
+

No playlists fetched yet.

+ +
+ ) : ( +
+ {playlists.map(pl => ( + + ))} +
+ )} +
+ ) + )} + {/* Video list */} - {loadingVideos ? ( + {tab !== "playlists" && (loadingVideos ? (
@@ -369,7 +502,7 @@ export default function ChannelPage() { )}
- )} + ))}
); }