Add popular fetch progress to Downloads page
- Track active background tasks in an in-memory dict with a lock - Expose GET /api/channels/tasks returning running task list - _fetch_popular_task updates done count as each video fetch completes - Downloads page polls /tasks every 2s and shows progress bars Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,5 @@
|
|||||||
import json
|
import json
|
||||||
|
import threading as _threading
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
@@ -14,6 +15,9 @@ from ..services import ytdlp
|
|||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
_tasks: dict = {}
|
||||||
|
_tasks_lock = _threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
class ChannelOut(BaseModel):
|
class ChannelOut(BaseModel):
|
||||||
id: int
|
id: int
|
||||||
@@ -286,6 +290,12 @@ def sync_all_channels(
|
|||||||
return {"indexing": len(channels)}
|
return {"indexing": len(channels)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/tasks")
|
||||||
|
def get_active_tasks(current_user: User = Depends(get_current_user)):
|
||||||
|
with _tasks_lock:
|
||||||
|
return list(_tasks.values())
|
||||||
|
|
||||||
|
|
||||||
@router.post("/mark-seen", status_code=204)
|
@router.post("/mark-seen", status_code=204)
|
||||||
def mark_channels_seen(
|
def mark_channels_seen(
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
@@ -652,11 +662,11 @@ def fetch_popular_videos(
|
|||||||
):
|
):
|
||||||
"""Fetch the channel's most popular videos from YouTube and index them."""
|
"""Fetch the channel's most popular videos from YouTube and index them."""
|
||||||
channel = _get_channel_or_404(db, channel_id)
|
channel = _get_channel_or_404(db, channel_id)
|
||||||
background_tasks.add_task(_fetch_popular_task, channel_id, channel.youtube_channel_id)
|
background_tasks.add_task(_fetch_popular_task, channel_id, channel.youtube_channel_id, channel.name or "")
|
||||||
return {"detail": "Fetching popular videos"}
|
return {"detail": "Fetching popular videos"}
|
||||||
|
|
||||||
|
|
||||||
def _fetch_popular_task(channel_id: int, youtube_channel_id: str):
|
def _fetch_popular_task(channel_id: int, youtube_channel_id: str, channel_name: str = ""):
|
||||||
"""Half-and-half popular fetch.
|
"""Half-and-half popular fetch.
|
||||||
|
|
||||||
Phase 1 (fast): flat-playlist crawl of the full channel → store any
|
Phase 1 (fast): flat-playlist crawl of the full channel → store any
|
||||||
@@ -742,15 +752,32 @@ def _fetch_popular_task(channel_id: int, youtube_channel_id: str):
|
|||||||
if not video_ids:
|
if not video_ids:
|
||||||
return
|
return
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=3) as pool:
|
task_id = f"popular-{channel_id}"
|
||||||
futures = {pool.submit(ytdlp.fetch_video_metadata, vid): vid for vid in video_ids}
|
with _tasks_lock:
|
||||||
results = {}
|
_tasks[task_id] = {
|
||||||
for future in as_completed(futures):
|
"id": task_id,
|
||||||
vid = futures[future]
|
"label": f"Popular fetch — {channel_name}" if channel_name else "Popular fetch",
|
||||||
try:
|
"total": len(video_ids),
|
||||||
results[vid] = future.result()
|
"done": 0,
|
||||||
except Exception:
|
"started_at": datetime.utcnow().isoformat(),
|
||||||
pass
|
}
|
||||||
|
|
||||||
|
results = {}
|
||||||
|
try:
|
||||||
|
with ThreadPoolExecutor(max_workers=3) as pool:
|
||||||
|
futures = {pool.submit(ytdlp.fetch_video_metadata, vid): vid for vid in video_ids}
|
||||||
|
for future in as_completed(futures):
|
||||||
|
vid = futures[future]
|
||||||
|
try:
|
||||||
|
results[vid] = future.result()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
with _tasks_lock:
|
||||||
|
if task_id in _tasks:
|
||||||
|
_tasks[task_id]["done"] += 1
|
||||||
|
finally:
|
||||||
|
with _tasks_lock:
|
||||||
|
_tasks.pop(task_id, None)
|
||||||
|
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ export const renameChannelGroup = (id, name) => api.patch(`/channels/groups/${id
|
|||||||
export const addChannelToGroup = (groupId, channelId) => api.post(`/channels/groups/${groupId}/channels/${channelId}`);
|
export const addChannelToGroup = (groupId, channelId) => api.post(`/channels/groups/${groupId}/channels/${channelId}`);
|
||||||
export const removeChannelFromGroup = (groupId, channelId) => api.delete(`/channels/groups/${groupId}/channels/${channelId}`);
|
export const removeChannelFromGroup = (groupId, channelId) => api.delete(`/channels/groups/${groupId}/channels/${channelId}`);
|
||||||
export const bulkChannelAction = (channel_ids, action) => api.post("/channels/bulk-action", { channel_ids, action });
|
export const bulkChannelAction = (channel_ids, action) => api.post("/channels/bulk-action", { channel_ids, action });
|
||||||
|
export const getActiveTasks = () => api.get("/channels/tasks");
|
||||||
|
|
||||||
// Videos
|
// Videos
|
||||||
export const homeFeed = (page = 0, limit = 25, mode = "ranked", duration = "") =>
|
export const homeFeed = (page = 0, limit = 25, mode = "ranked", duration = "") =>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { useState, useMemo } from "react";
|
import { useState, useMemo } from "react";
|
||||||
import { useNavigate } from "react-router-dom";
|
import { useNavigate } from "react-router-dom";
|
||||||
import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
|
import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
|
||||||
import { getDownloads, deleteDownload, deleteAllDownloads, restoreDownload } from "../api";
|
import { getDownloads, deleteDownload, deleteAllDownloads, restoreDownload, getActiveTasks } from "../api";
|
||||||
import SortPicker from "../components/SortPicker";
|
import SortPicker from "../components/SortPicker";
|
||||||
|
|
||||||
const HISTORY_SORTS = [
|
const HISTORY_SORTS = [
|
||||||
@@ -66,6 +66,12 @@ export default function DownloadsPage() {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const { data: activeTasks = [] } = useQuery({
|
||||||
|
queryKey: ["active-tasks"],
|
||||||
|
queryFn: () => getActiveTasks().then((r) => r.data),
|
||||||
|
refetchInterval: (query) => (query.state.data?.length > 0 ? 2000 : 5000),
|
||||||
|
});
|
||||||
|
|
||||||
const clearAllMut = useMutation({
|
const clearAllMut = useMutation({
|
||||||
mutationFn: deleteAllDownloads,
|
mutationFn: deleteAllDownloads,
|
||||||
onSuccess: () => {
|
onSuccess: () => {
|
||||||
@@ -127,6 +133,26 @@ export default function DownloadsPage() {
|
|||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{activeTasks.length > 0 && (
|
||||||
|
<section>
|
||||||
|
<h2 className="font-display font-semibold text-base text-zinc-400 mb-3">Background Tasks</h2>
|
||||||
|
<div className="flex flex-col gap-3">
|
||||||
|
{activeTasks.map((task) => {
|
||||||
|
const pct = task.total > 0 ? (task.done / task.total) * 100 : 0;
|
||||||
|
return (
|
||||||
|
<div key={task.id} className="bg-zinc-900 rounded-xl p-4">
|
||||||
|
<div className="flex items-center justify-between mb-1">
|
||||||
|
<p className="text-sm font-medium text-zinc-100">{task.label}</p>
|
||||||
|
<span className="text-xs text-zinc-400 tabular-nums">{task.done}/{task.total}</span>
|
||||||
|
</div>
|
||||||
|
<ProgressBar pct={pct} />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
)}
|
||||||
|
|
||||||
{active.length > 0 && (
|
{active.length > 0 && (
|
||||||
<section>
|
<section>
|
||||||
<h2 className="font-display font-semibold text-base text-zinc-400 mb-3">Active</h2>
|
<h2 className="font-display font-semibold text-base text-zinc-400 mb-3">Active</h2>
|
||||||
|
|||||||
Reference in New Issue
Block a user