Add scheduled sync, disk space awareness, and subtitle downloads
- auto-sync daemon: background thread checks every hour and syncs followed channels for users with sync_interval_hours set (6/12/24h options) - disk stats: /api/stats now returns total/used/free/download bytes; Stats page shows a disk usage bar - subtitles: subtitle_langs setting (e.g. "en,sv") passed through all download paths; yt-dlp writes .srt files alongside the video - Settings page: sync interval dropdown + subtitle languages input Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -71,6 +71,8 @@ def on_startup():
|
|||||||
"ALTER TABLE user_settings ADD COLUMN feed_weight_affinity REAL DEFAULT 5.0",
|
"ALTER TABLE user_settings ADD COLUMN feed_weight_affinity REAL DEFAULT 5.0",
|
||||||
"ALTER TABLE user_settings ADD COLUMN feed_weight_channel REAL DEFAULT 5.0",
|
"ALTER TABLE user_settings ADD COLUMN feed_weight_channel REAL DEFAULT 5.0",
|
||||||
"ALTER TABLE user_settings ADD COLUMN use_oauth2 INTEGER DEFAULT 0",
|
"ALTER TABLE user_settings ADD COLUMN use_oauth2 INTEGER DEFAULT 0",
|
||||||
|
"ALTER TABLE user_settings ADD COLUMN sync_interval_hours INTEGER DEFAULT 0",
|
||||||
|
"ALTER TABLE user_settings ADD COLUMN subtitle_langs TEXT DEFAULT ''",
|
||||||
"""CREATE TABLE IF NOT EXISTS search_history (
|
"""CREATE TABLE IF NOT EXISTS search_history (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
@@ -151,9 +153,47 @@ def on_startup():
|
|||||||
|
|
||||||
# Backfill descriptions for videos that don't have them yet (runs in background)
|
# Backfill descriptions for videos that don't have them yet (runs in background)
|
||||||
import threading
|
import threading
|
||||||
from .routers.channels import _enrich_missing_task
|
from .routers.channels import _enrich_missing_task, _index_channels_batch
|
||||||
threading.Thread(target=_enrich_missing_task, args=(3,), daemon=True).start()
|
threading.Thread(target=_enrich_missing_task, args=(3,), daemon=True).start()
|
||||||
|
|
||||||
|
def _auto_sync_daemon():
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from sqlalchemy import text as _text
|
||||||
|
while True:
|
||||||
|
time.sleep(3600)
|
||||||
|
try:
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
users_due = db.execute(
|
||||||
|
_text("SELECT user_id, sync_interval_hours FROM user_settings WHERE sync_interval_hours > 0")
|
||||||
|
).mappings().all()
|
||||||
|
for row in users_due:
|
||||||
|
uid = row["user_id"]
|
||||||
|
cutoff = datetime.utcnow() - timedelta(hours=row["sync_interval_hours"])
|
||||||
|
ch_ids = [
|
||||||
|
r["id"] for r in db.execute(
|
||||||
|
_text("""
|
||||||
|
SELECT c.id FROM channels c
|
||||||
|
JOIN user_channels uc ON c.id = uc.channel_id
|
||||||
|
WHERE uc.user_id = :uid AND uc.status = 'followed'
|
||||||
|
AND (c.crawled_at IS NULL OR c.crawled_at < :cutoff)
|
||||||
|
ORDER BY COALESCE(c.crawled_at, '1970-01-01') ASC
|
||||||
|
"""),
|
||||||
|
{"uid": uid, "cutoff": cutoff},
|
||||||
|
).mappings().all()
|
||||||
|
]
|
||||||
|
if ch_ids:
|
||||||
|
threading.Thread(
|
||||||
|
target=_index_channels_batch, args=(ch_ids, uid), daemon=True
|
||||||
|
).start()
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
threading.Thread(target=_auto_sync_daemon, daemon=True).start()
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/health")
|
@app.get("/api/health")
|
||||||
def health():
|
def health():
|
||||||
|
|||||||
@@ -122,6 +122,8 @@ class UserSettings(Base):
|
|||||||
feed_weight_affinity = Column(Float, default=5.0) # 0–10
|
feed_weight_affinity = Column(Float, default=5.0) # 0–10
|
||||||
feed_weight_channel = Column(Float, default=5.0) # 0–10
|
feed_weight_channel = Column(Float, default=5.0) # 0–10
|
||||||
use_oauth2 = Column(Boolean, default=False)
|
use_oauth2 = Column(Boolean, default=False)
|
||||||
|
sync_interval_hours = Column(Integer, default=0) # 0 = disabled, 6/12/24 = auto-sync interval
|
||||||
|
subtitle_langs = Column(String, default="") # "" = disabled, "en", "en,sv", etc.
|
||||||
|
|
||||||
|
|
||||||
class DiscoveryQueue(Base):
|
class DiscoveryQueue(Base):
|
||||||
|
|||||||
@@ -147,6 +147,7 @@ def _index_channel_task(channel_id: int, user_id: int):
|
|||||||
|
|
||||||
if channel_auto:
|
if channel_auto:
|
||||||
quality = user_settings.preferred_quality if user_settings else "best"
|
quality = user_settings.preferred_quality if user_settings else "best"
|
||||||
|
subtitle_langs = (user_settings.subtitle_langs or "") if user_settings else ""
|
||||||
from ..routers.downloads import _on_progress, _on_complete, _on_error
|
from ..routers.downloads import _on_progress, _on_complete, _on_error
|
||||||
for yt_id, vid_id in new_video_ids:
|
for yt_id, vid_id in new_video_ids:
|
||||||
existing_dl = db.query(Download).filter_by(
|
existing_dl = db.query(Download).filter_by(
|
||||||
@@ -159,7 +160,7 @@ def _index_channel_task(channel_id: int, user_id: int):
|
|||||||
import threading
|
import threading
|
||||||
t = threading.Thread(
|
t = threading.Thread(
|
||||||
target=ytdlp.start_download,
|
target=ytdlp.start_download,
|
||||||
args=(yt_id, dl.id, _on_progress, _on_complete, _on_error, quality),
|
args=(yt_id, dl.id, _on_progress, _on_complete, _on_error, quality, subtitle_langs),
|
||||||
daemon=True,
|
daemon=True,
|
||||||
)
|
)
|
||||||
t.start()
|
t.start()
|
||||||
|
|||||||
@@ -127,6 +127,7 @@ def create_download(
|
|||||||
user_settings = db.query(UserSettings).filter_by(user_id=current_user.id).first()
|
user_settings = db.query(UserSettings).filter_by(user_id=current_user.id).first()
|
||||||
default_quality = user_settings.preferred_quality if user_settings else "best"
|
default_quality = user_settings.preferred_quality if user_settings else "best"
|
||||||
quality = body.quality if body.quality in ytdlp.QUALITY_FORMATS else default_quality
|
quality = body.quality if body.quality in ytdlp.QUALITY_FORMATS else default_quality
|
||||||
|
subtitle_langs = (user_settings.subtitle_langs or "") if user_settings else ""
|
||||||
|
|
||||||
_DL_SELECT = """
|
_DL_SELECT = """
|
||||||
SELECT d.id, d.status, d.progress_percent, d.resolution,
|
SELECT d.id, d.status, d.progress_percent, d.resolution,
|
||||||
@@ -155,7 +156,7 @@ def create_download(
|
|||||||
ytdlp.start_download,
|
ytdlp.start_download,
|
||||||
video.youtube_video_id, dl.id,
|
video.youtube_video_id, dl.id,
|
||||||
_on_progress, _on_complete, _on_error,
|
_on_progress, _on_complete, _on_error,
|
||||||
quality,
|
quality, subtitle_langs,
|
||||||
)
|
)
|
||||||
|
|
||||||
row = db.execute(text(_DL_SELECT), {"id": dl.id}).mappings().first()
|
row = db.execute(text(_DL_SELECT), {"id": dl.id}).mappings().first()
|
||||||
@@ -190,6 +191,11 @@ def _get_quality(db, user_id: int) -> str:
|
|||||||
return s.preferred_quality if s else "best"
|
return s.preferred_quality if s else "best"
|
||||||
|
|
||||||
|
|
||||||
|
def _get_subtitle_langs(db, user_id: int) -> str:
|
||||||
|
s = db.query(UserSettings).filter_by(user_id=user_id).first()
|
||||||
|
return (s.subtitle_langs or "") if s else ""
|
||||||
|
|
||||||
|
|
||||||
@router.post("/channel/{channel_id}", status_code=202)
|
@router.post("/channel/{channel_id}", status_code=202)
|
||||||
def download_channel_videos(
|
def download_channel_videos(
|
||||||
channel_id: int,
|
channel_id: int,
|
||||||
@@ -198,6 +204,7 @@ def download_channel_videos(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
quality = _get_quality(db, current_user.id)
|
quality = _get_quality(db, current_user.id)
|
||||||
|
subtitle_langs = _get_subtitle_langs(db, current_user.id)
|
||||||
rows = db.execute(
|
rows = db.execute(
|
||||||
text("""
|
text("""
|
||||||
SELECT v.id, v.youtube_video_id
|
SELECT v.id, v.youtube_video_id
|
||||||
@@ -216,7 +223,7 @@ def download_channel_videos(
|
|||||||
db.flush()
|
db.flush()
|
||||||
background_tasks.add_task(
|
background_tasks.add_task(
|
||||||
ytdlp.start_download, row["youtube_video_id"], dl.id,
|
ytdlp.start_download, row["youtube_video_id"], dl.id,
|
||||||
_on_progress, _on_complete, _on_error, quality,
|
_on_progress, _on_complete, _on_error, quality, subtitle_langs,
|
||||||
)
|
)
|
||||||
count += 1
|
count += 1
|
||||||
db.commit()
|
db.commit()
|
||||||
@@ -230,6 +237,7 @@ def download_following_videos(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
quality = _get_quality(db, current_user.id)
|
quality = _get_quality(db, current_user.id)
|
||||||
|
subtitle_langs = _get_subtitle_langs(db, current_user.id)
|
||||||
rows = db.execute(
|
rows = db.execute(
|
||||||
text("""
|
text("""
|
||||||
SELECT v.id, v.youtube_video_id
|
SELECT v.id, v.youtube_video_id
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ class SettingsOut(BaseModel):
|
|||||||
feed_weight_affinity: float = 5.0
|
feed_weight_affinity: float = 5.0
|
||||||
feed_weight_channel: float = 5.0
|
feed_weight_channel: float = 5.0
|
||||||
use_oauth2: bool = False
|
use_oauth2: bool = False
|
||||||
|
sync_interval_hours: int = 0
|
||||||
|
subtitle_langs: str = ""
|
||||||
|
|
||||||
model_config = {"from_attributes": True}
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
@@ -55,6 +57,8 @@ class SettingsPatch(BaseModel):
|
|||||||
feed_weight_affinity: Optional[float] = Field(None, ge=0.0, le=10.0)
|
feed_weight_affinity: Optional[float] = Field(None, ge=0.0, le=10.0)
|
||||||
feed_weight_channel: Optional[float] = Field(None, ge=0.0, le=10.0)
|
feed_weight_channel: Optional[float] = Field(None, ge=0.0, le=10.0)
|
||||||
use_oauth2: Optional[bool] = None
|
use_oauth2: Optional[bool] = None
|
||||||
|
sync_interval_hours: Optional[int] = Field(None, ge=0, le=168)
|
||||||
|
subtitle_langs: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
def _get_or_create(db: Session, user_id: int) -> UserSettings:
|
def _get_or_create(db: Session, user_id: int) -> UserSettings:
|
||||||
@@ -123,6 +127,10 @@ def update_settings(
|
|||||||
if body.use_oauth2 is not None:
|
if body.use_oauth2 is not None:
|
||||||
s.use_oauth2 = body.use_oauth2
|
s.use_oauth2 = body.use_oauth2
|
||||||
ytdlp.set_oauth2(body.use_oauth2)
|
ytdlp.set_oauth2(body.use_oauth2)
|
||||||
|
if body.sync_interval_hours is not None:
|
||||||
|
s.sync_interval_hours = body.sync_interval_hours
|
||||||
|
if body.subtitle_langs is not None:
|
||||||
|
s.subtitle_langs = body.subtitle_langs.strip()
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(s)
|
db.refresh(s)
|
||||||
|
|||||||
@@ -1,8 +1,12 @@
|
|||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
||||||
from ..auth_utils import get_current_user
|
from ..auth_utils import get_current_user
|
||||||
|
from ..config import settings
|
||||||
from ..database import get_db
|
from ..database import get_db
|
||||||
from ..models import User, UserTagAffinity
|
from ..models import User, UserTagAffinity
|
||||||
|
|
||||||
@@ -120,6 +124,15 @@ def get_stats(
|
|||||||
{"uid": uid},
|
{"uid": uid},
|
||||||
).mappings().first()
|
).mappings().first()
|
||||||
|
|
||||||
|
try:
|
||||||
|
disk = shutil.disk_usage(settings.download_path)
|
||||||
|
download_bytes = sum(
|
||||||
|
e.stat().st_size for e in os.scandir(settings.download_path) if e.is_file()
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
disk = None
|
||||||
|
download_bytes = 0
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"total_watched": totals["total_watched"] or 0,
|
"total_watched": totals["total_watched"] or 0,
|
||||||
"total_watch_seconds": totals["total_watch_seconds"] or 0,
|
"total_watch_seconds": totals["total_watch_seconds"] or 0,
|
||||||
@@ -141,6 +154,12 @@ def get_stats(
|
|||||||
"total_liked": liked_count["n"] or 0,
|
"total_liked": liked_count["n"] or 0,
|
||||||
"top_categories": [dict(r) for r in top_categories],
|
"top_categories": [dict(r) for r in top_categories],
|
||||||
"taste_profile": [dict(r) for r in taste_profile],
|
"taste_profile": [dict(r) for r in taste_profile],
|
||||||
|
"disk": {
|
||||||
|
"total_bytes": disk.total if disk else None,
|
||||||
|
"free_bytes": disk.free if disk else None,
|
||||||
|
"used_bytes": disk.used if disk else None,
|
||||||
|
"download_bytes": download_bytes,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -621,17 +621,20 @@ def start_download(
|
|||||||
on_complete: Any,
|
on_complete: Any,
|
||||||
on_error: Any,
|
on_error: Any,
|
||||||
quality: str = "best",
|
quality: str = "best",
|
||||||
|
subtitle_langs: str = "",
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Start yt-dlp download in a background thread.
|
"""Start yt-dlp download in a background thread.
|
||||||
|
|
||||||
Uses a single progressive MP4 format so the file is playable as it downloads.
|
|
||||||
--no-part writes directly to the final filename (no .part rename at the end).
|
--no-part writes directly to the final filename (no .part rename at the end).
|
||||||
"""
|
"""
|
||||||
url = f"https://www.youtube.com/watch?v={video_id}"
|
url = f"https://www.youtube.com/watch?v={video_id}"
|
||||||
# Predictable output path — lets the player start before download finishes
|
|
||||||
output_template = str(Path(settings.download_path) / f"{video_id}.%(ext)s")
|
output_template = str(Path(settings.download_path) / f"{video_id}.%(ext)s")
|
||||||
|
|
||||||
fmt = QUALITY_FORMATS.get(quality, QUALITY_FORMATS["best"])
|
fmt = QUALITY_FORMATS.get(quality, QUALITY_FORMATS["best"])
|
||||||
|
subtitle_args = (
|
||||||
|
["--write-subs", "--write-auto-subs", "--sub-langs", subtitle_langs, "--convert-subs", "srt"]
|
||||||
|
if subtitle_langs else []
|
||||||
|
)
|
||||||
|
|
||||||
def _run_download():
|
def _run_download():
|
||||||
with _SEMAPHORE:
|
with _SEMAPHORE:
|
||||||
@@ -645,6 +648,7 @@ def start_download(
|
|||||||
"--no-part", "--no-mtime",
|
"--no-part", "--no-mtime",
|
||||||
"-o", output_template,
|
"-o", output_template,
|
||||||
"--newline", "--progress", "--no-colors",
|
"--newline", "--progress", "--no-colors",
|
||||||
|
*subtitle_args,
|
||||||
*cookie_args,
|
*cookie_args,
|
||||||
],
|
],
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
|
|||||||
@@ -650,6 +650,25 @@ export default function SettingsPage() {
|
|||||||
<DiagnosticSection />
|
<DiagnosticSection />
|
||||||
<SubscriptionImportSection />
|
<SubscriptionImportSection />
|
||||||
|
|
||||||
|
{/* Sync */}
|
||||||
|
<Section title="Sync">
|
||||||
|
<Row
|
||||||
|
label="Auto-sync interval"
|
||||||
|
hint="How often to automatically sync your followed channels in the background."
|
||||||
|
>
|
||||||
|
<select
|
||||||
|
value={s?.sync_interval_hours ?? 0}
|
||||||
|
onChange={(e) => set({ sync_interval_hours: Number(e.target.value) })}
|
||||||
|
className="bg-zinc-800 text-zinc-200 text-sm rounded-lg px-3 py-1.5 border border-zinc-700 focus:outline-none focus:border-accent"
|
||||||
|
>
|
||||||
|
<option value={0}>Off</option>
|
||||||
|
<option value={6}>Every 6 hours</option>
|
||||||
|
<option value={12}>Every 12 hours</option>
|
||||||
|
<option value={24}>Every 24 hours</option>
|
||||||
|
</select>
|
||||||
|
</Row>
|
||||||
|
</Section>
|
||||||
|
|
||||||
{/* Download quality */}
|
{/* Download quality */}
|
||||||
<Section title="Download quality">
|
<Section title="Download quality">
|
||||||
<Row
|
<Row
|
||||||
@@ -691,6 +710,18 @@ export default function SettingsPage() {
|
|||||||
onChange={(v) => set({ auto_download_on_sync: v })}
|
onChange={(v) => set({ auto_download_on_sync: v })}
|
||||||
/>
|
/>
|
||||||
</Row>
|
</Row>
|
||||||
|
<Row
|
||||||
|
label="Subtitle languages"
|
||||||
|
hint={'Download subtitles for these languages. e.g. "en" or "en,sv". Leave blank to skip.'}
|
||||||
|
>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={s?.subtitle_langs ?? ""}
|
||||||
|
onChange={(e) => set({ subtitle_langs: e.target.value })}
|
||||||
|
placeholder="en, sv, …"
|
||||||
|
className="bg-zinc-800 text-zinc-200 text-sm rounded-lg px-3 py-1.5 border border-zinc-700 focus:outline-none focus:border-accent w-36"
|
||||||
|
/>
|
||||||
|
</Row>
|
||||||
</Section>
|
</Section>
|
||||||
|
|
||||||
{/* Feed */}
|
{/* Feed */}
|
||||||
|
|||||||
@@ -11,6 +11,14 @@ function fmt(seconds) {
|
|||||||
return `${h}h ${m}m`;
|
return `${h}h ${m}m`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function fmtBytes(bytes) {
|
||||||
|
if (!bytes) return "0 B";
|
||||||
|
const units = ["B", "KB", "MB", "GB", "TB"];
|
||||||
|
let i = 0, v = bytes;
|
||||||
|
while (v >= 1024 && i < units.length - 1) { v /= 1024; i++; }
|
||||||
|
return `${v.toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
|
||||||
|
}
|
||||||
|
|
||||||
function StatCard({ label, value, sub }) {
|
function StatCard({ label, value, sub }) {
|
||||||
return (
|
return (
|
||||||
<div className="bg-zinc-900 rounded-2xl p-4 flex flex-col gap-1">
|
<div className="bg-zinc-900 rounded-2xl p-4 flex flex-col gap-1">
|
||||||
@@ -170,6 +178,29 @@ export default function Stats() {
|
|||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Disk usage */}
|
||||||
|
{data.disk?.total_bytes && (
|
||||||
|
<div className="bg-zinc-900 rounded-2xl p-5 flex flex-col gap-3">
|
||||||
|
<h2 className="text-xs font-semibold text-zinc-500 uppercase tracking-wider">Disk usage</h2>
|
||||||
|
<div className="flex flex-col gap-2">
|
||||||
|
<div className="flex justify-between text-sm">
|
||||||
|
<span className="text-zinc-400">Downloads</span>
|
||||||
|
<span className="text-zinc-300 font-mono">{fmtBytes(data.disk.download_bytes)}</span>
|
||||||
|
</div>
|
||||||
|
<div className="h-2 bg-zinc-800 rounded-full overflow-hidden">
|
||||||
|
<div
|
||||||
|
className="h-full bg-accent/70 rounded-full transition-all"
|
||||||
|
style={{ width: `${Math.min((data.disk.used_bytes / data.disk.total_bytes) * 100, 100)}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="flex justify-between text-[11px] text-zinc-600">
|
||||||
|
<span>{fmtBytes(data.disk.used_bytes)} used</span>
|
||||||
|
<span>{fmtBytes(data.disk.free_bytes)} free · {fmtBytes(data.disk.total_bytes)} total</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
{/* Taste profile */}
|
{/* Taste profile */}
|
||||||
{topTags.length > 0 && (
|
{topTags.length > 0 && (
|
||||||
<div className="bg-zinc-900 rounded-2xl p-5 flex flex-col gap-3">
|
<div className="bg-zinc-900 rounded-2xl p-5 flex flex-col gap-3">
|
||||||
|
|||||||
Reference in New Issue
Block a user