feat: incremental SSE scan, recursive NC traversal, custom folder support
- nc_scan.py: recursive collect_audio_files (fixes depth-1 bug); scan_band_folder yields ndjson events (progress/song/session/skipped/done) for streaming - songs.py: replace old flat scan with scan_band_folder; add GET nc-scan/stream endpoint using _member_from_request so ?token= auth works for fetch-based SSE - BandPage.tsx: scan button now consumes ndjson stream via fetch+ReadableStream; sessions/unattributed invalidated as each song/session event arrives - session.py: add extract_session_folder() for YYMMDD path extraction - rehearsal_session.py: get_or_create uses begin_nested() savepoint to handle races - band.py: add get_by_nc_folder_prefix() for custom nc_folder_path band lookup - internal.py: nc-upload falls back to prefix match when slug lookup fails - event_loop.py: remove hardcoded bands/ guard; let internal API handle filtering Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -81,6 +81,19 @@ class BandRepository(BaseRepository[Band]):
|
|||||||
result = await self.session.execute(stmt)
|
result = await self.session.execute(stmt)
|
||||||
return result.scalar_one_or_none()
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
|
async def get_by_nc_folder_prefix(self, path: str) -> Band | None:
|
||||||
|
"""Return the band whose nc_folder_path is a prefix of path."""
|
||||||
|
stmt = select(Band).where(Band.nc_folder_path.is_not(None))
|
||||||
|
result = await self.session.execute(stmt)
|
||||||
|
bands = result.scalars().all()
|
||||||
|
# Longest match wins (most specific prefix)
|
||||||
|
best: Band | None = None
|
||||||
|
for band in bands:
|
||||||
|
folder = band.nc_folder_path # type: ignore[union-attr]
|
||||||
|
if path.startswith(folder) and (best is None or len(folder) > len(best.nc_folder_path)): # type: ignore[arg-type]
|
||||||
|
best = band
|
||||||
|
return best
|
||||||
|
|
||||||
async def list_for_member(self, member_id: uuid.UUID) -> list[Band]:
|
async def list_for_member(self, member_id: uuid.UUID) -> list[Band]:
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Band)
|
select(Band)
|
||||||
|
|||||||
@@ -40,11 +40,19 @@ class RehearsalSessionRepository(BaseRepository[RehearsalSession]):
|
|||||||
existing = await self.get_by_band_and_date(band_id, session_date)
|
existing = await self.get_by_band_and_date(band_id, session_date)
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
return existing
|
return existing
|
||||||
|
try:
|
||||||
|
async with self.session.begin_nested():
|
||||||
return await self.create(
|
return await self.create(
|
||||||
band_id=band_id,
|
band_id=band_id,
|
||||||
date=datetime(session_date.year, session_date.month, session_date.day),
|
date=datetime(session_date.year, session_date.month, session_date.day),
|
||||||
nc_folder_path=nc_folder_path,
|
nc_folder_path=nc_folder_path,
|
||||||
)
|
)
|
||||||
|
except Exception:
|
||||||
|
# Another request raced us — fetch the row that now exists
|
||||||
|
existing = await self.get_by_band_and_date(band_id, session_date)
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
raise
|
||||||
|
|
||||||
async def list_for_band(self, band_id: uuid.UUID) -> list[tuple[RehearsalSession, int]]:
|
async def list_for_band(self, band_id: uuid.UUID) -> list[tuple[RehearsalSession, int]]:
|
||||||
"""Return (session, recording_count) tuples, newest date first."""
|
"""Return (session, recording_count) tuples, newest date first."""
|
||||||
|
|||||||
@@ -44,15 +44,20 @@ async def nc_upload(
|
|||||||
if Path(path).suffix.lower() not in AUDIO_EXTENSIONS:
|
if Path(path).suffix.lower() not in AUDIO_EXTENSIONS:
|
||||||
return {"status": "skipped", "reason": "not an audio file"}
|
return {"status": "skipped", "reason": "not an audio file"}
|
||||||
|
|
||||||
parts = path.split("/")
|
|
||||||
if len(parts) < 3 or parts[0] != "bands":
|
|
||||||
return {"status": "skipped", "reason": "path not under bands/"}
|
|
||||||
|
|
||||||
band_slug = parts[1]
|
|
||||||
band_repo = BandRepository(session)
|
band_repo = BandRepository(session)
|
||||||
band = await band_repo.get_by_slug(band_slug)
|
|
||||||
|
# Try slug-based lookup first (standard bands/{slug}/ layout)
|
||||||
|
parts = path.split("/")
|
||||||
|
band = None
|
||||||
|
if len(parts) >= 3 and parts[0] == "bands":
|
||||||
|
band = await band_repo.get_by_slug(parts[1])
|
||||||
|
|
||||||
|
# Fall back to prefix match for bands with custom nc_folder_path
|
||||||
if band is None:
|
if band is None:
|
||||||
log.warning("nc-upload: band slug '%s' not found in DB", band_slug)
|
band = await band_repo.get_by_nc_folder_prefix(path)
|
||||||
|
|
||||||
|
if band is None:
|
||||||
|
log.info("nc-upload: no band found for path '%s' — skipping", path)
|
||||||
return {"status": "skipped", "reason": "band not found"}
|
return {"status": "skipped", "reason": "band not found"}
|
||||||
|
|
||||||
# Determine song title and folder from path.
|
# Determine song title and folder from path.
|
||||||
|
|||||||
@@ -1,23 +1,24 @@
|
|||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import uuid
|
import uuid
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from rehearsalhub.db.engine import get_session
|
from rehearsalhub.db.engine import get_session, get_session_factory
|
||||||
from rehearsalhub.db.models import Member
|
from rehearsalhub.db.models import Member
|
||||||
from rehearsalhub.dependencies import get_current_member
|
from rehearsalhub.dependencies import get_current_member
|
||||||
from rehearsalhub.repositories.audio_version import AudioVersionRepository
|
from rehearsalhub.routers.versions import _member_from_request
|
||||||
from rehearsalhub.repositories.band import BandRepository
|
from rehearsalhub.repositories.band import BandRepository
|
||||||
from rehearsalhub.repositories.comment import CommentRepository
|
from rehearsalhub.repositories.comment import CommentRepository
|
||||||
from rehearsalhub.repositories.rehearsal_session import RehearsalSessionRepository
|
|
||||||
from rehearsalhub.repositories.song import SongRepository
|
from rehearsalhub.repositories.song import SongRepository
|
||||||
from rehearsalhub.schemas.comment import SongCommentCreate, SongCommentRead
|
from rehearsalhub.schemas.comment import SongCommentCreate, SongCommentRead
|
||||||
from rehearsalhub.schemas.song import SongCreate, SongRead, SongUpdate
|
from rehearsalhub.schemas.song import SongCreate, SongRead, SongUpdate
|
||||||
from rehearsalhub.services.band import BandService
|
from rehearsalhub.services.band import BandService
|
||||||
from rehearsalhub.services.session import parse_rehearsal_date
|
from rehearsalhub.services.nc_scan import scan_band_folder
|
||||||
from rehearsalhub.services.song import SongService
|
from rehearsalhub.services.song import SongService
|
||||||
from rehearsalhub.storage.nextcloud import NextcloudClient
|
from rehearsalhub.storage.nextcloud import NextcloudClient
|
||||||
|
|
||||||
@@ -137,6 +138,58 @@ async def create_song(
|
|||||||
return read
|
return read
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_band_and_assert_member(
|
||||||
|
band_id: uuid.UUID,
|
||||||
|
current_member: Member,
|
||||||
|
session: AsyncSession,
|
||||||
|
):
|
||||||
|
band_svc = BandService(session)
|
||||||
|
try:
|
||||||
|
await band_svc.assert_membership(band_id, current_member.id)
|
||||||
|
except PermissionError:
|
||||||
|
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Not a member")
|
||||||
|
band_repo = BandRepository(session)
|
||||||
|
band = await band_repo.get_by_id(band_id)
|
||||||
|
if band is None:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Band not found")
|
||||||
|
return band
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/bands/{band_id}/nc-scan/stream")
|
||||||
|
async def scan_nextcloud_stream(
|
||||||
|
band_id: uuid.UUID,
|
||||||
|
session: AsyncSession = Depends(get_session),
|
||||||
|
current_member: Member = Depends(_member_from_request),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
SSE endpoint: streams scan progress as newline-delimited JSON events.
|
||||||
|
Each event is a JSON object on its own line.
|
||||||
|
Accepts ?token= for EventSource clients that can't set headers.
|
||||||
|
"""
|
||||||
|
band = await _get_band_and_assert_member(band_id, current_member, session)
|
||||||
|
band_folder = band.nc_folder_path or f"bands/{band.slug}/"
|
||||||
|
nc = NextcloudClient.for_member(current_member)
|
||||||
|
member_id = current_member.id
|
||||||
|
|
||||||
|
async def event_generator():
|
||||||
|
async with get_session_factory()() as db:
|
||||||
|
try:
|
||||||
|
async for event in scan_band_folder(db, nc, band_id, band_folder, member_id):
|
||||||
|
yield json.dumps(event) + "\n"
|
||||||
|
if event.get("type") in ("song", "session"):
|
||||||
|
await db.commit()
|
||||||
|
except Exception as exc:
|
||||||
|
log.exception("SSE scan error for band %s", band_id)
|
||||||
|
yield json.dumps({"type": "error", "message": str(exc)}) + "\n"
|
||||||
|
finally:
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
return StreamingResponse(
|
||||||
|
event_generator(),
|
||||||
|
media_type="application/x-ndjson",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/bands/{band_id}/nc-scan", response_model=NcScanResult)
|
@router.post("/bands/{band_id}/nc-scan", response_model=NcScanResult)
|
||||||
async def scan_nextcloud(
|
async def scan_nextcloud(
|
||||||
band_id: uuid.UUID,
|
band_id: uuid.UUID,
|
||||||
@@ -144,170 +197,30 @@ async def scan_nextcloud(
|
|||||||
current_member: Member = Depends(get_current_member),
|
current_member: Member = Depends(get_current_member),
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Scan the band's Nextcloud folder for audio files and import any not yet
|
Blocking scan — collects all results then returns. Delegates to scan_band_folder.
|
||||||
registered as songs/versions. Idempotent — safe to call multiple times.
|
Prefer the SSE /nc-scan/stream endpoint for large folders.
|
||||||
"""
|
"""
|
||||||
band_svc = BandService(session)
|
band = await _get_band_and_assert_member(band_id, current_member, session)
|
||||||
try:
|
|
||||||
await band_svc.assert_membership(band_id, current_member.id)
|
|
||||||
except PermissionError:
|
|
||||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Not a member")
|
|
||||||
|
|
||||||
band_repo = BandRepository(session)
|
|
||||||
band = await band_repo.get_by_id(band_id)
|
|
||||||
if band is None:
|
|
||||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Band not found")
|
|
||||||
|
|
||||||
nc = NextcloudClient.for_member(current_member)
|
|
||||||
version_repo = AudioVersionRepository(session)
|
|
||||||
session_repo = RehearsalSessionRepository(session)
|
|
||||||
song_svc = SongService(session)
|
|
||||||
|
|
||||||
# dav_prefix to strip full WebDAV hrefs → user-relative paths
|
|
||||||
dav_prefix = f"/remote.php/dav/files/{nc._auth[0]}/"
|
|
||||||
|
|
||||||
def relative(href: str) -> str:
|
|
||||||
if href.startswith(dav_prefix):
|
|
||||||
return href[len(dav_prefix):]
|
|
||||||
return href.lstrip("/")
|
|
||||||
|
|
||||||
imported_songs: list[SongRead] = []
|
|
||||||
skipped_count = 0
|
|
||||||
band_folder = band.nc_folder_path or f"bands/{band.slug}/"
|
band_folder = band.nc_folder_path or f"bands/{band.slug}/"
|
||||||
|
nc = NextcloudClient.for_member(current_member)
|
||||||
|
|
||||||
log.info("NC scan START — band='%s' folder='%s' nc_user='%s'", band.slug, band_folder, nc._auth[0])
|
songs: list[SongRead] = []
|
||||||
|
stats = {"found": 0, "imported": 0, "skipped": 0}
|
||||||
|
|
||||||
try:
|
async for event in scan_band_folder(session, nc, band_id, band_folder, current_member.id):
|
||||||
items = await nc.list_folder(band_folder)
|
if event["type"] == "song":
|
||||||
except Exception as exc:
|
songs.append(SongRead(**event["song"]))
|
||||||
log.error("NC scan FAILED — could not list '%s': %s", band_folder, exc)
|
elif event["type"] == "done":
|
||||||
raise HTTPException(
|
stats = event["stats"]
|
||||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
elif event["type"] == "error":
|
||||||
detail=f"Cannot read Nextcloud folder '{band_folder}': {exc}",
|
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=event["message"])
|
||||||
)
|
|
||||||
|
|
||||||
log.info("NC scan — found %d top-level entries in '%s'", len(items), band_folder)
|
|
||||||
for item in items:
|
|
||||||
log.info(" entry href=%s → rel=%s", item.path, relative(item.path))
|
|
||||||
|
|
||||||
# Collect (nc_file_path, nc_folder, song_title, rehearsal_label) tuples.
|
|
||||||
# nc_folder is the directory that groups versions of the same song.
|
|
||||||
# For YYMMDD / dated rehearsal subfolders each file is its own song —
|
|
||||||
# the song title comes from the filename stem, not the folder name.
|
|
||||||
to_import: list[tuple[str, str, str, str | None]] = []
|
|
||||||
|
|
||||||
for item in items:
|
|
||||||
rel = relative(item.path)
|
|
||||||
if rel.endswith("/"):
|
|
||||||
dir_name = Path(rel.rstrip("/")).name
|
|
||||||
try:
|
|
||||||
sub_items = await nc.list_folder(rel)
|
|
||||||
except Exception as exc:
|
|
||||||
log.warning("NC scan — could not list subfolder '%s': %s", rel, exc)
|
|
||||||
continue
|
|
||||||
|
|
||||||
all_sub = [relative(s.path) for s in sub_items]
|
|
||||||
audio_files = [s for s in sub_items if Path(relative(s.path)).suffix.lower() in AUDIO_EXTENSIONS]
|
|
||||||
log.info(
|
|
||||||
"NC scan — subfolder '%s': %d entries total, %d audio files",
|
|
||||||
dir_name, len(all_sub), len(audio_files),
|
|
||||||
)
|
|
||||||
for s in sub_items:
|
|
||||||
sr = relative(s.path)
|
|
||||||
ext = Path(sr).suffix.lower()
|
|
||||||
if ext and ext not in AUDIO_EXTENSIONS:
|
|
||||||
log.info(" skip (not audio ext=%s): %s", ext, sr)
|
|
||||||
|
|
||||||
for sub in audio_files:
|
|
||||||
sub_rel = relative(sub.path)
|
|
||||||
song_title = Path(sub_rel).stem
|
|
||||||
song_folder = str(Path(sub_rel).parent) + "/"
|
|
||||||
rehearsal_label = dir_name
|
|
||||||
log.info(" queue for import: %s → title='%s' folder='%s'", sub_rel, song_title, song_folder)
|
|
||||||
to_import.append((sub_rel, song_folder, song_title, rehearsal_label))
|
|
||||||
else:
|
|
||||||
ext = Path(rel).suffix.lower()
|
|
||||||
if ext in AUDIO_EXTENSIONS:
|
|
||||||
folder = str(Path(rel).parent) + "/"
|
|
||||||
title = Path(rel).stem
|
|
||||||
log.info(" queue for import (root-level): %s → title='%s'", rel, title)
|
|
||||||
to_import.append((rel, folder, title, None))
|
|
||||||
elif ext:
|
|
||||||
log.info(" skip root-level (not audio ext=%s): %s", ext, rel)
|
|
||||||
|
|
||||||
log.info("NC scan — %d audio files queued for import", len(to_import))
|
|
||||||
|
|
||||||
song_repo = SongRepository(session)
|
|
||||||
from rehearsalhub.schemas.audio_version import AudioVersionCreate # noqa: PLC0415
|
|
||||||
|
|
||||||
for nc_file_path, nc_folder, song_title, rehearsal_label in to_import:
|
|
||||||
# Skip if this exact file version is already registered
|
|
||||||
try:
|
|
||||||
meta = await nc.get_file_metadata(nc_file_path)
|
|
||||||
etag = meta.etag
|
|
||||||
except Exception as exc:
|
|
||||||
log.warning("Could not fetch metadata for '%s': %s — skipping", nc_file_path, exc)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if etag and await version_repo.get_by_etag(etag):
|
|
||||||
log.debug("Skipping '%s' — etag already registered", nc_file_path)
|
|
||||||
skipped_count += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Resolve rehearsal session from YYMMDD folder segment
|
|
||||||
rehearsal_date = parse_rehearsal_date(nc_file_path)
|
|
||||||
rehearsal_session_id = None
|
|
||||||
if rehearsal_date:
|
|
||||||
rs = await session_repo.get_or_create(band_id, rehearsal_date, nc_folder)
|
|
||||||
rehearsal_session_id = rs.id
|
|
||||||
|
|
||||||
# Find or create song record
|
|
||||||
song = await song_repo.get_by_nc_folder_path(nc_folder)
|
|
||||||
if song is None:
|
|
||||||
song = await song_repo.get_by_title_and_band(band_id, song_title)
|
|
||||||
if song is None:
|
|
||||||
log.info("Creating new song '%s' (folder: %s)", song_title, nc_folder)
|
|
||||||
song = await song_repo.create(
|
|
||||||
band_id=band_id,
|
|
||||||
session_id=rehearsal_session_id,
|
|
||||||
title=song_title,
|
|
||||||
status="jam",
|
|
||||||
notes=None,
|
|
||||||
nc_folder_path=nc_folder,
|
|
||||||
created_by=current_member.id,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
log.info("Found existing song '%s' (id: %s)", song.title, song.id)
|
|
||||||
if rehearsal_session_id and song.session_id is None:
|
|
||||||
song = await song_repo.update(song, session_id=rehearsal_session_id)
|
|
||||||
|
|
||||||
await song_svc.register_version(
|
|
||||||
song.id,
|
|
||||||
AudioVersionCreate(
|
|
||||||
nc_file_path=nc_file_path,
|
|
||||||
nc_file_etag=etag,
|
|
||||||
format=Path(nc_file_path).suffix.lstrip(".").lower(),
|
|
||||||
file_size_bytes=meta.size if etag else None,
|
|
||||||
),
|
|
||||||
current_member.id,
|
|
||||||
)
|
|
||||||
|
|
||||||
read = SongRead.model_validate(song)
|
|
||||||
read.version_count = 1
|
|
||||||
imported_songs.append(read)
|
|
||||||
label_info = f" [rehearsal: {rehearsal_label}]" if rehearsal_label else ""
|
|
||||||
log.info("Imported '%s' as song '%s'%s", nc_file_path, song_title, label_info)
|
|
||||||
|
|
||||||
log.info(
|
|
||||||
"NC scan complete for '%s': %d imported, %d skipped (already registered)",
|
|
||||||
band_folder, len(imported_songs), skipped_count,
|
|
||||||
)
|
|
||||||
return NcScanResult(
|
return NcScanResult(
|
||||||
folder=band_folder,
|
folder=band_folder,
|
||||||
files_found=len(to_import),
|
files_found=stats["found"],
|
||||||
imported=len(imported_songs),
|
imported=stats["imported"],
|
||||||
skipped=skipped_count,
|
skipped=stats["skipped"],
|
||||||
songs=imported_songs,
|
songs=songs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
196
api/src/rehearsalhub/services/nc_scan.py
Normal file
196
api/src/rehearsalhub/services/nc_scan.py
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
"""Core nc-scan logic shared by the blocking and streaming endpoints."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import AsyncGenerator
|
||||||
|
from urllib.parse import unquote
|
||||||
|
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from rehearsalhub.db.models import Member
|
||||||
|
from rehearsalhub.repositories.audio_version import AudioVersionRepository
|
||||||
|
from rehearsalhub.repositories.rehearsal_session import RehearsalSessionRepository
|
||||||
|
from rehearsalhub.repositories.song import SongRepository
|
||||||
|
from rehearsalhub.schemas.audio_version import AudioVersionCreate
|
||||||
|
from rehearsalhub.schemas.song import SongRead
|
||||||
|
from rehearsalhub.services.session import extract_session_folder, parse_rehearsal_date
|
||||||
|
from rehearsalhub.services.song import SongService
|
||||||
|
from rehearsalhub.storage.nextcloud import NextcloudClient
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
AUDIO_EXTENSIONS = {".mp3", ".wav", ".flac", ".ogg", ".m4a", ".aac", ".opus"}
|
||||||
|
|
||||||
|
# Maximum folder depth to recurse into below the band root.
|
||||||
|
# Depth 0 = band root, 1 = YYMMDD folder, 2 = song subfolder, 3 = safety margin.
|
||||||
|
MAX_SCAN_DEPTH = 3
|
||||||
|
|
||||||
|
|
||||||
|
def _make_relative(dav_prefix: str):
|
||||||
|
"""Return a function that strips the WebDAV prefix and URL-decodes a href."""
|
||||||
|
def relative(href: str) -> str:
|
||||||
|
decoded = unquote(href)
|
||||||
|
if decoded.startswith(dav_prefix):
|
||||||
|
return decoded[len(dav_prefix):]
|
||||||
|
# Strip any leading slash for robustness
|
||||||
|
return decoded.lstrip("/")
|
||||||
|
return relative
|
||||||
|
|
||||||
|
|
||||||
|
async def collect_audio_files(
|
||||||
|
nc: NextcloudClient,
|
||||||
|
relative: object, # Callable[[str], str]
|
||||||
|
folder_path: str,
|
||||||
|
max_depth: int = MAX_SCAN_DEPTH,
|
||||||
|
_depth: int = 0,
|
||||||
|
) -> AsyncGenerator[str, None]:
|
||||||
|
"""
|
||||||
|
Recursively yield user-relative audio file paths under folder_path.
|
||||||
|
|
||||||
|
Handles any depth:
|
||||||
|
bands/slug/take.wav depth 0
|
||||||
|
bands/slug/231015/take.wav depth 1
|
||||||
|
bands/slug/231015/groove/take.wav depth 2 ← was broken before
|
||||||
|
"""
|
||||||
|
if _depth > max_depth:
|
||||||
|
log.debug("Max depth %d exceeded at '%s', stopping recursion", max_depth, folder_path)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
items = await nc.list_folder(folder_path)
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("Could not list folder '%s': %s", folder_path, exc)
|
||||||
|
return
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
"scan depth=%d folder='%s' entries=%d",
|
||||||
|
_depth, folder_path, len(items),
|
||||||
|
)
|
||||||
|
|
||||||
|
for item in items:
|
||||||
|
rel = relative(item.path) # type: ignore[operator]
|
||||||
|
if rel.endswith("/"):
|
||||||
|
# It's a subdirectory — recurse
|
||||||
|
log.info(" → subdir: %s", rel)
|
||||||
|
async for subpath in collect_audio_files(nc, relative, rel, max_depth, _depth + 1):
|
||||||
|
yield subpath
|
||||||
|
else:
|
||||||
|
ext = Path(rel).suffix.lower()
|
||||||
|
if ext in AUDIO_EXTENSIONS:
|
||||||
|
log.info(" → audio file: %s", rel)
|
||||||
|
yield rel
|
||||||
|
elif ext:
|
||||||
|
log.debug(" → skip (ext=%s): %s", ext, rel)
|
||||||
|
|
||||||
|
|
||||||
|
async def scan_band_folder(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
nc: NextcloudClient,
|
||||||
|
band_id,
|
||||||
|
band_folder: str,
|
||||||
|
member_id,
|
||||||
|
) -> AsyncGenerator[dict, None]:
|
||||||
|
"""
|
||||||
|
Async generator that scans band_folder and yields event dicts:
|
||||||
|
{"type": "progress", "message": str}
|
||||||
|
{"type": "song", "song": SongRead-dict, "is_new": bool}
|
||||||
|
{"type": "session", "session": {id, date, label}}
|
||||||
|
{"type": "skipped", "path": str, "reason": str}
|
||||||
|
{"type": "done", "stats": {found, imported, skipped}}
|
||||||
|
{"type": "error", "message": str}
|
||||||
|
"""
|
||||||
|
dav_prefix = f"/remote.php/dav/files/{nc._auth[0]}/"
|
||||||
|
relative = _make_relative(dav_prefix)
|
||||||
|
|
||||||
|
version_repo = AudioVersionRepository(db_session)
|
||||||
|
session_repo = RehearsalSessionRepository(db_session)
|
||||||
|
song_repo = SongRepository(db_session)
|
||||||
|
song_svc = SongService(db_session)
|
||||||
|
|
||||||
|
found = 0
|
||||||
|
imported = 0
|
||||||
|
skipped = 0
|
||||||
|
|
||||||
|
yield {"type": "progress", "message": f"Scanning {band_folder}…"}
|
||||||
|
|
||||||
|
async for nc_file_path in collect_audio_files(nc, relative, band_folder):
|
||||||
|
found += 1
|
||||||
|
song_folder = str(Path(nc_file_path).parent).rstrip("/") + "/"
|
||||||
|
song_title = Path(nc_file_path).stem
|
||||||
|
|
||||||
|
yield {"type": "progress", "message": f"Checking {Path(nc_file_path).name}…"}
|
||||||
|
|
||||||
|
# Fetch file metadata (etag + size) — one PROPFIND per file
|
||||||
|
try:
|
||||||
|
meta = await nc.get_file_metadata(nc_file_path)
|
||||||
|
etag = meta.etag
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("Metadata error for '%s': %s", nc_file_path, exc)
|
||||||
|
yield {"type": "skipped", "path": nc_file_path, "reason": f"metadata error: {exc}"}
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Skip if this exact version is already indexed
|
||||||
|
if etag and await version_repo.get_by_etag(etag):
|
||||||
|
log.info("Already registered (etag match): %s", nc_file_path)
|
||||||
|
skipped += 1
|
||||||
|
yield {"type": "skipped", "path": nc_file_path, "reason": "already registered"}
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Resolve or create a RehearsalSession from a YYMMDD folder segment
|
||||||
|
rehearsal_date = parse_rehearsal_date(nc_file_path)
|
||||||
|
rehearsal_session_id = None
|
||||||
|
if rehearsal_date:
|
||||||
|
session_folder = extract_session_folder(nc_file_path) or song_folder
|
||||||
|
rs = await session_repo.get_or_create(band_id, rehearsal_date, session_folder)
|
||||||
|
rehearsal_session_id = rs.id
|
||||||
|
yield {
|
||||||
|
"type": "session",
|
||||||
|
"session": {
|
||||||
|
"id": str(rs.id),
|
||||||
|
"date": rs.date.isoformat(),
|
||||||
|
"label": rs.label,
|
||||||
|
"nc_folder_path": rs.nc_folder_path,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Find or create the Song record
|
||||||
|
song = await song_repo.get_by_nc_folder_path(song_folder)
|
||||||
|
if song is None:
|
||||||
|
song = await song_repo.get_by_title_and_band(band_id, song_title)
|
||||||
|
is_new = song is None
|
||||||
|
if is_new:
|
||||||
|
log.info("Creating song '%s' folder='%s'", song_title, song_folder)
|
||||||
|
song = await song_repo.create(
|
||||||
|
band_id=band_id,
|
||||||
|
session_id=rehearsal_session_id,
|
||||||
|
title=song_title,
|
||||||
|
status="jam",
|
||||||
|
notes=None,
|
||||||
|
nc_folder_path=song_folder,
|
||||||
|
created_by=member_id,
|
||||||
|
)
|
||||||
|
elif rehearsal_session_id and song.session_id is None:
|
||||||
|
song = await song_repo.update(song, session_id=rehearsal_session_id)
|
||||||
|
|
||||||
|
# Register the audio version
|
||||||
|
await song_svc.register_version(
|
||||||
|
song.id,
|
||||||
|
AudioVersionCreate(
|
||||||
|
nc_file_path=nc_file_path,
|
||||||
|
nc_file_etag=etag,
|
||||||
|
format=Path(nc_file_path).suffix.lstrip(".").lower(),
|
||||||
|
file_size_bytes=meta.size,
|
||||||
|
),
|
||||||
|
member_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
imported += 1
|
||||||
|
read = SongRead.model_validate(song, update={"version_count": 1, "session_id": rehearsal_session_id})
|
||||||
|
yield {"type": "song", "song": read.model_dump(mode="json"), "is_new": is_new}
|
||||||
|
|
||||||
|
yield {
|
||||||
|
"type": "done",
|
||||||
|
"stats": {"found": found, "imported": imported, "skipped": skipped},
|
||||||
|
}
|
||||||
@@ -41,7 +41,20 @@ def parse_rehearsal_date(nc_file_path: str) -> date | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def nc_folder_for_path(nc_file_path: str) -> str:
|
def extract_session_folder(nc_file_path: str) -> str | None:
|
||||||
"""Return the parent directory of a file path, with trailing slash."""
|
"""
|
||||||
from pathlib import Path
|
Return the YYMMDD/YYYYMMDD folder path (with trailing slash) from a file path,
|
||||||
return str(Path(nc_file_path).parent).rstrip("/") + "/"
|
or None if no date segment is found.
|
||||||
|
|
||||||
|
e.g. "bands/slug/231015/groove/take.wav" → "bands/slug/231015/"
|
||||||
|
"bands/slug/take.wav" → None
|
||||||
|
"""
|
||||||
|
for pattern in (_YYYYMMDD_RE, _YYMMDD_RE):
|
||||||
|
m = pattern.search(nc_file_path)
|
||||||
|
if m:
|
||||||
|
idx = m.start(1)
|
||||||
|
# Walk back to the preceding slash (or start)
|
||||||
|
start = nc_file_path.rfind("/", 0, idx) + 1
|
||||||
|
end = m.end(1)
|
||||||
|
return nc_file_path[:end].rstrip("/") + "/"
|
||||||
|
return None
|
||||||
|
|||||||
@@ -66,11 +66,6 @@ def normalize_nc_path(raw_path: str, username: str) -> str:
|
|||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
def is_band_audio_path(path: str) -> bool:
|
|
||||||
"""True if the user-relative path is inside a band folder."""
|
|
||||||
parts = path.strip("/").split("/")
|
|
||||||
return len(parts) >= 2 and parts[0] == "bands"
|
|
||||||
|
|
||||||
|
|
||||||
def extract_nc_file_path(activity: dict[str, Any]) -> str | None:
|
def extract_nc_file_path(activity: dict[str, Any]) -> str | None:
|
||||||
"""Extract the server-relative file path from an activity event."""
|
"""Extract the server-relative file path from an activity event."""
|
||||||
@@ -140,10 +135,6 @@ async def poll_once(nc_client: NextcloudWatcherClient, settings: WatcherSettings
|
|||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not is_band_audio_path(nc_path):
|
|
||||||
log.info(" → skip: path not inside a bands/ folder")
|
|
||||||
continue
|
|
||||||
|
|
||||||
if activity_type not in _UPLOAD_TYPES and subject not in _UPLOAD_SUBJECTS:
|
if activity_type not in _UPLOAD_TYPES and subject not in _UPLOAD_SUBJECTS:
|
||||||
log.info(
|
log.info(
|
||||||
" → skip: type=%r subject=%r is not a file upload event",
|
" → skip: type=%r subject=%r is not a file upload event",
|
||||||
|
|||||||
@@ -60,6 +60,8 @@ export function BandPage() {
|
|||||||
const [showCreate, setShowCreate] = useState(false);
|
const [showCreate, setShowCreate] = useState(false);
|
||||||
const [title, setTitle] = useState("");
|
const [title, setTitle] = useState("");
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [scanning, setScanning] = useState(false);
|
||||||
|
const [scanProgress, setScanProgress] = useState<string | null>(null);
|
||||||
const [scanMsg, setScanMsg] = useState<string | null>(null);
|
const [scanMsg, setScanMsg] = useState<string | null>(null);
|
||||||
const [inviteLink, setInviteLink] = useState<string | null>(null);
|
const [inviteLink, setInviteLink] = useState<string | null>(null);
|
||||||
const [editingFolder, setEditingFolder] = useState(false);
|
const [editingFolder, setEditingFolder] = useState(false);
|
||||||
@@ -123,22 +125,65 @@ export function BandPage() {
|
|||||||
onError: (err) => setError(err instanceof Error ? err.message : "Failed to create song"),
|
onError: (err) => setError(err instanceof Error ? err.message : "Failed to create song"),
|
||||||
});
|
});
|
||||||
|
|
||||||
const scanMutation = useMutation({
|
async function startScan() {
|
||||||
mutationFn: () => api.post<NcScanResult>(`/bands/${bandId}/nc-scan`, {}),
|
if (scanning || !bandId) return;
|
||||||
onSuccess: (result) => {
|
setScanning(true);
|
||||||
|
setScanMsg(null);
|
||||||
|
setScanProgress("Starting scan…");
|
||||||
|
|
||||||
|
const token = localStorage.getItem("rh_token");
|
||||||
|
const url = `/api/v1/bands/${bandId}/nc-scan/stream${token ? `?token=${encodeURIComponent(token)}` : ""}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const resp = await fetch(url);
|
||||||
|
if (!resp.ok || !resp.body) {
|
||||||
|
const text = await resp.text().catch(() => resp.statusText);
|
||||||
|
throw new Error(text || `HTTP ${resp.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const reader = resp.body.getReader();
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
let buf = "";
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
buf += decoder.decode(value, { stream: true });
|
||||||
|
const lines = buf.split("\n");
|
||||||
|
buf = lines.pop() ?? "";
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
if (!line.trim()) continue;
|
||||||
|
let event: Record<string, unknown>;
|
||||||
|
try { event = JSON.parse(line); } catch { continue; }
|
||||||
|
|
||||||
|
if (event.type === "progress") {
|
||||||
|
setScanProgress(event.message as string);
|
||||||
|
} else if (event.type === "song" || event.type === "session") {
|
||||||
qc.invalidateQueries({ queryKey: ["sessions", bandId] });
|
qc.invalidateQueries({ queryKey: ["sessions", bandId] });
|
||||||
qc.invalidateQueries({ queryKey: ["songs-unattributed", bandId] });
|
qc.invalidateQueries({ queryKey: ["songs-unattributed", bandId] });
|
||||||
if (result.imported > 0) {
|
} else if (event.type === "done") {
|
||||||
setScanMsg(`Imported ${result.imported} new song${result.imported !== 1 ? "s" : ""} from ${result.folder} (${result.skipped} already registered).`);
|
const s = event.stats as { found: number; imported: number; skipped: number };
|
||||||
} else if (result.files_found === 0) {
|
if (s.imported > 0) {
|
||||||
setScanMsg(`No audio files found in ${result.folder}.`);
|
setScanMsg(`Imported ${s.imported} new song${s.imported !== 1 ? "s" : ""} (${s.skipped} already registered).`);
|
||||||
|
} else if (s.found === 0) {
|
||||||
|
setScanMsg("No audio files found.");
|
||||||
} else {
|
} else {
|
||||||
setScanMsg(`All ${result.files_found} file${result.files_found !== 1 ? "s" : ""} in ${result.folder} already registered.`);
|
setScanMsg(`All ${s.found} file${s.found !== 1 ? "s" : ""} already registered.`);
|
||||||
}
|
}
|
||||||
setTimeout(() => setScanMsg(null), 6000);
|
setTimeout(() => setScanMsg(null), 6000);
|
||||||
},
|
} else if (event.type === "error") {
|
||||||
onError: (err) => setScanMsg(err instanceof Error ? err.message : "Scan failed"),
|
setScanMsg(`Scan error: ${event.message}`);
|
||||||
});
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
setScanMsg(err instanceof Error ? err.message : "Scan failed");
|
||||||
|
} finally {
|
||||||
|
setScanning(false);
|
||||||
|
setScanProgress(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const inviteMutation = useMutation({
|
const inviteMutation = useMutation({
|
||||||
mutationFn: () => api.post<BandInvite>(`/bands/${bandId}/invites`, {}),
|
mutationFn: () => api.post<BandInvite>(`/bands/${bandId}/invites`, {}),
|
||||||
@@ -306,11 +351,11 @@ export function BandPage() {
|
|||||||
<h2 style={{ color: "var(--text)", margin: 0, fontSize: 16 }}>Recordings</h2>
|
<h2 style={{ color: "var(--text)", margin: 0, fontSize: 16 }}>Recordings</h2>
|
||||||
<div style={{ display: "flex", gap: 8 }}>
|
<div style={{ display: "flex", gap: 8 }}>
|
||||||
<button
|
<button
|
||||||
onClick={() => scanMutation.mutate()}
|
onClick={startScan}
|
||||||
disabled={scanMutation.isPending}
|
disabled={scanning}
|
||||||
style={{ background: "none", border: "1px solid var(--border)", borderRadius: 6, color: "var(--teal)", cursor: "pointer", padding: "6px 14px", fontSize: 12 }}
|
style={{ background: "none", border: "1px solid var(--border)", borderRadius: 6, color: "var(--teal)", cursor: "pointer", padding: "6px 14px", fontSize: 12 }}
|
||||||
>
|
>
|
||||||
{scanMutation.isPending ? "Scanning…" : "⟳ Scan Nextcloud"}
|
{scanning ? "Scanning…" : "⟳ Scan Nextcloud"}
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
onClick={() => { setShowCreate(!showCreate); setError(null); }}
|
onClick={() => { setShowCreate(!showCreate); setError(null); }}
|
||||||
@@ -321,6 +366,11 @@ export function BandPage() {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{scanning && scanProgress && (
|
||||||
|
<div style={{ background: "var(--bg-subtle)", border: "1px solid var(--border)", borderRadius: 6, color: "var(--text-muted)", fontSize: 12, padding: "8px 14px", marginBottom: 8, fontFamily: "monospace" }}>
|
||||||
|
{scanProgress}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
{scanMsg && (
|
{scanMsg && (
|
||||||
<div style={{ background: "var(--teal-bg)", border: "1px solid var(--teal)", borderRadius: 6, color: "var(--teal)", fontSize: 12, padding: "8px 14px", marginBottom: 12 }}>
|
<div style={{ background: "var(--teal-bg)", border: "1px solid var(--teal)", borderRadius: 6, color: "var(--teal)", fontSize: 12, padding: "8px 14px", marginBottom: 12 }}>
|
||||||
{scanMsg}
|
{scanMsg}
|
||||||
|
|||||||
Reference in New Issue
Block a user