sschuhmann/rehearshalhub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: resolve job-not-found race and YYMMDD scan folder structure

Browse Source 
Race condition (worker "Job not found in DB"):
- RedisJobQueue.enqueue() was pushing job IDs to Redis immediately after
  flush() but before the API transaction committed, so the worker would
  read an ID that didn't exist yet in the DB from its own session.
- Fix: defer the Redis rpush until after session.commit() via a pending-
  push list drained by get_session() after each successful commit.
- Worker: drain stale Redis queue entries on startup to clear any IDs
  left over from previously uncommitted transactions.
- Worker: add 3-attempt retry with 200ms sleep when a job is not found,
  as a safety net for any remaining propagation edge cases.

NC scan folder structure (YYMMDD rehearsal subfolders):
- Previously used dir_name as song title for all files in a subdirectory,
  meaning every file got the folder name (e.g. "231015") as its title.
- Fix: derive song title from Path(sub_rel).stem so each audio file gets
  its own name; use the file's parent path as nc_folder for version grouping.
- Rehearsal folder name stored in song.notes as "Rehearsal: YYMMDD".
- Added structured logging throughout the scan: entries found, per-folder
  file counts, skip/create/import decisions, and final summary count.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Steffen Schuhmann
2026-03-28 21:58:46 +01:00
parent f7be1b994d
commit b28472c32f
4 changed files with 103 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
api/src/rehearsalhub/routers/songs.py
View File

@@ -99,61 +99,85 @@ async def scan_nextcloud(
return href.lstrip("/")
imported: list[SongRead] = []
band_folder = band.nc_folder_path or f"bands/{band.slug}/"
log.info("Starting NC scan for band '%s' in folder '%s'", band.slug, band_folder)
try:
items = await nc.list_folder(band.nc_folder_path or f"bands/{band.slug}/")
items = await nc.list_folder(band_folder)
except Exception as exc:
raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Nextcloud unreachable: {exc}")
# Collect (nc_file_path, song_folder_rel, song_title) tuples
to_import: list[tuple[str, str, str]] = []
log.info("Found %d top-level entries in '%s'", len(items), band_folder)
# Collect (nc_file_path, nc_folder, song_title, rehearsal_label) tuples.
# nc_folder is the directory that groups versions of the same song.
# For YYMMDD / dated rehearsal subfolders each file is its own song —
# the song title comes from the filename stem, not the folder name.
to_import: list[tuple[str, str, str, str | None]] = []
for item in items:
rel = relative(item.path)
if rel.endswith("/"):
# It's a subdirectory — scan one level deeper
dir_name = Path(rel.rstrip("/")).name
try:
sub_items = await nc.list_folder(rel)
except Exception:
except Exception as exc:
log.warning("Could not list subfolder '%s': %s", rel, exc)
continue
dir_name = Path(rel.rstrip("/")).name
for sub in sub_items:
audio_files = [s for s in sub_items if Path(relative(s.path)).suffix.lower() in AUDIO_EXTENSIONS]
log.info("Subfolder '%s': %d audio files found", dir_name, len(audio_files))
for sub in audio_files:
sub_rel = relative(sub.path)
if Path(sub_rel).suffix.lower() in AUDIO_EXTENSIONS:
to_import.append((sub_rel, rel, dir_name))
song_title = Path(sub_rel).stem
# Each file in a rehearsal folder is its own song,
# grouped under its own sub-subfolder path for version tracking.
song_folder = str(Path(sub_rel).parent) + "/"
rehearsal_label = dir_name # e.g. "231015" or "2023-10-15"
to_import.append((sub_rel, song_folder, song_title, rehearsal_label))
else:
if Path(rel).suffix.lower() in AUDIO_EXTENSIONS:
folder = str(Path(rel).parent) + "/"
title = Path(rel).stem
to_import.append((rel, folder, title))
to_import.append((rel, folder, title, None))
for nc_file_path, nc_folder, song_title in to_import:
# Skip if version already registered by etag
log.info("NC scan: %d audio files to evaluate for import", len(to_import))
song_repo = SongRepository(session)
from rehearsalhub.schemas.audio_version import AudioVersionCreate # noqa: PLC0415
for nc_file_path, nc_folder, song_title, rehearsal_label in to_import:
# Skip if this exact file version is already registered
try:
meta = await nc.get_file_metadata(nc_file_path)
etag = meta.etag
except Exception:
etag = None
if etag and await version_repo.get_by_etag(etag):
except Exception as exc:
log.warning("Could not fetch metadata for '%s': %s — skipping", nc_file_path, exc)
continue
# Find or create song
song_repo = SongRepository(session)
if etag and await version_repo.get_by_etag(etag):
log.debug("Skipping '%s' — etag already registered", nc_file_path)
continue
# Find or create song record
song = await song_repo.get_by_nc_folder_path(nc_folder)
if song is None:
song = await song_repo.get_by_title_and_band(band_id, song_title)
if song is None:
log.info("Creating new song '%s' (folder: %s)", song_title, nc_folder)
song = await song_repo.create(
band_id=band_id,
title=song_title,
status="jam",
notes=None,
notes=f"Rehearsal: {rehearsal_label}" if rehearsal_label else None,
nc_folder_path=nc_folder,
created_by=current_member.id,
)
else:
log.info("Found existing song '%s' (id: %s)", song.title, song.id)
from rehearsalhub.schemas.audio_version import AudioVersionCreate # noqa: PLC0415
await song_svc.register_version(
song.id,
AudioVersionCreate(
@@ -168,8 +192,10 @@ async def scan_nextcloud(
read = SongRead.model_validate(song)
read.version_count = 1
imported.append(read)
log.info("Imported %s as song '%s'", nc_file_path, song_title)
label_info = f" [rehearsal: {rehearsal_label}]" if rehearsal_label else ""
log.info("Imported '%s' as song '%s'%s", nc_file_path, song_title, label_info)
log.info("NC scan complete: %d new versions imported", len(imported))
return imported