feat: stem storage format, MP3 128 kbps by default (#783)

app/api/jobs.py

@@ -63,6 +63,7 @@
     get_discogs_token,
     get_max_duration_sec,
 )
+from app.core.stem_files import stem_path, stem_url
 from app.core.stems_location import is_relocating
 from app.pipeline import discogs, jobqueue
 from app.pipeline.analyze import dynamic_range_db, measure_loudness
@@ -103,6 +104,7 @@
 )
 from app.pipeline.lyrics_retime import saved_view as saved_retime_view
 from app.pipeline.runner import _pipeline_lock
+from app.pipeline.stem_storage import store_split_stems
 from app.pipeline.vocal_split import split_vocals
 from app.pipeline.work_lookup import find_work, might_have_work
 
@@ -501,7 +503,7 @@ def _loudness_sources(job_id: str) -> list[Path]:
     stems_dir = (JOBS_DIR / job_id / "stems").resolve()
     if not stems_dir.is_relative_to(JOBS_DIR.resolve()):
         return []
-    return [p for p in (stems_dir / f"{n}.wav" for n in STEM_NAMES) if p.is_file()]
+    return [p for p in (stem_path(stems_dir, n) for n in STEM_NAMES) if p.is_file()]
 
 
 def _upgrade_loudness(job: Job) -> None:
@@ -805,6 +807,12 @@ async def start_vocal_split(job_id: str) -> Response:
     try:
         async with _pipeline_lock:
             new_names = await asyncio.to_thread(split_vocals, job, stems_dir)
+            # The peaks are read from the new WAVs first; then the two are kept
+            # the way the song keeps its stems (#783). Neither raises.
+            rms_values = await asyncio.to_thread(
+                merge_stem_peaks, stems_dir, ["vocals", *new_names]
+            )
+            await asyncio.to_thread(store_split_stems, job, stems_dir, new_names)
     except Exception as e:
         cause = classify_failure("\n".join([*(getattr(e, "tail", None) or []), str(e)]))
         logger.warning("[%s] vocal split failed: %s", job_id, e, exc_info=True)
@@ -817,11 +825,10 @@ async def start_vocal_split(job_id: str) -> Response:
     existing = {s["name"] for s in job.stems}
     for name in new_names:
         if name not in existing:
-            job.stems.append({"name": name, "url": f"/api/jobs/{job_id}/stems/{name}.wav"})
-    # "vocals" rides along so presence_for_split can recover the scale the base
-    # stems were normalised against; without it the two new cards would have no
-    # percentage to show.
-    rms_values = merge_stem_peaks(stems_dir, ["vocals", *new_names])
+            job.stems.append({"name": name, "url": stem_url(job_id, stems_dir, name)})
+    # "vocals" rode along in merge_stem_peaks above so presence_for_split can
+    # recover the scale the base stems were normalised against; without it the
+    # two new cards would have no percentage to show.
     extra_presence = presence_for_split(rms_values, job.stem_presence)
     if extra_presence:
         job.stem_presence = {**(job.stem_presence or {}), **extra_presence}
@@ -1195,7 +1202,7 @@ def _fingerprint_audio(job: Job) -> list[Path]:
     stems_root = (JOBS_DIR / job.id / "stems").resolve()
     stems = []
     for name in STEM_NAMES:
-        path = (stems_root / f"{name}.wav").resolve()
+        path = stem_path(stems_root, name).resolve()
         if path.is_file() and path.is_relative_to(JOBS_DIR.resolve()):
             stems.append(path)
     return stems
@@ -1560,7 +1567,7 @@ def _retime_ready(job_dir: Path, lyrics: dict | None) -> None:
             raise HTTPException(status_code=404, detail="no lyrics")
     if not lyric_pairs(lyrics):
         raise HTTPException(status_code=409, detail="lyrics have no lines")
-    vocals = (job_dir / "stems" / "vocals.wav").resolve()
+    vocals = stem_path(job_dir / "stems", "vocals").resolve()
     if not vocals.is_relative_to(JOBS_DIR.resolve()) or not vocals.is_file():
         raise HTTPException(status_code=409, detail="no vocals stem")
 

app/api/stems.py

@@ -35,6 +35,7 @@
 )
 from app.core.registry import get as registry_get
 from app.core.settings import get_export_sample_rate
+from app.core.stem_files import stem_ext, stem_media_type, stem_path
 from app.core.takes import take_dir
 from app.pipeline.audio_stats import vocal_envelope
 from app.pipeline.click_render import cache_key as click_cache_key
@@ -349,7 +350,8 @@ def _validate_stem_path(job_id: str, name: str):
     job = registry_get(job_id)
     if job is None or job.status != "done":
         raise HTTPException(status_code=404, detail="job not ready")
-    path = (JOBS_DIR / job_id / "stems" / f"{name}.wav").resolve()
+    # WAV or MP3, whichever this song keeps (#783).
+    path = stem_path(JOBS_DIR / job_id / "stems", name).resolve()
     if not path.is_file() or not path.is_relative_to(JOBS_DIR.resolve()):
         raise HTTPException(status_code=404, detail="stem not found")
     return path
@@ -1044,7 +1046,7 @@ def _expand_render_lanes(
         # pointing at whichever component was tried last.
         paths: list[Path] = []
         for comp in components:
-            comp_path = (JOBS_DIR / job_id / "stems" / f"{comp}.wav").resolve()
+            comp_path = stem_path(JOBS_DIR / job_id / "stems", comp).resolve()
             if not comp_path.is_file() or not comp_path.is_relative_to(JOBS_DIR.resolve()):
                 paths = []
                 break
@@ -1071,7 +1073,7 @@ def _job_original_set(job_id: str) -> frozenset[str] | None:
     """The stems the job's own original.wav holds (make_original_track: every
     base stem not selected), or None when it has no such file."""
     job = registry_get(job_id)
-    path = (JOBS_DIR / job_id / "stems" / "original.wav").resolve()
+    path = stem_path(JOBS_DIR / job_id / "stems", "original").resolve()
     if job is None or not path.is_file() or not path.is_relative_to(JOBS_DIR.resolve()):
         return None
     selected = set(job.selected_stems)
@@ -1128,7 +1130,7 @@ def _expand_mode_lanes(
             continue
         paths: list[Path] = []
         for comp in components:
-            comp_path = (JOBS_DIR / job_id / "stems" / f"{comp}.wav").resolve()
+            comp_path = stem_path(JOBS_DIR / job_id / "stems", comp).resolve()
             if not comp_path.is_file() or not comp_path.is_relative_to(JOBS_DIR.resolve()):
                 paths = []
                 break
@@ -1498,7 +1500,7 @@ async def get_vocal_envelope(job_id: str) -> Response:
     if job is None or job.status != "done":
         raise HTTPException(status_code=404, detail="job not ready")
     stems_dir = (JOBS_DIR / job_id / "stems").resolve()
-    src = stems_dir / "vocals.wav"
+    src = stem_path(stems_dir, "vocals")
     dest = stems_dir / _VOCAL_ENVELOPE_FILE
     jobs_root = JOBS_DIR.resolve()
     if not src.is_file() or not src.is_relative_to(jobs_root) or not dest.is_relative_to(jobs_root):
@@ -1557,12 +1559,16 @@ async def get_stem(
     click_accent: int = Query(default=-1, ge=-1, le=32, description="-1 auto, 0 off, N per bar"),
     click_gain: float = Query(default=0.6, ge=0, le=4, description="Click level"),
 ) -> FileResponse | StreamingResponse:
-    """Download a WAV stem. Optional ?start=&end= trims to a time region."""
+    """A stem, as the song stores it: WAV, or MP3 for a song stored as MP3
+    (#783), with its own type and file name whatever this URL says. Optional
+    ?start=&end= trims to a time region, always rendered as WAV."""
     path = _validate_stem_path(job_id, name)
 
     if start is None and end is None:
         return FileResponse(
-            path, media_type="audio/wav", filename=_stem_download_name(job_id, name, "wav")
+            path,
+            media_type=stem_media_type(path),
+            filename=_stem_download_name(job_id, name, stem_ext(path)),
         )
 
     if start is None or end is None or start >= end:
@@ -2211,17 +2217,22 @@ def _arcname(prefix: str, name: str, ext: str) -> str:
 def _build_stems_zip(
     sources: list[tuple[str, Path]], fmt: str, dest: Path, prefix: str = ""
 ) -> None:
-    """Blocking: write the stems into a ZIP. WAV files are stored as-is; MP3,
-    FLAC, and OGG are transcoded per stem via ffmpeg. ZIP_STORED throughout - audio doesn't
-    meaningfully compress, and STORED keeps the build fast. Runs in a thread."""
-    if fmt == "wav":
+    """Blocking: write the stems into a ZIP. A file already in `fmt` is stored
+    as-is (a WAV in a WAV zip, and an MP3-stored stem in an MP3 zip, which is
+    not encoded a second time); anything else is transcoded per stem via
+    ffmpeg. ZIP_STORED throughout - audio doesn't meaningfully compress, and
+    STORED keeps the build fast. Runs in a thread."""
+    if all(p.suffix.lower() == f".{fmt}" for _name, p in sources):
         with zipfile.ZipFile(dest, "w", zipfile.ZIP_STORED) as zf:
             for name, p in sources:
-                zf.write(p, arcname=_arcname(prefix, name, "wav"))
+                zf.write(p, arcname=_arcname(prefix, name, fmt))
         return
     encode = _ENCODE_ARGS[fmt]
     with tempfile.TemporaryDirectory() as td, zipfile.ZipFile(dest, "w", zipfile.ZIP_STORED) as zf:
         for name, p in sources:
+            if p.suffix.lower() == f".{fmt}":
+                zf.write(p, arcname=_arcname(prefix, name, fmt))
+                continue
             out = os.path.join(td, f"{name}.{fmt}")
             cmd = [
                 ffmpeg_executable(),
@@ -2292,7 +2303,7 @@ async def get_all_stems_zip(
 
     sources: list[tuple[str, Path]] = []
     for name in wanted:
-        p = (stems_dir / f"{name}.wav").resolve()
+        p = stem_path(stems_dir, name).resolve()
         if p.is_file() and p.is_relative_to(jobs_root):
             sources.append((name, p))
     labels = {t.get("id"): t.get("label") for t in job.takes}

app/api/takes.py

@@ -91,6 +91,7 @@
 from app.core.registry import get as registry_get
 from app.core.registry import persist as registry_persist
 from app.core.settings import get_export_sample_rate
+from app.core.stem_files import stem_path
 from app.core.takes import (
     camera_file,
     make_track,
@@ -268,7 +269,9 @@ def _reference_stem(job_id: str) -> Path | None:
     if not stems_dir.is_relative_to(JOBS_DIR.resolve()):
         return None
     for name in (*STEM_NAMES, *EXTRA_STEM_NAMES, "original"):
-        path = stems_dir / f"{name}.wav"
+        # WAV or MP3 (#783): an MP3 stem decodes to the WAV's exact frame
+        # count, which stem_storage verified before keeping it.
+        path = stem_path(stems_dir, name)
         if path.is_file():
             return path
     return None

app/core/config.py

@@ -403,6 +403,19 @@ def js_solver_available() -> bool:
 # A backing track without one stem (app/pipeline/backing.py): a sum of five
 # WAVs, seconds for a song; the ceiling only guards a stuck disk.
 TIMEOUT_BACKING = _env_int("STEMDECK_TIMEOUT_BACKING", 120)
+
+# How a finished song's stems are kept on disk (#783): "wav" (16-bit PCM, as
+# separated) or "mp3" (CBR libmp3lame at one of STEM_MP3_KBPS_CHOICES). The
+# default for an install with no saved choice and no env var. One line to
+# change. 128 kbps is the maintainer's choice: stems summed again in the
+# mixer carry the coding noise of each one.
+DEFAULT_STEM_STORAGE: tuple[str, int] = ("mp3", 128)
+STEM_FORMAT_CHOICES: tuple[str, ...] = ("wav", "mp3")
+STEM_MP3_KBPS_CHOICES: tuple[int, ...] = (128, 192, 256)
+# The MP3 step at the end of the pipeline (app/pipeline/stem_storage.py): one
+# ffmpeg per stem, a few seconds each. Terminated when ffmpeg is silent this
+# long; the song then keeps its WAVs.
+TIMEOUT_STEM_TRANSCODE_STALL = _env_int("STEMDECK_TIMEOUT_STEM_TRANSCODE_STALL", 120)
 # Video render of a take (libx264 + AAC). Slower than any audio export: a
 # 5-minute 1080p30 encode runs about 1x to 3x realtime on a desktop CPU.
 TIMEOUT_VIDEO_RENDER = _env_int("STEMDECK_TIMEOUT_VIDEO_RENDER", 3600)

app/core/models.py

@@ -338,6 +338,13 @@ class Job:
     # Wall-clock seconds per pipeline stage ({"download": 12.3, ...}); written
     # to metadata.json and the one-line completion summary in the log.
     stage_timings: dict[str, float] | None = None
+    # How this song's stems are kept on disk (#783): "wav" or "mp3", set when
+    # the job finishes from the Stem storage format setting of that moment,
+    # with the MP3 bitrate (kbps) or None. None for a song finished before the
+    # setting existed, whose stems are WAV. Readers do not branch on it: they
+    # resolve each file through app.core.stem_files, which looks at the disk.
+    stem_format: str | None = None
+    stem_mp3_kbps: int | None = None
     # On-demand lead/backing vocal split (#275) -- a post-hoc action on an
     # already-"done" job, not part of the main pipeline. "none" until the user
     # asks for it; "error" leaves the job's base stems untouched (see
@@ -431,6 +438,8 @@ def to_state(self) -> dict[str, Any]:
             "compute_device": self.compute_device,
             "gpu_fallback": self.gpu_fallback,
             "stage_timings": self.stage_timings,
+            "stem_format": self.stem_format,
+            "stem_mp3_kbps": self.stem_mp3_kbps,
             "vocal_split": self.vocal_split,
             "trashed_at": self.trashed_at,
             "favorite": self.favorite,

app/core/registry.py

@@ -14,6 +14,7 @@
 from app.core.config import DEMUCS_MODEL, EXTRA_STEM_NAMES, JOB_ID_RE, LYRICS_FILE, STEM_NAMES
 from app.core.models import Job, clean_artist, clean_audio_tags, clean_identity, clean_work
 from app.core.settings import quarantine_corrupt
+from app.core.stem_files import stem_exists, stem_url
 from app.core.takes import reconcile_tracks, recover_takes
 
 logger = logging.getLogger("stemdeck.registry")
@@ -566,22 +567,23 @@ def _recover_done_job(job_dir: Path) -> Job | None:
     stems_dir = job_dir / "stems"
     if not stems_dir.is_dir():
         return None
+    # WAV or MP3, whichever the song was stored as (#783).
     stems = [
-        {"name": name, "url": f"/api/jobs/{job_dir.name}/stems/{name}.wav"}
+        {"name": name, "url": stem_url(job_dir.name, stems_dir, name)}
         for name in ("original", *STEM_NAMES, *EXTRA_STEM_NAMES)
-        if (stems_dir / f"{name}.wav").is_file()
+        if stem_exists(stems_dir, name)
     ]
     if not stems:
         return None
     mix_url = None
-    if (stems_dir / "mix.wav").is_file():
-        mix_url = f"/api/jobs/{job_dir.name}/stems/mix.wav"
+    if stem_exists(stems_dir, "mix"):
+        mix_url = stem_url(job_dir.name, stems_dir, "mix")
     selected = [stem["name"] for stem in stems if stem["name"] in STEM_NAMES] or list(STEM_NAMES)
     # A restart between the split finishing and its next registry persist
     # would otherwise report the job as never split -- derive from disk the
     # same way `stems` above does, so the recovered library entry doesn't
     # regress (#275).
-    has_split = all((stems_dir / f"{name}.wav").is_file() for name in EXTRA_STEM_NAMES)
+    has_split = all(stem_exists(stems_dir, name) for name in EXTRA_STEM_NAMES)
     vocal_split = "done" if has_split else "none"
     meta_path = job_dir / "metadata.json"
     meta: dict = {}
@@ -633,6 +635,10 @@ def _recover_done_job(job_dir: Path) -> Job | None:
         # The file is the record: metadata.json predates it for most jobs.
         has_lyrics=(job_dir / LYRICS_FILE).is_file(),
         vocal_split=vocal_split,
+        stem_format=meta.get("stem_format") if meta.get("stem_format") in ("wav", "mp3") else None,
+        stem_mp3_kbps=meta.get("stem_mp3_kbps")
+        if isinstance(meta.get("stem_mp3_kbps"), int)
+        else None,
         # Takes are files first: their folders survive whatever the registry
         # lost, so the lanes come back with the job.
         takes=recover_takes(job_dir),

app/core/settings.py

@@ -18,6 +18,8 @@
 - `discogs_token`     - the user's Discogs token, for band profiles.
 - `transcribe_lyrics` - Whisper lyrics when none are found: auto | on | off.
 - `video_encoder`     - H.264 encoder of the take render: auto | cpu | an encoder.
+- `stem_format`       - how new songs' stems are stored: wav | mp3 (#783).
+- `stem_mp3_kbps`     - the MP3 bitrate when stem_format is mp3: 128 | 192 | 256.
 
 Defaults fall back to the config.py constants (which honor their env vars), so
 nothing changes until the user overrides a value.
@@ -36,8 +38,11 @@
 
 from app.core.config import (
     DATA_DIR,
+    DEFAULT_STEM_STORAGE,
     MAX_DURATION_SEC,
     PLAYLIST_MAX_ITEMS,
+    STEM_FORMAT_CHOICES,
+    STEM_MP3_KBPS_CHOICES,
     VIDEO_ENCODER_CHOICES,
     VIDEO_MAX_HEIGHT,
     available_torch_devices,
@@ -635,6 +640,73 @@ def set_separation_quality(value: str) -> str:
         return choice
 
 
+# ── stem_format / stem_mp3_kbps ──
+# How a song's stems are stored once it finishes (#783): "wav" or "mp3" at a
+# CBR bitrate. Read when a job finishes (app/pipeline/stem_storage.py), so a
+# change applies to the next song without a restart; songs already in the
+# library keep whatever they were stored as. STEMDECK_STEM_FORMAT and
+# STEMDECK_STEM_MP3_KBPS seed the defaults for Docker and Unraid; a choice
+# saved in Settings wins over them, as for separation_quality.
+def _default_stem_format() -> str:
+    env = os.environ.get("STEMDECK_STEM_FORMAT", "").strip().lower()
+    return env if env in STEM_FORMAT_CHOICES else DEFAULT_STEM_STORAGE[0]
+
+
+def _default_stem_mp3_kbps() -> int:
+    raw = os.environ.get("STEMDECK_STEM_MP3_KBPS", "").strip().lower().removesuffix("k")
+    try:
+        kbps = int(raw)
+    except ValueError:
+        return DEFAULT_STEM_STORAGE[1]
+    return kbps if kbps in STEM_MP3_KBPS_CHOICES else DEFAULT_STEM_STORAGE[1]
+
+
+def get_stem_format() -> str:
+    with _LOCK:
+        v = _ensure().get("stem_format")
+        return v if isinstance(v, str) and v in STEM_FORMAT_CHOICES else _default_stem_format()
+
+
+def get_stem_mp3_kbps() -> int:
+    with _LOCK:
+        v = _ensure().get("stem_mp3_kbps")
+        ok = isinstance(v, int) and not isinstance(v, bool) and v in STEM_MP3_KBPS_CHOICES
+        return v if ok else _default_stem_mp3_kbps()
+
+
+def check_stem_format(value: object) -> str:
+    """The whitelisted format for ``value``, or ValueError (never echoing it)."""
+    choice = value.strip().lower() if isinstance(value, str) else ""
+    if choice not in STEM_FORMAT_CHOICES:
+        raise ValueError("stem_format must be one of: " + ", ".join(STEM_FORMAT_CHOICES))
+    return STEM_FORMAT_CHOICES[STEM_FORMAT_CHOICES.index(choice)]
+
+
+def check_stem_mp3_kbps(value: object) -> int:
+    """The whitelisted bitrate for ``value``, or ValueError (never echoing it)."""
+    kbps = value if isinstance(value, int) and not isinstance(value, bool) else None
+    if kbps not in STEM_MP3_KBPS_CHOICES:
+        raise ValueError(
+            "stem_mp3_kbps must be one of: " + ", ".join(map(str, STEM_MP3_KBPS_CHOICES))
+        )
+    return kbps
+
+
+def set_stem_storage(fmt: object = None, kbps: object = None) -> tuple[str, int]:
+    """Persist either or both, after both are validated: a request with one
+    bad value changes nothing. Returns what now applies."""
+    choice = check_stem_format(fmt) if fmt is not None else None
+    rate = check_stem_mp3_kbps(kbps) if kbps is not None else None
+    with _LOCK:
+        if choice is not None:
+            _ensure()["stem_format"] = choice
+        if rate is not None:
+            _ensure()["stem_mp3_kbps"] = rate
+        if choice is not None or rate is not None:
+            _save()
+        return get_stem_format(), get_stem_mp3_kbps()
+
+
 # ── video_encoder ──
 # The H.264 encoder of the take render (app/pipeline/video_encoder.py): "auto"
 # (default) uses a GPU encoder that passed a real test encode on this machine,