From 3180ed5819768440f25d6e94e1463393ad943bea Mon Sep 17 00:00:00 2001 From: Thales <> Date: Fri, 25 Sep 2026 09:41:09 +0100 Subject: [PATCH] Keep yt-dlp's search extractors in packaged builds Packaging prunes yt-dlp down to YouTube and SoundCloud and writes a new extractor registry. That registry left out YoutubeSearchIE, YoutubeSearchURLIE and SoundcloudSearchIE, so every search in a packaged build failed with "No suitable extractor found for URL ytsearch8:...". Source runs never prune, which is why it only showed up in releases. The build-time verify now fails if search is not registered, and two tests check the app's own extractor allowlists and search targets against a pruned tree, so a new allowlist entry cannot get ahead of the prune again. Fixes #692 --- scripts/prune_ytdlp_extractors.py | 21 ++++++++++- tests/test_prune_ytdlp_extractors.py | 56 +++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/scripts/prune_ytdlp_extractors.py b/scripts/prune_ytdlp_extractors.py index 1d8ca421..4407b760 100644 --- a/scripts/prune_ytdlp_extractors.py +++ b/scripts/prune_ytdlp_extractors.py @@ -74,14 +74,27 @@ # The classes _extractors.py must expose. Kept explicit so that a yt-dlp release # renaming one fails here, loudly, instead of at a user's first download. +# +# Every IE_NAME that app/ passes as allowed_extractors must be exported here, +# or yt-dlp finds no extractor for it. A module surviving the prune is not +# enough: only what this registry imports is registered. The search classes +# were missing, and search failed in every packaged build while working from +# source (#692). tests/test_prune_ytdlp_extractors.py checks the app's lists. EXPORTS = { "soundcloud": ( "SoundcloudIE", "SoundcloudPlaylistIE", + "SoundcloudSearchIE", "SoundcloudSetIE", "SoundcloudUserIE", ), - "youtube": ("YoutubeIE", "YoutubePlaylistIE", "YoutubeTabIE"), + "youtube": ( + "YoutubeIE", + "YoutubePlaylistIE", + "YoutubeSearchIE", + "YoutubeSearchURLIE", + "YoutubeTabIE", + ), "generic": ("GenericIE",), } @@ -180,6 +193,12 @@ def verify(site_packages: pathlib.Path) -> None: "'https://youtu.be/dQw4w9WgXcQ'), 'youtu.be URL no longer matches'\n" "assert get_info_extractor('Soundcloud').suitable(" "'https://soundcloud.com/artist/track'), 'SoundCloud URL no longer matches'\n" + # Search is registered separately from the URL extractors, and missing + # it passes every check above (#692). + "assert get_info_extractor('YoutubeSearch').suitable(" + "'ytsearch1:x'), 'YouTube search no longer registered'\n" + "assert get_info_extractor('SoundcloudSearch').suitable(" + "'scsearch1:x'), 'SoundCloud search no longer registered'\n" "print(' verified:', ', '.join(names))\n" ) result = subprocess.run( diff --git a/tests/test_prune_ytdlp_extractors.py b/tests/test_prune_ytdlp_extractors.py index 7eadb4f0..ad7765b8 100644 --- a/tests/test_prune_ytdlp_extractors.py +++ b/tests/test_prune_ytdlp_extractors.py @@ -10,6 +10,7 @@ from __future__ import annotations +import json import os import pathlib import shutil @@ -39,7 +40,7 @@ def pruned(tmp_path_factory) -> pathlib.Path: return site -def _run_in(site: pathlib.Path, code: str) -> subprocess.CompletedProcess: +def _run_in(site: pathlib.Path, code: str, *args: str) -> subprocess.CompletedProcess: """Run code with the pruned tree ahead of the installed one on sys.path. Inheriting the environment rather than replacing it: a bare env breaks @@ -49,7 +50,7 @@ def _run_in(site: pathlib.Path, code: str) -> subprocess.CompletedProcess: env = dict(os.environ) env["PYTHONPATH"] = str(site) return subprocess.run( - [sys.executable, "-c", code], + [sys.executable, "-c", code, *args], capture_output=True, text=True, env=env, @@ -84,6 +85,57 @@ def test_youtube_and_soundcloud_still_match(pruned: pathlib.Path) -> None: assert proc.returncode == 0, proc.stdout + proc.stderr +def _app_allowlists() -> list[list[str]]: + from app.pipeline.download import _ALLOWED_EXTRACTORS, _ALLOWED_PLAYLIST_EXTRACTORS + from app.pipeline.search import _EXTRACTORS + + return [_ALLOWED_EXTRACTORS, _ALLOWED_PLAYLIST_EXTRACTORS, *_EXTRACTORS.values()] + + +def test_every_extractor_the_app_allows_is_registered(pruned: pathlib.Path) -> None: + """An allowlisted name the registry does not export matches nothing. + + Search failed in every packaged build for exactly this reason while working + from source, where nothing is pruned (#692). Reads the app's own lists, so + a new allowlist entry cannot get ahead of the prune. + """ + wanted = {name for allowed in _app_allowlists() for name in allowed} + proc = _run_in( + pruned, + "from yt_dlp.extractor import gen_extractor_classes\n" + "print(','.join(c.IE_NAME.lower() for c in gen_extractor_classes()))\n", + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + names = set(proc.stdout.strip().split(",")) + assert wanted <= names, f"allowlisted but not shipped: {sorted(wanted - names)}" + + +def test_every_search_target_finds_an_extractor(pruned: pathlib.Path) -> None: + """What search.py actually hands yt-dlp, under the allowlist it uses. + + Offline: yt-dlp picks the extractor before any request, which is where + #692 failed with "No suitable extractor found for URL ytsearch8:...". + """ + from app.pipeline.search import _EXTRACTORS, _target + + cases = [ + (_target(source, kind, "purple rain", 8), allowed) + for (source, kind), allowed in _EXTRACTORS.items() + ] + proc = _run_in( + pruned, + "import json, sys\n" + "from yt_dlp import YoutubeDL\n" + "for url, allowed in json.loads(sys.argv[1]):\n" + " ydl = YoutubeDL({'quiet': True, 'allowed_extractors': allowed})\n" + " if not any(ie.suitable(url) for ie in ydl._ies.values()):\n" + " print('no extractor for', url)\n", + json.dumps(cases), + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert proc.stdout == "", proc.stdout + + def test_the_registry_holds_only_what_ships(pruned: pathlib.Path) -> None: proc = _run_in( pruned,