From f7cbc885c190b01dd090aafb976b40877ddc0ebe Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Sun, 16 Aug 2026 03:17:58 +0100 Subject: [PATCH] fix(docker): migrate retained SearXNG settings (#6055) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(docker): migrate retained SearXNG settings Retained nonempty SearXNG settings can miss defaults required by newer pinned images while bypassing the entrypoint's narrow regeneration checks. Add an atomic PyYAML-aware migration to all Compose variants. Preserve existing inheritance choices, custom content, secrets, ownership, and mode while inserting only the missing top-level default-inheritance key. Validated with 39 focused and adjacent tests, compile checks, and fresh and retained pinned-image HTTP 200 gates. Full repository CI remains for the PR. * fix(docker): chmod the settings temp file before chowning it The Compose cap set is `cap_drop: ALL` plus CHOWN/SETGID/SETUID/DAC_OVERRIDE and carries no FOWNER, and searxng's own entrypoint chowns /etc/searxng to searxng:searxng, so every retained settings file belongs to that user by the second boot. Chowning the temporary file first left root unable to chmod it, so the migration exited 1 and `set -eu` killed the container before `exec /usr/local/searxng/entrypoint.sh` — SearXNG never started and odysseus blocked on its healthcheck. Swap the two calls so the chmod lands while the temporary file is still root-owned, and cover the ordering with a test that refuses the chmod once the chown has happened, the way the kernel does. * fix(docker): let searxng boot when the settings migration fails The migration runs under `set -eu`, so any settings file it cannot parse or rewrite took the container down instead of merely going unmigrated. A symlinked /etc/searxng/settings.yml is enough: the migration refuses a non-regular file and searxng, which reads through the symlink perfectly well, never got to start. Guard the call with `|| true` in all three Compose variants. The failure still prints its reason on stderr, and searxng is left to report anything genuinely wrong with the file. --------- Co-authored-by: Léo --- docker-compose.gpu-amd.yml | 5 + docker-compose.gpu-nvidia.yml | 5 + docker-compose.yml | 5 + scripts/migrate_searxng_settings.py | 166 ++++++++++++ tests/test_searxng_settings_migration.py | 323 +++++++++++++++++++++++ 5 files changed, 504 insertions(+) create mode 100644 scripts/migrate_searxng_settings.py create mode 100644 tests/test_searxng_settings_migration.py diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml index 53b0d8d63..85d872b58 100644 --- a/docker-compose.gpu-amd.yml +++ b/docker-compose.gpu-amd.yml @@ -130,12 +130,17 @@ services: fi sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml fi + # Advisory: a settings file the migration cannot parse or rewrite must + # not be what stops searxng from booting. It explains itself on stderr + # and we carry on, letting searxng report anything genuinely wrong. + /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true exec /usr/local/searxng/entrypoint.sh ports: - "127.0.0.1:8080:8080" volumes: - searxng-data:/etc/searxng - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z + - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z environment: - SEARXNG_BASE_URL=http://localhost:8080/ - SEARXNG_SECRET=${SEARXNG_SECRET:-} diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml index 2b7f11f44..a367be001 100644 --- a/docker-compose.gpu-nvidia.yml +++ b/docker-compose.gpu-nvidia.yml @@ -133,12 +133,17 @@ services: fi sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml fi + # Advisory: a settings file the migration cannot parse or rewrite must + # not be what stops searxng from booting. It explains itself on stderr + # and we carry on, letting searxng report anything genuinely wrong. + /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true exec /usr/local/searxng/entrypoint.sh ports: - "127.0.0.1:8080:8080" volumes: - searxng-data:/etc/searxng - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z + - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z environment: - SEARXNG_BASE_URL=http://localhost:8080/ - SEARXNG_SECRET=${SEARXNG_SECRET:-} diff --git a/docker-compose.yml b/docker-compose.yml index c23d51318..39774fb53 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -111,12 +111,17 @@ services: fi sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml fi + # Advisory: a settings file the migration cannot parse or rewrite must + # not be what stops searxng from booting. It explains itself on stderr + # and we carry on, letting searxng report anything genuinely wrong. + /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true exec /usr/local/searxng/entrypoint.sh ports: - "127.0.0.1:8080:8080" volumes: - searxng-data:/etc/searxng - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z + - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z environment: - SEARXNG_BASE_URL=http://localhost:8080/ - SEARXNG_SECRET=${SEARXNG_SECRET:-} diff --git a/scripts/migrate_searxng_settings.py b/scripts/migrate_searxng_settings.py new file mode 100644 index 000000000..4b58e2efc --- /dev/null +++ b/scripts/migrate_searxng_settings.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Make retained SearXNG settings inherit defaults without replacing them.""" + +from __future__ import annotations + +import os +import stat +import sys +import tempfile +from pathlib import Path + +import yaml +from yaml.nodes import MappingNode +from yaml.tokens import BlockMappingStartToken, FlowMappingStartToken + + +_UTF8_BOM = b"\xef\xbb\xbf" + + +def _parse_root_mapping(text: str) -> tuple[MappingNode | None, dict]: + """Parse settings with the same safe YAML semantics SearXNG uses.""" + try: + loaded = yaml.safe_load(text) + node = yaml.compose(text, Loader=yaml.SafeLoader) + except yaml.YAMLError: + raise ValueError("settings file is not valid single-document YAML") from None + + if loaded is None and node is None: + return None, {} + if not isinstance(loaded, dict) or not isinstance(node, MappingNode): + raise ValueError("settings root is not a mapping") + return node, loaded + + +def _flow_mapping_start(text: str) -> int: + """Return the root flow mapping's opening-brace character offset.""" + try: + for token in yaml.scan(text, Loader=yaml.SafeLoader): + if isinstance(token, FlowMappingStartToken): + return token.start_mark.index + except yaml.YAMLError: + pass + raise ValueError("flow-style settings mapping has no opening brace") + + +def _newline_for(contents: bytes) -> bytes: + first_lf = contents.find(b"\n") + if first_lf > 0 and contents[first_lf - 1 : first_lf + 1] == b"\r\n": + return b"\r\n" + return b"\n" + + +def _block_mapping_position(text: str, root: MappingNode | None) -> tuple[int, int]: + """Return a safe character offset and indent for a root block mapping key.""" + if root is None: + return len(text), 0 + + try: + for token in yaml.scan(text, Loader=yaml.SafeLoader): + if not isinstance(token, BlockMappingStartToken): + continue + line_start = token.start_mark.index - token.start_mark.column + if not text[line_start : token.start_mark.index].strip(): + return line_start, token.start_mark.column + return root.end_mark.index, token.start_mark.column + except yaml.YAMLError: + pass + return root.end_mark.index, root.start_mark.column + + +def _add_block_default_inheritance( + contents: bytes, text: str, root: MappingNode | None +) -> bytes: + newline = _newline_for(contents) + character_offset, indent_width = _block_mapping_position(text, root) + bom_length = len(_UTF8_BOM) if contents.startswith(_UTF8_BOM) else 0 + offset = bom_length + len(text[:character_offset].encode("utf-8")) + separator = b"" + if offset not in (0, bom_length) and not contents[:offset].endswith((b"\n", b"\r")): + separator = newline + addition = ( + separator + + b" " * indent_width + + b"use_default_settings: true" + + newline + ) + return contents[:offset] + addition + contents[offset:] + + +def migrate_settings(path: Path) -> bool: + """Add the missing inheritance key atomically; return whether the file changed.""" + source_stat = path.lstat() + if not stat.S_ISREG(source_stat.st_mode): + raise ValueError(f"settings path is not a regular file: {path}") + + contents = path.read_bytes() + if not contents: + return False + + text = contents.decode("utf-8-sig") + root, loaded = _parse_root_mapping(text) + if "use_default_settings" in loaded: + return False + + if root is not None and root.flow_style: + start = _flow_mapping_start(text) + bom_length = len(_UTF8_BOM) if contents.startswith(_UTF8_BOM) else 0 + offset = bom_length + len(text[: start + 1].encode("utf-8")) + separator = b", " if root.value else b"" + updated = ( + contents[:offset] + + b"use_default_settings: true" + + separator + + contents[offset:] + ) + else: + updated = _add_block_default_inheritance(contents, text, root) + fd, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.odysseus-", dir=path.parent + ) + temporary = Path(temporary_name) + try: + # chmod before chown: the Compose cap set is `cap_drop: ALL` plus + # CHOWN/SETGID/SETUID/DAC_OVERRIDE, with no FOWNER. Once the temporary + # file belongs to searxng:searxng — which every retained settings file + # does, because searxng's entrypoint chowns /etc/searxng — root can no + # longer chmod it and the migration dies with EPERM. + os.fchmod(fd, stat.S_IMODE(source_stat.st_mode)) + os.fchown(fd, source_stat.st_uid, source_stat.st_gid) + with os.fdopen(fd, "wb") as handle: + fd = -1 + handle.write(updated) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + if fd >= 0: + os.close(fd) + temporary.unlink(missing_ok=True) + return True + + +def main(argv: list[str]) -> int: + if len(argv) > 2: + print(f"usage: {Path(argv[0]).name} [settings.yml]", file=sys.stderr) + return 2 + + path = Path(argv[1]) if len(argv) == 2 else Path("/etc/searxng/settings.yml") + try: + changed = migrate_settings(path) + except (OSError, UnicodeError, ValueError) as exc: + print(f"SearXNG settings migration failed: {exc}", file=sys.stderr) + return 1 + + if changed: + print("Added use_default_settings inheritance to retained SearXNG settings") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/tests/test_searxng_settings_migration.py b/tests/test_searxng_settings_migration.py new file mode 100644 index 000000000..75cef7775 --- /dev/null +++ b/tests/test_searxng_settings_migration.py @@ -0,0 +1,323 @@ +import importlib.util +import os +import stat +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parent.parent +MIGRATION = ROOT / "scripts" / "migrate_searxng_settings.py" +COMPOSE_FILES = ( + ROOT / "docker-compose.yml", + ROOT / "docker-compose.gpu-amd.yml", + ROOT / "docker-compose.gpu-nvidia.yml", +) + + +def _run(path: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(MIGRATION), str(path)], + capture_output=True, + check=False, + text=True, + ) + + +def _load_migration_module(): + spec = importlib.util.spec_from_file_location("searxng_settings_migration", MIGRATION) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_retained_settings_gain_defaults_without_changing_custom_content(tmp_path): + settings = tmp_path / "settings.yml" + retained = ( + b"# retained deployment settings\n" + b"server:\n" + b' secret_key: "representative-retained-secret"\n' + b"search:\n" + b" safe_search: 1\n" + b" formats:\n" + b" - html\n" + b" - json\n" + b"ui:\n" + b" static_use_hash: true\n" + ) + settings.write_bytes(retained) + settings.chmod(0o640) + before = settings.stat() + + first = _run(settings) + + assert first.returncode == 0, first.stderr + assert "representative-retained-secret" not in first.stdout + first.stderr + assert settings.read_bytes() == ( + b"# retained deployment settings\n" + b"use_default_settings: true\n" + + retained.removeprefix(b"# retained deployment settings\n") + ) + after = settings.stat() + assert stat.S_IMODE(after.st_mode) == 0o640 + assert (after.st_uid, after.st_gid) == (before.st_uid, before.st_gid) + + migrated = settings.read_bytes() + second = _run(settings) + + assert second.returncode == 0, second.stderr + assert settings.read_bytes() == migrated + assert second.stdout == "" + + +def test_fresh_generated_settings_are_left_byte_identical(tmp_path): + settings = tmp_path / "settings.yml" + generated = (ROOT / "config" / "searxng" / "settings.yml").read_bytes().replace( + b"__SEARXNG_SECRET__", b"representative-generated-secret" + ) + settings.write_bytes(generated) + + result = _run(settings) + + assert result.returncode == 0, result.stderr + assert settings.read_bytes() == generated + assert result.stdout == "" + + +@pytest.mark.parametrize( + "key", + ( + b"use_default_settings: false\n", + b"use_default_settings:\n engines:\n keep_only:\n - brave\n", + b"'use_default_settings': true\n", + b'"use_default_settings": true\n', + ), +) +def test_explicit_top_level_setting_is_not_overridden(tmp_path, key): + settings = tmp_path / "settings.yml" + original = key + b"server:\n secret_key: retained\n" + settings.write_bytes(original) + + result = _run(settings) + + assert result.returncode == 0, result.stderr + assert settings.read_bytes() == original + + +def test_key_is_inserted_inside_explicit_yaml_document(tmp_path): + settings = tmp_path / "settings.yml" + original = b"\xef\xbb\xbf# header\r\n---\r\nserver:\r\n secret_key: retained\r\n" + settings.write_bytes(original) + + result = _run(settings) + + assert result.returncode == 0, result.stderr + assert settings.read_bytes() == ( + b"\xef\xbb\xbf# header\r\n---\r\nuse_default_settings: true\r\n" + b"server:\r\n secret_key: retained\r\n" + ) + + +def test_indented_root_mapping_keeps_its_existing_indent(tmp_path): + settings = tmp_path / "settings.yml" + original = b" server:\n secret_key: retained\n search:\n safe_search: 1\n" + settings.write_bytes(original) + + result = _run(settings) + + assert result.returncode == 0, result.stderr + assert settings.read_bytes() == b" use_default_settings: true\n" + original + + +@pytest.mark.parametrize( + "property_line", + (b"!!map\n", b"&settings\n", b"--- !!map\n"), +) +def test_block_mapping_properties_stay_attached_to_the_root(tmp_path, property_line): + settings = tmp_path / "settings.yml" + mapping = b"server:\n secret_key: retained\nsearch:\n safe_search: 1\n" + original = property_line + mapping + settings.write_bytes(original) + original_data = yaml.safe_load(original) + + result = _run(settings) + + assert result.returncode == 0, result.stderr + expected = property_line + b"use_default_settings: true\n" + mapping + assert settings.read_bytes() == expected + migrated_data = yaml.safe_load(settings.read_bytes()) + assert migrated_data.pop("use_default_settings") is True + assert migrated_data == original_data + + +def test_flow_mapping_gains_default_inheritance_without_reformatting(tmp_path): + settings = tmp_path / "settings.yml" + original = b"{server: {secret_key: retained}, search: {safe_search: 1}}\n" + settings.write_bytes(original) + + result = _run(settings) + + assert result.returncode == 0, result.stderr + assert settings.read_bytes() == ( + b"{use_default_settings: true, " + original.removeprefix(b"{") + ) + + +@pytest.mark.parametrize( + ("original", "expected"), + ( + (b"{}\n", b"{use_default_settings: true}\n"), + ( + b"{server: {use_default_settings: false, secret_key: retained}}\n", + b"{use_default_settings: true, " + b"server: {use_default_settings: false, secret_key: retained}}\n", + ), + ( + b"!!map {server: {secret_key: retained}}\n", + b"!!map {use_default_settings: true, server: {secret_key: retained}}\n", + ), + ), +) +def test_other_flow_mapping_shapes_gain_only_the_root_key(tmp_path, original, expected): + settings = tmp_path / "settings.yml" + settings.write_bytes(original) + + result = _run(settings) + + assert result.returncode == 0, result.stderr + assert settings.read_bytes() == expected + + +@pytest.mark.parametrize( + "original", + ( + b"{use_default_settings: true, server: {secret_key: retained}}\n", + b'{"use_default_settings": {engines: {keep_only: [brave]}}}\n', + b"{use_default_settings: true, server: {secret_key: abc#def}}\n", + b"{use_default_settings: true, server: {secret_key: 'ab''cd'}}\n", + ), +) +def test_flow_mapping_with_existing_setting_is_left_untouched(tmp_path, original): + settings = tmp_path / "settings.yml" + settings.write_bytes(original) + + result = _run(settings) + + assert result.returncode == 0, result.stderr + assert settings.read_bytes() == original + + +@pytest.mark.parametrize( + "original", + ( + b"%YAML 1.1\n---\nuse_default_settings: true\n" + b"server:\n secret_key: retained\n", + b"use_default_settings: true\nserver:\n secret_key: retained\n...\n", + ), +) +def test_valid_document_metadata_with_existing_key_is_left_untouched( + tmp_path, original +): + settings = tmp_path / "settings.yml" + settings.write_bytes(original) + + result = _run(settings) + + assert result.returncode == 0, result.stderr + assert settings.read_bytes() == original + + +def test_invalid_utf8_is_not_replaced(tmp_path): + settings = tmp_path / "settings.yml" + original = b"server:\n secret_key: \xff\n" + settings.write_bytes(original) + before = os.stat(settings) + + result = _run(settings) + + after = os.stat(settings) + assert result.returncode == 1 + assert settings.read_bytes() == original + assert after.st_ino == before.st_ino + + +def test_temporary_file_is_chmodded_before_it_is_chowned(tmp_path, monkeypatch): + # The Compose cap set is `cap_drop: ALL` plus CHOWN/SETGID/SETUID/ + # DAC_OVERRIDE and carries no FOWNER, and searxng's entrypoint chowns + # /etc/searxng to searxng:searxng, so every retained settings file is owned + # by that user. Chowning the temporary file first therefore makes the chmod + # that follows fail with EPERM, and `set -eu` in the Compose entrypoint + # turns that into a container that never starts. The guard below refuses + # the chmod once the chown has landed, the way the kernel does. + migration = _load_migration_module() + settings = tmp_path / "settings.yml" + settings.write_bytes(b"server:\n secret_key: retained\n") + settings.chmod(0o640) + calls = [] + real_fchmod = migration.os.fchmod + real_fchown = migration.os.fchown + + def guarded_fchmod(fd, mode): + if "fchown" in calls: + raise PermissionError(1, "Operation not permitted") + calls.append("fchmod") + return real_fchmod(fd, mode) + + def recording_fchown(fd, uid, gid): + calls.append("fchown") + return real_fchown(fd, uid, gid) + + monkeypatch.setattr(migration.os, "fchmod", guarded_fchmod) + monkeypatch.setattr(migration.os, "fchown", recording_fchown) + + assert migration.migrate_settings(settings) is True + + assert calls == ["fchmod", "fchown"] + assert stat.S_IMODE(settings.stat().st_mode) == 0o640 + assert settings.read_bytes() == ( + b"use_default_settings: true\nserver:\n secret_key: retained\n" + ) + + +def test_replace_failure_preserves_original_and_removes_temporary_file( + tmp_path, monkeypatch +): + migration = _load_migration_module() + settings = tmp_path / "settings.yml" + original = b"server:\n secret_key: retained\n" + settings.write_bytes(original) + before = settings.stat() + + def fail_replace(_source, _destination): + raise OSError("injected replace failure") + + monkeypatch.setattr(migration.os, "replace", fail_replace) + + with pytest.raises(OSError, match="injected replace failure"): + migration.migrate_settings(settings) + + after = settings.stat() + assert settings.read_bytes() == original + assert after.st_ino == before.st_ino + assert list(tmp_path.iterdir()) == [settings] + + +@pytest.mark.parametrize("compose_file", COMPOSE_FILES, ids=lambda path: path.name) +def test_compose_runs_migration_for_all_variants(compose_file): + text = compose_file.read_text(encoding="utf-8") + + # The `|| true` is load-bearing: the entrypoint runs under `set -eu`, so + # without it a settings file the migration cannot parse or rewrite stops + # searxng from starting at all instead of merely going unmigrated. + assert ( + "/usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py " + "/etc/searxng/settings.yml || true" in text + ) + assert ( + "./scripts/migrate_searxng_settings.py:" + "/tmp/migrate-searxng-settings.py:ro,z" in text + )