fix(cookbook): preserve endpoints across model launches (#6969)

* fix(cookbook): match stale endpoints by launched model, not display name

After registering an LLM endpoint, the Cookbook deleted every other local-*
endpoint with the same display name at a different URL. The display name is
the repo short name, which repos from different orgs share (for example
meta-llama/Llama-3.1-8B-Instruct and unsloth/Llama-3.1-8B-Instruct), so
launching one deleted the endpoint of the other while its server was running.

Match stale endpoints by the full model id they were last launched for,
which is the last entry of their pinned_models. Reusing an endpoint now
moves the launched model to the end of its pins so that entry stays current.

* fix(cookbook): guard endpoint cleanup by serve ownership

Preserve independently launched models instead of sweeping endpoints by display name or mutable model pins.

Bind Cookbook endpoint registrations to their serve sessions and prevent stale watchdogs or scheduled stops from deleting newer registrations.

Reuse the existing endpoint deletion contract to preserve settings, preferences, session cleanup, and cache invalidation.

Cover LLM and image endpoint lifecycle regressions.

---------

Co-authored-by: Alexandre Teixeira <general@alexandreteixeira.dev>

tests/test_cookbook_endpoint_registration.py

@@ -1,5 +1,9 @@
 from pathlib import Path
 
+import pytest
+
+from tests.helpers.database import disposable_database
+
 
 ROOT = Path(__file__).resolve().parents[1]
 COOKBOOK_RUNNING = ROOT / "static" / "js" / "cookbookRunning.js"
@@ -28,3 +32,416 @@ def test_cookbook_advertised_bind_urls_keep_connectable_host():
     assert "function _endpointFromAdvertisedUrl" in src
     assert "_isAnyBindHost(u.hostname) ? currentHost" in src
     assert "host = u.hostname || host;" not in src
+
+
+@pytest.fixture
+def endpoint_db(tmp_path, monkeypatch):
+    import core.database as database
+    import routes.model_routes as model_routes
+
+    with disposable_database(tmp_path) as factory:
+        monkeypatch.setattr(database, "SessionLocal", factory)
+        monkeypatch.setattr(model_routes, "_probe_endpoint", lambda *args, **kwargs: None)
+        yield factory
+
+
+def _launch_llm(
+    repo_id: str,
+    port: int,
+    serve_session_id: str | None = None,
+) -> str:
+    import routes.cookbook_routes as cookbook_routes
+    from routes.cookbook_helpers import ServeRequest
+
+    router = cookbook_routes.setup_cookbook_routes()
+    serve = next(
+        route.endpoint for route in router.routes
+        if route.path == "/api/model/serve" and "POST" in route.methods
+    )
+    cells = dict(zip(serve.__code__.co_freevars, serve.__closure__ or ()))
+    register = cells["_auto_register_llm_endpoint"].cell_contents
+    endpoint_id = register(
+        ServeRequest(repo_id=repo_id, cmd=f"vllm serve {repo_id} --port {port}"),
+        None,
+        serve_session_id=serve_session_id,
+    )
+    assert endpoint_id
+    return endpoint_id
+
+
+def _endpoint(factory, endpoint_id: str):
+    import core.database as database
+
+    with factory() as db:
+        return db.get(database.ModelEndpoint, endpoint_id)
+
+
+@pytest.mark.parametrize("reuse", [False, True])
+def test_launch_keeps_endpoint_of_other_model_with_same_short_name(endpoint_db, reuse):
+    if reuse:
+        mirror = _launch_llm("unsloth/Llama-3.1-8B-Instruct", 8002)
+    original = _launch_llm("meta-llama/Llama-3.1-8B-Instruct", 8001)
+
+    launched = _launch_llm("unsloth/Llama-3.1-8B-Instruct", 8002)
+
+    if reuse:
+        assert launched == mirror
+    assert _endpoint(endpoint_db, original) is not None
+    assert _endpoint(endpoint_db, launched) is not None
+
+
+@pytest.mark.parametrize("reuse", [False, True])
+def test_relaunch_on_another_port_preserves_existing_endpoint(endpoint_db, reuse):
+    if reuse:
+        current = _launch_llm("meta-llama/Llama-3.1-8B-Instruct", 8002)
+    stale = _launch_llm("Qwen/Qwen3-8B", 8001)
+
+    launched = _launch_llm("Qwen/Qwen3-8B", 8002)
+
+    if reuse:
+        assert launched == current
+    assert _endpoint(endpoint_db, stale) is not None
+    assert _endpoint(endpoint_db, launched) is not None
+
+
+def test_sweep_ignores_endpoint_now_serving_a_different_model(endpoint_db):
+    endpoint = _launch_llm("Qwen/Qwen3-8B", 8001)
+    _launch_llm("meta-llama/Llama-3.1-8B", 8001)
+    _launch_llm("Qwen/Qwen3-8B", 8001)
+
+    _launch_llm("meta-llama/Llama-3.1-8B", 8002)
+
+    assert _endpoint(endpoint_db, endpoint) is not None
+
+
+def test_sweep_ignores_endpoints_not_created_by_cookbook(endpoint_db):
+    import core.database as database
+
+    with endpoint_db() as db:
+        db.add(database.ModelEndpoint(
+            id="manual", name="Qwen3-8B", base_url="http://localhost:9000/v1",
+            pinned_models='["Qwen/Qwen3-8B"]',
+        ))
+        db.commit()
+
+    _launch_llm("Qwen/Qwen3-8B", 8002)
+
+    assert _endpoint(endpoint_db, "manual") is not None
+
+
+def test_manual_pin_order_does_not_change_on_relaunch(endpoint_db):
+    import json
+    from core.database import ModelEndpoint
+
+    endpoint_id = _launch_llm("organization-a/Model-A", 8001)
+
+    with endpoint_db() as db:
+        endpoint = db.get(ModelEndpoint, endpoint_id)
+        endpoint.pinned_models = json.dumps([
+            "organization-a/Model-A",
+            "organization-b/Model-B",
+        ])
+        db.commit()
+
+    _launch_llm("organization-a/Model-A", 8001)
+
+    with endpoint_db() as db:
+        endpoint = db.get(ModelEndpoint, endpoint_id)
+        assert json.loads(endpoint.pinned_models) == [
+            "organization-a/Model-A",
+            "organization-b/Model-B",
+        ]
+
+
+def test_manual_pins_cannot_trigger_endpoint_deletion(endpoint_db):
+    import json
+    from core.database import ModelEndpoint
+
+    original = _launch_llm("organization-a/Model-A", 8001)
+
+    with endpoint_db() as db:
+        endpoint = db.get(ModelEndpoint, original)
+        endpoint.pinned_models = json.dumps([
+            "organization-a/Model-A",
+            "organization-b/Model-B",
+        ])
+        db.commit()
+
+    _launch_llm("organization-b/Model-B", 8002)
+
+    assert _endpoint(endpoint_db, original) is not None
+
+
+def test_unavailable_model_probe_never_deletes_another_endpoint(
+    endpoint_db, monkeypatch
+):
+    import routes.model_routes as model_routes
+
+    original = _launch_llm("organization-a/Model-A", 8001)
+
+    monkeypatch.setattr(
+        model_routes,
+        "_probe_endpoint",
+        lambda *args, **kwargs: [],
+    )
+
+    _launch_llm("organization-a/Model-A", 8002)
+
+    assert _endpoint(endpoint_db, original) is not None
+
+
+@pytest.mark.asyncio
+async def test_old_watchdog_cannot_delete_reused_endpoint(
+    endpoint_db,
+    monkeypatch,
+    endpoint_http_api,
+):
+    import asyncio
+    import urllib.request
+    import routes.cookbook_routes as cookbook_routes
+
+    original = _launch_llm(
+        "organization-a/Model-A",
+        8001,
+        serve_session_id="serve-old",
+    )
+
+    reused = _launch_llm(
+        "organization-b/Model-B",
+        8001,
+        serve_session_id="serve-new",
+    )
+
+    assert original == reused
+
+    with endpoint_db() as db:
+        from core.database import ModelEndpoint
+
+        endpoint = db.get(ModelEndpoint, reused)
+        assert endpoint.cookbook_serve_session_id == "serve-new"
+
+    router = cookbook_routes.setup_cookbook_routes()
+    serve = next(
+        route.endpoint
+        for route in router.routes
+        if route.path == "/api/model/serve"
+        and "POST" in route.methods
+    )
+
+    cells = dict(
+        zip(
+            serve.__code__.co_freevars,
+            serve.__closure__ or (),
+        )
+    )
+
+    watchdog = cells["_serve_crash_watchdog"].cell_contents
+
+    class FakeProcess:
+        async def communicate(self):
+            return (
+                b"=== Process exited with code 1 ===\n",
+                b"",
+            )
+
+    async def fake_subprocess(*args, **kwargs):
+        return FakeProcess()
+
+    async def fake_sleep(seconds):
+        return None
+
+    def unavailable(*args, **kwargs):
+        raise OSError("Server still warming up")
+
+    monkeypatch.setattr(
+        asyncio,
+        "create_subprocess_exec",
+        fake_subprocess,
+    )
+    monkeypatch.setattr(asyncio, "sleep", fake_sleep)
+    monkeypatch.setattr(
+        urllib.request,
+        "urlopen",
+        unavailable,
+    )
+
+    await watchdog(
+        endpoint_id=original,
+        session_id="serve-old",
+        remote=None,
+        ssh_port=None,
+        is_windows=False,
+    )
+
+    assert _endpoint(endpoint_db, reused) is not None
+
+    # The current serve's own crash watchdog must still be able
+    # to remove its endpoint when it actually fails.
+    await watchdog(
+        endpoint_id=reused,
+        session_id="serve-new",
+        remote=None,
+        ssh_port=None,
+        is_windows=False,
+    )
+
+    assert _endpoint(endpoint_db, reused) is None
+
+
+def test_cookbook_serve_session_migration_is_idempotent(
+    tmp_path,
+    monkeypatch,
+):
+    from sqlalchemy import create_engine, inspect, text
+    import core.database as database
+
+    engine = create_engine(
+        f"sqlite:///{tmp_path / 'legacy-endpoints.db'}"
+    )
+
+    try:
+        with engine.begin() as conn:
+            conn.execute(
+                text(
+                    "CREATE TABLE model_endpoints "
+                    "(id VARCHAR PRIMARY KEY)"
+                )
+            )
+
+        monkeypatch.setattr(database, "engine", engine)
+
+        database._migrate_add_cookbook_serve_session_id_column()
+        database._migrate_add_cookbook_serve_session_id_column()
+
+        columns = {
+            column["name"]
+            for column in inspect(engine).get_columns(
+                "model_endpoints"
+            )
+        }
+
+        assert "cookbook_serve_session_id" in columns
+
+    finally:
+        engine.dispose()
+
+
+@pytest.fixture
+def endpoint_http_api(endpoint_db, monkeypatch):
+    """Route deletion through a disposable in-process FastAPI app."""
+    import httpx
+    from fastapi import FastAPI
+    import routes.model_routes as model_routes
+    import routes.prefs_routes as prefs_routes
+    import core.constants as constants
+    import src.cookbook_serve_lifecycle as lifecycle
+
+    monkeypatch.setattr(model_routes, "SessionLocal", endpoint_db)
+    monkeypatch.setattr(model_routes, "require_admin", lambda request: None)
+
+    settings = {}
+    preferences = {}
+    monkeypatch.setattr(model_routes, "_load_settings", lambda: settings)
+    monkeypatch.setattr(model_routes, "_save_settings", lambda value: settings.update(value))
+    monkeypatch.setattr(prefs_routes, "_load", lambda: preferences)
+    monkeypatch.setattr(prefs_routes, "_save", lambda value: preferences.update(value))
+
+    # Endpoint deletion must not touch the user's running sessions.
+    import src.ai_interaction as ai_interaction
+    monkeypatch.setattr(ai_interaction, "get_session_manager", lambda: None)
+
+    app = FastAPI()
+    router = model_routes.setup_model_routes(None)
+    app.include_router(router)
+
+    original_client = httpx.AsyncClient
+    transport = httpx.ASGITransport(app=app)
+
+    def client_factory(*args, **kwargs):
+        kwargs["transport"] = transport
+        return original_client(*args, **kwargs)
+
+    monkeypatch.setattr(httpx, "AsyncClient", client_factory)
+    monkeypatch.setattr(constants, "internal_api_base", lambda: "http://testserver")
+    monkeypatch.setattr(lifecycle, "internal_api_base", lambda: "http://testserver")
+
+    return {"settings": settings, "preferences": preferences, "router": router}
+
+
+@pytest.mark.asyncio
+async def test_ownership_condition_preserves_settings_on_mismatch(
+    endpoint_db, endpoint_http_api,
+):
+    import httpx
+    original = _launch_llm("organization-a/Model-A", 8001, "serve-old")
+    reused = _launch_llm("organization-b/Model-B", 8001, "serve-new")
+    assert reused == original
+    settings = endpoint_http_api["settings"]
+    settings.update({"default_endpoint_id": reused, "default_model": "organization-b/Model-B"})
+    async with httpx.AsyncClient() as client:
+        response = await client.delete(
+            f"http://testserver/api/model-endpoints/{reused}",
+            params={"expected_cookbook_serve_session_id": "serve-old"},
+        )
+    assert response.status_code == 409
+    assert _endpoint(endpoint_db, reused) is not None
+    assert settings["default_endpoint_id"] == reused
+
+
+@pytest.mark.asyncio
+async def test_owned_deletion_clears_settings_and_model_cache(
+    endpoint_db, endpoint_http_api,
+):
+    import httpx
+    from core.database import Session as DbSession
+
+    endpoint_id = _launch_llm("organization-a/Model-A", 8001, "serve-current")
+    settings = endpoint_http_api["settings"]
+    settings.update({
+        "default_endpoint_id": endpoint_id,
+        "default_model": "organization-a/Model-A",
+        "foreground_model_fallbacks": [{"endpoint_id": endpoint_id, "model": "x"}],
+    })
+    with endpoint_db() as db:
+        db.add(DbSession(
+            id="owned-delete-session", name="Test", owner=None,
+            endpoint_url="http://localhost:8001/v1",
+            endpoint_id=endpoint_id, model="organization-a/Model-A",
+            headers={"Authorization": "Bearer test-only"},
+        ))
+        db.commit()
+
+    # Seed the actual model listing cache captured by the route closure.
+    router = endpoint_http_api["router"]
+    route = next(r for r in router.routes if r.path == "/api/model-endpoints/{ep_id}" and "DELETE" in r.methods)
+    closure = dict(zip(route.endpoint.__code__.co_freevars, route.endpoint.__closure__ or ()))
+    invalidate = closure["_invalidate_models_cache"].cell_contents
+    cache_closure = dict(zip(invalidate.__code__.co_freevars, invalidate.__closure__ or ()))
+    models_cache = cache_closure["_models_cache"].cell_contents
+    models_cache["test"] = {"data": [], "time": 0}
+
+    async with httpx.AsyncClient() as client:
+        response = await client.delete(
+            f"http://testserver/api/model-endpoints/{endpoint_id}",
+            params={"expected_cookbook_serve_session_id": "serve-current"},
+        )
+    assert response.status_code == 200, response.text
+    assert _endpoint(endpoint_db, endpoint_id) is None
+    assert settings["default_endpoint_id"] == ""
+    assert settings["foreground_model_fallbacks"] == []
+    assert not models_cache
+    with endpoint_db() as db:
+        sess = db.get(DbSession, "owned-delete-session")
+        assert sess.headers == {}
+
+
+@pytest.mark.asyncio
+async def test_ordinary_admin_delete_does_not_require_serve_owner(
+    endpoint_db, endpoint_http_api,
+):
+    import httpx
+    endpoint_id = _launch_llm("organization-a/Model-A", 8001, "serve-new")
+    async with httpx.AsyncClient() as client:
+        response = await client.delete(f"http://testserver/api/model-endpoints/{endpoint_id}")
+    assert response.status_code == 200, response.text
+    assert _endpoint(endpoint_db, endpoint_id) is None

tests/test_cookbook_serve_lifecycle.py

@@ -4,6 +4,13 @@ import pytest
 
 from src import cookbook_serve_lifecycle as lifecycle
 
+from tests.test_cookbook_endpoint_registration import (
+    endpoint_db,
+    endpoint_http_api,
+    _endpoint,
+    _launch_llm,
+)
+
 
 @pytest.mark.asyncio
 async def test_tick_persists_only_successfully_stopped_serves(tmp_path, monkeypatch):
@@ -48,3 +55,219 @@ async def test_tick_persists_only_successfully_stopped_serves(tmp_path, monkeypa
     assert tasks["stop-succeeds"]["_scheduledStopAtMs"] is None
     assert tasks["stop-fails"]["status"] == "running"
     assert tasks["stop-fails"]["_scheduledStopAtMs"] == 0
+
+
+
+@pytest.mark.asyncio
+async def test_scheduled_stop_preserves_endpoint_reused_by_newer_serve(
+    endpoint_db,
+    endpoint_http_api,
+):
+    original = _launch_llm(
+        "organization-a/Model-A",
+        8001,
+        serve_session_id="serve-old",
+    )
+
+    reused = _launch_llm(
+        "organization-b/Model-B",
+        8001,
+        serve_session_id="serve-new",
+    )
+
+    assert original == reused
+
+    await lifecycle._delete_endpoint_for_task({
+        "_endpointId": original,
+        "sessionId": "serve-old",
+        "type": "serve",
+    })
+
+    assert _endpoint(endpoint_db, reused) is not None
+
+    await lifecycle._delete_endpoint_for_task({
+        "_endpointId": reused,
+        "sessionId": "serve-new",
+        "type": "serve",
+    })
+
+    assert _endpoint(endpoint_db, reused) is None
+
+
+@pytest.mark.asyncio
+async def test_scheduled_stop_preserves_legacy_endpoint_without_owner(
+    endpoint_db,
+    endpoint_http_api,
+):
+    endpoint_id = _launch_llm(
+        "organization-a/Model-A",
+        8002,
+    )
+
+    await lifecycle._delete_endpoint_for_task({
+        "_endpointId": endpoint_id,
+        "sessionId": "serve-old",
+        "type": "serve",
+    })
+
+    assert _endpoint(endpoint_db, endpoint_id) is not None
+
+
+@pytest.mark.asyncio
+async def test_scheduled_stop_cannot_delete_manual_endpoint(
+    endpoint_db,
+    endpoint_http_api,
+):
+    from core.database import ModelEndpoint
+
+    with endpoint_db() as db:
+        db.add(ModelEndpoint(
+            id="manual-endpoint",
+            name="Manual",
+            base_url="http://localhost:9000/v1",
+            cookbook_serve_session_id="serve-old",
+        ))
+        db.commit()
+
+    await lifecycle._delete_endpoint_for_task({
+        "_endpointId": "manual-endpoint",
+        "sessionId": "serve-old",
+        "type": "serve",
+    })
+
+    assert _endpoint(endpoint_db, "manual-endpoint") is not None
+
+
+@pytest.mark.asyncio
+async def test_scheduled_stop_requires_session_identity(endpoint_db, endpoint_http_api):
+    endpoint_id = _launch_llm(
+        "organization-a/Model-A",
+        8003,
+        serve_session_id="serve-current",
+    )
+
+    await lifecycle._delete_endpoint_for_task({
+        "_endpointId": endpoint_id,
+        "type": "serve",
+    })
+
+    assert _endpoint(endpoint_db, endpoint_id) is not None
+
+
+
+def _launch_image(repo_id, port, serve_session_id):
+    import routes.cookbook_routes as cookbook_routes
+    from routes.cookbook_helpers import ServeRequest
+
+    router = cookbook_routes.setup_cookbook_routes()
+
+    serve = next(
+        route.endpoint
+        for route in router.routes
+        if route.path == "/api/model/serve"
+        and "POST" in route.methods
+    )
+
+    cells = dict(
+        zip(
+            serve.__code__.co_freevars,
+            serve.__closure__ or (),
+        )
+    )
+
+    register = cells["_auto_register_image_endpoint"].cell_contents
+
+    endpoint_id = register(
+        ServeRequest(
+            repo_id=repo_id,
+            cmd=f"python scripts/diffusion_server.py --port {port}",
+        ),
+        None,
+        serve_session_id=serve_session_id,
+    )
+
+    assert endpoint_id
+    return endpoint_id
+
+
+@pytest.mark.asyncio
+async def test_scheduled_stop_deletes_owned_image_endpoint(
+    endpoint_db,
+    monkeypatch,
+    endpoint_http_api,
+):
+    import src.settings as settings
+    from core.database import ModelEndpoint
+
+    monkeypatch.setattr(settings, "load_settings", lambda: {})
+    monkeypatch.setattr(
+        settings,
+        "save_settings",
+        lambda value: None,
+    )
+
+    endpoint_id = _launch_image(
+        "organization-a/Image-A",
+        8101,
+        "serve-image-a",
+    )
+
+    assert endpoint_id.startswith("img-")
+
+    with endpoint_db() as db:
+        endpoint = db.get(ModelEndpoint, endpoint_id)
+        assert endpoint.cookbook_serve_session_id == "serve-image-a"
+
+    await lifecycle._delete_endpoint_for_task({
+        "_endpointId": endpoint_id,
+        "sessionId": "serve-image-a",
+        "type": "serve",
+    })
+
+    assert _endpoint(endpoint_db, endpoint_id) is None
+
+
+@pytest.mark.asyncio
+async def test_old_image_serve_cannot_delete_reused_endpoint(
+    endpoint_db,
+    monkeypatch,
+    endpoint_http_api,
+):
+    import src.settings as settings
+
+    monkeypatch.setattr(settings, "load_settings", lambda: {})
+    monkeypatch.setattr(
+        settings,
+        "save_settings",
+        lambda value: None,
+    )
+
+    original = _launch_image(
+        "organization-a/Image-A",
+        8102,
+        "serve-old",
+    )
+
+    reused = _launch_image(
+        "organization-b/Image-B",
+        8102,
+        "serve-new",
+    )
+
+    assert original == reused
+
+    await lifecycle._delete_endpoint_for_task({
+        "_endpointId": original,
+        "sessionId": "serve-old",
+        "type": "serve",
+    })
+
+    assert _endpoint(endpoint_db, reused) is not None
+
+    await lifecycle._delete_endpoint_for_task({
+        "_endpointId": reused,
+        "sessionId": "serve-new",
+        "type": "serve",
+    })
+
+    assert _endpoint(endpoint_db, reused) is None