"""Regression tests for #1538 — Nous Portal model picker should live-fetch the full catalog (~30 models) instead of returning the four-entry static list. Background ---------- Settings → Default Model showed only four Nous models (Claude Opus 4.6, Claude Sonnet 4.6, GPT-5.4 Mini, Gemini 3.1 Pro Preview) because ``_build_available_models_uncached()`` fell through to the generic ``pid in _PROVIDER_MODELS`` branch and returned ``copy.deepcopy(_PROVIDER_MODELS["nous"])``. The actual Nous Portal catalog has ~30 models live — including the latest Anthropic 4.7 family, GPT-5.5, Gemini 3.1 Pro/Flash, Kimi K2.6, MiniMax M2.7, several Xiaomi/Tencent/StepFun entries. Fix --- A dedicated ``elif pid == "nous":`` branch in ``_build_available_models_uncached()`` mirroring the Ollama Cloud pattern: live-fetch via ``hermes_cli.models.provider_model_ids("nous")``, prefix every id with ``@nous:`` to match the existing routing convention, fall back to the curated static list when ``hermes_cli`` is unavailable. """ from __future__ import annotations import sys import types import api.config as config import api.profiles as profiles # Sample Nous catalog used in the live-fetch test. Mirrors the shape returned # by hermes_cli.models.provider_model_ids("nous") (see #1538 issue body). SAMPLE_NOUS_LIVE_IDS = [ "moonshotai/kimi-k2.6", "xiaomi/mimo-v2.5-pro", "anthropic/claude-opus-4.7", "anthropic/claude-opus-4.6", "anthropic/claude-sonnet-4.6", "anthropic/claude-haiku-4.5", "openai/gpt-5.5", "openai/gpt-5.4-mini", "openai/gpt-5.3-codex", "google/gemini-3-pro-preview", "google/gemini-3.1-pro-preview", "google/gemini-3.1-flash-lite-preview", "qwen/qwen3.5-plus-02-15", "minimax/minimax-m2.7", "z-ai/glm-5.1", "x-ai/grok-4.20-beta", "tencent/hy3-preview", "stepfun/step-3.5-flash", "nvidia/nemotron-3-super-120b-a12b", "arcee-ai/trinity-large-thinking", ] def _install_fake_hermes_cli(monkeypatch, *, nous_ids=None, raise_on_lookup=False): """Install fake ``hermes_cli`` modules so detection sees Nous as authenticated and ``provider_model_ids("nous")`` returns the desired catalog. Mirrors :func:`tests.test_issue1420_lmstudio_provider_env_var._install_fake_hermes_cli` but specialised for Nous detection (Nous is OAuth so the env-var path is not used — we drive detection via ``hermes_cli.auth.list_auth_providers``). """ fake_pkg = types.ModuleType("hermes_cli") fake_pkg.__path__ = [] fake_models = types.ModuleType("hermes_cli.models") fake_models.list_available_providers = lambda: [] if raise_on_lookup: def _raise(_pid): raise RuntimeError("simulated hermes_cli failure") fake_models.provider_model_ids = _raise else: ids = list(nous_ids) if nous_ids is not None else [] fake_models.provider_model_ids = lambda pid: ids if pid == "nous" else [] fake_auth = types.ModuleType("hermes_cli.auth") def _list_auth_providers(): return [{"id": "nous", "authenticated": True}] def _get_auth_status(pid): return {"logged_in": True, "key_source": ""} if pid == "nous" else {} fake_auth.list_auth_providers = _list_auth_providers fake_auth.get_auth_status = _get_auth_status monkeypatch.setitem(sys.modules, "hermes_cli", fake_pkg) monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_models) monkeypatch.setitem(sys.modules, "hermes_cli.auth", fake_auth) monkeypatch.delitem(sys.modules, "agent.credential_pool", raising=False) monkeypatch.delitem(sys.modules, "agent", raising=False) config.invalidate_models_cache() def _swap_in_test_config(extra_cfg): """Snapshot config.cfg, replace with a minimal test config; return restore-fn.""" old_cfg = dict(config.cfg) old_mtime = config._cfg_mtime config.cfg.clear() config.cfg["model"] = {} config.cfg.update(extra_cfg) try: config._cfg_mtime = config.Path(config._get_config_path()).stat().st_mtime except Exception: config._cfg_mtime = 0.0 def _restore(): config.cfg.clear() config.cfg.update(old_cfg) config._cfg_mtime = old_mtime return _restore def _scrub_provider_env(monkeypatch): """Drop every provider env var so detection only sees what we install via the fake hermes_cli stubs (not unrelated keys leaked from the runner).""" for var in ( "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY", "DEEPSEEK_API_KEY", "XAI_API_KEY", "GROQ_API_KEY", "MISTRAL_API_KEY", "OPENROUTER_API_KEY", "OLLAMA_CLOUD_API_KEY", "OLLAMA_API_KEY", "GLM_API_KEY", "KIMI_API_KEY", "MOONSHOT_API_KEY", "MINIMAX_API_KEY", "MINIMAX_CN_API_KEY", "XIAOMI_API_KEY", "OPENCODE_ZEN_API_KEY", "OPENCODE_GO_API_KEY", "NOUS_API_KEY", "NVIDIA_API_KEY", "LM_API_KEY", "LMSTUDIO_API_KEY", ): monkeypatch.delenv(var, raising=False) class TestNousLiveCatalog: """When the Nous live catalog is available, the dropdown must surface it in full (>=20 entries) — not the four-entry static fallback (#1538).""" def test_nous_models_live_fetch_when_hermes_cli_available(self, monkeypatch, tmp_path): _scrub_provider_env(monkeypatch) _install_fake_hermes_cli(monkeypatch, nous_ids=SAMPLE_NOUS_LIVE_IDS) monkeypatch.setattr(profiles, "get_active_hermes_home", lambda: tmp_path) restore = _swap_in_test_config({"model": {"provider": "nous"}}) try: data = config.get_available_models() nous_groups = [g for g in data.get("groups", []) if g.get("provider_id") == "nous"] assert len(nous_groups) == 1, ( f"Expected exactly one Nous group, got {len(nous_groups)}: " f"{[g.get('provider_id') for g in data.get('groups', [])]}" ) models = nous_groups[0]["models"] assert len(models) >= 20, ( f"Live-fetched Nous catalog should expose >=20 entries, got " f"{len(models)}. The dispatch branch fell through to the four-entry " f"static list — pre-#1538 behaviour." ) finally: restore() def test_nous_model_ids_carry_at_nous_prefix(self, monkeypatch, tmp_path): _scrub_provider_env(monkeypatch) _install_fake_hermes_cli(monkeypatch, nous_ids=SAMPLE_NOUS_LIVE_IDS) monkeypatch.setattr(profiles, "get_active_hermes_home", lambda: tmp_path) restore = _swap_in_test_config({"model": {"provider": "nous"}}) try: data = config.get_available_models() nous_group = next(g for g in data["groups"] if g["provider_id"] == "nous") for m in nous_group["models"]: assert m["id"].startswith("@nous:"), ( f"Every Nous model id must start with '@nous:' so " f"resolve_model_provider routes through the explicit-provider-hint " f"branch (matches the static-list invariant from " f"tests/test_nous_portal_routing.py). Got: {m['id']!r}" ) finally: restore() def test_nous_labels_carry_via_nous_suffix(self, monkeypatch, tmp_path): _scrub_provider_env(monkeypatch) _install_fake_hermes_cli(monkeypatch, nous_ids=SAMPLE_NOUS_LIVE_IDS) monkeypatch.setattr(profiles, "get_active_hermes_home", lambda: tmp_path) restore = _swap_in_test_config({"model": {"provider": "nous"}}) try: data = config.get_available_models() nous_group = next(g for g in data["groups"] if g["provider_id"] == "nous") for m in nous_group["models"]: assert m["label"].endswith(" (via Nous)"), ( f"Every Nous live-fetched label must end with ' (via Nous)' so " f"the user can distinguish them from same-named direct-provider " f"entries (e.g. 'Claude Opus 4.7' via direct Anthropic). " f"Got: {m['label']!r}" ) finally: restore() def test_nous_live_catalog_includes_recent_models(self, monkeypatch, tmp_path): """Sanity: the recent-flagship models from the user's bug report (Claude Opus 4.7, GPT-5.5, Kimi K2.6) must reach the dropdown.""" _scrub_provider_env(monkeypatch) _install_fake_hermes_cli(monkeypatch, nous_ids=SAMPLE_NOUS_LIVE_IDS) monkeypatch.setattr(profiles, "get_active_hermes_home", lambda: tmp_path) restore = _swap_in_test_config({"model": {"provider": "nous"}}) try: data = config.get_available_models() nous_group = next(g for g in data["groups"] if g["provider_id"] == "nous") ids = {m["id"] for m in nous_group["models"]} for required in ( "@nous:anthropic/claude-opus-4.7", "@nous:openai/gpt-5.5", "@nous:moonshotai/kimi-k2.6", "@nous:google/gemini-3.1-pro-preview", "@nous:minimax/minimax-m2.7", ): assert required in ids, ( f"{required} missing from live-fetched Nous catalog. Either " f"the hermes_cli dispatch is broken or the @nous: prefix is " f"missing." ) finally: restore() class TestNousStaticFallback: """When ``hermes_cli`` is not importable or its lookup raises, we fall back to the curated four-entry static list — never empty.""" def test_static_fallback_when_hermes_cli_raises(self, monkeypatch, tmp_path): _scrub_provider_env(monkeypatch) _install_fake_hermes_cli(monkeypatch, raise_on_lookup=True) monkeypatch.setattr(profiles, "get_active_hermes_home", lambda: tmp_path) restore = _swap_in_test_config({"model": {"provider": "nous"}}) try: data = config.get_available_models() nous_groups = [g for g in data.get("groups", []) if g.get("provider_id") == "nous"] assert nous_groups, ( "Nous group must still appear when hermes_cli fails — the " "branch should fall back to the curated static list." ) models = nous_groups[0]["models"] assert len(models) == 4, ( f"Static fallback should expose exactly the four curated entries " f"in _PROVIDER_MODELS['nous']. Got {len(models)}: " f"{[m['id'] for m in models]}" ) for m in models: assert m["id"].startswith("@nous:"), m["id"] finally: restore() class TestFormatNousLabel: """Unit tests for the label formatter helper.""" def test_strips_vendor_namespace(self): from api.config import _format_nous_label assert _format_nous_label("anthropic/claude-opus-4.7") == "Claude Opus 4.7 (via Nous)" assert _format_nous_label("openai/gpt-5.4-mini") == "GPT 5.4 Mini (via Nous)" def test_handles_missing_vendor(self): from api.config import _format_nous_label # Defensive: id without slash should still render a sane label. assert _format_nous_label("kimi-k2.6") == "Kimi K2.6 (via Nous)" def test_handles_variant_after_colon(self): from api.config import _format_nous_label # Variant rendered in parentheses, mirroring _format_ollama_label. out = _format_nous_label("minimax/minimax-m2.5:free") assert out.endswith(" (via Nous)") assert "Free" in out assert "MiniMax M2.5" in out def test_minimax_renders_mixed_case(self): from api.config import _format_nous_label # Live wire returns lowercase 'minimax/minimax-...' but the curated # convention is mixed-case 'MiniMax'. assert _format_nous_label("minimax/minimax-m2.7").startswith("MiniMax M2.7") def test_label_always_ends_with_via_nous_suffix(self): from api.config import _format_nous_label for sample in [ "anthropic/claude-opus-4.7", "openai/gpt-5.5", "google/gemini-3.1-pro-preview", "moonshotai/kimi-k2.6", "z-ai/glm-5.1", "stepfun/step-3.5-flash", ]: assert _format_nous_label(sample).endswith(" (via Nous)"), sample class TestStaticListPreservedAsFallback: """The curated ``_PROVIDER_MODELS['nous']`` entry stays as the static fallback; existing routing invariants from :mod:`tests.test_nous_portal_routing` must remain valid.""" def test_static_list_present(self): from api.config import _PROVIDER_MODELS assert _PROVIDER_MODELS.get("nous"), ( "The curated static Nous list must remain in _PROVIDER_MODELS as " "a fallback for environments where hermes_cli is unavailable." ) def test_static_list_keeps_at_nous_prefix(self): # Keep parity with tests/test_nous_portal_routing.py — ensures the # static fallback path produces correctly-routable ids when used. from api.config import _PROVIDER_MODELS for m in _PROVIDER_MODELS["nous"]: assert m["id"].startswith("@nous:"), m["id"]