From 6e98053595dc32d30c07e37d80a7b8e9f42f8201 Mon Sep 17 00:00:00 2001 From: Avinash Balakrishnan Date: Wed, 24 Dec 2025 12:05:31 -0800 Subject: [PATCH 01/16] Adding a fix to pass `reasoning_effort` in conditionally --- mellea/backends/openai.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index ba825753..d3df90dd 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -631,15 +631,20 @@ async def _generate_from_chat_context_standard( formatted_tools = convert_tools_to_json(tools) use_tools = len(formatted_tools) > 0 + # Build optional reasoning parameters + reasoning_params = {} + if thinking is not None: + reasoning_params["reasoning_effort"] = thinking + chat_response: Coroutine[ Any, Any, ChatCompletion | openai.AsyncStream[ChatCompletionChunk] ] = self._async_client.chat.completions.create( model=self._hf_model_id, messages=conversation, # type: ignore - reasoning_effort=thinking, # type: ignore tools=formatted_tools if use_tools else None, # type: ignore # parallel_tool_calls=False, # We only support calling one tool per turn. But we do the choosing on our side so we leave this False. **extra_params, + **reasoning_params, # type: ignore **self._make_backend_specific_and_remove( model_opts, is_chat_context=ctx.is_chat_context ), From ab9e56b0650ea1ea5117983993834a8f35caf9fe Mon Sep 17 00:00:00 2001 From: Avinash Balakrishnan Date: Wed, 24 Dec 2025 12:06:23 -0800 Subject: [PATCH 02/16] adding tests --- test/backends/test_openai_ollama.py | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/backends/test_openai_ollama.py b/test/backends/test_openai_ollama.py index 1848287c..965abc52 100644 --- a/test/backends/test_openai_ollama.py +++ b/test/backends/test_openai_ollama.py @@ -216,6 +216,46 @@ async def get_client_async(): assert len(backend._client_cache.cache.values()) == 2 +async def test_reasoning_effort_conditional_passing(backend): + """Test that reasoning_effort is only passed to API when not None.""" + from unittest.mock import AsyncMock, MagicMock, patch + + ctx = ChatContext() + ctx = ctx.add(CBlock(value="Test")) + + mock_response = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[0].message = MagicMock() + mock_response.choices[0].message.content = "Response" + mock_response.choices[0].message.role = "assistant" + + # Test 1: reasoning_effort should NOT be passed when not specified + with patch.object( + backend._async_client.chat.completions, "create", new_callable=AsyncMock + ) as mock_create: + mock_create.return_value = mock_response + await backend.generate_from_chat_context( + CBlock(value="Hi"), ctx, model_options={} + ) + call_kwargs = mock_create.call_args.kwargs + assert "reasoning_effort" not in call_kwargs, ( + "reasoning_effort should not be passed when not specified" + ) + + # Test 2: reasoning_effort SHOULD be passed when specified + with patch.object( + backend._async_client.chat.completions, "create", new_callable=AsyncMock + ) as mock_create: + mock_create.return_value = mock_response + await backend.generate_from_chat_context( + CBlock(value="Hi"), ctx, model_options={ModelOption.THINKING: "medium"} + ) + call_kwargs = mock_create.call_args.kwargs + assert call_kwargs.get("reasoning_effort") == "medium", ( + "reasoning_effort should be passed with correct value when specified" + ) + + if __name__ == "__main__": import pytest From 0862cf02709c045618930cf3c4cab8045914774f Mon Sep 17 00:00:00 2001 From: Nathan Fulton Date: Fri, 26 Dec 2025 16:36:07 -0600 Subject: [PATCH 03/16] Fixes #274 --- mellea/backends/openai.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index d3df90dd..ae84b916 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -152,16 +152,21 @@ def __init__( ) self._hf_model_id = model_id.hf_model_name - if base_url is None: - self._base_url = "http://localhost:11434/v1" # ollama - else: - self._base_url = base_url if api_key is None: + FancyLogger.get_logger().warning( + "You are using an OpenAI backend with no api_key. Because no API key was provided, mellea assumes you intend to use the openai-compatible interface to your local ollama instance. If you intend to use OpenAI's platform you must specify your API key when instantiating your Mellea session/backend object." + ) + self._base_url: str | None = "http://localhost:11434/v1" # ollama self._api_key = "ollama" else: + self._base_url = base_url self._api_key = api_key - self._server_type = _server_type(self._base_url) + self._server_type: _ServerType = ( + _server_type(self._base_url) + if self._base_url is not None + else _ServerType.OPENAI + ) # type: ignore self._openai_client_kwargs = self.filter_openai_client_kwargs(**kwargs) @@ -981,6 +986,9 @@ def apply_chat_template(self, chat: list[dict[str, str]]): from transformers import AutoTokenizer if not hasattr(self, "_tokenizer"): + assert self._base_url, ( + "The OpenAI Platform does not support adapters. You must specify a _base_url when using adapters." + ) match _server_type(self._base_url): case _ServerType.LOCALHOST: self._tokenizer: "PreTrainedTokenizer" = ( # noqa: UP037 From 29481f70dc9f3a347a4bd1cf334b6762d7298d5b Mon Sep 17 00:00:00 2001 From: Nathan Fulton Date: Fri, 26 Dec 2025 17:05:28 -0600 Subject: [PATCH 04/16] Adds GPT 5.1 model identifier. --- mellea/backends/model_ids.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mellea/backends/model_ids.py b/mellea/backends/model_ids.py index 96b8afee..3ffdb4b8 100644 --- a/mellea/backends/model_ids.py +++ b/mellea/backends/model_ids.py @@ -17,6 +17,7 @@ class ModelIdentifier: ollama_name: str | None = None watsonx_name: str | None = None mlx_name: str | None = None + openai_name: str | None = None hf_tokenizer_name: str | None = None # if None, is the same as hf_model_name @@ -134,9 +135,9 @@ class ModelIdentifier: QWEN3_14B = ModelIdentifier(hf_model_name="Qwen/Qwen3-14B", ollama_name="qwen3:14b") -###################### -#### OpenAI models ### -###################### +########################### +#### OpenAI open models ### +########################### OPENAI_GPT_OSS_20B = ModelIdentifier( hf_model_name="openai/gpt-oss-20b", ollama_name="gpt-oss:20b" @@ -145,6 +146,12 @@ class ModelIdentifier: hf_model_name="openai/gpt-oss-120b", ollama_name="gpt-oss:120b" ) +########################### +#### OpenAI prop models ### +########################### + +OPENAI_GPT_5_1 = ModelIdentifier(openai_name="gpt-5.1") + ##################### #### Misc models #### ##################### From 1426ee95dd0d003aed11432c0f1a88702998462c Mon Sep 17 00:00:00 2001 From: Nathan Fulton Date: Fri, 26 Dec 2025 17:05:50 -0600 Subject: [PATCH 05/16] Changes OpenAI Backend default model_id to GPT 5.1. This default is changed because the default base_url is also changed. --- mellea/backends/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index ae84b916..4b722614 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -72,7 +72,7 @@ class OpenAIBackend(FormatterBackend, AdapterMixin): def __init__( self, - model_id: str | ModelIdentifier = model_ids.IBM_GRANITE_4_MICRO_3B, + model_id: str | ModelIdentifier = model_ids.OPENAI_GPT_5_1, formatter: Formatter | None = None, base_url: str | None = None, model_options: dict | None = None, From c11fbef41464027b693e9c0f19c9b62c9cdebc00 Mon Sep 17 00:00:00 2001 From: Nathan Fulton Date: Fri, 26 Dec 2025 17:26:29 -0600 Subject: [PATCH 06/16] Fixes bug: GenSlots did not work with OpenAI platform. The OpenAI response_format only accepts a limited set of schemas and will error out with a 400 if you do not follow their guidelines. One of these guidelines is that additionalProperties is set and is set to False. This commit monkey-patches the response_format provided to OpenAI platform backends, and leaves other OpenAI-"compatible" backends with the existing default behavior. This is a debatable choice. See https://community.openai.com/t/api-rejects-valid-json-schema/906163 and https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat --- mellea/backends/openai.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 4b722614..88386e50 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -603,14 +603,31 @@ async def _generate_from_chat_context_standard( extra_params: dict[str, Any] = {} if _format is not None: - extra_params["response_format"] = { - "type": "json_schema", - "json_schema": { - "name": _format.__name__, - "schema": _format.model_json_schema(), - "strict": True, - }, - } + monkey_patched_response_schema = _format.model_json_schema() + monkey_patched_response_schema["additionalProperties"] = False + if self._server_type == _ServerType.OPENAI: + extra_params["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": _format.__name__, + "schema": monkey_patched_response_schema, + "strict": True, + # "additionalProperties": False, + }, + } + print(extra_params["response_format"]) + else: + FancyLogger().get_logger().warning( + "Mellea assumes you are NOT using the OpenAI platform, and that other model providers have less strict requirements on support JSON schemas passed into `format=`. If you encounter a server-side error following this message, then you found an exception to this assumption. Please open an issue at github.com/generative_computing/mellea with this stack trace and your inference engine / model provider." + ) + extra_params["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": _format.__name__, + "schema": _format.model_json_schema(), + "strict": True, + }, + } # Append tool call information if applicable. tools: dict[str, Callable] = dict() From 0bde6ec4b94c71e1f80848c37330e31196701dc1 Mon Sep 17 00:00:00 2001 From: Nathan Fulton Date: Fri, 26 Dec 2025 17:32:49 -0600 Subject: [PATCH 07/16] Adds inline documentation for OpenAI model options monkey patching. --- mellea/backends/openai.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 88386e50..c9bae94e 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -603,16 +603,24 @@ async def _generate_from_chat_context_standard( extra_params: dict[str, Any] = {} if _format is not None: - monkey_patched_response_schema = _format.model_json_schema() - monkey_patched_response_schema["additionalProperties"] = False if self._server_type == _ServerType.OPENAI: + # The OpenAI platform requires that additionalProperties=False on all response_format schemas. + # However, not all schemas generates by Mellea include additionalProperties. + # GenerativeSlot, in particular, does not add this property. + # The easiest way to address this disparity between OpenAI and other inference providers is to + # monkey-patch the response format exactly when we are actually using the OpenAI server. + # + # This only addresses the additionalProperties=False constraint. + # Other constraints we should be checking/patching are described here: + # https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat + monkey_patched_response_schema = _format.model_json_schema() + monkey_patched_response_schema["additionalProperties"] = False extra_params["response_format"] = { "type": "json_schema", "json_schema": { "name": _format.__name__, "schema": monkey_patched_response_schema, "strict": True, - # "additionalProperties": False, }, } print(extra_params["response_format"]) From 4d87c830771d2d0032167d610d58f54e488087ac Mon Sep 17 00:00:00 2001 From: Nathan Fulton Date: Fri, 26 Dec 2025 17:36:38 -0600 Subject: [PATCH 08/16] removes debug print stmt. --- mellea/backends/openai.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index c9bae94e..7c0b30ec 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -623,7 +623,6 @@ async def _generate_from_chat_context_standard( "strict": True, }, } - print(extra_params["response_format"]) else: FancyLogger().get_logger().warning( "Mellea assumes you are NOT using the OpenAI platform, and that other model providers have less strict requirements on support JSON schemas passed into `format=`. If you encounter a server-side error following this message, then you found an exception to this assumption. Please open an issue at github.com/generative_computing/mellea with this stack trace and your inference engine / model provider." From f87f86ba011a940b9d981b3aafa4c41aca04fc86 Mon Sep 17 00:00:00 2001 From: Avinash Balakrishnan Date: Mon, 5 Jan 2026 08:46:44 -0800 Subject: [PATCH 09/16] adding a comment about reasoning_effort in openai sdk --- mellea/backends/openai.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index d3df90dd..530070a6 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -632,6 +632,7 @@ async def _generate_from_chat_context_standard( use_tools = len(formatted_tools) > 0 # Build optional reasoning parameters + # NOTE: the openai SDK doesn't like it if you pass `reasoning_effort` param to a non-reasoning model e.g. gpt4o reasoning_params = {} if thinking is not None: reasoning_params["reasoning_effort"] = thinking From a94205da5bdbacfa740c030a78c274357ad81355 Mon Sep 17 00:00:00 2001 From: Avinash Balakrishnan Date: Tue, 6 Jan 2026 09:57:49 -0800 Subject: [PATCH 10/16] removing all instances of hf_model_id in openai backend --- mellea/backends/openai.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 7d68ba58..609ec0f5 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -142,15 +142,14 @@ def __init__( self.default_to_constraint_checking_alora = default_to_constraint_checking_alora - self._model_id = model_id match model_id: case str(): - self._hf_model_id = model_id + self._model_id = model_id case ModelIdentifier(): - assert model_id.hf_model_name is not None, ( - "model_id is None. This can also happen if the ModelIdentifier has no hf_model_id name set." + assert model_id.openai_name is not None, ( + "model_id is None. This can also happen if the ModelIdentifier has no `openai_name` name set." ) - self._hf_model_id = model_id.hf_model_name + self._model_id = model_id.openai_name if api_key is None: FancyLogger.get_logger().warning( @@ -669,7 +668,7 @@ async def _generate_from_chat_context_standard( chat_response: Coroutine[ Any, Any, ChatCompletion | openai.AsyncStream[ChatCompletionChunk] ] = self._async_client.chat.completions.create( - model=self._hf_model_id, + model=self._model_id, messages=conversation, # type: ignore tools=formatted_tools if use_tools else None, # type: ignore # parallel_tool_calls=False, # We only support calling one tool per turn. But we do the choosing on our side so we leave this False. @@ -842,7 +841,7 @@ async def generate_from_raw( try: completion_response: Completion = ( await self._async_client.completions.create( - model=self._hf_model_id, + model=self._model_id, prompt=prompts, extra_body=extra_body, **self._make_backend_specific_and_remove( @@ -895,7 +894,10 @@ async def generate_from_raw( @property def base_model_name(self): """Returns the base_model_id of the model used by the backend. For example, `granite-3.3-8b-instruct` for `ibm-granite/granite-3.3-8b-instruct`.""" - return self._hf_model_id.split("/")[1] + if "/" in self._model_id: + return self._model_id.split("/")[1] + else: + return self._model_id def add_adapter(self, adapter: OpenAIAdapter): """Adds the given adapter to the backend. Must not have been added to a different backend.""" @@ -1017,7 +1019,7 @@ def apply_chat_template(self, chat: list[dict[str, str]]): match _server_type(self._base_url): case _ServerType.LOCALHOST: self._tokenizer: "PreTrainedTokenizer" = ( # noqa: UP037 - AutoTokenizer.from_pretrained(self._hf_model_id) + AutoTokenizer.from_pretrained(self._model_id) ) case _ServerType.OPENAI: raise Exception( From 1e7c1b474ca4cf4db9fdc1e69aafe7acb0bbca11 Mon Sep 17 00:00:00 2001 From: Avinash Balakrishnan Date: Tue, 6 Jan 2026 11:24:47 -0800 Subject: [PATCH 11/16] removing apply_chat_template and adding assertions for env variable --- mellea/backends/openai.py | 48 +++++++++++++++------------------------ 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 609ec0f5..77d1995f 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -6,6 +6,7 @@ import functools import inspect import json +import os from collections.abc import Callable, Coroutine from copy import deepcopy from enum import Enum @@ -151,15 +152,24 @@ def __init__( ) self._model_id = model_id.openai_name - if api_key is None: - FancyLogger.get_logger().warning( - "You are using an OpenAI backend with no api_key. Because no API key was provided, mellea assumes you intend to use the openai-compatible interface to your local ollama instance. If you intend to use OpenAI's platform you must specify your API key when instantiating your Mellea session/backend object." + # Use provided parameters or fall back to environment variables + self._api_key = api_key or os.getenv("OPENAI_API_KEY") + self._base_url = base_url or os.getenv("OPENAI_BASE_URL") + + # Validate that we have the required configuration + if self._api_key is None: + raise ValueError( + "OPENAI_API_KEY is required but not set. Please either:\n" + " 1. Set the environment variable: export OPENAI_API_KEY='your-key-here'\n" + " 2. Pass it as a parameter: OpenAIBackend(api_key='your-key-here')" + ) + + if self._base_url is None: + raise ValueError( + "OPENAI_BASE_URL is required but not set. Please either:\n" + " 1. Set the environment variable: export OPENAI_BASE_URL=\n" + " 2. Pass it as a parameter: OpenAIBackend(base_url=)" ) - self._base_url: str | None = "http://localhost:11434/v1" # ollama - self._api_key = "ollama" - else: - self._base_url = base_url - self._api_key = api_key self._server_type: _ServerType = ( _server_type(self._base_url) @@ -1007,25 +1017,3 @@ def list_adapters(self) -> list[str]: :returns: list of adapter names that are currently registered with this backend """ return list(self._loaded_adapters.keys()) - - def apply_chat_template(self, chat: list[dict[str, str]]): - """Apply the chat template for the model, if such a model is available (e.g., when it can deduce the huggingface model id).""" - from transformers import AutoTokenizer - - if not hasattr(self, "_tokenizer"): - assert self._base_url, ( - "The OpenAI Platform does not support adapters. You must specify a _base_url when using adapters." - ) - match _server_type(self._base_url): - case _ServerType.LOCALHOST: - self._tokenizer: "PreTrainedTokenizer" = ( # noqa: UP037 - AutoTokenizer.from_pretrained(self._model_id) - ) - case _ServerType.OPENAI: - raise Exception( - "apply_chat_template is called while targeting a server at openai.com. " - "This is not supported --- openai.com does not support Activated Lora. " - "Use a locally served vllm instance. " - ) - - return self._tokenizer.apply_chat_template(chat, tokenize=False) From a695cb4cc666287dc028aff6cbfa77497222640c Mon Sep 17 00:00:00 2001 From: Avinash Balakrishnan Date: Tue, 6 Jan 2026 11:28:08 -0800 Subject: [PATCH 12/16] adding some tests for param checking --- test/backends/test_openai_ollama.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/backends/test_openai_ollama.py b/test/backends/test_openai_ollama.py index 965abc52..9c79785d 100644 --- a/test/backends/test_openai_ollama.py +++ b/test/backends/test_openai_ollama.py @@ -1,6 +1,7 @@ # test/rits_backend_tests/test_openai_integration.py import asyncio import os +from unittest.mock import patch import openai import pydantic @@ -256,6 +257,49 @@ async def test_reasoning_effort_conditional_passing(backend): ) +def test_api_key_and_base_url_from_parameters(): + """Test that API key and base URL can be set via parameters.""" + backend = OpenAIBackend( + model_id="gpt-4", api_key="test-api-key", base_url="https://api.test.com/v1" + ) + assert backend._api_key == "test-api-key" + assert backend._base_url == "https://api.test.com/v1" + + +def test_api_key_and_base_url_from_env_variables(): + """Test that API key and base URL fall back to environment variables.""" + with patch.dict( + os.environ, + {"OPENAI_API_KEY": "env-api-key", "OPENAI_BASE_URL": "https://api.env.com/v1"}, + ): + backend = OpenAIBackend(model_id="gpt-4") + assert backend._api_key == "env-api-key" + assert backend._base_url == "https://api.env.com/v1" + + +def test_parameter_overrides_env_variable(): + """Test that explicit parameters override environment variables.""" + with patch.dict( + os.environ, + {"OPENAI_API_KEY": "env-api-key", "OPENAI_BASE_URL": "https://api.env.com/v1"}, + ): + backend = OpenAIBackend( + model_id="gpt-4", + api_key="param-api-key", + base_url="https://api.param.com/v1", + ) + assert backend._api_key == "param-api-key" + assert backend._base_url == "https://api.param.com/v1" + + +def test_missing_api_key_raises_error(): + """Test that missing API key raises ValueError with helpful message.""" + with patch.dict(os.environ, {}, clear=True): + with pytest.raises(ValueError) as exc_info: + OpenAIBackend(model_id="gpt-4", base_url="https://api.test.com/v1") + assert "OPENAI_API_KEY is required but not set" in str(exc_info.value) + + if __name__ == "__main__": import pytest From 41a0c62ad3c9ee98727b6552844b0a9f26613eaa Mon Sep 17 00:00:00 2001 From: Avinash Balakrishnan Date: Tue, 6 Jan 2026 12:27:15 -0800 Subject: [PATCH 13/16] changing env variable handling logic. --- mellea/backends/openai.py | 12 ++++++------ test/backends/test_openai_ollama.py | 15 +++------------ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 77d1995f..6061a358 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -153,20 +153,20 @@ def __init__( self._model_id = model_id.openai_name # Use provided parameters or fall back to environment variables - self._api_key = api_key or os.getenv("OPENAI_API_KEY") - self._base_url = base_url or os.getenv("OPENAI_BASE_URL") + self._api_key = api_key + self._base_url = base_url # Validate that we have the required configuration - if self._api_key is None: + if self._api_key is None and os.getenv("OPENAI_API_KEY") is None: raise ValueError( - "OPENAI_API_KEY is required but not set. Please either:\n" + "OPENAI_API_KEY or api_key is required but not set. Please either:\n" " 1. Set the environment variable: export OPENAI_API_KEY='your-key-here'\n" " 2. Pass it as a parameter: OpenAIBackend(api_key='your-key-here')" ) - if self._base_url is None: + if self._base_url is None and os.getenv("OPENAI_BASE_URL") is None: raise ValueError( - "OPENAI_BASE_URL is required but not set. Please either:\n" + "OPENAI_BASE_URL or base_url is required but not set. Please either:\n" " 1. Set the environment variable: export OPENAI_BASE_URL=\n" " 2. Pass it as a parameter: OpenAIBackend(base_url=)" ) diff --git a/test/backends/test_openai_ollama.py b/test/backends/test_openai_ollama.py index 9c79785d..57ca3281 100644 --- a/test/backends/test_openai_ollama.py +++ b/test/backends/test_openai_ollama.py @@ -266,17 +266,6 @@ def test_api_key_and_base_url_from_parameters(): assert backend._base_url == "https://api.test.com/v1" -def test_api_key_and_base_url_from_env_variables(): - """Test that API key and base URL fall back to environment variables.""" - with patch.dict( - os.environ, - {"OPENAI_API_KEY": "env-api-key", "OPENAI_BASE_URL": "https://api.env.com/v1"}, - ): - backend = OpenAIBackend(model_id="gpt-4") - assert backend._api_key == "env-api-key" - assert backend._base_url == "https://api.env.com/v1" - - def test_parameter_overrides_env_variable(): """Test that explicit parameters override environment variables.""" with patch.dict( @@ -297,7 +286,9 @@ def test_missing_api_key_raises_error(): with patch.dict(os.environ, {}, clear=True): with pytest.raises(ValueError) as exc_info: OpenAIBackend(model_id="gpt-4", base_url="https://api.test.com/v1") - assert "OPENAI_API_KEY is required but not set" in str(exc_info.value) + assert "OPENAI_API_KEY or api_key is required but not set" in str( + exc_info.value + ) if __name__ == "__main__": From c905843639dc4d82ec4bdb285623d43d5cfe7b97 Mon Sep 17 00:00:00 2001 From: Avinash Balakrishnan Date: Tue, 6 Jan 2026 12:44:07 -0800 Subject: [PATCH 14/16] base_url check is now a warning --- mellea/backends/openai.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 6061a358..27058b46 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -165,10 +165,9 @@ def __init__( ) if self._base_url is None and os.getenv("OPENAI_BASE_URL") is None: - raise ValueError( - "OPENAI_BASE_URL or base_url is required but not set. Please either:\n" - " 1. Set the environment variable: export OPENAI_BASE_URL=\n" - " 2. Pass it as a parameter: OpenAIBackend(base_url=)" + FancyLogger.get_logger().warning( + "OPENAI_BASE_URL or base_url is not set. Please either:\n" + "The openai SDK is going to assume that the base_url is `https://api.openai.com/v1`" ) self._server_type: _ServerType = ( From 0a7747ad139d905e99d8f037393cb0a7d1075ce0 Mon Sep 17 00:00:00 2001 From: jakelorocco <59755218+jakelorocco@users.noreply.github.com> Date: Tue, 6 Jan 2026 15:46:26 -0500 Subject: [PATCH 15/16] fix: change warning message in openai.py --- mellea/backends/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 27058b46..65ab544a 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -166,7 +166,7 @@ def __init__( if self._base_url is None and os.getenv("OPENAI_BASE_URL") is None: FancyLogger.get_logger().warning( - "OPENAI_BASE_URL or base_url is not set. Please either:\n" + "OPENAI_BASE_URL or base_url is not set.\n" "The openai SDK is going to assume that the base_url is `https://api.openai.com/v1`" ) From d0ecfc794c6d0d519b78c2ca4fce9800e3333383 Mon Sep 17 00:00:00 2001 From: Avinash Balakrishnan Date: Tue, 6 Jan 2026 13:12:53 -0800 Subject: [PATCH 16/16] marking test as qualitative cause it's causing timeouts in github actions(I think) --- test/stdlib_intrinsics/test_rag/test_rag.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/stdlib_intrinsics/test_rag/test_rag.py b/test/stdlib_intrinsics/test_rag/test_rag.py index 016b78a0..47b13e02 100644 --- a/test/stdlib_intrinsics/test_rag/test_rag.py +++ b/test/stdlib_intrinsics/test_rag/test_rag.py @@ -184,6 +184,7 @@ def test_answer_relevance(backend): assert result == answer +@pytest.mark.qualitative def test_answer_relevance_classifier(backend): """Verify that the first phase of the answer relevance flow behaves as expectee.""" context, answer, docs = _read_input_json("answer_relevance.json")