diff --git a/README.md b/README.md index abefd18..94fcf23 100644 --- a/README.md +++ b/README.md @@ -167,9 +167,9 @@ Each target specifies: - name: azure_base provider: azure settings: - endpoint: "AZURE_OPEN_AI_ENDPOINT" - api_key: "AZURE_OPEN_AI_API_KEY" - model: "LLM_MODEL" + endpoint: "AZURE_OPENAI_ENDPOINT" + api_key: "AZURE_OPENAI_API_KEY" + model: "AZURE_DEPLOYMENT_NAME" ``` **Anthropic targets:** @@ -178,7 +178,7 @@ Each target specifies: provider: anthropic settings: api_key: "ANTHROPIC_API_KEY" - model: "LLM_MODEL" + model: "ANTHROPIC_MODEL" ``` **VS Code targets:** diff --git a/docs/examples/simple/.bbeval/targets.yaml b/docs/examples/simple/.bbeval/targets.yaml index 1cca8b9..64e29c4 100644 --- a/docs/examples/simple/.bbeval/targets.yaml +++ b/docs/examples/simple/.bbeval/targets.yaml @@ -15,15 +15,15 @@ # The base LLM provider needs no extra settings, as the model is # defined in the .env file. settings: - endpoint: AZURE_OPEN_AI_ENDPOINT - api_key: AZURE_OPEN_AI_API_KEY - model: LLM_MODEL + endpoint: AZURE_OPENAI_ENDPOINT + api_key: AZURE_OPENAI_API_KEY + model: AZURE_DEPLOYMENT_NAME - name: default provider: azure # The base LLM provider needs no extra settings, as the model is # defined in the .env file. settings: - endpoint: AZURE_OPEN_AI_ENDPOINT - api_key: AZURE_OPEN_AI_API_KEY - model: LLM_MODEL + endpoint: AZURE_OPENAI_ENDPOINT + api_key: AZURE_OPENAI_API_KEY + model: AZURE_DEPLOYMENT_NAME diff --git a/docs/examples/simple/.env.template b/docs/examples/simple/.env.template index 5bc52ae..e11194a 100644 --- a/docs/examples/simple/.env.template +++ b/docs/examples/simple/.env.template @@ -6,9 +6,9 @@ PROVIDER=azure # or anthropic # Azure OpenAI Configuration # These are the default environment variable names used in the provided targets.yaml -AZURE_OPEN_AI_ENDPOINT=https://your-endpoint.openai.azure.com/ -AZURE_OPEN_AI_API_KEY=your-api-key-here -LLM_MODEL=gpt-5-chat # or your deployment name +AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/ +AZURE_OPENAI_API_KEY=your-api-key-here +AZURE_DEPLOYMENT_NAME=gpt-5-chat # or your deployment name # Anthropic Configuration (if using Anthropic provider) ANTHROPIC_API_KEY=your-anthropic-api-key-here diff --git a/pyproject.toml b/pyproject.toml index e2f902e..5e7c1d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "bbeval" -version = "0.1.9" +version = "0.1.10" description = "A lightweight, extensible evaluator for testing model-generated responses" readme = "README.md" requires-python = ">=3.12" diff --git a/src/bbeval/cli.py b/src/bbeval/cli.py index 188ea5a..5889ef6 100644 --- a/src/bbeval/cli.py +++ b/src/bbeval/cli.py @@ -97,13 +97,13 @@ def create_judge_model(target: Dict, targets: List[Dict], model: str, verbose: b print(f" No judge_target specified, falling back to Azure") judge_provider = "azure" judge_settings = { - 'endpoint': 'AZURE_OPEN_AI_ENDPOINT', - 'api_key': 'AZURE_OPEN_AI_API_KEY' + 'endpoint': 'AZURE_OPENAI_ENDPOINT', + 'api_key': 'AZURE_OPENAI_API_KEY' } judge_model = model # Check if Azure credentials are available for fallback - if not os.getenv('AZURE_OPEN_AI_ENDPOINT') or not os.getenv('AZURE_OPEN_AI_API_KEY'): + if not os.getenv('AZURE_OPENAI_ENDPOINT') or not os.getenv('AZURE_OPENAI_API_KEY'): if verbose: print(f" Azure credentials not found, using mock judge") judge_provider = "mock" @@ -494,7 +494,13 @@ def run_evaluation(test_file: str, provider = target['provider'] settings = target.get('settings') # For DSPy configuration, we still need a model parameter (but not for results) - model = os.getenv('LLM_MODEL', 'gpt-4') + # Get model from target settings, resolving env var if needed + model = settings.get('model', 'AZURE_DEPLOYMENT_NAME') if settings else 'AZURE_DEPLOYMENT_NAME' + if isinstance(model, str) and model in os.environ: + model = os.getenv(model, 'gpt-4') + elif isinstance(model, str): + # Fallback if model string is not an env var name + model = 'gpt-4' # Configure model if dry_run: diff --git a/src/diagnostics/azure_deployment_diag.py b/src/diagnostics/azure_deployment_diag.py index 7506600..0d4a916 100644 --- a/src/diagnostics/azure_deployment_diag.py +++ b/src/diagnostics/azure_deployment_diag.py @@ -12,9 +12,9 @@ load_dotenv() # Get Azure OpenAI configuration -endpoint = os.getenv("AZURE_OPEN_AI_ENDPOINT") -api_key = os.getenv("AZURE_OPEN_AI_API_KEY") -model = os.getenv("LLM_MODEL", "gpt-5-chat") +endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") +api_key = os.getenv("AZURE_OPENAI_API_KEY") +model = os.getenv("AZURE_DEPLOYMENT_NAME", "gpt-5-chat") print(f"Endpoint: {endpoint}") print(f"API Key: {'***' + api_key[-4:] if api_key else 'None'}")