From f483f05ea51cf59e38bfc6f82348dfe1a7ace5a2 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 24 Nov 2025 13:16:04 +1100 Subject: [PATCH 1/5] add user prompt --- apps/cli/src/commands/init/index.ts | 46 +++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/apps/cli/src/commands/init/index.ts b/apps/cli/src/commands/init/index.ts index 15a80f3..1d88e0f 100644 --- a/apps/cli/src/commands/init/index.ts +++ b/apps/cli/src/commands/init/index.ts @@ -1,5 +1,6 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; +import * as readline from "node:readline/promises"; import { TemplateManager } from "../../templates/index.js"; @@ -7,18 +8,57 @@ export interface InitCommandOptions { targetPath?: string; } +async function promptYesNo(message: string): Promise { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + try { + const answer = await rl.question(`${message} (y/N): `); + return answer.toLowerCase() === "y" || answer.toLowerCase() === "yes"; + } finally { + rl.close(); + } +} + export async function initCommand(options: InitCommandOptions = {}): Promise { const targetPath = path.resolve(options.targetPath ?? "."); const githubDir = path.join(targetPath, ".github"); + // Get templates + const templates = TemplateManager.getTemplates(); + + // Check if any files already exist + const existingFiles: string[] = []; + if (existsSync(githubDir)) { + for (const template of templates) { + const targetFilePath = path.join(githubDir, template.path); + if (existsSync(targetFilePath)) { + existingFiles.push(path.relative(targetPath, targetFilePath)); + } + } + } + + // If files exist, prompt user + if (existingFiles.length > 0) { + console.log("We detected an existing setup:"); + existingFiles.forEach((file) => console.log(` - ${file}`)); + console.log(); + + const shouldReplace = await promptYesNo("Do you want to replace these files?"); + if (!shouldReplace) { + console.log("\nInit cancelled. No files were changed."); + return; + } + console.log(); + } + // Create .github directory if it doesn't exist if (!existsSync(githubDir)) { mkdirSync(githubDir, { recursive: true }); } - // Get templates - const templates = TemplateManager.getTemplates(); - // Copy each template to .github for (const template of templates) { const targetFilePath = path.join(githubDir, template.path); From 90dc290fd97d16aa2036032013b41b01ea65e197 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 24 Nov 2025 13:19:05 +1100 Subject: [PATCH 2/5] update eval-build prompt --- apps/cli/src/templates/eval-build.prompt.md | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/cli/src/templates/eval-build.prompt.md b/apps/cli/src/templates/eval-build.prompt.md index ad62888..e347b38 100644 --- a/apps/cli/src/templates/eval-build.prompt.md +++ b/apps/cli/src/templates/eval-build.prompt.md @@ -13,6 +13,7 @@ description: 'Apply when writing evals in YAML format' - Message fields: `role` (required), `content` (required) - Message roles: `system`, `user`, `assistant`, `tool` - Content types: `text` (inline), `file` (relative or absolute path) +- Attachments (type: `file`) should default to the `user` role - File paths must start with "/" for absolute paths (e.g., "/prompts/file.md") ## Example From 1e2d77cede0d6329c28cc9fca38b4b4790347615 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 24 Nov 2025 14:03:54 +1100 Subject: [PATCH 3/5] add config.yaml and targets.yaml to init subcommand --- apps/cli/src/commands/init/index.ts | 47 +++++++++++++++--- apps/cli/src/templates/.env.template | 35 +++++++++++++ apps/cli/src/templates/config.yaml | 16 ++++++ apps/cli/src/templates/index.ts | 52 +++++++++++++++++++ apps/cli/src/templates/targets.yaml | 74 ++++++++++++++++++++++++++++ apps/cli/test/init.test.ts | 59 ++++++++++++++++++++++ apps/cli/tsup.config.ts | 5 +- 7 files changed, 281 insertions(+), 7 deletions(-) create mode 100644 apps/cli/src/templates/.env.template create mode 100644 apps/cli/src/templates/config.yaml create mode 100644 apps/cli/src/templates/targets.yaml diff --git a/apps/cli/src/commands/init/index.ts b/apps/cli/src/commands/init/index.ts index 1d88e0f..eba84c1 100644 --- a/apps/cli/src/commands/init/index.ts +++ b/apps/cli/src/commands/init/index.ts @@ -25,20 +25,30 @@ async function promptYesNo(message: string): Promise { export async function initCommand(options: InitCommandOptions = {}): Promise { const targetPath = path.resolve(options.targetPath ?? "."); const githubDir = path.join(targetPath, ".github"); + const agentvDir = path.join(targetPath, ".agentv"); // Get templates - const templates = TemplateManager.getTemplates(); + const githubTemplates = TemplateManager.getTemplates(); + const agentvTemplates = TemplateManager.getAgentvTemplates(); // Check if any files already exist const existingFiles: string[] = []; if (existsSync(githubDir)) { - for (const template of templates) { + for (const template of githubTemplates) { const targetFilePath = path.join(githubDir, template.path); if (existsSync(targetFilePath)) { existingFiles.push(path.relative(targetPath, targetFilePath)); } } } + if (existsSync(agentvDir)) { + for (const template of agentvTemplates) { + const targetFilePath = path.join(agentvDir, template.path); + if (existsSync(targetFilePath)) { + existingFiles.push(path.relative(targetPath, targetFilePath)); + } + } + } // If files exist, prompt user if (existingFiles.length > 0) { @@ -59,8 +69,13 @@ export async function initCommand(options: InitCommandOptions = {}): Promise console.log(` - ${t.path}`)); - console.log("\nYou can now create eval files using the schema and prompt templates."); + githubTemplates.forEach((t) => console.log(` - ${t.path}`)); + console.log(`\nFiles installed to ${path.relative(targetPath, agentvDir)}:`); + agentvTemplates.forEach((t) => console.log(` - ${t.path}`)); + console.log("\nYou can now:"); + console.log(" 1. Edit .agentv/.env with your API credentials"); + console.log(" 2. Configure targets in .agentv/targets.yaml"); + console.log(" 3. Create eval files using the schema and prompt templates"); } diff --git a/apps/cli/src/templates/.env.template b/apps/cli/src/templates/.env.template new file mode 100644 index 0000000..6d9c0dd --- /dev/null +++ b/apps/cli/src/templates/.env.template @@ -0,0 +1,35 @@ +# Example environment configuration for AgentV +# Copy this file to .env and fill in your credentials + +# Model Provider Selection (Optional - can be configured via targets.yaml) +PROVIDER=azure + +# Azure OpenAI Configuration +# These are the default environment variable names used in the provided targets.yaml +AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/ +AZURE_OPENAI_API_KEY=your-api-key-here +AZURE_DEPLOYMENT_NAME=gpt-4o + +# Anthropic Configuration (if using Anthropic provider) +ANTHROPIC_API_KEY=your-anthropic-api-key-here + +# VS Code Workspace Paths for Execution Targets +# Note: Using forward slashes is recommended for paths in .env files +# to avoid issues with escape characters. +PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace + +# CLI provider sample (used by the local_cli target) +PROJECT_ROOT=D:/GitHub/your-username/agentv/docs/examples/simple +LOCAL_AGENT_TOKEN=your-cli-token + +# Codex CLI Configuration +# Either OPENAI_API_KEY or CODEX_API_KEY must be set before running codex targets +OPENAI_API_KEY=your-openai-or-codex-key +CODEX_API_KEY= +CODEX_PROFILE=default +CODEX_MODEL=gpt-4o-mini +CODEX_APPROVAL_PRESET=auto +CODEX_CLI_PATH=C:/Program Files/Codex/bin/codex.exe +CODEX_WORKSPACE_DIR=C:/Temp/agentv-codex +# Optional override if your codex config lives outside the default ~/.codex/config +CODEX_CONFIG_PATH= diff --git a/apps/cli/src/templates/config.yaml b/apps/cli/src/templates/config.yaml new file mode 100644 index 0000000..33577b3 --- /dev/null +++ b/apps/cli/src/templates/config.yaml @@ -0,0 +1,16 @@ +$schema: agentv-config-v2 + +# Customize which files are treated as guidelines vs regular file content + +# Custom guideline patterns: +guideline_patterns: + - "**/*.instructions.md" + - "**/instructions/**" + - "**/*.prompt.md" + - "**/prompts/**" + +# Notes: +# - Patterns use standard glob syntax (via micromatch library) +# - Paths are normalized to forward slashes for cross-platform compatibility +# - Only files matching these patterns are loaded as guidelines +# - All other files referenced in eval cases are treated as regular file content diff --git a/apps/cli/src/templates/index.ts b/apps/cli/src/templates/index.ts index c9a5259..b9121c2 100644 --- a/apps/cli/src/templates/index.ts +++ b/apps/cli/src/templates/index.ts @@ -36,6 +36,18 @@ export class TemplateManager { path.join(templatesDir, "config-schema.json"), "utf-8" ); + const targetsYaml = readFileSync( + path.join(templatesDir, "targets.yaml"), + "utf-8" + ); + const configYaml = readFileSync( + path.join(templatesDir, "config.yaml"), + "utf-8" + ); + const envTemplate = readFileSync( + path.join(templatesDir, ".env.template"), + "utf-8" + ); return [ { @@ -52,4 +64,44 @@ export class TemplateManager { }, ]; } + + static getAgentvTemplates(): Template[] { + // Get templates for .agentv directory + const currentDir = path.dirname(fileURLToPath(import.meta.url)); + + let templatesDir: string; + if (currentDir.includes(path.sep + "dist")) { + templatesDir = path.join(currentDir, "templates"); + } else { + templatesDir = currentDir; + } + + const targetsYaml = readFileSync( + path.join(templatesDir, "targets.yaml"), + "utf-8" + ); + const configYaml = readFileSync( + path.join(templatesDir, "config.yaml"), + "utf-8" + ); + const envTemplate = readFileSync( + path.join(templatesDir, ".env.template"), + "utf-8" + ); + + return [ + { + path: "targets.yaml", + content: targetsYaml, + }, + { + path: "config.yaml", + content: configYaml, + }, + { + path: ".env", + content: envTemplate, + }, + ]; + } } diff --git a/apps/cli/src/templates/targets.yaml b/apps/cli/src/templates/targets.yaml new file mode 100644 index 0000000..0372cb2 --- /dev/null +++ b/apps/cli/src/templates/targets.yaml @@ -0,0 +1,74 @@ +$schema: agentv-targets-v2 + +# A list of all supported evaluation targets for the project. +# Each target defines a provider and its specific configuration. +# Actual values for paths/keys are stored in the local .env file. + +targets: + - name: vscode_projectx + provider: vscode + settings: + # VS Code provider requires a workspace template path. + workspace_template: PROJECTX_WORKSPACE_PATH + # This key makes the judge model configurable and is good practice + judge_target: azure_base + + - name: vscode_insiders_projectx + provider: vscode-insiders + settings: + # VS Code Insiders provider (preview version) requires a workspace template path. + workspace_template: PROJECTX_WORKSPACE_PATH + # This key makes the judge model configurable and is good practice + judge_target: azure_base + + - name: azure_base + provider: azure + # The base LLM provider needs no extra settings, as the model is + # defined in the .env file. + settings: + endpoint: AZURE_OPENAI_ENDPOINT + api_key: AZURE_OPENAI_API_KEY + model: AZURE_DEPLOYMENT_NAME + + - name: default + provider: azure + # The base LLM provider needs no extra settings, as the model is + # defined in the .env file. + settings: + endpoint: AZURE_OPENAI_ENDPOINT + api_key: AZURE_OPENAI_API_KEY + model: AZURE_DEPLOYMENT_NAME + + - name: local_cli + provider: cli + judge_target: azure_base + settings: + # Passes the fully rendered prompt and any attached files to a local Python script + # NOTE: Do not add quotes around {PROMPT} or {FILES} - they are already shell-escaped + command_template: uv run ./mock_cli.py --prompt {PROMPT} {FILES} + # Format for each file in {FILES}. {path} and {basename} are automatically shell-escaped, so no quotes needed + files_format: --file {path} + # Optional working directory and env overrides resolved from .env + cwd: CLI_EVALS_DIR + env: + API_TOKEN: LOCAL_AGENT_TOKEN + timeout_seconds: 30 + healthcheck: + type: command + command_template: uv run ./mock_cli.py --healthcheck + + - name: codex_cli + provider: codex + judge_target: azure_base + settings: + # Uses the Codex CLI (defaults to `codex` on PATH) + # executable: CODEX_CLI_PATH # Optional: override executable path + # args: # Optional additional CLI arguments + # - --profile + # - CODEX_PROFILE + # - --model + # - CODEX_MODEL + # - --ask-for-approval + # - CODEX_APPROVAL_PRESET + timeout_seconds: 180 + cwd: CODEX_WORKSPACE_DIR # Where scratch workspaces are created diff --git a/apps/cli/test/init.test.ts b/apps/cli/test/init.test.ts index 8a780c9..9786c38 100644 --- a/apps/cli/test/init.test.ts +++ b/apps/cli/test/init.test.ts @@ -29,6 +29,13 @@ describe("init command", () => { expect(existsSync(githubDir)).toBe(true); }); + it("should create .agentv directory if it doesn't exist", async () => { + await initCommand({ targetPath: TEST_DIR }); + + const agentvDir = path.join(TEST_DIR, ".agentv"); + expect(existsSync(agentvDir)).toBe(true); + }); + it("should create prompt template file", async () => { await initCommand({ targetPath: TEST_DIR }); @@ -55,6 +62,41 @@ describe("init command", () => { expect(schema.properties.evalcases).toBeDefined(); }); + it("should create targets.yaml file", async () => { + await initCommand({ targetPath: TEST_DIR }); + + const targetsFile = path.join(TEST_DIR, ".agentv", "targets.yaml"); + expect(existsSync(targetsFile)).toBe(true); + + const content = readFileSync(targetsFile, "utf-8"); + expect(content).toContain("$schema: agentv-targets-v2"); + expect(content).toContain("targets:"); + expect(content).toContain("azure_base"); + }); + + it("should create config.yaml file", async () => { + await initCommand({ targetPath: TEST_DIR }); + + const configFile = path.join(TEST_DIR, ".agentv", "config.yaml"); + expect(existsSync(configFile)).toBe(true); + + const content = readFileSync(configFile, "utf-8"); + expect(content).toContain("$schema: agentv-config-v2"); + expect(content).toContain("guideline_patterns:"); + }); + + it("should create .env file", async () => { + await initCommand({ targetPath: TEST_DIR }); + + const envFile = path.join(TEST_DIR, ".agentv", ".env"); + expect(existsSync(envFile)).toBe(true); + + const content = readFileSync(envFile, "utf-8"); + expect(content).toContain("AZURE_OPENAI_ENDPOINT"); + expect(content).toContain("AZURE_OPENAI_API_KEY"); + expect(content).toContain("PROJECTX_WORKSPACE_PATH"); + }); + it("should work when .github directory already exists", async () => { const githubDir = path.join(TEST_DIR, ".github"); mkdirSync(githubDir, { recursive: true }); @@ -68,12 +110,29 @@ describe("init command", () => { expect(existsSync(schemaFile)).toBe(true); }); + it("should work when .agentv directory already exists", async () => { + const agentvDir = path.join(TEST_DIR, ".agentv"); + mkdirSync(agentvDir, { recursive: true }); + + await initCommand({ targetPath: TEST_DIR }); + + const targetsFile = path.join(agentvDir, "targets.yaml"); + const configFile = path.join(agentvDir, "config.yaml"); + const envFile = path.join(agentvDir, ".env"); + + expect(existsSync(targetsFile)).toBe(true); + expect(existsSync(configFile)).toBe(true); + expect(existsSync(envFile)).toBe(true); + }); + it("should default to current directory when no path provided", async () => { // Just test that it defaults to "." when no path is provided // We can't test process.chdir in vitest workers await initCommand({ targetPath: TEST_DIR }); const githubDir = path.join(TEST_DIR, ".github"); + const agentvDir = path.join(TEST_DIR, ".agentv"); expect(existsSync(githubDir)).toBe(true); + expect(existsSync(agentvDir)).toBe(true); }); }); diff --git a/apps/cli/tsup.config.ts b/apps/cli/tsup.config.ts index 4478fdd..a6ca82f 100644 --- a/apps/cli/tsup.config.ts +++ b/apps/cli/tsup.config.ts @@ -25,7 +25,10 @@ export default defineConfig({ const templates = [ "eval-build.prompt.md", "eval-schema.json", - "config-schema.json" + "config-schema.json", + "targets.yaml", + "config.yaml", + ".env.template" ]; for (const file of templates) { From f530fc292615319c9b211b5a0993766d53437da1 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 24 Nov 2025 14:55:38 +1100 Subject: [PATCH 4/5] Refactor template system --- apps/cli/src/commands/init/index.ts | 2 +- .../src/templates/{ => agentv}/.env.template | 12 -- .../src/templates/{ => agentv}/config.yaml | 0 .../src/templates/{ => agentv}/targets.yaml | 0 .../{ => github/contexts}/config-schema.json | 0 .../{ => github/contexts}/eval-schema.json | 0 .../{ => github/prompts}/eval-build.prompt.md | 0 apps/cli/src/templates/index.ts | 116 +++++------------- apps/cli/tsup.config.ts | 32 ++--- .../core/src/evaluation/providers/index.ts | 5 +- .../core/src/evaluation/providers/preread.ts | 2 +- 11 files changed, 48 insertions(+), 121 deletions(-) rename apps/cli/src/templates/{ => agentv}/.env.template (68%) rename apps/cli/src/templates/{ => agentv}/config.yaml (100%) rename apps/cli/src/templates/{ => agentv}/targets.yaml (100%) rename apps/cli/src/templates/{ => github/contexts}/config-schema.json (100%) rename apps/cli/src/templates/{ => github/contexts}/eval-schema.json (100%) rename apps/cli/src/templates/{ => github/prompts}/eval-build.prompt.md (100%) diff --git a/apps/cli/src/commands/init/index.ts b/apps/cli/src/commands/init/index.ts index eba84c1..3e07ca5 100644 --- a/apps/cli/src/commands/init/index.ts +++ b/apps/cli/src/commands/init/index.ts @@ -28,7 +28,7 @@ export async function initCommand(options: InitCommandOptions = {}): Promise { - const templatesDir = path.join("dist", "templates"); - if (!existsSync(templatesDir)) { - mkdirSync(templatesDir, { recursive: true }); - } + const srcTemplatesDir = path.join("src", "templates"); + const distTemplatesDir = path.join("dist", "templates"); - // Copy template files - const templates = [ - "eval-build.prompt.md", - "eval-schema.json", - "config-schema.json", - "targets.yaml", - "config.yaml", - ".env.template" - ]; - - for (const file of templates) { - copyFileSync( - path.join("src", "templates", file), - path.join(templatesDir, file) - ); - } + // Copy entire templates directory structure recursively + cpSync(srcTemplatesDir, distTemplatesDir, { + recursive: true, + filter: (src) => { + // Skip index.ts and any TypeScript files + return !src.endsWith(".ts"); + } + }); console.log("✓ Template files copied to dist/templates"); }, diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 0e1fc53..a9d1c21 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -1,5 +1,6 @@ import { AnthropicProvider, AzureProvider, GeminiProvider } from "./ax.js"; import { CliProvider } from "./cli.js"; +import { CodexProvider } from "./codex.js"; import { MockProvider } from "./mock.js"; import type { ResolvedTarget } from "./targets.js"; import { resolveTargetDefinition } from "./targets.js"; @@ -15,13 +16,9 @@ export type { TargetDefinition, } from "./types.js"; -import { CodexProvider } from "./codex.js"; -import type { CodexResolvedConfig } from "./targets.js"; - export type { AnthropicResolvedConfig, AzureResolvedConfig, - CodexResolvedConfig, CliResolvedConfig, GeminiResolvedConfig, MockResolvedConfig, diff --git a/packages/core/src/evaluation/providers/preread.ts b/packages/core/src/evaluation/providers/preread.ts index 2bb9680..186c4e0 100644 --- a/packages/core/src/evaluation/providers/preread.ts +++ b/packages/core/src/evaluation/providers/preread.ts @@ -1,7 +1,7 @@ import path from "node:path"; -import { isGuidelineFile } from "../yaml-parser.js"; import type { ProviderRequest } from "./types.js"; +import { isGuidelineFile } from "../yaml-parser.js"; export interface PromptDocumentOptions { readonly guidelinePatterns?: readonly string[]; From 695f9d7850d8984f5a10dd5a2fdd8f083953ed7b Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 24 Nov 2025 14:57:53 +1100 Subject: [PATCH 5/5] chore: bump version to 0.5.1 --- apps/cli/package.json | 2 +- packages/core/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/cli/package.json b/apps/cli/package.json index 14df1e5..0182ac3 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "agentv", - "version": "0.5.0", + "version": "0.5.1", "description": "CLI entry point for AgentV", "type": "module", "repository": { diff --git a/packages/core/package.json b/packages/core/package.json index 369ff51..170bf82 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@agentv/core", - "version": "0.5.0", + "version": "0.5.1", "description": "Primitive runtime components for AgentV", "type": "module", "repository": {