diff --git a/apps/cli/package.json b/apps/cli/package.json index 14df1e5..0182ac3 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "agentv", - "version": "0.5.0", + "version": "0.5.1", "description": "CLI entry point for AgentV", "type": "module", "repository": { diff --git a/apps/cli/src/commands/init/index.ts b/apps/cli/src/commands/init/index.ts index 15a80f3..3e07ca5 100644 --- a/apps/cli/src/commands/init/index.ts +++ b/apps/cli/src/commands/init/index.ts @@ -1,5 +1,6 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; +import * as readline from "node:readline/promises"; import { TemplateManager } from "../../templates/index.js"; @@ -7,20 +8,74 @@ export interface InitCommandOptions { targetPath?: string; } +async function promptYesNo(message: string): Promise { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + try { + const answer = await rl.question(`${message} (y/N): `); + return answer.toLowerCase() === "y" || answer.toLowerCase() === "yes"; + } finally { + rl.close(); + } +} + export async function initCommand(options: InitCommandOptions = {}): Promise { const targetPath = path.resolve(options.targetPath ?? "."); const githubDir = path.join(targetPath, ".github"); + const agentvDir = path.join(targetPath, ".agentv"); + + // Get templates + const githubTemplates = TemplateManager.getGithubTemplates(); + const agentvTemplates = TemplateManager.getAgentvTemplates(); + + // Check if any files already exist + const existingFiles: string[] = []; + if (existsSync(githubDir)) { + for (const template of githubTemplates) { + const targetFilePath = path.join(githubDir, template.path); + if (existsSync(targetFilePath)) { + existingFiles.push(path.relative(targetPath, targetFilePath)); + } + } + } + if (existsSync(agentvDir)) { + for (const template of agentvTemplates) { + const targetFilePath = path.join(agentvDir, template.path); + if (existsSync(targetFilePath)) { + existingFiles.push(path.relative(targetPath, targetFilePath)); + } + } + } + + // If files exist, prompt user + if (existingFiles.length > 0) { + console.log("We detected an existing setup:"); + existingFiles.forEach((file) => console.log(` - ${file}`)); + console.log(); + + const shouldReplace = await promptYesNo("Do you want to replace these files?"); + if (!shouldReplace) { + console.log("\nInit cancelled. No files were changed."); + return; + } + console.log(); + } // Create .github directory if it doesn't exist if (!existsSync(githubDir)) { mkdirSync(githubDir, { recursive: true }); } - // Get templates - const templates = TemplateManager.getTemplates(); + // Create .agentv directory if it doesn't exist + if (!existsSync(agentvDir)) { + mkdirSync(agentvDir, { recursive: true }); + } - // Copy each template to .github - for (const template of templates) { + // Copy each .github template + for (const template of githubTemplates) { const targetFilePath = path.join(githubDir, template.path); const targetDirPath = path.dirname(targetFilePath); @@ -34,8 +89,28 @@ export async function initCommand(options: InitCommandOptions = {}): Promise console.log(` - ${t.path}`)); - console.log("\nYou can now create eval files using the schema and prompt templates."); + githubTemplates.forEach((t) => console.log(` - ${t.path}`)); + console.log(`\nFiles installed to ${path.relative(targetPath, agentvDir)}:`); + agentvTemplates.forEach((t) => console.log(` - ${t.path}`)); + console.log("\nYou can now:"); + console.log(" 1. Edit .agentv/.env with your API credentials"); + console.log(" 2. Configure targets in .agentv/targets.yaml"); + console.log(" 3. Create eval files using the schema and prompt templates"); } diff --git a/apps/cli/src/templates/agentv/.env.template b/apps/cli/src/templates/agentv/.env.template new file mode 100644 index 0000000..4eccc0c --- /dev/null +++ b/apps/cli/src/templates/agentv/.env.template @@ -0,0 +1,23 @@ +# Example environment configuration for AgentV +# Copy this file to .env and fill in your credentials + +# Model Provider Selection (Optional - can be configured via targets.yaml) +PROVIDER=azure + +# Azure OpenAI Configuration +# These are the default environment variable names used in the provided targets.yaml +AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/ +AZURE_OPENAI_API_KEY=your-api-key-here +AZURE_DEPLOYMENT_NAME=gpt-4o + +# Anthropic Configuration (if using Anthropic provider) +ANTHROPIC_API_KEY=your-anthropic-api-key-here + +# VS Code Workspace Paths for Execution Targets +# Note: Using forward slashes is recommended for paths in .env files +# to avoid issues with escape characters. +PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace + +# CLI provider sample (used by the local_cli target) +PROJECT_ROOT=D:/GitHub/your-username/agentv/docs/examples/simple +LOCAL_AGENT_TOKEN=your-cli-token diff --git a/apps/cli/src/templates/agentv/config.yaml b/apps/cli/src/templates/agentv/config.yaml new file mode 100644 index 0000000..33577b3 --- /dev/null +++ b/apps/cli/src/templates/agentv/config.yaml @@ -0,0 +1,16 @@ +$schema: agentv-config-v2 + +# Customize which files are treated as guidelines vs regular file content + +# Custom guideline patterns: +guideline_patterns: + - "**/*.instructions.md" + - "**/instructions/**" + - "**/*.prompt.md" + - "**/prompts/**" + +# Notes: +# - Patterns use standard glob syntax (via micromatch library) +# - Paths are normalized to forward slashes for cross-platform compatibility +# - Only files matching these patterns are loaded as guidelines +# - All other files referenced in eval cases are treated as regular file content diff --git a/apps/cli/src/templates/agentv/targets.yaml b/apps/cli/src/templates/agentv/targets.yaml new file mode 100644 index 0000000..0372cb2 --- /dev/null +++ b/apps/cli/src/templates/agentv/targets.yaml @@ -0,0 +1,74 @@ +$schema: agentv-targets-v2 + +# A list of all supported evaluation targets for the project. +# Each target defines a provider and its specific configuration. +# Actual values for paths/keys are stored in the local .env file. + +targets: + - name: vscode_projectx + provider: vscode + settings: + # VS Code provider requires a workspace template path. + workspace_template: PROJECTX_WORKSPACE_PATH + # This key makes the judge model configurable and is good practice + judge_target: azure_base + + - name: vscode_insiders_projectx + provider: vscode-insiders + settings: + # VS Code Insiders provider (preview version) requires a workspace template path. + workspace_template: PROJECTX_WORKSPACE_PATH + # This key makes the judge model configurable and is good practice + judge_target: azure_base + + - name: azure_base + provider: azure + # The base LLM provider needs no extra settings, as the model is + # defined in the .env file. + settings: + endpoint: AZURE_OPENAI_ENDPOINT + api_key: AZURE_OPENAI_API_KEY + model: AZURE_DEPLOYMENT_NAME + + - name: default + provider: azure + # The base LLM provider needs no extra settings, as the model is + # defined in the .env file. + settings: + endpoint: AZURE_OPENAI_ENDPOINT + api_key: AZURE_OPENAI_API_KEY + model: AZURE_DEPLOYMENT_NAME + + - name: local_cli + provider: cli + judge_target: azure_base + settings: + # Passes the fully rendered prompt and any attached files to a local Python script + # NOTE: Do not add quotes around {PROMPT} or {FILES} - they are already shell-escaped + command_template: uv run ./mock_cli.py --prompt {PROMPT} {FILES} + # Format for each file in {FILES}. {path} and {basename} are automatically shell-escaped, so no quotes needed + files_format: --file {path} + # Optional working directory and env overrides resolved from .env + cwd: CLI_EVALS_DIR + env: + API_TOKEN: LOCAL_AGENT_TOKEN + timeout_seconds: 30 + healthcheck: + type: command + command_template: uv run ./mock_cli.py --healthcheck + + - name: codex_cli + provider: codex + judge_target: azure_base + settings: + # Uses the Codex CLI (defaults to `codex` on PATH) + # executable: CODEX_CLI_PATH # Optional: override executable path + # args: # Optional additional CLI arguments + # - --profile + # - CODEX_PROFILE + # - --model + # - CODEX_MODEL + # - --ask-for-approval + # - CODEX_APPROVAL_PRESET + timeout_seconds: 180 + cwd: CODEX_WORKSPACE_DIR # Where scratch workspaces are created diff --git a/apps/cli/src/templates/config-schema.json b/apps/cli/src/templates/github/contexts/config-schema.json similarity index 100% rename from apps/cli/src/templates/config-schema.json rename to apps/cli/src/templates/github/contexts/config-schema.json diff --git a/apps/cli/src/templates/eval-schema.json b/apps/cli/src/templates/github/contexts/eval-schema.json similarity index 100% rename from apps/cli/src/templates/eval-schema.json rename to apps/cli/src/templates/github/contexts/eval-schema.json diff --git a/apps/cli/src/templates/eval-build.prompt.md b/apps/cli/src/templates/github/prompts/eval-build.prompt.md similarity index 98% rename from apps/cli/src/templates/eval-build.prompt.md rename to apps/cli/src/templates/github/prompts/eval-build.prompt.md index ad62888..e347b38 100644 --- a/apps/cli/src/templates/eval-build.prompt.md +++ b/apps/cli/src/templates/github/prompts/eval-build.prompt.md @@ -13,6 +13,7 @@ description: 'Apply when writing evals in YAML format' - Message fields: `role` (required), `content` (required) - Message roles: `system`, `user`, `assistant`, `tool` - Content types: `text` (inline), `file` (relative or absolute path) +- Attachments (type: `file`) should default to the `user` role - File paths must start with "/" for absolute paths (e.g., "/prompts/file.md") ## Example diff --git a/apps/cli/src/templates/index.ts b/apps/cli/src/templates/index.ts index c9a5259..0213db7 100644 --- a/apps/cli/src/templates/index.ts +++ b/apps/cli/src/templates/index.ts @@ -1,4 +1,4 @@ -import { readFileSync } from "node:fs"; +import { readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -8,48 +8,52 @@ export interface Template { } export class TemplateManager { - static getTemplates(): Template[] { - // Resolve templates directory: - // - In production (dist): templates are at dist/templates/ - // - In development (src): templates are at src/templates/ + static getGithubTemplates(): Template[] { + return this.getTemplatesFromDir("github"); + } + + static getAgentvTemplates(): Template[] { + return this.getTemplatesFromDir("agentv"); + } + + private static getTemplatesFromDir(subdir: string): Template[] { const currentDir = path.dirname(fileURLToPath(import.meta.url)); // Check if we're running from dist or src let templatesDir: string; if (currentDir.includes(path.sep + "dist")) { // Production: templates are at dist/templates/ - templatesDir = path.join(currentDir, "templates"); + templatesDir = path.join(currentDir, "templates", subdir); } else { // Development: templates are at src/templates/ (same directory as this file) - templatesDir = currentDir; + templatesDir = path.join(currentDir, subdir); } - - const evalBuildPrompt = readFileSync( - path.join(templatesDir, "eval-build.prompt.md"), - "utf-8" - ); - const evalSchema = readFileSync( - path.join(templatesDir, "eval-schema.json"), - "utf-8" - ); - const configSchema = readFileSync( - path.join(templatesDir, "config-schema.json"), - "utf-8" - ); - - return [ - { - path: "prompts/eval-build.prompt.md", - content: evalBuildPrompt, - }, - { - path: "contexts/eval-schema.json", - content: evalSchema, - }, - { - path: "contexts/config-schema.json", - content: configSchema, - }, - ]; + + return this.readTemplatesRecursively(templatesDir, ""); + } + + private static readTemplatesRecursively(dir: string, relativePath: string): Template[] { + const templates: Template[] = []; + const entries = readdirSync(dir); + + for (const entry of entries) { + const fullPath = path.join(dir, entry); + const stat = statSync(fullPath); + const entryRelativePath = relativePath ? path.join(relativePath, entry) : entry; + + if (stat.isDirectory()) { + // Recursively read subdirectories + templates.push(...this.readTemplatesRecursively(fullPath, entryRelativePath)); + } else { + // Read file content + const content = readFileSync(fullPath, "utf-8"); + templates.push({ + path: entryRelativePath.split(path.sep).join("/"), // Normalize to forward slashes + content, + }); + } + } + + return templates; } } diff --git a/apps/cli/test/init.test.ts b/apps/cli/test/init.test.ts index 8a780c9..9786c38 100644 --- a/apps/cli/test/init.test.ts +++ b/apps/cli/test/init.test.ts @@ -29,6 +29,13 @@ describe("init command", () => { expect(existsSync(githubDir)).toBe(true); }); + it("should create .agentv directory if it doesn't exist", async () => { + await initCommand({ targetPath: TEST_DIR }); + + const agentvDir = path.join(TEST_DIR, ".agentv"); + expect(existsSync(agentvDir)).toBe(true); + }); + it("should create prompt template file", async () => { await initCommand({ targetPath: TEST_DIR }); @@ -55,6 +62,41 @@ describe("init command", () => { expect(schema.properties.evalcases).toBeDefined(); }); + it("should create targets.yaml file", async () => { + await initCommand({ targetPath: TEST_DIR }); + + const targetsFile = path.join(TEST_DIR, ".agentv", "targets.yaml"); + expect(existsSync(targetsFile)).toBe(true); + + const content = readFileSync(targetsFile, "utf-8"); + expect(content).toContain("$schema: agentv-targets-v2"); + expect(content).toContain("targets:"); + expect(content).toContain("azure_base"); + }); + + it("should create config.yaml file", async () => { + await initCommand({ targetPath: TEST_DIR }); + + const configFile = path.join(TEST_DIR, ".agentv", "config.yaml"); + expect(existsSync(configFile)).toBe(true); + + const content = readFileSync(configFile, "utf-8"); + expect(content).toContain("$schema: agentv-config-v2"); + expect(content).toContain("guideline_patterns:"); + }); + + it("should create .env file", async () => { + await initCommand({ targetPath: TEST_DIR }); + + const envFile = path.join(TEST_DIR, ".agentv", ".env"); + expect(existsSync(envFile)).toBe(true); + + const content = readFileSync(envFile, "utf-8"); + expect(content).toContain("AZURE_OPENAI_ENDPOINT"); + expect(content).toContain("AZURE_OPENAI_API_KEY"); + expect(content).toContain("PROJECTX_WORKSPACE_PATH"); + }); + it("should work when .github directory already exists", async () => { const githubDir = path.join(TEST_DIR, ".github"); mkdirSync(githubDir, { recursive: true }); @@ -68,12 +110,29 @@ describe("init command", () => { expect(existsSync(schemaFile)).toBe(true); }); + it("should work when .agentv directory already exists", async () => { + const agentvDir = path.join(TEST_DIR, ".agentv"); + mkdirSync(agentvDir, { recursive: true }); + + await initCommand({ targetPath: TEST_DIR }); + + const targetsFile = path.join(agentvDir, "targets.yaml"); + const configFile = path.join(agentvDir, "config.yaml"); + const envFile = path.join(agentvDir, ".env"); + + expect(existsSync(targetsFile)).toBe(true); + expect(existsSync(configFile)).toBe(true); + expect(existsSync(envFile)).toBe(true); + }); + it("should default to current directory when no path provided", async () => { // Just test that it defaults to "." when no path is provided // We can't test process.chdir in vitest workers await initCommand({ targetPath: TEST_DIR }); const githubDir = path.join(TEST_DIR, ".github"); + const agentvDir = path.join(TEST_DIR, ".agentv"); expect(existsSync(githubDir)).toBe(true); + expect(existsSync(agentvDir)).toBe(true); }); }); diff --git a/apps/cli/tsup.config.ts b/apps/cli/tsup.config.ts index 4478fdd..da3095b 100644 --- a/apps/cli/tsup.config.ts +++ b/apps/cli/tsup.config.ts @@ -1,4 +1,4 @@ -import { copyFileSync, mkdirSync, existsSync } from "node:fs"; +import { cpSync, existsSync } from "node:fs"; import path from "node:path"; import { defineConfig } from "tsup"; @@ -16,24 +16,17 @@ export default defineConfig({ external: ["micromatch"], // Copy template files after build onSuccess: async () => { - const templatesDir = path.join("dist", "templates"); - if (!existsSync(templatesDir)) { - mkdirSync(templatesDir, { recursive: true }); - } + const srcTemplatesDir = path.join("src", "templates"); + const distTemplatesDir = path.join("dist", "templates"); - // Copy template files - const templates = [ - "eval-build.prompt.md", - "eval-schema.json", - "config-schema.json" - ]; - - for (const file of templates) { - copyFileSync( - path.join("src", "templates", file), - path.join(templatesDir, file) - ); - } + // Copy entire templates directory structure recursively + cpSync(srcTemplatesDir, distTemplatesDir, { + recursive: true, + filter: (src) => { + // Skip index.ts and any TypeScript files + return !src.endsWith(".ts"); + } + }); console.log("✓ Template files copied to dist/templates"); }, diff --git a/packages/core/package.json b/packages/core/package.json index 369ff51..170bf82 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@agentv/core", - "version": "0.5.0", + "version": "0.5.1", "description": "Primitive runtime components for AgentV", "type": "module", "repository": { diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 0e1fc53..a9d1c21 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -1,5 +1,6 @@ import { AnthropicProvider, AzureProvider, GeminiProvider } from "./ax.js"; import { CliProvider } from "./cli.js"; +import { CodexProvider } from "./codex.js"; import { MockProvider } from "./mock.js"; import type { ResolvedTarget } from "./targets.js"; import { resolveTargetDefinition } from "./targets.js"; @@ -15,13 +16,9 @@ export type { TargetDefinition, } from "./types.js"; -import { CodexProvider } from "./codex.js"; -import type { CodexResolvedConfig } from "./targets.js"; - export type { AnthropicResolvedConfig, AzureResolvedConfig, - CodexResolvedConfig, CliResolvedConfig, GeminiResolvedConfig, MockResolvedConfig, diff --git a/packages/core/src/evaluation/providers/preread.ts b/packages/core/src/evaluation/providers/preread.ts index 2bb9680..186c4e0 100644 --- a/packages/core/src/evaluation/providers/preread.ts +++ b/packages/core/src/evaluation/providers/preread.ts @@ -1,7 +1,7 @@ import path from "node:path"; -import { isGuidelineFile } from "../yaml-parser.js"; import type { ProviderRequest } from "./types.js"; +import { isGuidelineFile } from "../yaml-parser.js"; export interface PromptDocumentOptions { readonly guidelinePatterns?: readonly string[];