feat: add user confirmation to init subcommand

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentv",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": "CLI entry point for AgentV",
   "type": "module",
   "repository": {

apps/cli/src/commands/init/index.ts

@@ -1,26 +1,81 @@
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import path from "node:path";
+import * as readline from "node:readline/promises";
 
 import { TemplateManager } from "../../templates/index.js";
 
 export interface InitCommandOptions {
   targetPath?: string;
 }
 
+async function promptYesNo(message: string): Promise<boolean> {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  try {
+    const answer = await rl.question(`${message} (y/N): `);
+    return answer.toLowerCase() === "y" || answer.toLowerCase() === "yes";
+  } finally {
+    rl.close();
+  }
+}
+
 export async function initCommand(options: InitCommandOptions = {}): Promise<void> {
   const targetPath = path.resolve(options.targetPath ?? ".");
   const githubDir = path.join(targetPath, ".github");
+  const agentvDir = path.join(targetPath, ".agentv");
+
+  // Get templates
+  const githubTemplates = TemplateManager.getGithubTemplates();
+  const agentvTemplates = TemplateManager.getAgentvTemplates();
+
+  // Check if any files already exist
+  const existingFiles: string[] = [];
+  if (existsSync(githubDir)) {
+    for (const template of githubTemplates) {
+      const targetFilePath = path.join(githubDir, template.path);
+      if (existsSync(targetFilePath)) {
+        existingFiles.push(path.relative(targetPath, targetFilePath));
+      }
+    }
+  }
+  if (existsSync(agentvDir)) {
+    for (const template of agentvTemplates) {
+      const targetFilePath = path.join(agentvDir, template.path);
+      if (existsSync(targetFilePath)) {
+        existingFiles.push(path.relative(targetPath, targetFilePath));
+      }
+    }
+  }
+
+  // If files exist, prompt user
+  if (existingFiles.length > 0) {
+    console.log("We detected an existing setup:");
+    existingFiles.forEach((file) => console.log(`  - ${file}`));
+    console.log();
+
+    const shouldReplace = await promptYesNo("Do you want to replace these files?");
+    if (!shouldReplace) {
+      console.log("\nInit cancelled. No files were changed.");
+      return;
+    }
+    console.log();
+  }
 
   // Create .github directory if it doesn't exist
   if (!existsSync(githubDir)) {
     mkdirSync(githubDir, { recursive: true });
   }
 
-  // Get templates
-  const templates = TemplateManager.getTemplates();
+  // Create .agentv directory if it doesn't exist
+  if (!existsSync(agentvDir)) {
+    mkdirSync(agentvDir, { recursive: true });
+  }
 
-  // Copy each template to .github
-  for (const template of templates) {
+  // Copy each .github template
+  for (const template of githubTemplates) {
     const targetFilePath = path.join(githubDir, template.path);
     const targetDirPath = path.dirname(targetFilePath);
 
@@ -34,8 +89,28 @@ export async function initCommand(options: InitCommandOptions = {}): Promise<voi
     console.log(`Created ${path.relative(targetPath, targetFilePath)}`);
   }
 
+  // Copy each .agentv template
+  for (const template of agentvTemplates) {
+    const targetFilePath = path.join(agentvDir, template.path);
+    const targetDirPath = path.dirname(targetFilePath);
+
+    // Create directory if needed
+    if (!existsSync(targetDirPath)) {
+      mkdirSync(targetDirPath, { recursive: true });
+    }
+
+    // Write file
+    writeFileSync(targetFilePath, template.content, "utf-8");
+    console.log(`Created ${path.relative(targetPath, targetFilePath)}`);
+  }
+
   console.log("\nAgentV initialized successfully!");
   console.log(`\nFiles installed to ${path.relative(targetPath, githubDir)}:`);
-  templates.forEach((t) => console.log(`  - ${t.path}`));
-  console.log("\nYou can now create eval files using the schema and prompt templates.");
+  githubTemplates.forEach((t) => console.log(`  - ${t.path}`));
+  console.log(`\nFiles installed to ${path.relative(targetPath, agentvDir)}:`);
+  agentvTemplates.forEach((t) => console.log(`  - ${t.path}`));
+  console.log("\nYou can now:");
+  console.log("  1. Edit .agentv/.env with your API credentials");
+  console.log("  2. Configure targets in .agentv/targets.yaml");
+  console.log("  3. Create eval files using the schema and prompt templates");
 }

apps/cli/src/templates/agentv/.env.template

@@ -0,0 +1,23 @@
+# Example environment configuration for AgentV
+# Copy this file to .env and fill in your credentials
+
+# Model Provider Selection (Optional - can be configured via targets.yaml)
+PROVIDER=azure
+
+# Azure OpenAI Configuration
+# These are the default environment variable names used in the provided targets.yaml
+AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/
+AZURE_OPENAI_API_KEY=your-api-key-here
+AZURE_DEPLOYMENT_NAME=gpt-4o
+
+# Anthropic Configuration (if using Anthropic provider)
+ANTHROPIC_API_KEY=your-anthropic-api-key-here
+
+# VS Code Workspace Paths for Execution Targets
+# Note: Using forward slashes is recommended for paths in .env files 
+# to avoid issues with escape characters.
+PROJECTX_WORKSPACE_PATH=C:/Users/your-username/OneDrive - Company Pty Ltd/sample.code-workspace
+
+# CLI provider sample (used by the local_cli target)
+PROJECT_ROOT=D:/GitHub/your-username/agentv/docs/examples/simple
+LOCAL_AGENT_TOKEN=your-cli-token

apps/cli/src/templates/agentv/config.yaml

@@ -0,0 +1,16 @@
+$schema: agentv-config-v2
+
+# Customize which files are treated as guidelines vs regular file content
+
+# Custom guideline patterns:
+guideline_patterns:
+  - "**/*.instructions.md"
+  - "**/instructions/**"
+  - "**/*.prompt.md"
+  - "**/prompts/**"
+
+# Notes:
+# - Patterns use standard glob syntax (via micromatch library)
+# - Paths are normalized to forward slashes for cross-platform compatibility
+# - Only files matching these patterns are loaded as guidelines
+# - All other files referenced in eval cases are treated as regular file content

apps/cli/src/templates/agentv/targets.yaml

@@ -0,0 +1,74 @@
+$schema: agentv-targets-v2
+
+# A list of all supported evaluation targets for the project.
+# Each target defines a provider and its specific configuration.
+# Actual values for paths/keys are stored in the local .env file.
+
+targets:
+  - name: vscode_projectx
+    provider: vscode
+    settings:
+      # VS Code provider requires a workspace template path.
+      workspace_template: PROJECTX_WORKSPACE_PATH
+    # This key makes the judge model configurable and is good practice
+    judge_target: azure_base
+
+  - name: vscode_insiders_projectx
+    provider: vscode-insiders
+    settings:
+      # VS Code Insiders provider (preview version) requires a workspace template path.
+      workspace_template: PROJECTX_WORKSPACE_PATH
+    # This key makes the judge model configurable and is good practice
+    judge_target: azure_base
+
+  - name: azure_base
+    provider: azure
+    # The base LLM provider needs no extra settings, as the model is
+    # defined in the .env file.
+    settings:
+      endpoint: AZURE_OPENAI_ENDPOINT
+      api_key: AZURE_OPENAI_API_KEY
+      model: AZURE_DEPLOYMENT_NAME
+
+  - name: default
+    provider: azure
+    # The base LLM provider needs no extra settings, as the model is
+    # defined in the .env file.
+    settings:
+      endpoint: AZURE_OPENAI_ENDPOINT
+      api_key: AZURE_OPENAI_API_KEY
+      model: AZURE_DEPLOYMENT_NAME
+
+  - name: local_cli
+    provider: cli
+    judge_target: azure_base
+    settings:
+      # Passes the fully rendered prompt and any attached files to a local Python script
+      # NOTE: Do not add quotes around {PROMPT} or {FILES} - they are already shell-escaped
+      command_template: uv run ./mock_cli.py --prompt {PROMPT} {FILES}
+      # Format for each file in {FILES}. {path} and {basename} are automatically shell-escaped, so no quotes needed
+      files_format: --file {path}
+      # Optional working directory and env overrides resolved from .env
+      cwd: CLI_EVALS_DIR
+      env:
+        API_TOKEN: LOCAL_AGENT_TOKEN
+      timeout_seconds: 30
+      healthcheck:
+        type: command
+        command_template: uv run ./mock_cli.py --healthcheck
+
+  - name: codex_cli
+    provider: codex
+    judge_target: azure_base
+    settings:
+      # Uses the Codex CLI (defaults to `codex` on PATH)
+      # executable: CODEX_CLI_PATH        # Optional: override executable path
+      # args:                             # Optional additional CLI arguments
+      #   - --profile
+      #   - CODEX_PROFILE
+      #   - --model
+      #   - CODEX_MODEL
+      #   - --ask-for-approval
+      #   - CODEX_APPROVAL_PRESET
+      timeout_seconds: 180
+      cwd: CODEX_WORKSPACE_DIR            # Where scratch workspaces are created

apps/cli/src/templates/eval-build.prompt.md → apps/cli/src/templates/github/prompts/eval-build.prompt.md

@@ -13,6 +13,7 @@ description: 'Apply when writing evals in YAML format'
 - Message fields: `role` (required), `content` (required)
 - Message roles: `system`, `user`, `assistant`, `tool`
 - Content types: `text` (inline), `file` (relative or absolute path)
+- Attachments (type: `file`) should default to the `user` role
 - File paths must start with "/" for absolute paths (e.g., "/prompts/file.md")
 
 ## Example

apps/cli/src/templates/index.ts

@@ -1,4 +1,4 @@
-import { readFileSync } from "node:fs";
+import { readFileSync, readdirSync, statSync } from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
@@ -8,48 +8,52 @@ export interface Template {
 }
 
 export class TemplateManager {
-  static getTemplates(): Template[] {
-    // Resolve templates directory:
-    // - In production (dist): templates are at dist/templates/
-    // - In development (src): templates are at src/templates/
+  static getGithubTemplates(): Template[] {
+    return this.getTemplatesFromDir("github");
+  }
+
+  static getAgentvTemplates(): Template[] {
+    return this.getTemplatesFromDir("agentv");
+  }
+
+  private static getTemplatesFromDir(subdir: string): Template[] {
     const currentDir = path.dirname(fileURLToPath(import.meta.url));
 
     // Check if we're running from dist or src
     let templatesDir: string;
     if (currentDir.includes(path.sep + "dist")) {
       // Production: templates are at dist/templates/
-      templatesDir = path.join(currentDir, "templates");
+      templatesDir = path.join(currentDir, "templates", subdir);
     } else {
       // Development: templates are at src/templates/ (same directory as this file)
-      templatesDir = currentDir;
+      templatesDir = path.join(currentDir, subdir);
     }
-
-    const evalBuildPrompt = readFileSync(
-      path.join(templatesDir, "eval-build.prompt.md"),
-      "utf-8"
-    );
-    const evalSchema = readFileSync(
-      path.join(templatesDir, "eval-schema.json"),
-      "utf-8"
-    );
-    const configSchema = readFileSync(
-      path.join(templatesDir, "config-schema.json"),
-      "utf-8"
-    );
-
-    return [
-      {
-        path: "prompts/eval-build.prompt.md",
-        content: evalBuildPrompt,
-      },
-      {
-        path: "contexts/eval-schema.json",
-        content: evalSchema,
-      },
-      {
-        path: "contexts/config-schema.json",
-        content: configSchema,
-      },
-    ];
+
+    return this.readTemplatesRecursively(templatesDir, "");
+  }
+
+  private static readTemplatesRecursively(dir: string, relativePath: string): Template[] {
+    const templates: Template[] = [];
+    const entries = readdirSync(dir);
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry);
+      const stat = statSync(fullPath);
+      const entryRelativePath = relativePath ? path.join(relativePath, entry) : entry;
+
+      if (stat.isDirectory()) {
+        // Recursively read subdirectories
+        templates.push(...this.readTemplatesRecursively(fullPath, entryRelativePath));
+      } else {
+        // Read file content
+        const content = readFileSync(fullPath, "utf-8");
+        templates.push({
+          path: entryRelativePath.split(path.sep).join("/"), // Normalize to forward slashes
+          content,
+        });
+      }
+    }
+
+    return templates;
   }
 }