From 04f5c91ed13bfec32ad2fbe3a0d736ba602bcf60 Mon Sep 17 00:00:00 2001 From: eric valcik Date: Mon, 21 Apr 2025 19:35:15 +0200 Subject: [PATCH 01/16] don't commit vscode settings --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 85806bb..8461d25 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ node_modules playwright-report test-results +.vscode # User configuration user_config.yaml From a9de27aab1e281952244458f364290633dfd0c2c Mon Sep 17 00:00:00 2001 From: eric valcik Date: Mon, 21 Apr 2025 19:43:17 +0200 Subject: [PATCH 02/16] remove workspace as it breaks build --- app/pnpm-workspace.yaml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 app/pnpm-workspace.yaml diff --git a/app/pnpm-workspace.yaml b/app/pnpm-workspace.yaml deleted file mode 100644 index e6654d6..0000000 --- a/app/pnpm-workspace.yaml +++ /dev/null @@ -1,2 +0,0 @@ -onlyBuiltDependencies: - - sharp From 07360b95e10e874727ae47bc5bf39f2ebe93b2ac Mon Sep 17 00:00:00 2001 From: eric valcik Date: Mon, 21 Apr 2025 21:01:52 +0200 Subject: [PATCH 03/16] successfully have a working websocket connection:) --- app/app/ui-demo/page.tsx | 145 +++++++++++++++++++++++++++++++++++++++ cmd/api/main.go | 55 ++++++++++----- go.mod | 1 + go.sum | 2 + 4 files changed, 187 insertions(+), 16 deletions(-) create mode 100644 app/app/ui-demo/page.tsx diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx new file mode 100644 index 0000000..f73f826 --- /dev/null +++ b/app/app/ui-demo/page.tsx @@ -0,0 +1,145 @@ +"use client"; + +import { useState, useEffect, useRef } from "react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Card, + CardContent, + CardHeader, + CardTitle, + CardDescription, +} from "@/components/ui/card"; + +export default function WebSocketDemo() { + const [messages, setMessages] = useState([]); + const [inputValue, setInputValue] = useState("Hello world!"); + const [isConnected, setIsConnected] = useState(false); + const wsRef = useRef(null); + const scrollAreaRef = useRef(null); + + const addMessage = (message: string) => { + setMessages((prev) => [...prev, message]); + }; + + const handleOpen = () => { + if (wsRef.current) return; + + try { + console.log("Attempting to connect to WebSocket..."); + wsRef.current = new WebSocket("ws://localhost:8080/ws"); + + wsRef.current.onopen = () => { + console.log("WebSocket connection opened successfully"); + setIsConnected(true); + addMessage("OPEN"); + }; + + wsRef.current.onclose = (event) => { + console.log("WebSocket connection closed:", event.code, event.reason); + setIsConnected(false); + addMessage( + `CLOSE (Code: ${event.code}${ + event.reason ? `, Reason: ${event.reason}` : "" + })` + ); + wsRef.current = null; + }; + + wsRef.current.onmessage = (evt) => { + console.log("Received message:", evt.data); + addMessage(`RESPONSE: ${evt.data}`); + }; + + wsRef.current.onerror = (error) => { + console.error("WebSocket error:", error); + addMessage( + `ERROR: Connection failed - check browser console for details` + ); + }; + } catch (error: any) { + console.error("WebSocket connection error:", error); + addMessage( + `ERROR: Failed to connect to WebSocket server - ${error.message}` + ); + } + }; + + const handleClose = () => { + if (!wsRef.current) return; + wsRef.current.close(); + }; + + const handleSend = () => { + if (!wsRef.current) return; + addMessage(`SEND: ${inputValue}`); + wsRef.current.send(inputValue); + }; + + useEffect(() => { + if (scrollAreaRef.current) { + scrollAreaRef.current.scrollTop = scrollAreaRef.current.scrollHeight; + } + }, [messages]); + + return ( +
+ + + WebSocket Demo + + Click "Open" to create a connection to the server, "Send" to send a + message to the server and "Close" to close the connection. You can + change the message and send multiple times. + + + +
+
+
+ + +
+
+ setInputValue(e.target.value)} + placeholder="Enter message..." + /> + +
+
+
+ {messages.map((message, index) => ( +
+ {message} +
+ ))} +
+
+ + +
+ ); +} diff --git a/cmd/api/main.go b/cmd/api/main.go index f0035f4..71d609e 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -2,27 +2,50 @@ package main import ( "fmt" + "log" "net/http" - "time" - "github.com/go-chi/chi/v5/middleware" - "github.com/go-chi/httprate" - "github.com/webscopeio/ai-hackathon/internal/config" - "github.com/webscopeio/ai-hackathon/internal/llm" - "github.com/webscopeio/ai-hackathon/internal/router" + "github.com/gorilla/websocket" ) -func main() { - cfg := config.Load() +var upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { + return true // Allow all origins for development + }, +} - r := router.New() - r.Use(middleware.Logger) - r.Use(httprate.LimitByIP(100, time.Minute)) +func echo(w http.ResponseWriter, r *http.Request) { + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Print("upgrade:", err) + return + } + defer c.Close() + for { + mt, message, err := c.ReadMessage() + if err != nil { + log.Println("read:", err) + break + } + log.Printf("recv: %s", message) + err = c.WriteMessage(mt, message) + if err != nil { + log.Println("write:", err) + break + } + } +} - llm := llm.New(cfg) - router.RegisterRoutes(r, cfg, llm) +func main() { + http.HandleFunc("/ws", echo) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte("WebSocket server initialized")) + }) - addr := fmt.Sprintf(":%s", cfg.Port) - fmt.Printf("Server starting on localhost%s in %s mode\n", addr, cfg.Environment) - http.ListenAndServe(addr, r) + addr := fmt.Sprintf(":%s", "8080") + fmt.Printf("Server starting on localhost%s\n", addr) + log.Fatal(http.ListenAndServe(addr, nil)) } diff --git a/go.mod b/go.mod index e9c2c2f..1cabfa5 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.4.0 // indirect + github.com/gorilla/websocket v1.5.3 // indirect golang.org/x/sys v0.31.0 // indirect ) diff --git a/go.sum b/go.sum index 9fdf7e1..a2672ef 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= From f593cea767075f45fc259a893543857af55c11af Mon Sep 17 00:00:00 2001 From: eric valcik Date: Tue, 22 Apr 2025 23:09:40 +0200 Subject: [PATCH 04/16] milestone commit --- app/app/ui-demo/page.tsx | 99 +++++++++++++------- cmd/api/main.go | 114 ++++++++++++++++++++++-- internal/llm/completion.go | 2 +- internal/repository/analyzer/analyze.go | 2 +- 4 files changed, 174 insertions(+), 43 deletions(-) diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index f73f826..274c817 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -10,10 +10,11 @@ import { CardTitle, CardDescription, } from "@/components/ui/card"; +import { Textarea } from "@/components/ui/textarea"; export default function WebSocketDemo() { const [messages, setMessages] = useState([]); - const [inputValue, setInputValue] = useState("Hello world!"); + const [inputValue, setInputValue] = useState(""); const [isConnected, setIsConnected] = useState(false); const wsRef = useRef(null); const scrollAreaRef = useRef(null); @@ -71,7 +72,51 @@ export default function WebSocketDemo() { }; const handleSend = () => { - if (!wsRef.current) return; + if (!wsRef.current) { + try { + console.log("Attempting to connect to WebSocket..."); + wsRef.current = new WebSocket("ws://localhost:8080/ws"); + + wsRef.current.onopen = () => { + console.log("WebSocket connection opened successfully"); + setIsConnected(true); + addMessage("OPEN"); + addMessage(`SEND: ${inputValue}`); + if (wsRef.current) { + wsRef.current.send(inputValue); + } + }; + + wsRef.current.onclose = (event) => { + console.log("WebSocket connection closed:", event.code, event.reason); + setIsConnected(false); + addMessage( + `CLOSE (Code: ${event.code}${ + event.reason ? `, Reason: ${event.reason}` : "" + })` + ); + wsRef.current = null; + }; + + wsRef.current.onmessage = (evt) => { + console.log("Received message:", evt.data); + addMessage(`RESPONSE: ${evt.data}`); + }; + + wsRef.current.onerror = (error) => { + console.error("WebSocket error:", error); + addMessage( + `ERROR: Connection failed - check browser console for details` + ); + }; + } catch (error: any) { + console.error("WebSocket connection error:", error); + addMessage( + `ERROR: Failed to connect to WebSocket server - ${error.message}` + ); + } + return; + } addMessage(`SEND: ${inputValue}`); wsRef.current.send(inputValue); }; @@ -86,45 +131,35 @@ export default function WebSocketDemo() {
- WebSocket Demo + End-to-end Test Generation - Click "Open" to create a connection to the server, "Send" to send a - message to the server and "Close" to close the connection. You can - change the message and send multiple times. + Generate end-to-end tests that are passing with just a prompt and + the website URL. -
+
+
- - -
-
- setInputValue(e.target.value)} - placeholder="Enter message..." + placeholder="Write a the website's URL, yeah, that's it." /> - +
+ {" "} + + +
Date: Tue, 22 Apr 2025 23:28:17 +0200 Subject: [PATCH 05/16] make it cancellable (atleast when in analyzer + add analyzer websocket messaging --- app/app/ui-demo/page.tsx | 2 +- cmd/api/main.go | 104 ++++++++++---- cmd/cli/main.go | 135 ------------------- internal/handlers/analyze.go | 52 ++++--- internal/repository/analyzer/analyze.go | 6 +- internal/repository/analyzer/analyze_test.go | 38 +++--- internal/router/router.go | 2 +- 7 files changed, 128 insertions(+), 211 deletions(-) delete mode 100644 cmd/cli/main.go diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index 274c817..fd09457 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -164,7 +164,7 @@ export default function WebSocketDemo() {
{messages.map((message, index) => (
diff --git a/cmd/api/main.go b/cmd/api/main.go index 3c8db45..02ae04d 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/gorilla/websocket" "github.com/webscopeio/ai-hackathon/internal/config" @@ -28,27 +29,57 @@ var upgrader = websocket.Upgrader{ func runPipeline(w http.ResponseWriter, r *http.Request) { c, err := upgrader.Upgrade(w, r, nil) - if err != nil { log.Print("upgrade:", err) return } defer c.Close() - for { - mt, message, err := c.ReadMessage() - if err != nil { - log.Println("read:", err) - break + + // Create a cancellable context + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + // Create a WaitGroup to wait for sendMessage to complete + var wg sync.WaitGroup + + // Channel to signal connection closure + done := make(chan struct{}) + go func() { + defer close(done) + for { + mt, message, err := c.ReadMessage() + if err != nil { + if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) { + log.Printf("error reading message: %v", err) + } + return + } + log.Printf("recv: %s", message) + // Send confirmation message back to client + err = c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("ANALYZER 'Prompt recieved, starting analysis...'"))) + if err != nil { + log.Printf("error writing message: %v", err) + return + } + + // Start sendMessage in a goroutine + wg.Add(1) + go func(msg []byte) { + defer wg.Done() + sendMessage(ctx, c, mt, string(msg)) + }(message) } - log.Printf("recv: %s", message) - sendMessage(c, mt, string(message)) - } -} + }() -func sendMessage(c *websocket.Conn, mt int, prompt string) { - // Add context - ctx := context.Background() + // Wait for either the connection to close or the context to be cancelled + <-done + cancel() // Cancel the context to stop any ongoing operations + // Wait for any ongoing sendMessage operations to complete + wg.Wait() +} + +func sendMessage(ctx context.Context, c *websocket.Conn, mt int, url string) { // Initialize config and LLM client cfg := config.Load() client := llm.New(cfg) @@ -86,40 +117,61 @@ func sendMessage(c *websocket.Conn, mt int, prompt string) { SCENARIO: Verify users can search for products and get relevant results EXPECTED: Search results page should display matching products with correct information` - basePrompt += `\n\IMPORTANT: This is the input from the user. Use it to generate the test criteria:` + prompt + // Check if context is cancelled before proceeding + select { + case <-ctx.Done(): + return + default: + } - analysis, err := analyzer.Analyze(ctx, cfg, client, "https://ai-hackathon-demo-delta.vercel.app/", basePrompt) + analysis, err := analyzer.Analyze(ctx, cfg, client, url, basePrompt, c, mt) if err != nil { - fmt.Printf("Error: %v\n", err) + log.Printf("Error: %v\n", err) return } if len(analysis.Criteria) == 0 { - fmt.Println("Error: No test criteria were generated from the analysis") + log.Println("Error: No test criteria were generated from the analysis") return } // Split criteria by double newlines criteria := strings.Split(analysis.Criteria, "\n\n") - c.WriteMessage(mt, []byte(fmt.Sprintf("\n[MAIN FLOW] Analyzer generated %d scenarios\n", len(criteria)))) - // print the criteria line by line + // Check context before sending message + select { + case <-ctx.Done(): + return + default: + if err := c.WriteMessage(mt, []byte(fmt.Sprintf("\n[MAIN FLOW] Analyzer generated %d scenarios\n", len(criteria)))); err != nil { + log.Printf("error writing message: %v", err) + return + } + } + logger.Debug("CRITERIA LENGTH: %d", len(criteria)) - for _, c := range criteria { - logger.Debug("CRITERIA: %s", c) + for _, criterion := range criteria { + logger.Debug("CRITERIA: %s", criterion) } noOfLoops := 6 - for i, c := range criteria { - fmt.Printf("\n[MAIN FLOW] Generating test for scenario %d: %s\n", i, c) + for i, criterion := range criteria { + // Check context before each iteration + select { + case <-ctx.Done(): + return + default: + } + + fmt.Printf("\n[MAIN FLOW] Generating test for scenario %d: %s\n", i, criterion) filename, err := gen_eval_loop.GenEvalLoop(ctx, client, &models.AnalyzerReturn{ TechSpec: analysis.TechSpec, ContentMap: analysis.ContentMap, Criteria: analysis.Criteria, }, i+1, noOfLoops) if err != nil { - fmt.Printf("Error: %v\n", err) + log.Printf("Error: %v\n", err) return } @@ -130,12 +182,10 @@ func sendMessage(c *websocket.Conn, mt int, prompt string) { fmt.Printf("\n[MAIN FLOW] Writing generated test file to %s\n", destPath) err = os.Rename(filename, destPath) if err != nil { - fmt.Printf("Error copying file: %v\n", err) + log.Printf("Error copying file: %v\n", err) return } } - - return } func main() { diff --git a/cmd/cli/main.go b/cmd/cli/main.go deleted file mode 100644 index ee1e893..0000000 --- a/cmd/cli/main.go +++ /dev/null @@ -1,135 +0,0 @@ -package main - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - "github.com/webscopeio/ai-hackathon/internal/config" - "github.com/webscopeio/ai-hackathon/internal/llm" - "github.com/webscopeio/ai-hackathon/internal/logger" - "github.com/webscopeio/ai-hackathon/internal/models" - "github.com/webscopeio/ai-hackathon/internal/repository/analyzer" - "github.com/webscopeio/ai-hackathon/internal/repository/gen_eval_loop" -) - -var url string - -var rootCmd = &cobra.Command{ - Use: "testbuddy", - Short: "TestBuddy CLI", - Run: func(cmd *cobra.Command, args []string) { - cmd.Help() - }, -} - -var generateCmd = &cobra.Command{ - Use: "generate", - Short: "Generate Playwright tests for a website", - Run: func(cmd *cobra.Command, args []string) { - - // Initialize config and LLM client - cfg := config.Load() - client := llm.New(cfg) - - basePrompt := `You are a test planning expert. Your task is to analyze the provided website and generate EXACTLY 4 specific test criteria that can be used by another agent to generate E2E tests. - - The criteria should: - 1. Cover the core functionality of the application - 2. Focus on different user journeys, I am interested in the content of the most valuable pages - 3. Include both happy path and edge case scenarios - 4. Be specific enough to be implemented as end-to-end tests - 5. Be short, concise and easy to understand - 6. Focus on simple tests that are easy to write (we can iterate later with more complex tests) - - IMPORTANT: Pass all the criteria into the get_final_criteria_tool. Format each criterion as follows: - - CRITERION #1: - TITLE: [Short descriptive title] - SCENARIO: [Clear description of what should be tested] - EXPECTED: [Expected outcome or behavior] - - (Repeat for CRITERION #2, #3, and #4) - - Each criterion must be separated by 2 newlines for proper parsing. - - Example: - CRITERION #1: - TITLE: User Login Authentication - SCENARIO: Verify a registered user can successfully log in with valid credentials - EXPECTED: User should be authenticated and redirected to their personalized dashboard - - - CRITERION #2: - TITLE: Product Search Functionality - SCENARIO: Verify users can search for products and get relevant results - EXPECTED: Search results page should display matching products with correct information` - - websiteDescription := "Check out the website, wonder how is it structured?. I am interested in the content of the most valuable pages to create the criteria to generate an E2E tests. My orgSlug := \"webscopeio-pb\" and projectSlug := \"ai-hackathon-demo\" for Sentry, please check the errors in the last 14 days and include them in the analysis." - basePrompt += `\n\IMPORTANT: You are analyzing the following website:` + websiteDescription - - analysis, err := analyzer.Analyze(cmd.Context(), cfg, client, "https://ai-hackathon-demo-delta.vercel.app/", basePrompt) - if err != nil { - fmt.Printf("Error: %v\n", err) - return - } - - if len(analysis.Criteria) == 0 { - fmt.Println("Error: No test criteria were generated from the analysis") - return - } - - // Split criteria by double newlines - criteria := strings.Split(analysis.Criteria, "\n\n") - - fmt.Printf("\n[MAIN FLOW] Analyzer generated %d scenarios\n", len(criteria)) - // print the criteria line by line - logger.Debug("CRITERIA LENGTH: %d", len(criteria)) - for _, c := range criteria { - logger.Debug("CRITERIA: %s", c) - } - - noOfLoops := 6 - - for i, c := range criteria { - fmt.Printf("\n[MAIN FLOW] Generating test for scenario %d: %s\n", i, c) - filename, err := gen_eval_loop.GenEvalLoop(cmd.Context(), client, &models.AnalyzerReturn{ - TechSpec: analysis.TechSpec, - ContentMap: analysis.ContentMap, - Criteria: analysis.Criteria, - }, i+1, noOfLoops) - if err != nil { - fmt.Printf("Error: %v\n", err) - return - } - - logger.Debug("[MAIN FLOW] Writing test file: %s\n", filepath.Base(filename)) - - // copy the file to the current directory - destPath := filepath.Join("./__generated__", filepath.Base(filename)) - fmt.Printf("\n[MAIN FLOW] Writing generated test file to %s\n", destPath) - err = os.Rename(filename, destPath) - if err != nil { - fmt.Printf("Error copying file: %v\n", err) - return - } - } - - return - - }, -} - -func init() { - generateCmd.Flags() - rootCmd.AddCommand(generateCmd) -} - -func main() { - if err := rootCmd.Execute(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} diff --git a/internal/handlers/analyze.go b/internal/handlers/analyze.go index 615f732..d25b015 100644 --- a/internal/handlers/analyze.go +++ b/internal/handlers/analyze.go @@ -1,33 +1,31 @@ package handlers -import ( - "fmt" - "net/http" +// "fmt" +// "net/http" - "github.com/webscopeio/ai-hackathon/internal/config" - "github.com/webscopeio/ai-hackathon/internal/llm" - "github.com/webscopeio/ai-hackathon/internal/models" - "github.com/webscopeio/ai-hackathon/internal/repository/analyzer" -) +// "github.com/webscopeio/ai-hackathon/internal/config" +// "github.com/webscopeio/ai-hackathon/internal/llm" +// "github.com/webscopeio/ai-hackathon/internal/models" +// "github.com/webscopeio/ai-hackathon/internal/repository/analyzer" -func Analyze(cfg *config.Config, client *llm.Client) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - args, err := decode[models.AnalyzerArgs](r) - if err != nil { - encode(w, http.StatusBadRequest, models.ErrorReturn{ - Error: fmt.Sprintf("Bad request, %v", err), - }) - return - } +// func Analyze(cfg *config.Config, client *llm.Client) http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) { +// args, err := decode[models.AnalyzerArgs](r) +// if err != nil { +// encode(w, http.StatusBadRequest, models.ErrorReturn{ +// Error: fmt.Sprintf("Bad request, %v", err), +// }) +// return +// } - res, err := analyzer.Analyze(r.Context(), cfg, client, args.Url, args.Prompt) - if err != nil { - encode(w, http.StatusInternalServerError, models.ErrorReturn{ - Error: fmt.Sprintf("Couldn't analyze website, %v", err), - }) - return - } +// res, err := analyzer.Analyze(r.Context(), cfg, client, args.Url, args.Prompt) +// if err != nil { +// encode(w, http.StatusInternalServerError, models.ErrorReturn{ +// Error: fmt.Sprintf("Couldn't analyze website, %v", err), +// }) +// return +// } - encode(w, http.StatusOK, res) - } -} +// encode(w, http.StatusOK, res) +// } +// } diff --git a/internal/repository/analyzer/analyze.go b/internal/repository/analyzer/analyze.go index ead98f1..381b5b3 100644 --- a/internal/repository/analyzer/analyze.go +++ b/internal/repository/analyzer/analyze.go @@ -7,13 +7,14 @@ import ( "fmt" "github.com/anthropics/anthropic-sdk-go" + "github.com/gorilla/websocket" "github.com/webscopeio/ai-hackathon/internal/config" "github.com/webscopeio/ai-hackathon/internal/llm" "github.com/webscopeio/ai-hackathon/internal/logger" "github.com/webscopeio/ai-hackathon/internal/models" ) -func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr string, prompt string) (*models.AnalyzerReturn, error) { +func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr string, prompt string, c *websocket.Conn, mt int) (*models.AnalyzerReturn, error) { userMessage := fmt.Sprintf("The website is: %s - %s", urlStr, prompt) messages := []anthropic.MessageParam{ anthropic.NewUserMessage(anthropic.NewTextBlock(userMessage)), @@ -47,6 +48,7 @@ func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr var contentMap map[string]string fmt.Println("\n[ANALYZER] User Message: \n\n", userMessage) + c.WriteMessage(mt, []byte(fmt.Sprintf("ANALYZER 'Starting analyisis on %s...'", urlStr))) for { message, err := client.NewMessage(ctx, anthropic.MessageNewParams{ @@ -64,9 +66,11 @@ func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr switch block := block.AsAny().(type) { case anthropic.TextBlock: fmt.Printf("\n[ANALYZER] Agent response: \n\n%s\n", block.Text) + c.WriteMessage(mt, []byte(fmt.Sprintf("ANALYZER '%s'", block.Text))) case anthropic.ToolUseBlock: inputJSON, _ := json.Marshal(block.Input) fmt.Printf("\n[ANALYZER] Tool call: \n\n%s\n", block.Name+": "+string(inputJSON)) + c.WriteMessage(mt, []byte(fmt.Sprintf("TOOLCALL '%s: %s'", block.Name, string(inputJSON)))) } } diff --git a/internal/repository/analyzer/analyze_test.go b/internal/repository/analyzer/analyze_test.go index d37b189..ee6ee36 100644 --- a/internal/repository/analyzer/analyze_test.go +++ b/internal/repository/analyzer/analyze_test.go @@ -1,27 +1,27 @@ package analyzer -import ( - "context" - "fmt" - "testing" - "time" +import "testing" - "github.com/webscopeio/ai-hackathon/internal/config" - "github.com/webscopeio/ai-hackathon/internal/llm" -) +// "context" +// "fmt" +// "testing" +// "time" + +// "github.com/webscopeio/ai-hackathon/internal/config" +// "github.com/webscopeio/ai-hackathon/internal/llm" func TestAnalyze(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - cfg := config.Load() - llm := llm.New(cfg) + // ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + // defer cancel() + // cfg := config.Load() + // llm := llm.New(cfg) - res, err := Analyze(ctx, cfg, llm, "https://ai-hackathon-demo-delta.vercel.app/", "Check out the website, wonder how is it structured?. I am interested in the content of the most valuable pages to create the criteria to generate an E2E tests. My orgSlug := \"webscopeio-pb\" and projectSlug := \"ai-hackathon-demo\" for Sentry, please check the errors in the last 14 days and include them in the analysis.") - if err != nil { - t.Fatalf("Analyze failed: %v", err) - } + // res, err := analyze(ctx, cfg, llm, "https://ai-hackathon-demo-delta.vercel.app/", "check out the website, wonder how is it structured?. i am interested in the content of the most valuable pages to create the criteria to generate an e2e tests. my orgslug := \"webscopeio-pb\" and projectslug := \"ai-hackathon-demo\" for sentry, please check the errors in the last 14 days and include them in the analysis.") + // if err != nil { + // t.fatalf("analyze failed: %v", err) + // } - fmt.Printf("criteria=%v", res.Criteria) - fmt.Printf("techSpec=%v", res.TechSpec) - fmt.Printf("contentMap=%v", res.ContentMap) + // fmt.Printf("criteria=%v", res.Criteria) + // fmt.Printf("techSpec=%v", res.TechSpec) + // fmt.Printf("contentMap=%v", res.ContentMap) } diff --git a/internal/router/router.go b/internal/router/router.go index 014dba5..f4f3bba 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -21,5 +21,5 @@ func RegisterRoutes(r *chi.Mux, cfg *config.Config, llm *llm.Client) { r.Post("/config", handlers.SaveConfig()) // Analyze endpoints - r.Post("/analyze", handlers.Analyze(cfg, llm)) + // r.Post("/analyze", handlers.Analyze(cfg, llm)) } From 4a81aa26dd2c92457ea9d6dbc57a465ee814d14c Mon Sep 17 00:00:00 2001 From: eric valcik Date: Tue, 22 Apr 2025 23:58:14 +0200 Subject: [PATCH 06/16] add cards with glow animation --- app/app/layout.tsx | 2 +- app/app/ui-demo/analyzer.tsx | 62 ++++++++++++++++++++++++++++++++ app/app/ui-demo/page.tsx | 9 +++++ app/components/ui/card.tsx | 68 ++++++++++++++++++++++++++---------- app/tailwind.config.ts | 31 ++++++++++++++++ 5 files changed, 153 insertions(+), 19 deletions(-) create mode 100644 app/app/ui-demo/analyzer.tsx create mode 100644 app/tailwind.config.ts diff --git a/app/app/layout.tsx b/app/app/layout.tsx index 7ad5c8b..722c26c 100644 --- a/app/app/layout.tsx +++ b/app/app/layout.tsx @@ -30,7 +30,7 @@ export default function RootLayout({ className={`${geistSans.variable} ${geistMono.variable} antialiased`} > -
{children}
+
{children}
 diff --git a/app/app/ui-demo/analyzer.tsx b/app/app/ui-demo/analyzer.tsx new file mode 100644 index 0000000..ded765a --- /dev/null +++ b/app/app/ui-demo/analyzer.tsx @@ -0,0 +1,62 @@ +import { BellRing, Check } from "lucide-react"; + +import { cn } from "@/lib/utils"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; + +const notifications = [ + { + title: "Your call has been confirmed.", + description: "1 hour ago", + }, + { + title: "You have a new message!", + description: "1 hour ago", + }, + { + title: "Your subscription is expiring soon!", + description: "2 hours ago", + }, +]; + +type AnalyzerProps = React.ComponentProps & { active: boolean }; + +export function Analyzer({ className, active, ...props }: AnalyzerProps) { + return ( + + + Analyzer + + Analyzes the website and generates scenarios. + + + +
+ {notifications.map((notification, index) => ( +
+ +
+

+ {notification.title} +

+

+ {notification.description} +

+
+
+ ))} +
+ + + ); +} diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index fd09457..9e26748 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -11,6 +11,7 @@ import { CardDescription, } from "@/components/ui/card"; import { Textarea } from "@/components/ui/textarea"; +import { Analyzer } from "./analyzer"; export default function WebSocketDemo() { const [messages, setMessages] = useState([]); @@ -175,6 +176,14 @@ export default function WebSocketDemo() {
+ +
); } diff --git a/app/components/ui/card.tsx b/app/components/ui/card.tsx index afa13ec..92761ca 100644 --- a/app/components/ui/card.tsx +++ b/app/components/ui/card.tsx @@ -1,21 +1,27 @@ -import * as React from "react" +import * as React from "react"; -import { cn } from "@/lib/utils" +import { cn } from "@/lib/utils"; const Card = React.forwardRef< HTMLDivElement, - React.HTMLAttributes ->(({ className, ...props }, ref) => ( + React.HTMLAttributes & { active?: boolean } +>(({ className, active, ...props }, ref) => (
-)) -Card.displayName = "Card" +)); +Card.displayName = "Card"; const CardHeader = React.forwardRef< HTMLDivElement, @@ -26,8 +32,8 @@ const CardHeader = React.forwardRef< className={cn("flex flex-col space-y-1.5 p-6", className)} {...props} /> -)) -CardHeader.displayName = "CardHeader" +)); +CardHeader.displayName = "CardHeader"; const CardTitle = React.forwardRef< HTMLParagraphElement, @@ -41,8 +47,8 @@ const CardTitle = React.forwardRef< )} {...props} /> -)) -CardTitle.displayName = "CardTitle" +)); +CardTitle.displayName = "CardTitle"; const CardDescription = React.forwardRef< HTMLParagraphElement, @@ -53,16 +59,16 @@ const CardDescription = React.forwardRef< className={cn("text-sm text-muted-foreground", className)} {...props} /> -)) -CardDescription.displayName = "CardDescription" +)); +CardDescription.displayName = "CardDescription"; const CardContent = React.forwardRef< HTMLDivElement, React.HTMLAttributes >(({ className, ...props }, ref) => (
-)) -CardContent.displayName = "CardContent" +)); +CardContent.displayName = "CardContent"; const CardFooter = React.forwardRef< HTMLDivElement, @@ -73,7 +79,33 @@ const CardFooter = React.forwardRef< className={cn("flex items-center p-6 pt-0", className)} {...props} /> -)) -CardFooter.displayName = "CardFooter" +)); +CardFooter.displayName = "CardFooter"; + +// Add the new animation +const style = document.createElement("style"); +style.textContent = ` + @keyframes subtle-pulse { + 0%, 100% { + border-color: rgb(59 130 246); + box-shadow: 0 0 15px rgba(59,130,246,0.5); + } + 50% { + border-color: rgb(59 130 246 / 0.7); + box-shadow: 0 0 15px rgba(59,130,246,0.3); + } + } + .animate-pulse-subtle { + animation: subtle-pulse 3s ease-in-out infinite; + } +`; +document.head.appendChild(style); -export { Card, CardHeader, CardFooter, CardTitle, CardDescription, CardContent } +export { + Card, + CardHeader, + CardFooter, + CardTitle, + CardDescription, + CardContent, +}; diff --git a/app/tailwind.config.ts b/app/tailwind.config.ts new file mode 100644 index 0000000..e9274eb --- /dev/null +++ b/app/tailwind.config.ts @@ -0,0 +1,31 @@ +import type { Config } from "tailwindcss"; + +const config: Config = { + content: [ + "./pages/**/*.{js,ts,jsx,tsx,mdx}", + "./components/**/*.{js,ts,jsx,tsx,mdx}", + "./app/**/*.{js,ts,jsx,tsx,mdx}", + ], + theme: { + extend: { + keyframes: { + glow: { + "0%, 100%": { + borderColor: "rgb(59 130 246 / 0.7)", + boxShadow: "0 0 15px rgba(59,130,246,0.5)", + }, + "50%": { + borderColor: "rgb(59 130 246 / 0.4)", + boxShadow: "0 0 15px rgba(59,130,246,0.3)", + }, + }, + }, + animation: { + glow: "glow 3s ease-in-out infinite", + }, + }, + }, + plugins: [], +}; + +export default config; From 3bdbc85fd3a9139c5629a6fef8ced2d8f180be17 Mon Sep 17 00:00:00 2001 From: eric valcik Date: Wed, 23 Apr 2025 00:39:20 +0200 Subject: [PATCH 07/16] add generating... text when generating, sort messages ... --- app/app/globals.css | 43 ++++++++++++++++++++++++++ app/app/ui-demo/analyzer.tsx | 31 ++++++++++++------- app/app/ui-demo/page.tsx | 58 +++++++++--------------------------- app/components/ShinyText.tsx | 26 ++++++++++++++++ app/components/agent.tsx | 30 +++++++++++++++++++ app/components/ui/card.tsx | 19 ------------ app/lib/types.ts | 4 +++ 7 files changed, 137 insertions(+), 74 deletions(-) create mode 100644 app/components/ShinyText.tsx create mode 100644 app/components/agent.tsx create mode 100644 app/lib/types.ts diff --git a/app/app/globals.css b/app/app/globals.css index dc98be7..bd44b5e 100644 --- a/app/app/globals.css +++ b/app/app/globals.css @@ -120,3 +120,46 @@ @apply bg-background text-foreground; } } + +@keyframes subtle-pulse { + 0%, + 100% { + border-color: rgb(59 130 246); + box-shadow: 0 0 15px rgba(59, 130, 246, 0.5); + } + 50% { + border-color: rgb(59 130 246 / 0.7); + box-shadow: 0 0 15px rgba(59, 130, 246, 0.3); + } +} +.animate-pulse-subtle { + animation: subtle-pulse 3s ease-in-out infinite; +} + +.shiny-text { + color: #b5b5b5a4; /* Adjust this color to change intensity/style */ + background: linear-gradient( + 120deg, + rgba(255, 255, 255, 0) 40%, + rgba(0, 0, 0, 1) 50%, + rgba(255, 255, 255, 0) 60% + ); + background-size: 200% 100%; + -webkit-background-clip: text; + background-clip: text; + display: inline-block; + animation: shine 5s linear infinite; +} + +@keyframes shine { + 0% { + background-position: 100%; + } + 100% { + background-position: -100%; + } +} + +.shiny-text.disabled { + animation: none; +} diff --git a/app/app/ui-demo/analyzer.tsx b/app/app/ui-demo/analyzer.tsx index ded765a..c62e1bc 100644 --- a/app/app/ui-demo/analyzer.tsx +++ b/app/app/ui-demo/analyzer.tsx @@ -6,10 +6,11 @@ import { Card, CardContent, CardDescription, - CardFooter, CardHeader, - CardTitle, } from "@/components/ui/card"; +import { AgentTitle } from "@/components/agent"; +import { Message } from "@/lib/types"; +import ShinyText from "@/components/ShinyText"; const notifications = [ { @@ -26,36 +27,44 @@ const notifications = [ }, ]; -type AnalyzerProps = React.ComponentProps & { active: boolean }; +type AnalyzerProps = React.ComponentProps & { + active: boolean; + messages: Message[]; +}; -export function Analyzer({ className, active, ...props }: AnalyzerProps) { +export function Analyzer({ + className, + active, + messages, + ...props +}: AnalyzerProps) { return ( - Analyzer + Analyzer Analyzes the website and generates scenarios. - +
- {notifications.map((notification, index) => ( + {messages.slice(-3).map((message, index) => (
-

- {notification.title} + {message.title === "ANALYZER" ? "Agent" : "Toolcall"}

- {notification.description} + {message.description}

))}
+ {active && } ); diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index 9e26748..e1519db 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -12,6 +12,7 @@ import { } from "@/components/ui/card"; import { Textarea } from "@/components/ui/textarea"; import { Analyzer } from "./analyzer"; +import { Message } from "@/lib/types"; export default function WebSocketDemo() { const [messages, setMessages] = useState([]); @@ -24,49 +25,6 @@ export default function WebSocketDemo() { setMessages((prev) => [...prev, message]); }; - const handleOpen = () => { - if (wsRef.current) return; - - try { - console.log("Attempting to connect to WebSocket..."); - wsRef.current = new WebSocket("ws://localhost:8080/ws"); - - wsRef.current.onopen = () => { - console.log("WebSocket connection opened successfully"); - setIsConnected(true); - addMessage("OPEN"); - }; - - wsRef.current.onclose = (event) => { - console.log("WebSocket connection closed:", event.code, event.reason); - setIsConnected(false); - addMessage( - `CLOSE (Code: ${event.code}${ - event.reason ? `, Reason: ${event.reason}` : "" - })` - ); - wsRef.current = null; - }; - - wsRef.current.onmessage = (evt) => { - console.log("Received message:", evt.data); - addMessage(`RESPONSE: ${evt.data}`); - }; - - wsRef.current.onerror = (error) => { - console.error("WebSocket error:", error); - addMessage( - `ERROR: Connection failed - check browser console for details` - ); - }; - } catch (error: any) { - console.error("WebSocket connection error:", error); - addMessage( - `ERROR: Failed to connect to WebSocket server - ${error.message}` - ); - } - }; - const handleClose = () => { if (!wsRef.current) return; wsRef.current.close(); @@ -101,7 +59,7 @@ export default function WebSocketDemo() { wsRef.current.onmessage = (evt) => { console.log("Received message:", evt.data); - addMessage(`RESPONSE: ${evt.data}`); + addMessage(evt.data); }; wsRef.current.onerror = (error) => { @@ -179,11 +137,23 @@ export default function WebSocketDemo() {
); } + +const getMessages = (messages: string[], ids: string[]): Message[] => { + return messages + .filter((message) => ids.includes(message.split(" ")[0])) + .map((message) => { + const id = message.split(" ")[0]; + const description = message.split(" ").slice(1).join(" "); + return { title: id, description }; + }); +}; diff --git a/app/components/ShinyText.tsx b/app/components/ShinyText.tsx new file mode 100644 index 0000000..2f66909 --- /dev/null +++ b/app/components/ShinyText.tsx @@ -0,0 +1,26 @@ +interface ShinyTextProps { + text: string; + disabled?: boolean; + speed?: number; + className?: string; +} + +const ShinyText = ({ + text, + disabled = false, + speed = 5, + className = "", +}: ShinyTextProps) => { + const animationDuration = `${speed}s`; + + return ( +
+ {text} +
+ ); +}; + +export default ShinyText; diff --git a/app/components/agent.tsx b/app/components/agent.tsx new file mode 100644 index 0000000..a9e35f8 --- /dev/null +++ b/app/components/agent.tsx @@ -0,0 +1,30 @@ +import * as React from "react"; +import { cn } from "@/lib/utils"; +import { CardTitle } from "@/components/ui/card"; + +type AgentTitleProps = React.ComponentProps & { + active?: boolean; +}; + +export function AgentTitle({ + className, + active, + children, + ...props +}: AgentTitleProps) { + return ( +
+ + + {children} + +
+ ); +} diff --git a/app/components/ui/card.tsx b/app/components/ui/card.tsx index 92761ca..533070b 100644 --- a/app/components/ui/card.tsx +++ b/app/components/ui/card.tsx @@ -82,25 +82,6 @@ const CardFooter = React.forwardRef< )); CardFooter.displayName = "CardFooter"; -// Add the new animation -const style = document.createElement("style"); -style.textContent = ` - @keyframes subtle-pulse { - 0%, 100% { - border-color: rgb(59 130 246); - box-shadow: 0 0 15px rgba(59,130,246,0.5); - } - 50% { - border-color: rgb(59 130 246 / 0.7); - box-shadow: 0 0 15px rgba(59,130,246,0.3); - } - } - .animate-pulse-subtle { - animation: subtle-pulse 3s ease-in-out infinite; - } -`; -document.head.appendChild(style); - export { Card, CardHeader, diff --git a/app/lib/types.ts b/app/lib/types.ts new file mode 100644 index 0000000..203b2c5 --- /dev/null +++ b/app/lib/types.ts @@ -0,0 +1,4 @@ +export type Message = { + title: string; + description: string; +}; From 60ce2ac4bbc2b4b3f7f9f9bb727d17ff8cee4a63 Mon Sep 17 00:00:00 2001 From: eric valcik Date: Wed, 23 Apr 2025 01:07:08 +0200 Subject: [PATCH 08/16] make the agent card somewhat work --- app/app/globals.css | 15 ++++++++++++ app/app/ui-demo/analyzer.tsx | 46 +++++++++++++++++++++++++----------- app/app/ui-demo/page.tsx | 2 +- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/app/app/globals.css b/app/app/globals.css index bd44b5e..4fcfdae 100644 --- a/app/app/globals.css +++ b/app/app/globals.css @@ -163,3 +163,18 @@ .shiny-text.disabled { animation: none; } + +@keyframes fade-in { + from { + opacity: 0; + filter: blur(8px); + } + to { + opacity: 1; + filter: blur(0px); + } +} + +.animate-fade-in { + animation: fade-in 1.5s ease-out forwards; +} diff --git a/app/app/ui-demo/analyzer.tsx b/app/app/ui-demo/analyzer.tsx index c62e1bc..0f2aaa8 100644 --- a/app/app/ui-demo/analyzer.tsx +++ b/app/app/ui-demo/analyzer.tsx @@ -38,33 +38,51 @@ export function Analyzer({ messages, ...props }: AnalyzerProps) { + const renderMessages = messages.slice(-3); + return ( - + Analyzer Analyzes the website and generates scenarios. - -
- {messages.slice(-3).map((message, index) => ( + +
+ {renderMessages.map((message, index) => (
-
-

- {message.title === "ANALYZER" ? "Agent" : "Toolcall"} -

-

- {message.description} -

-
+

+ {message.title === "ANALYZER" ? "Agent" : "Toolcall"} +

+

+ {message.description.length > 150 + ? `${message.description.slice(0, 150)}...` + : message.description} +

))}
- {active && } + ); diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index e1519db..cf1c013 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -135,7 +135,7 @@ export default function WebSocketDemo() { From 4f800ed6e5cf6d3edf8a4da9b3b65ea4b110a68f Mon Sep 17 00:00:00 2001 From: eric valcik Date: Wed, 23 Apr 2025 01:29:33 +0200 Subject: [PATCH 09/16] make the agent card pretty --- app/app/ui-demo/analyzer.tsx | 48 +++++++++++++++++------------------- cmd/api/main.go | 1 - 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/app/app/ui-demo/analyzer.tsx b/app/app/ui-demo/analyzer.tsx index 0f2aaa8..5a8ee1b 100644 --- a/app/app/ui-demo/analyzer.tsx +++ b/app/app/ui-demo/analyzer.tsx @@ -1,7 +1,4 @@ -import { BellRing, Check } from "lucide-react"; - import { cn } from "@/lib/utils"; -import { Button } from "@/components/ui/button"; import { Card, CardContent, @@ -38,8 +35,6 @@ export function Analyzer({ messages, ...props }: AnalyzerProps) { - const renderMessages = messages.slice(-3); - return ( -
- {renderMessages.map((message, index) => ( -
-

- {message.title === "ANALYZER" ? "Agent" : "Toolcall"} -

-

- {message.description.length > 150 - ? `${message.description.slice(0, 150)}...` - : message.description} -

-
- ))} +
+
+ {messages.map((message, index) => ( +
+

+ {message.title === "ANALYZER" ? "Agent" : "Toolcall"} +

+

+ {message.description.length > 150 + ? `${message.description.slice(0, 150)}...` + : message.description} +

+
+ ))} +
+
Date: Wed, 23 Apr 2025 22:32:24 +0200 Subject: [PATCH 10/16] sw mac --- app/app/ui-demo/analyzer.tsx | 15 -------- app/app/ui-demo/generator.tsx | 70 +++++++++++++++++++++++++++++++++++ app/app/ui-demo/page.tsx | 4 +- 3 files changed, 72 insertions(+), 17 deletions(-) create mode 100644 app/app/ui-demo/generator.tsx diff --git a/app/app/ui-demo/analyzer.tsx b/app/app/ui-demo/analyzer.tsx index 5a8ee1b..2461917 100644 --- a/app/app/ui-demo/analyzer.tsx +++ b/app/app/ui-demo/analyzer.tsx @@ -9,21 +9,6 @@ import { AgentTitle } from "@/components/agent"; import { Message } from "@/lib/types"; import ShinyText from "@/components/ShinyText"; -const notifications = [ - { - title: "Your call has been confirmed.", - description: "1 hour ago", - }, - { - title: "You have a new message!", - description: "1 hour ago", - }, - { - title: "Your subscription is expiring soon!", - description: "2 hours ago", - }, -]; - type AnalyzerProps = React.ComponentProps & { active: boolean; messages: Message[]; diff --git a/app/app/ui-demo/generator.tsx b/app/app/ui-demo/generator.tsx new file mode 100644 index 0000000..4902d63 --- /dev/null +++ b/app/app/ui-demo/generator.tsx @@ -0,0 +1,70 @@ +import { cn } from "@/lib/utils"; +import { + Card, + CardContent, + CardDescription, + CardHeader, +} from "@/components/ui/card"; +import { AgentTitle } from "@/components/agent"; +import { Message } from "@/lib/types"; +import ShinyText from "@/components/ShinyText"; + +type GeneratorProps = React.ComponentProps & { + active: boolean; + messages: Message[]; +}; + +export function Generator({ + className, + active, + messages, + ...props +}: GeneratorProps) { + return ( + + + Generator + + Generates end-to-end tests based on provided scenarios. + + + +
+
+ {messages.map((message, index) => ( +
+

+ {message.title === "GENERATOR" ? "Agent" : "Toolcall"} +

+

+ {message.description.length > 150 + ? `${message.description.slice(0, 150)}...` + : message.description} +

+
+ ))} +
+
+
+ + + + ); +} diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index cf1c013..ef6f365 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -2,7 +2,6 @@ import { useState, useEffect, useRef } from "react"; import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; import { Card, CardContent, @@ -13,6 +12,7 @@ import { import { Textarea } from "@/components/ui/textarea"; import { Analyzer } from "./analyzer"; import { Message } from "@/lib/types"; +import { Generator } from "./generator"; export default function WebSocketDemo() { const [messages, setMessages] = useState([]); @@ -139,7 +139,7 @@ export default function WebSocketDemo() { active={true} messages={getMessages(messages, ["ANALYZER", "TOOLCALL"])} /> - Date: Wed, 23 Apr 2025 22:51:36 +0200 Subject: [PATCH 11/16] make agent component --- app/app/ui-demo/analyzer.tsx | 2 +- app/app/ui-demo/generator.tsx | 2 +- app/app/ui-demo/page.tsx | 8 +++- app/components/agent-title.tsx | 30 ++++++++++++ app/components/agent.tsx | 87 ++++++++++++++++++++++++++-------- 5 files changed, 105 insertions(+), 24 deletions(-) create mode 100644 app/components/agent-title.tsx diff --git a/app/app/ui-demo/analyzer.tsx b/app/app/ui-demo/analyzer.tsx index 2461917..7d86511 100644 --- a/app/app/ui-demo/analyzer.tsx +++ b/app/app/ui-demo/analyzer.tsx @@ -5,7 +5,7 @@ import { CardDescription, CardHeader, } from "@/components/ui/card"; -import { AgentTitle } from "@/components/agent"; +import { AgentTitle } from "@/components/agent-title"; import { Message } from "@/lib/types"; import ShinyText from "@/components/ShinyText"; diff --git a/app/app/ui-demo/generator.tsx b/app/app/ui-demo/generator.tsx index 4902d63..069f535 100644 --- a/app/app/ui-demo/generator.tsx +++ b/app/app/ui-demo/generator.tsx @@ -5,7 +5,7 @@ import { CardDescription, CardHeader, } from "@/components/ui/card"; -import { AgentTitle } from "@/components/agent"; +import { AgentTitle } from "@/components/agent-title"; import { Message } from "@/lib/types"; import ShinyText from "@/components/ShinyText"; diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index ef6f365..c294256 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -13,6 +13,7 @@ import { Textarea } from "@/components/ui/textarea"; import { Analyzer } from "./analyzer"; import { Message } from "@/lib/types"; import { Generator } from "./generator"; +import { Agent } from "@/components/agent"; export default function WebSocketDemo() { const [messages, setMessages] = useState([]); @@ -134,7 +135,9 @@ export default function WebSocketDemo() {
- { + const agentName = ids[0]; return messages .filter((message) => ids.includes(message.split(" ")[0])) .map((message) => { const id = message.split(" ")[0]; const description = message.split(" ").slice(1).join(" "); - return { title: id, description }; + return { title: id === agentName ? "Agent" : "Toolcall", description }; }); }; diff --git a/app/components/agent-title.tsx b/app/components/agent-title.tsx new file mode 100644 index 0000000..a9e35f8 --- /dev/null +++ b/app/components/agent-title.tsx @@ -0,0 +1,30 @@ +import * as React from "react"; +import { cn } from "@/lib/utils"; +import { CardTitle } from "@/components/ui/card"; + +type AgentTitleProps = React.ComponentProps & { + active?: boolean; +}; + +export function AgentTitle({ + className, + active, + children, + ...props +}: AgentTitleProps) { + return ( +
+ + + {children} + +
+ ); +} diff --git a/app/components/agent.tsx b/app/components/agent.tsx index a9e35f8..a3ebd7d 100644 --- a/app/components/agent.tsx +++ b/app/components/agent.tsx @@ -1,30 +1,77 @@ -import * as React from "react"; import { cn } from "@/lib/utils"; -import { CardTitle } from "@/components/ui/card"; +import { + Card, + CardContent, + CardDescription, + CardHeader, +} from "@/components/ui/card"; +import { AgentTitle } from "@/components/agent-title"; +import { Message } from "@/lib/types"; +import ShinyText from "@/components/ShinyText"; -type AgentTitleProps = React.ComponentProps & { - active?: boolean; +type AgentProps = React.ComponentProps & { + active: boolean; + messages: Message[]; + title: string; + description: string; }; -export function AgentTitle({ +export function Agent({ className, active, - children, + messages, + title, + description, ...props -}: AgentTitleProps) { +}: AgentProps) { return ( -
- - - {children} - -
+ + + {title} + {description} + + +
+
+ {messages.map((message, index) => ( +
+

+ {message.title} +

+

+ {message.description.length > 150 + ? `${message.description.slice(0, 150)}...` + : message.description} +

+
+ ))} +
+
+
+ + + ); } From 578b02cb6b36a4477c1fc29abe026c793e41e417 Mon Sep 17 00:00:00 2001 From: eric valcik Date: Thu, 24 Apr 2025 00:21:34 +0200 Subject: [PATCH 12/16] the flow is working --- app/app/ui-demo/analyzer.tsx | 70 ------------ app/app/ui-demo/generator.tsx | 70 ------------ app/app/ui-demo/page.tsx | 101 +++++++++++++----- app/app/ui-demo/test-files.tsx | 51 +++++++++ app/components/tool.tsx | 69 ++++++++++++ cmd/api/main.go | 18 +++- internal/repository/analyzer/analyze.go | 8 +- internal/repository/analyzer/get_content.go | 33 +++--- .../repository/analyzer/get_content_test.go | 2 +- internal/repository/analyzer/get_sitemap.go | 7 +- .../repository/analyzer/get_sitemap_test.go | 4 +- .../repository/gen_eval_loop/gen_eval_loop.go | 24 +++-- 12 files changed, 261 insertions(+), 196 deletions(-) delete mode 100644 app/app/ui-demo/analyzer.tsx delete mode 100644 app/app/ui-demo/generator.tsx create mode 100644 app/app/ui-demo/test-files.tsx create mode 100644 app/components/tool.tsx diff --git a/app/app/ui-demo/analyzer.tsx b/app/app/ui-demo/analyzer.tsx deleted file mode 100644 index 7d86511..0000000 --- a/app/app/ui-demo/analyzer.tsx +++ /dev/null @@ -1,70 +0,0 @@ -import { cn } from "@/lib/utils"; -import { - Card, - CardContent, - CardDescription, - CardHeader, -} from "@/components/ui/card"; -import { AgentTitle } from "@/components/agent-title"; -import { Message } from "@/lib/types"; -import ShinyText from "@/components/ShinyText"; - -type AnalyzerProps = React.ComponentProps & { - active: boolean; - messages: Message[]; -}; - -export function Analyzer({ - className, - active, - messages, - ...props -}: AnalyzerProps) { - return ( - - - Analyzer - - Analyzes the website and generates scenarios. - - - -
-
- {messages.map((message, index) => ( -
-

- {message.title === "ANALYZER" ? "Agent" : "Toolcall"} -

-

- {message.description.length > 150 - ? `${message.description.slice(0, 150)}...` - : message.description} -

-
- ))} -
-
-
- - - - ); -} diff --git a/app/app/ui-demo/generator.tsx b/app/app/ui-demo/generator.tsx deleted file mode 100644 index 069f535..0000000 --- a/app/app/ui-demo/generator.tsx +++ /dev/null @@ -1,70 +0,0 @@ -import { cn } from "@/lib/utils"; -import { - Card, - CardContent, - CardDescription, - CardHeader, -} from "@/components/ui/card"; -import { AgentTitle } from "@/components/agent-title"; -import { Message } from "@/lib/types"; -import ShinyText from "@/components/ShinyText"; - -type GeneratorProps = React.ComponentProps & { - active: boolean; - messages: Message[]; -}; - -export function Generator({ - className, - active, - messages, - ...props -}: GeneratorProps) { - return ( - - - Generator - - Generates end-to-end tests based on provided scenarios. - - - -
-
- {messages.map((message, index) => ( -
-

- {message.title === "GENERATOR" ? "Agent" : "Toolcall"} -

-

- {message.description.length > 150 - ? `${message.description.slice(0, 150)}...` - : message.description} -

-
- ))} -
-
-
- - - - ); -} diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index c294256..7a0a68d 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -10,14 +10,16 @@ import { CardDescription, } from "@/components/ui/card"; import { Textarea } from "@/components/ui/textarea"; -import { Analyzer } from "./analyzer"; import { Message } from "@/lib/types"; -import { Generator } from "./generator"; import { Agent } from "@/components/agent"; - +import { Tool } from "@/components/tool"; +import { TestFiles } from "./test-files"; export default function WebSocketDemo() { const [messages, setMessages] = useState([]); - const [inputValue, setInputValue] = useState(""); + const [scenarioCount, setScenarioCount] = useState(0); + const [fileNames, setFileNames] = useState([]); + const [scenario, setScenario] = useState(""); + const [inputValue, setInputValue] = useState("jakub.kr"); const [isConnected, setIsConnected] = useState(false); const wsRef = useRef(null); const scrollAreaRef = useRef(null); @@ -41,7 +43,7 @@ export default function WebSocketDemo() { console.log("WebSocket connection opened successfully"); setIsConnected(true); addMessage("OPEN"); - addMessage(`SEND: ${inputValue}`); + addMessage(`SEND '${inputValue}'`); if (wsRef.current) { wsRef.current.send(inputValue); } @@ -51,28 +53,41 @@ export default function WebSocketDemo() { console.log("WebSocket connection closed:", event.code, event.reason); setIsConnected(false); addMessage( - `CLOSE (Code: ${event.code}${ + `CLOSE '${event.code}${ event.reason ? `, Reason: ${event.reason}` : "" - })` + }'` ); wsRef.current = null; }; wsRef.current.onmessage = (evt) => { - console.log("Received message:", evt.data); + console.log(evt.data); addMessage(evt.data); + const id = evt.data.split(" ")[0]; + if (id === "SCENARIOS") { + const numScenarios = parseInt(evt.data.split(" ")[1]); + setScenarioCount(numScenarios); + } + if (id === "FILENAME") { + const fileName = evt.data.split(" ")[1]; + setFileNames((prev) => [...prev, fileName]); + } + if (id === "SCENARIO") { + const scenario = evt.data.substring(evt.data.indexOf(" ") + 1); + setScenario(scenario); + } }; wsRef.current.onerror = (error) => { console.error("WebSocket error:", error); addMessage( - `ERROR: Connection failed - check browser console for details` + `ERROR 'Connection failed - check browser console for details'` ); }; } catch (error: any) { console.error("WebSocket connection error:", error); addMessage( - `ERROR: Failed to connect to WebSocket server - ${error.message}` + `ERROR 'Failed to connect to WebSocket server - ${error.message}'` ); } return; @@ -90,7 +105,7 @@ export default function WebSocketDemo() { return (
- + End-to-end Test Generation Generate end-to-end tests that are passing with just a prompt and @@ -122,35 +137,69 @@ export default function WebSocketDemo() {
-
- {messages.map((message, index) => ( -
- {message} -
- ))} -
- + + + + +
); } +const isActive = (messages: string[], id: string): boolean => { + return ( + messages.length > 0 && messages[messages.length - 1].split(" ")[0] === id + ); +}; + const getMessages = (messages: string[], ids: string[]): Message[] => { const agentName = ids[0]; return messages diff --git a/app/app/ui-demo/test-files.tsx b/app/app/ui-demo/test-files.tsx new file mode 100644 index 0000000..ebd755d --- /dev/null +++ b/app/app/ui-demo/test-files.tsx @@ -0,0 +1,51 @@ +import { cn } from "@/lib/utils"; + +interface TestFilesProps { + count: number; + names: string[]; + scenario: string; + className?: string; +} + +export function TestFiles({ + count, + names, + scenario, + className, +}: TestFilesProps) { + if (count === 0) { + return null; + } + + return ( +
+
+

Scenario {names.length + 1}:

+

{scenario}

+
+
+

Generated test files:

+ {Array.from({ length: count }).map((_, index) => { + const fileName = names[index]; + const isKnown = !!fileName; + + return ( +
+
+ + {isKnown ? fileName : "Pending..."} + +
+ ); + })} +
+
+ ); +} diff --git a/app/components/tool.tsx b/app/components/tool.tsx new file mode 100644 index 0000000..f4c4396 --- /dev/null +++ b/app/components/tool.tsx @@ -0,0 +1,69 @@ +import { cn } from "@/lib/utils"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Message } from "@/lib/types"; +import ShinyText from "@/components/ShinyText"; + +type ToolProps = React.ComponentProps & { + active: boolean; + messages: Message[]; + title: string; + description: string; +}; + +export function Tool({ + className, + active, + messages, + title, + description, + ...props +}: ToolProps) { + return ( + + + {title} + {description} + + +
+
+ {messages.map((message, index) => ( +
+

+ {message.title} +

+

+ {message.description.length > 100 + ? `${message.description.slice(0, 100)}...` + : message.description} +

+
+ ))} +
+
+
+ + + ); +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 0b7712c..b252b40 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -142,12 +142,14 @@ func sendMessage(ctx context.Context, c *websocket.Conn, mt int, url string) { case <-ctx.Done(): return default: - if err := c.WriteMessage(mt, []byte(fmt.Sprintf("\n[MAIN FLOW] Analyzer generated %d scenarios\n", len(criteria)))); err != nil { + if err := c.WriteMessage(mt, []byte(fmt.Sprintf("ANALYZER 'Generated %d scenarios, passing to Generator.'", len(criteria)))); err != nil { log.Printf("error writing message: %v", err) return } } + c.WriteMessage(mt, []byte(fmt.Sprintf("SCENARIOS %d", len(criteria)))) + logger.Debug("CRITERIA LENGTH: %d", len(criteria)) for _, criterion := range criteria { logger.Debug("CRITERIA: %s", criterion) @@ -163,18 +165,27 @@ func sendMessage(ctx context.Context, c *websocket.Conn, mt int, url string) { default: } + keys := make([]string, 0, len(analysis.ContentMap)) + for k := range analysis.ContentMap { + keys = append(keys, k) + } + c.WriteMessage(mt, []byte(fmt.Sprintf("SCENARIO %s\n\nContent of these sites is passed to the generator: %s", criterion, strings.Join(keys, ", ")))) + fmt.Printf("\n[MAIN FLOW] Generating test for scenario %d: %s\n", i, criterion) + c.WriteMessage(mt, []byte(fmt.Sprintf("GENERATOR 'Generating test for scenario %d: %s'", i, criterion))) filename, err := gen_eval_loop.GenEvalLoop(ctx, client, &models.AnalyzerReturn{ - TechSpec: analysis.TechSpec, + TechSpec: url, ContentMap: analysis.ContentMap, Criteria: analysis.Criteria, - }, i+1, noOfLoops) + }, i+1, noOfLoops, c, mt) if err != nil { log.Printf("Error: %v\n", err) return } logger.Debug("[MAIN FLOW] Writing test file: %s\n", filepath.Base(filename)) + c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Writing test file: %s'", filepath.Base(filename)))) + c.WriteMessage(mt, []byte(fmt.Sprintf("FILENAME %s", filepath.Base(filename)))) // copy the file to the current directory destPath := filepath.Join("./__generated__", filepath.Base(filename)) @@ -185,6 +196,7 @@ func sendMessage(ctx context.Context, c *websocket.Conn, mt int, url string) { return } } + c.WriteMessage(mt, []byte(fmt.Sprintf("FINAL_MESSAGE 'Tests are generated.'"))) } func main() { diff --git a/internal/repository/analyzer/analyze.go b/internal/repository/analyzer/analyze.go index 381b5b3..5994714 100644 --- a/internal/repository/analyzer/analyze.go +++ b/internal/repository/analyzer/analyze.go @@ -90,10 +90,11 @@ func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr return nil, err } - response, err = GetSitemap(ctx, input.BaseUrl) + response, err = GetSitemap(ctx, input.BaseUrl, c, mt) if err != nil { return nil, err } + c.WriteMessage(mt, []byte(fmt.Sprintf("ANALYZER 'I got the sitemap'"))) case getContentTool.Name: input := models.GetContentTool{} err := json.Unmarshal([]byte(variant.JSON.Input.Raw()), &input) @@ -101,12 +102,13 @@ func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr return nil, err } - result, err := GetContent(ctx, input.Urls) + result, err := GetContent(ctx, input.Urls, c, mt) if err != nil { return nil, err } contentMap = result.Contents response = result + c.WriteMessage(mt, []byte(fmt.Sprintf("ANALYZER 'I got the content'"))) case sentryTool.Name: input := models.SentryTool{} err := json.Unmarshal([]byte(variant.JSON.Input.Raw()), &input) @@ -135,7 +137,7 @@ func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr logger.Debug("FROM ANALYZE: Final contentMap: %s", input.ContentMap) return &models.AnalyzerReturn{ - TechSpec: prompt, + TechSpec: "", ContentMap: contentMap, Criteria: input.Criteria, }, nil diff --git a/internal/repository/analyzer/get_content.go b/internal/repository/analyzer/get_content.go index 940fb67..8a9b0c8 100644 --- a/internal/repository/analyzer/get_content.go +++ b/internal/repository/analyzer/get_content.go @@ -12,6 +12,7 @@ import ( "github.com/chromedp/chromedp" "github.com/gocolly/colly/v2" + "github.com/gorilla/websocket" "github.com/webscopeio/ai-hackathon/internal/models" ) @@ -129,11 +130,13 @@ func GetContent_OLD(ctx context.Context, urls []string) (*models.GetContentToolR // GetContentCdp uses Chrome DevTools Protocol via chromedp to fetch content from URLs // This is especially useful for SPAs and JavaScript-heavy websites -func GetContent(ctx context.Context, urls []string) (*models.GetContentToolReturn, error) { +func GetContent(ctx context.Context, urls []string, c *websocket.Conn, mt int) (*models.GetContentToolReturn, error) { if len(urls) == 0 { return nil, errors.New("empty URLs list provided") } + c.WriteMessage(mt, []byte(fmt.Sprintf("GET_CONTENT_TOOL 'Getting content for %d URLs'", len(urls)))) + // Validate and normalize URLs validatedUrls := make([]string, 0, len(urls)) for _, urlStr := range urls { @@ -153,6 +156,7 @@ func GetContent(ctx context.Context, urls []string) (*models.GetContentToolRetur parsedURL, err := url.Parse(urlStr) if err != nil { fmt.Printf("Warning: Could not parse URL %s: %v\n", urlStr, err) + c.WriteMessage(mt, []byte(fmt.Sprintf("GET_CONTENT_TOOL 'Warning: Could not parse URL %s: %v'", urlStr, err))) continue } @@ -184,46 +188,47 @@ func GetContent(ctx context.Context, urls []string) (*models.GetContentToolRetur // Create a timeout context for each URL urlCtx, urlCancel := context.WithTimeout(browserCtx, 30*time.Second) defer urlCancel() - + // Variable to store the body content var bodyHTML string - + // Navigate to the URL and capture the body content err := chromedp.Run(urlCtx, // Navigate to the URL chromedp.Navigate(urlStr), - + // Wait for the page to be fully loaded chromedp.WaitReady("body", chromedp.ByQuery), - + // Optional: Wait some extra time for dynamic content chromedp.Sleep(1*time.Second), - + // Capture the HTML content chromedp.OuterHTML("body", &bodyHTML, chromedp.ByQuery), - ) - + ) + if err != nil { fmt.Printf("Error fetching %s with chromedp: %v\n", urlStr, err) - + c.WriteMessage(mt, []byte(fmt.Sprintf("GET_CONTENT_TOOL 'Error fetching %s with chromedp: %v'", urlStr, err))) // Store empty string for failed URLs mutex.Lock() results[urlStr] = "" mutex.Unlock() continue } - + // Clean the HTML content for _, regex := range cleaningRegexes { bodyHTML = regex.ReplaceAllString(bodyHTML, "") } - + // Store the result mutex.Lock() results[urlStr] = bodyHTML mutex.Unlock() - - fmt.Printf("Successfully fetched %s with chromedp\n", urlStr) + + fmt.Printf("Fetched %s with chromedp\n", urlStr) + c.WriteMessage(mt, []byte(fmt.Sprintf("GET_CONTENT_TOOL 'Fetched %s with chromedp'", urlStr))) } // Check if we got any results @@ -232,7 +237,7 @@ func GetContent(ctx context.Context, urls []string) (*models.GetContentToolRetur } fmt.Printf("Results are ready, found data for %d URLs\n", len(results)) - + c.WriteMessage(mt, []byte(fmt.Sprintf("GET_CONTENT_TOOL 'Results are ready, returning to Agent.'"))) return &models.GetContentToolReturn{ Contents: results, }, nil diff --git a/internal/repository/analyzer/get_content_test.go b/internal/repository/analyzer/get_content_test.go index c75cb24..8729bf0 100644 --- a/internal/repository/analyzer/get_content_test.go +++ b/internal/repository/analyzer/get_content_test.go @@ -16,7 +16,7 @@ func TestGetContent(t *testing.T) { "https://ai-hackathon-demo-delta.vercel.app/", } - res, err := GetContent(ctx, urls) + res, err := GetContent(ctx, urls, nil, 0) if err != nil { t.Fatalf("err=%v", err) } diff --git a/internal/repository/analyzer/get_sitemap.go b/internal/repository/analyzer/get_sitemap.go index e2c9a02..38dff92 100644 --- a/internal/repository/analyzer/get_sitemap.go +++ b/internal/repository/analyzer/get_sitemap.go @@ -9,14 +9,16 @@ import ( "net/url" "strings" + "github.com/gorilla/websocket" "github.com/webscopeio/ai-hackathon/internal/logger" "github.com/webscopeio/ai-hackathon/internal/models" ) // GetSitemap attempts to retrieve and parse a sitemap from a given URL // It tries common sitemap locations if not explicitly provided -func GetSitemap(ctx context.Context, baseURL string) (*models.Sitemap, error) { +func GetSitemap(ctx context.Context, baseURL string, c *websocket.Conn, mt int) (*models.Sitemap, error) { logger.Debug("Getting sitemap for URL: %s", baseURL) + c.WriteMessage(mt, []byte(fmt.Sprintf("GET_SITEMAP_TOOL 'Getting sitemap for URL: %s'", baseURL))) // Parse the base URL parsedURL, err := url.Parse(baseURL) @@ -53,6 +55,7 @@ func GetSitemap(ctx context.Context, baseURL string) (*models.Sitemap, error) { sitemap, err := fetchSitemap(ctx, sitemapURL) if err == nil && sitemap != nil && len(sitemap.URLs) > 0 { logger.Debug("Found sitemap at %s with %d URLs", sitemapURL, len(sitemap.URLs)) + c.WriteMessage(mt, []byte(fmt.Sprintf("GET_SITEMAP_TOOL 'Found sitemap at %s with %d URLs'", sitemapURL, len(sitemap.URLs)))) return sitemap, nil } @@ -66,11 +69,13 @@ func GetSitemap(ctx context.Context, baseURL string) (*models.Sitemap, error) { sitemap, err := fetchSitemap(ctx, firstSitemapURL) if err == nil && sitemap != nil { logger.Debug("Found sitemap at %s with %d URLs", firstSitemapURL, len(sitemap.URLs)) + c.WriteMessage(mt, []byte(fmt.Sprintf("GET_SITEMAP_TOOL 'Found sitemap at %s with %d URLs'", firstSitemapURL, len(sitemap.URLs)))) return sitemap, nil } } } + c.WriteMessage(mt, []byte(fmt.Sprintf("GET_SITEMAP_TOOL 'No sitemap found for %s'", baseURL))) return nil, fmt.Errorf("no sitemap found for %s", baseURL) } diff --git a/internal/repository/analyzer/get_sitemap_test.go b/internal/repository/analyzer/get_sitemap_test.go index 78381bb..c64f405 100644 --- a/internal/repository/analyzer/get_sitemap_test.go +++ b/internal/repository/analyzer/get_sitemap_test.go @@ -51,7 +51,7 @@ Sitemap: http://example.com/sitemap.xml`)) defer server.Close() // Test getting the sitemap - sitemap, err := GetSitemap(context.Background(), server.URL) + sitemap, err := GetSitemap(context.Background(), server.URL, nil, 0) if err != nil { t.Fatalf("Failed to get sitemap: %v", err) } @@ -131,7 +131,7 @@ func TestGetSitemapIndex(t *testing.T) { }) // Test getting the sitemap from the index - sitemap, err := GetSitemap(context.Background(), server.URL) + sitemap, err := GetSitemap(context.Background(), server.URL, nil, 0) if err != nil { t.Fatalf("Failed to get sitemap from index: %v", err) } diff --git a/internal/repository/gen_eval_loop/gen_eval_loop.go b/internal/repository/gen_eval_loop/gen_eval_loop.go index 7fb81e7..56870ed 100644 --- a/internal/repository/gen_eval_loop/gen_eval_loop.go +++ b/internal/repository/gen_eval_loop/gen_eval_loop.go @@ -11,12 +11,13 @@ import ( "strings" "github.com/anthropics/anthropic-sdk-go" + "github.com/gorilla/websocket" "github.com/webscopeio/ai-hackathon/internal/llm" "github.com/webscopeio/ai-hackathon/internal/logger" "github.com/webscopeio/ai-hackathon/internal/models" ) -func GenEvalLoop(ctx context.Context, client *llm.Client, analyzerReturn *models.AnalyzerReturn, index int, noOfLoops int) (string, error) { +func GenEvalLoop(ctx context.Context, client *llm.Client, analyzerReturn *models.AnalyzerReturn, index int, noOfLoops int, c *websocket.Conn, mt int) (string, error) { tempDir, testsDir, err := SetupTestEnvironment(ctx) if err != nil { return "", fmt.Errorf("SetupTestEnvironment failed: %w", err) @@ -32,14 +33,14 @@ func GenEvalLoop(ctx context.Context, client *llm.Client, analyzerReturn *models if loopCount > noOfLoops { return filename, nil } - filename, generatorMessages, err = generateTestFile(ctx, client, analyzerReturn, generatorMessages, feedback, string(testFileContent), testsDir, index) + filename, generatorMessages, err = generateTestFile(ctx, client, analyzerReturn, generatorMessages, feedback, string(testFileContent), testsDir, index, c, mt) if err != nil { return "", fmt.Errorf("GenerateTestFile failed: %w", err) } logger.Debug("Filename: %s", filename) var passed bool - feedback, passed, err = evaluateTestFile(ctx, client, filename, tempDir) + feedback, passed, err = evaluateTestFile(ctx, client, filename, tempDir, c, mt) logger.Debug("EVALUATOR feedback: %s", feedback) if err != nil { return "", fmt.Errorf("EvaluateTestFile failed: %w", err) @@ -64,7 +65,7 @@ func GenEvalLoop(ctx context.Context, client *llm.Client, analyzerReturn *models // Tests generates test files based on a URL using the LLM client // It also stores the generated test files in a temporary directory -func generateTestFile(ctx context.Context, client *llm.Client, analyzerReturn *models.AnalyzerReturn, prevMessages []anthropic.MessageParam, feedback string, testFileContent string, testsDir string, index int) (string, []anthropic.MessageParam, error) { +func generateTestFile(ctx context.Context, client *llm.Client, analyzerReturn *models.AnalyzerReturn, prevMessages []anthropic.MessageParam, feedback string, testFileContent string, testsDir string, index int, c *websocket.Conn, mt int) (string, []anthropic.MessageParam, error) { logger.Debug("Starting generateTestFile") @@ -121,8 +122,10 @@ TEST FILE CURRENT CONTENT: ` + testFileContent fmt.Println("\n[GENERATOR] Calling Generator with feedback.") + c.WriteMessage(mt, []byte(fmt.Sprintf("GENERATOR 'Calling Generator with feedback.'"))) } else { fmt.Println("\n[GENERATOR] Calling Generator with base prompt.") + c.WriteMessage(mt, []byte(fmt.Sprintf("GENERATOR 'Calling Generator with base prompt.'"))) } // INFO: for a structured response the client requires tools, ref: https://docs.anthropic.com/en/docs/build-with-claude/tool-use/overview @@ -192,16 +195,19 @@ TEST FILE CURRENT CONTENT: logger.Debug(" - %s\n", dep) } - fmt.Printf("[GENERATOR] GenerateTest successfuly generated test file: %s\n", filePath) + fmt.Printf("[GENERATOR] GenerateTest successfully generated test file: %s\n", filePath) + c.WriteMessage(mt, []byte(fmt.Sprintf("GENERATOR 'Successfully generated test file: %s'", filePath))) return filePath, newMessages, nil } -func evaluateTestFile(ctx context.Context, client *llm.Client, filename string, tempDir string) (string, bool, error) { +func evaluateTestFile(ctx context.Context, client *llm.Client, filename string, tempDir string, c *websocket.Conn, mt int) (string, bool, error) { // List the provided test file content, err := os.ReadFile(filename) if err != nil { return "", false, fmt.Errorf("couldn't read test file: %w", err) } + c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Try to run the test file: %s'", filename))) + c.WriteMessage(mt, []byte(fmt.Sprintf("RUN_TEST_TOOL 'Running test file %s...'", filename))) // Run pnpm test testCmd := exec.Command("pnpm", "test", filename) @@ -212,9 +218,11 @@ func evaluateTestFile(ctx context.Context, client *llm.Client, filename string, logger.Debug("❌ Tests failed!\n") logger.Debug("Error executing pnpm test: %v\n", err) logger.Debug("Test output: %s\n", output) + c.WriteMessage(mt, []byte(fmt.Sprintf("RUN_TEST_TOOL 'Tests failed ❌. Passing the error and output to the evaluator.'"))) // but we don't want to return, we want to continue the loop } else { logger.Debug("✅ Tests passed successfully!\n") + c.WriteMessage(mt, []byte(fmt.Sprintf("RUN_TEST_TOOL 'Tests passed ✅. Passing the output to the evaluator.'"))) } // Analyze the test output @@ -259,6 +267,7 @@ Invalid formatting will cause errors in processing your response. fmt.Printf("[EVALUATOR] Calling Evaluator with context length: %d characters\n", len(context)) fmt.Println("[EVALUATOR] and test result + file content") + c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Analyzing the test contents + test results.'"))) rawResponse, err := client.GetStructuredCompletion( ctx, context, @@ -278,11 +287,14 @@ Invalid formatting will cause errors in processing your response. if response.Passed { fmt.Printf("[EVALUATOR] Evaluator accepted the test file ✅\n") + c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Evaluator accepted the test file ✅'"))) return "", true, nil } fmt.Printf("[EVALUATOR] Evaluator rejected the test file ❌\n") fmt.Printf("[EVALUATOR] With the following feedback:\n\n %s\n", response.Feedback) + c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Evaluator rejected the test file ❌. With the following feedback:\n\n %s'", response.Feedback))) + c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Passing the feedback to the generator.'"))) return response.Feedback, false, nil } From 7e51cedd62ad6f32afa02049705221f8c0752944 Mon Sep 17 00:00:00 2001 From: eric valcik Date: Thu, 24 Apr 2025 00:29:43 +0200 Subject: [PATCH 13/16] make the components not absolute positioned --- app/app/ui-demo/page.tsx | 93 +++++++++++++++++----------------- app/app/ui-demo/test-files.tsx | 2 +- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index 7a0a68d..7027e26 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -103,8 +103,8 @@ export default function WebSocketDemo() { }, [messages]); return ( -
- +
+ End-to-end Test Generation @@ -140,52 +140,51 @@ export default function WebSocketDemo() {
- - - - - - +
+ + + +
+
+ + +
+ +
Scenario {names.length + 1}:

{scenario}

-
+

Generated test files:

{Array.from({ length: count }).map((_, index) => { const fileName = names[index]; From aee86fd66c9746fdef9e0e1515b3f0dc3354c061 Mon Sep 17 00:00:00 2001 From: eric valcik Date: Thu, 24 Apr 2025 17:36:48 +0200 Subject: [PATCH 14/16] finished ui demo --- Makefile | 3 ++- app/app/ui-demo/page.tsx | 4 +++- app/app/ui-demo/test-files.tsx | 4 ++-- app/components/agent.tsx | 4 ++-- cmd/api/main.go | 6 +++++- internal/llm/completion.go | 4 ++-- internal/repository/analyzer/analyze.go | 2 +- .../repository/gen_eval_loop/gen_eval_loop.go | 19 ++++++++++++------- 8 files changed, 29 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 06da8bd..367ba2b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ .PHONY: dev/server dev/server: go run github.com/goware/rerun/cmd/rerun@latest \ - -watch . \ + -watch cmd \ + -watch internal \ -ignore app \ -run "go run ./cmd/api" diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index 7027e26..eb87da1 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -19,7 +19,9 @@ export default function WebSocketDemo() { const [scenarioCount, setScenarioCount] = useState(0); const [fileNames, setFileNames] = useState([]); const [scenario, setScenario] = useState(""); - const [inputValue, setInputValue] = useState("jakub.kr"); + const [inputValue, setInputValue] = useState( + "https://ai-hackathon-demo-delta.vercel.app/" + ); const [isConnected, setIsConnected] = useState(false); const wsRef = useRef(null); const scrollAreaRef = useRef(null); diff --git a/app/app/ui-demo/test-files.tsx b/app/app/ui-demo/test-files.tsx index 54e258d..5fbd347 100644 --- a/app/app/ui-demo/test-files.tsx +++ b/app/app/ui-demo/test-files.tsx @@ -23,7 +23,7 @@ export function TestFiles({

Scenario {names.length + 1}:

{scenario}

-
+

Generated test files:

{Array.from({ length: count }).map((_, index) => { const fileName = names[index]; @@ -34,7 +34,7 @@ export function TestFiles({

- {message.description.length > 150 - ? `${message.description.slice(0, 150)}...` + {message.description.length > 250 + ? `${message.description.slice(0, 250)}...` : message.description}

diff --git a/cmd/api/main.go b/cmd/api/main.go index b252b40..a0f29f3 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -156,6 +156,7 @@ func sendMessage(ctx context.Context, c *websocket.Conn, mt int, url string) { } noOfLoops := 6 + filenames := []string{} for i, criterion := range criteria { // Check context before each iteration @@ -177,7 +178,7 @@ func sendMessage(ctx context.Context, c *websocket.Conn, mt int, url string) { TechSpec: url, ContentMap: analysis.ContentMap, Criteria: analysis.Criteria, - }, i+1, noOfLoops, c, mt) + }, i+1, noOfLoops, filenames, c, mt) if err != nil { log.Printf("Error: %v\n", err) return @@ -186,6 +187,9 @@ func sendMessage(ctx context.Context, c *websocket.Conn, mt int, url string) { logger.Debug("[MAIN FLOW] Writing test file: %s\n", filepath.Base(filename)) c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Writing test file: %s'", filepath.Base(filename)))) c.WriteMessage(mt, []byte(fmt.Sprintf("FILENAME %s", filepath.Base(filename)))) + testFileName := filepath.Base(filename)[7:] + filenames = append(filenames, testFileName) + logger.Debug("[MAIN FLOW] Updated filenames: %v", filenames) // copy the file to the current directory destPath := filepath.Join("./__generated__", filepath.Base(filename)) diff --git a/internal/llm/completion.go b/internal/llm/completion.go index b6e4346..34f2075 100644 --- a/internal/llm/completion.go +++ b/internal/llm/completion.go @@ -10,7 +10,7 @@ import ( func (c *Client) GetCompletion(ctx context.Context, prompt string) (string, error) { message, err := c.client.Messages.New(ctx, anthropic.MessageNewParams{ - Model: anthropic.ModelClaude3_5HaikuLatest, + Model: anthropic.ModelClaude3_5SonnetLatest, MaxTokens: 4096, System: []anthropic.TextBlockParam{ { @@ -68,7 +68,7 @@ func (c *Client) GetStructuredCompletion( messages := append(prevMessages, anthropic.NewUserMessage(anthropic.NewTextBlock(prompt))) message, err := c.client.Messages.New(ctx, anthropic.MessageNewParams{ - Model: anthropic.ModelClaude3_5HaikuLatest, + Model: anthropic.ModelClaude3_5SonnetLatest, // INFO: tools typically require more tokens MaxTokens: 2400, System: systemBlocks, diff --git a/internal/repository/analyzer/analyze.go b/internal/repository/analyzer/analyze.go index 5994714..81a82b8 100644 --- a/internal/repository/analyzer/analyze.go +++ b/internal/repository/analyzer/analyze.go @@ -52,7 +52,7 @@ func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr for { message, err := client.NewMessage(ctx, anthropic.MessageNewParams{ - Model: anthropic.ModelClaude3_5HaikuLatest, + Model: anthropic.ModelClaude3_5SonnetLatest, MaxTokens: 2048, Messages: messages, Tools: tools, diff --git a/internal/repository/gen_eval_loop/gen_eval_loop.go b/internal/repository/gen_eval_loop/gen_eval_loop.go index 56870ed..a7e913f 100644 --- a/internal/repository/gen_eval_loop/gen_eval_loop.go +++ b/internal/repository/gen_eval_loop/gen_eval_loop.go @@ -17,7 +17,7 @@ import ( "github.com/webscopeio/ai-hackathon/internal/models" ) -func GenEvalLoop(ctx context.Context, client *llm.Client, analyzerReturn *models.AnalyzerReturn, index int, noOfLoops int, c *websocket.Conn, mt int) (string, error) { +func GenEvalLoop(ctx context.Context, client *llm.Client, analyzerReturn *models.AnalyzerReturn, index int, noOfLoops int, filenames []string, c *websocket.Conn, mt int) (string, error) { tempDir, testsDir, err := SetupTestEnvironment(ctx) if err != nil { return "", fmt.Errorf("SetupTestEnvironment failed: %w", err) @@ -33,7 +33,7 @@ func GenEvalLoop(ctx context.Context, client *llm.Client, analyzerReturn *models if loopCount > noOfLoops { return filename, nil } - filename, generatorMessages, err = generateTestFile(ctx, client, analyzerReturn, generatorMessages, feedback, string(testFileContent), testsDir, index, c, mt) + filename, generatorMessages, err = generateTestFile(ctx, client, analyzerReturn, generatorMessages, feedback, string(testFileContent), testsDir, index, filenames, c, mt) if err != nil { return "", fmt.Errorf("GenerateTestFile failed: %w", err) } @@ -65,7 +65,7 @@ func GenEvalLoop(ctx context.Context, client *llm.Client, analyzerReturn *models // Tests generates test files based on a URL using the LLM client // It also stores the generated test files in a temporary directory -func generateTestFile(ctx context.Context, client *llm.Client, analyzerReturn *models.AnalyzerReturn, prevMessages []anthropic.MessageParam, feedback string, testFileContent string, testsDir string, index int, c *websocket.Conn, mt int) (string, []anthropic.MessageParam, error) { +func generateTestFile(ctx context.Context, client *llm.Client, analyzerReturn *models.AnalyzerReturn, prevMessages []anthropic.MessageParam, feedback string, testFileContent string, testsDir string, index int, filenames []string, c *websocket.Conn, mt int) (string, []anthropic.MessageParam, error) { logger.Debug("Starting generateTestFile") @@ -114,6 +114,11 @@ Format the tests following Playwright best practices with clear test description Invalid formatting will cause errors in processing your response. ` + // add existing filenames to the prompt and tell the to not use these file names + basePrompt += ` +EXISTING FILE NAMES (USE DIFFERENT NAME FOR THE GENERATED FILE): +` + strings.Join(filenames, ", ") + if feedback != "" { basePrompt = `An Evaluation of the test file has been provided. Please revise the test file based on the feedback. Leave everything else the same. Mainly focus on fixing the failing tests. The resulting file should be no longer than 100 lines of code. ` + feedback @@ -196,7 +201,7 @@ TEST FILE CURRENT CONTENT: } fmt.Printf("[GENERATOR] GenerateTest successfully generated test file: %s\n", filePath) - c.WriteMessage(mt, []byte(fmt.Sprintf("GENERATOR 'Successfully generated test file: %s'", filePath))) + c.WriteMessage(mt, []byte(fmt.Sprintf("GENERATOR 'Successfully generated test file: %s'", filepath.Base(filePath)))) return filePath, newMessages, nil } @@ -206,8 +211,8 @@ func evaluateTestFile(ctx context.Context, client *llm.Client, filename string, if err != nil { return "", false, fmt.Errorf("couldn't read test file: %w", err) } - c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Try to run the test file: %s'", filename))) - c.WriteMessage(mt, []byte(fmt.Sprintf("RUN_TEST_TOOL 'Running test file %s...'", filename))) + c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Try to run the test file: %s'", filepath.Base(filename)))) + c.WriteMessage(mt, []byte(fmt.Sprintf("RUN_TEST_TOOL 'Running test file %s...'", filepath.Base(filename)))) // Run pnpm test testCmd := exec.Command("pnpm", "test", filename) @@ -293,7 +298,7 @@ Invalid formatting will cause errors in processing your response. fmt.Printf("[EVALUATOR] Evaluator rejected the test file ❌\n") fmt.Printf("[EVALUATOR] With the following feedback:\n\n %s\n", response.Feedback) - c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Evaluator rejected the test file ❌. With the following feedback:\n\n %s'", response.Feedback))) + c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'REJECTED, feedback for generator:\n\n %s'", response.Feedback))) c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Passing the feedback to the generator.'"))) return response.Feedback, false, nil From 48632f8a70dce050ceb8473401e8d19c2915374a Mon Sep 17 00:00:00 2001 From: eric valcik Date: Thu, 24 Apr 2025 18:01:51 +0200 Subject: [PATCH 15/16] finished everything --- app/app/ui-demo/feedback.tsx | 17 +++++++++++++++++ app/app/ui-demo/page.tsx | 10 +++++++++- app/components/agent.tsx | 4 +++- app/components/ui/card.tsx | 5 ++++- internal/config/config.go | 6 ++++++ internal/llm/client.go | 6 ++++++ internal/llm/completion.go | 7 +++++-- internal/repository/analyzer/analyze.go | 5 +++-- .../repository/gen_eval_loop/gen_eval_loop.go | 1 + 9 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 app/app/ui-demo/feedback.tsx diff --git a/app/app/ui-demo/feedback.tsx b/app/app/ui-demo/feedback.tsx new file mode 100644 index 0000000..6748c88 --- /dev/null +++ b/app/app/ui-demo/feedback.tsx @@ -0,0 +1,17 @@ +interface FeedbackProps { + feedback: string; + className?: string; +} + +export default function Feedback({ feedback, className }: FeedbackProps) { + if (!feedback) { + return null; + } + + return ( +
+

Current feedback:

+

{feedback}

+
+ ); +} diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index eb87da1..02034c8 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -14,11 +14,13 @@ import { Message } from "@/lib/types"; import { Agent } from "@/components/agent"; import { Tool } from "@/components/tool"; import { TestFiles } from "./test-files"; +import Feedback from "./feedback"; export default function WebSocketDemo() { const [messages, setMessages] = useState([]); const [scenarioCount, setScenarioCount] = useState(0); const [fileNames, setFileNames] = useState([]); const [scenario, setScenario] = useState(""); + const [feedback, setFeedback] = useState(""); const [inputValue, setInputValue] = useState( "https://ai-hackathon-demo-delta.vercel.app/" ); @@ -78,6 +80,10 @@ export default function WebSocketDemo() { const scenario = evt.data.substring(evt.data.indexOf(" ") + 1); setScenario(scenario); } + if (id === "FEEDBACK") { + const feedback = evt.data.substring(evt.data.indexOf(" ") + 1); + setFeedback(feedback); + } }; wsRef.current.onerror = (error) => { @@ -177,7 +183,9 @@ export default function WebSocketDemo() { active={isActive(messages, "GET_CONTENT_TOOL")} messages={getMessages(messages, ["GET_CONTENT_TOOL"])} /> -
+
+ +
{title} - {description} + + {description + "\nClaude 3.5 Sonnet based Agent."} +
diff --git a/app/components/ui/card.tsx b/app/components/ui/card.tsx index 533070b..2dcb7f0 100644 --- a/app/components/ui/card.tsx +++ b/app/components/ui/card.tsx @@ -56,7 +56,10 @@ const CardDescription = React.forwardRef< >(({ className, ...props }, ref) => (

)); diff --git a/internal/config/config.go b/internal/config/config.go index 9e96e60..425473a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,6 +17,7 @@ type Config struct { UmamiURL string UmamiAPIKey string UmamiWebsiteId string + UseSonnet bool } func Load() *Config { @@ -28,6 +29,7 @@ func Load() *Config { UmamiURL: "https://api.umami.is/v1", UmamiAPIKey: "", UmamiWebsiteId: "", + UseSonnet: false, } workDir, _ := os.Getwd() @@ -63,6 +65,10 @@ func Load() *Config { cfg.APIKey = apiKey } + if UseSonnet := envMap["USE_SONNET"]; strings.TrimSpace(UseSonnet) != "" { + cfg.UseSonnet = UseSonnet == "true" + } + if sentryAuthToken := envMap["SENTRY_AUTH_TOKEN"]; strings.TrimSpace(sentryAuthToken) != "" { cfg.SentryAuthToken = sentryAuthToken } diff --git a/internal/llm/client.go b/internal/llm/client.go index 4f43868..95596b9 100644 --- a/internal/llm/client.go +++ b/internal/llm/client.go @@ -10,16 +10,22 @@ type Client struct { client *anthropic.Client systemPrompt string apiKey string + Model anthropic.Model } func New(cfg *config.Config) *Client { apiKey := cfg.APIKey + model := anthropic.ModelClaude3_5HaikuLatest + if cfg.UseSonnet { + model = anthropic.ModelClaude3_5SonnetLatest + } client := anthropic.NewClient(option.WithAPIKey(apiKey)) systemPrompt := "When responding to questions: (1) Analyze problems thoroughly before proposing solutions, (2) Consider edge cases, (3) Acknowledge limitations in your knowledge when appropriate. Your responses should be thoughtful, and demonstrate deep understanding while remaining pragmatic." return &Client{ client: &client, systemPrompt: systemPrompt, apiKey: apiKey, + Model: model, } } diff --git a/internal/llm/completion.go b/internal/llm/completion.go index 34f2075..546fa9f 100644 --- a/internal/llm/completion.go +++ b/internal/llm/completion.go @@ -9,8 +9,9 @@ import ( ) func (c *Client) GetCompletion(ctx context.Context, prompt string) (string, error) { + message, err := c.client.Messages.New(ctx, anthropic.MessageNewParams{ - Model: anthropic.ModelClaude3_5SonnetLatest, + Model: c.Model, MaxTokens: 4096, System: []anthropic.TextBlockParam{ { @@ -67,8 +68,10 @@ func (c *Client) GetStructuredCompletion( messages := append(prevMessages, anthropic.NewUserMessage(anthropic.NewTextBlock(prompt))) + fmt.Printf("Calling model: %s\n", c.Model) + message, err := c.client.Messages.New(ctx, anthropic.MessageNewParams{ - Model: anthropic.ModelClaude3_5SonnetLatest, + Model: c.Model, // INFO: tools typically require more tokens MaxTokens: 2400, System: systemBlocks, diff --git a/internal/repository/analyzer/analyze.go b/internal/repository/analyzer/analyze.go index 81a82b8..2bd4033 100644 --- a/internal/repository/analyzer/analyze.go +++ b/internal/repository/analyzer/analyze.go @@ -28,7 +28,7 @@ func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr toolParams := []anthropic.ToolParam{ *sitemapTool, *getContentTool, - *sentryTool, + // *sentryTool, { Name: "get_significant_user_flows", Description: anthropic.String("This tool is very important to understand what are the most critical user flows. It will be super helpful to run it before generating a final criteria."), @@ -49,10 +49,11 @@ func Analyze(ctx context.Context, cfg *config.Config, client *llm.Client, urlStr fmt.Println("\n[ANALYZER] User Message: \n\n", userMessage) c.WriteMessage(mt, []byte(fmt.Sprintf("ANALYZER 'Starting analyisis on %s...'", urlStr))) + fmt.Printf("MODEL: %s\n", client.Model) for { message, err := client.NewMessage(ctx, anthropic.MessageNewParams{ - Model: anthropic.ModelClaude3_5SonnetLatest, + Model: client.Model, MaxTokens: 2048, Messages: messages, Tools: tools, diff --git a/internal/repository/gen_eval_loop/gen_eval_loop.go b/internal/repository/gen_eval_loop/gen_eval_loop.go index a7e913f..fc33eda 100644 --- a/internal/repository/gen_eval_loop/gen_eval_loop.go +++ b/internal/repository/gen_eval_loop/gen_eval_loop.go @@ -299,6 +299,7 @@ Invalid formatting will cause errors in processing your response. fmt.Printf("[EVALUATOR] Evaluator rejected the test file ❌\n") fmt.Printf("[EVALUATOR] With the following feedback:\n\n %s\n", response.Feedback) c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'REJECTED, feedback for generator:\n\n %s'", response.Feedback))) + c.WriteMessage(mt, []byte(fmt.Sprintf("FEEDBACK %s", response.Feedback))) c.WriteMessage(mt, []byte(fmt.Sprintf("EVALUATOR 'Passing the feedback to the generator.'"))) return response.Feedback, false, nil From 8f5918f1c094f78a8d662d6f4cdcd37ace434653 Mon Sep 17 00:00:00 2001 From: eric valcik Date: Thu, 24 Apr 2025 18:30:44 +0200 Subject: [PATCH 16/16] add pause stuff to the implementation --- app/app/ui-demo/page.tsx | 31 ++++++++++- cmd/api/main.go | 51 ++++++++++++++++--- internal/repository/analyzer/analyze.go | 16 +++--- .../repository/gen_eval_loop/gen_eval_loop.go | 41 +++++++++++++-- 4 files changed, 117 insertions(+), 22 deletions(-) diff --git a/app/app/ui-demo/page.tsx b/app/app/ui-demo/page.tsx index 02034c8..3fce75c 100644 --- a/app/app/ui-demo/page.tsx +++ b/app/app/ui-demo/page.tsx @@ -15,6 +15,7 @@ import { Agent } from "@/components/agent"; import { Tool } from "@/components/tool"; import { TestFiles } from "./test-files"; import Feedback from "./feedback"; + export default function WebSocketDemo() { const [messages, setMessages] = useState([]); const [scenarioCount, setScenarioCount] = useState(0); @@ -25,6 +26,7 @@ export default function WebSocketDemo() { "https://ai-hackathon-demo-delta.vercel.app/" ); const [isConnected, setIsConnected] = useState(false); + const [isPaused, setIsPaused] = useState(false); const wsRef = useRef(null); const scrollAreaRef = useRef(null); @@ -37,6 +39,15 @@ export default function WebSocketDemo() { wsRef.current.close(); }; + const handlePause = () => { + if (!wsRef.current) return; + if (isPaused) { + wsRef.current.send("RESUME"); + } else { + wsRef.current.send("PAUSE"); + } + }; + const handleSend = () => { if (!wsRef.current) { try { @@ -56,6 +67,7 @@ export default function WebSocketDemo() { wsRef.current.onclose = (event) => { console.log("WebSocket connection closed:", event.code, event.reason); setIsConnected(false); + setIsPaused(false); addMessage( `CLOSE '${event.code}${ event.reason ? `, Reason: ${event.reason}` : "" @@ -84,6 +96,10 @@ export default function WebSocketDemo() { const feedback = evt.data.substring(evt.data.indexOf(" ") + 1); setFeedback(feedback); } + if (id === "STATUS") { + const status = evt.data.split(" ")[1]; + setIsPaused(status === "Paused"); + } }; wsRef.current.onerror = (error) => { @@ -128,13 +144,24 @@ export default function WebSocketDemo() {