[WIP] Multi-modal Optimizer + Context for Optimization

.github/workflows/ci.yml

@@ -1,26 +1,26 @@
-name: CI
+name: Full CI (with Ollama)
 
 on:
   push:
-    branches: [ main, dev, experimental, ci-multi ]
-  pull_request:
-    branches: [ main, dev, experimental, ci-multi ]
+    branches: [ main ]
+  # Manual trigger for testing or major releases
+  workflow_dispatch:
 
 jobs:
-  test:
+  full-test:
     runs-on: ubuntu-latest
     timeout-minutes: 180
 
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
 
-    # 1) Restore any cached Ollama data (~2 GB)
+    # 1) Restore any cached Ollama data
     - name: Restore Ollama cache
       uses: actions/cache@v4
       with:
         path: ~/.ollama
-        key: qwen3-4b-gguf-v1
+        key: qwen3-vl-2b-v1
 
     # 2) Install Ollama
     - name: Install Ollama
@@ -43,13 +43,13 @@ jobs:
       run: |
         sudo systemctl enable --now ollama
 
-    # 5) Pull the phi4-mini:3.8b model (uses cache if present)
-    - name: Pull phi4-mini:3.8b model
-      run: ollama pull phi4-mini:3.8b
+    # 5) Pull the qwen3-vl:2b model - supports multimodal/vision with OpenAI-compatible API
+    - name: Pull qwen3-vl:2b model (multimodal)
+      run: ollama pull qwen3-vl:2b
 
     # 6) Set up Python & install dependencies
     - uses: actions/setup-python@v5
-      with: { python-version: "3.10" }
+      with: { python-version: "3.13" }
     - name: Install Python deps
       run: |
         pip install -e .
@@ -60,13 +60,13 @@ jobs:
       run: |
         echo "OPENAI_API_KEY=ollama"             >> $GITHUB_ENV
         echo "OPENAI_API_BASE=http://localhost:11434/v1" >> $GITHUB_ENV
-        echo "TRACE_LITELLM_MODEL=openai/phi4-mini:3.8b" >> $GITHUB_ENV
+        echo "TRACE_LITELLM_MODEL=openai/qwen3-vl:2b" >> $GITHUB_ENV
 
     # 8) Run all Trace unit tests
     - name: Run unit tests
       run: pytest tests/unit_tests/
 
     # 9) Run basic tests for each optimizer (some will fail due to the small LLM model chosen for free GitHub CI)
-    - name: Run optimizers test suite
-      run: pytest tests/llm_optimizers_tests/test_optimizer.py || true 
-      continue-on-error: true
+#    - name: Run optimizers test suite
+#      run: pytest tests/llm_optimizers_tests/test_optimizer.py || true
+#      continue-on-error: true

.github/workflows/python-app.yml

@@ -19,10 +19,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.10
+    - name: Set up Python 3.13
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: "3.13"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/unit-tests.yml

@@ -0,0 +1,30 @@
+name: Unit Tests
+
+on:
+  push:
+    branches: [ experimental, dev ]
+  pull_request:
+    branches: [ main, experimental, dev ]
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.13"
+
+    - name: Install Python dependencies
+      run: |
+        pip install -e .
+        pip install pytest numpy
+
+    - name: Run unit tests
+      run: pytest tests/unit_tests/ -v
+

docs/tutorials/minibatch.ipynb

@@ -601,11 +601,11 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[Step 1] \u001b[92mAverage test score: 1.0\u001b[0m\n",
+      "[Step 1] \u001B[92mAverage test score: 1.0\u001B[0m\n",
       "Epoch: 0. Iteration: 1\n",
       "[Step 1] Instantaneous train score: 1.0\n",
       "[Step 1] Average train score: 1.0\n",
-      "[Step 1] \u001b[91mParameter: str:20: You're a helpful agent\u001b[0m\n"
+      "[Step 1] \u001B[91mParameter: str:20: You're a helpful agent\u001B[0m\n"
      ]
     },
     {
@@ -641,11 +641,11 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[Step 2] \u001b[92mAverage test score: 1.0\u001b[0m\n",
+      "[Step 2] \u001B[92mAverage test score: 1.0\u001B[0m\n",
       "Epoch: 0. Iteration: 2\n",
       "[Step 2] Instantaneous train score: 1.0\n",
       "[Step 2] Average train score: 1.0\n",
-      "[Step 2] \u001b[91mParameter: str:20: You're a helpful agent\u001b[0m\n"
+      "[Step 2] \u001B[91mParameter: str:20: You're a helpful agent\u001B[0m\n"
      ]
     },
     {
@@ -677,11 +677,11 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[Step 3] \u001b[92mAverage test score: 1.0\u001b[0m\n",
+      "[Step 3] \u001B[92mAverage test score: 1.0\u001B[0m\n",
       "Epoch: 0. Iteration: 3\n",
       "[Step 3] Instantaneous train score: 1.0\n",
       "[Step 3] Average train score: 1.0\n",
-      "[Step 3] \u001b[91mParameter: str:20: You're a helpful agent\u001b[0m\n"
+      "[Step 3] \u001B[91mParameter: str:20: You're a helpful agent\u001B[0m\n"
      ]
     },
     {
@@ -714,11 +714,11 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[Step 4] \u001b[92mAverage test score: 1.0\u001b[0m\n",
+      "[Step 4] \u001B[92mAverage test score: 1.0\u001B[0m\n",
       "Epoch: 0. Iteration: 4\n",
       "[Step 4] Instantaneous train score: 1.0\n",
       "[Step 4] Average train score: 1.0\n",
-      "[Step 4] \u001b[91mParameter: str:20: You're a helpful agent\u001b[0m\n"
+      "[Step 4] \u001B[91mParameter: str:20: You're a helpful agent\u001B[0m\n"
      ]
     },
     {
@@ -751,11 +751,11 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[Step 5] \u001b[92mAverage test score: 1.0\u001b[0m\n",
+      "[Step 5] \u001B[92mAverage test score: 1.0\u001B[0m\n",
       "Epoch: 0. Iteration: 5\n",
       "[Step 5] Instantaneous train score: 1.0\n",
       "[Step 5] Average train score: 1.0\n",
-      "[Step 5] \u001b[91mParameter: str:20: You're a helpful agent\u001b[0m\n",
+      "[Step 5] \u001B[91mParameter: str:20: You're a helpful agent\u001B[0m\n",
       "FINISHED TRAINING\n"
      ]
     },
@@ -831,4 +831,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

opto/features/flows/compose.py

@@ -1,6 +1,7 @@
 import opto.trace as trace
 from typing import Union, get_type_hints, Any, Dict, List, Optional
 from opto.utils.llm import AbstractModel, LLM
+from opto.features.flows.types import MultiModalPayload, QueryModel
 import contextvars
 
 """
@@ -157,6 +158,7 @@ def __init__(self,
             system_prompt: The system prompt to use for LLM calls. If None and the class has a docstring, the docstring will be used.
             llm: The LLM model to use for inference
             chat_history_on: if on, maintain chat history for multi-turn conversations
+            model_name: override the default name of the model
         """
         if system_prompt is None:
             system_prompt = "You are a helpful assistant."
@@ -178,7 +180,9 @@ def __init__(self,
         self.model_name = model_name if model_name else f"{self.__class__.__name__}{len(current_llm_sessions)}"
         current_llm_sessions.append(1)  # just a marker
 
-    def forward(self, user_query: str, chat_history_on: Optional[bool] = None) -> str:
+    def forward(self, user_query: str,
+                payload: Optional[MultiModalPayload] = None,
+                chat_history_on: Optional[bool] = None) -> str:
         """This function takes user_query as input, and returns the response from the LLM, with the system prompt prepended.
         This method will always save chat history.
 
@@ -187,17 +191,19 @@ def forward(self, user_query: str, chat_history_on: Optional[bool] = None) -> st
         If chat_history_on is True, the chat history will be included in the LLM input.
 
         Args:
-            user_query: The user query to send to the LLM
+            user_query: The user query to send to the LLM. This should be a string containing the user's input or question.
 
         Returns:
             str: For direct pattern
         """
         chat_history_on = self.chat_history_on if chat_history_on is None else chat_history_on
 
+        user_message = QueryModel(query=user_query, multimodal_payload=payload).query
+
         messages = [{"role": "system", "content": self.system_prompt.data}]
         if chat_history_on:
             messages.extend(self.chat_history.get_messages())
-        messages.append({"role": "user", "content": user_query})
+        messages.append({"role": "user", "content": user_message})
 
         response = self.llm(messages=messages)
 
@@ -225,5 +231,7 @@ def call_llm(*messages) -> str:
 
         return response_node
 
-    def chat(self, user_query: str) -> str:
-        return self.forward(user_query)
+    def chat(self, user_query: str, payload: Optional[MultiModalPayload] = None, chat_history_on: Optional[bool] = None) -> str:
+        """Note that chat/forward always assumes it's a single turn of the conversation. History/context management will be accomplished
+           through other APIs"""
+        return self.forward(user_query, payload, chat_history_on)