allow eval run to use files on HL workspace

Author: Andrei Bratu

src/humanloop/evals/run.py

@@ -35,6 +35,7 @@
 )
 
 from humanloop import EvaluatorResponse, FlowResponse, PromptResponse, ToolResponse
+from humanloop.agents.client import AgentsClient
 from humanloop.core.api_error import ApiError
 from humanloop.context import (
     EvaluationContext,
@@ -100,7 +101,7 @@
 RESET = "\033[0m"
 
 
-CLIENT_TYPE = TypeVar("CLIENT_TYPE", PromptsClient, ToolsClient, FlowsClient, EvaluatorsClient)
+CLIENT_TYPE = TypeVar("CLIENT_TYPE", PromptsClient, ToolsClient, FlowsClient, EvaluatorsClient, AgentsClient)
 
 
 def run_eval(
@@ -135,14 +136,35 @@ def run_eval(
                 % (type_.capitalize(), decorator_type.capitalize())
             )
 
+    # Check if file exists on Humanloop before trying to upsert
+    file_exists = False
     try:
-        hl_file = _upsert_file(file=file_, type=type_, client=client)
-    except ValidationError as e:
-        sys.stdout.write(f"{RED}Error in your `file` argument:\n\n{e}{RESET}")
-        return []
-    except Exception as e:
-        sys.stdout.write(f"{RED}Error in your `file` argument:\n\n{e}{RESET}")
-        return []
+        path_ = file_.get("path")
+        hl_file = client.files.retrieve_by_path(path=path_)
+        if hl_file.type != type_:
+            sys.stdout.write(
+                f"{RED}Error in your `file` argument:\n\nThe file type {hl_file.type} does not "
+                f"match the type of the File at {path_}: {type_}.{RESET}"
+            )
+            return []
+        file_exists = True
+        sys.stdout.write(f"{CYAN}Using the File definition from Humanloop workspace: {path_}{RESET}")
+    except ApiError as e:
+        if e.status_code == 404:
+            # Let's create the file
+            pass
+        else:
+            sys.stdout.write(f"{RED}Error in your `file` argument:\n\n{e}{RESET}")
+            return []
+    if not file_exists:
+        try:
+            hl_file = _upsert_file(file=file_, type=type_, client=client)
+        except ValidationError as e:
+            sys.stdout.write(f"{RED}Error in your `file` argument:\n\n{e}{RESET}")
+            return []
+        except Exception as e:
+            sys.stdout.write(f"{RED}Error in your `file` argument:\n\n{e}{RESET}")
+            return []
     try:
         hl_dataset = _upsert_dataset(dataset=dataset, client=client)
     except Exception as e:
@@ -494,7 +516,7 @@ def _file_or_file_inside_hl_utility(file: File) -> File:
 
 
 def _get_file_type(file: File) -> FileType:
-    # Determine the `type` of the `file` to Evaluate - if not `type` provided, default to `flow`
+    """Return the type of the File to be evaluated or default to `flow`."""
     try:
         type_ = typing.cast(FileType, file.pop("type"))  # type: ignore [arg-type, misc]
         sys.stdout.write(
@@ -508,7 +530,7 @@ def _get_file_type(file: File) -> FileType:
 
 
 def _get_file_callable(file: File, type_: FileType) -> Optional[Callable]:
-    # Get the `callable` from the `file` to Evaluate
+    """Get the callable of the File to be evaluated, or throw if None was provided for Flows."""
     function_ = typing.cast(Optional[Callable], file.pop("callable", None))
     if function_ is None:
         if type_ == "flow":
@@ -549,6 +571,9 @@ def _upsert_file(
     elif type == "evaluator":
         hl_file = client.evaluators.upsert(**file_dict)  # type: ignore [arg-type, assignment]
 
+    elif type == "agent":
+        hl_file = client.agents.upsert(**file_dict)  # type: ignore [arg-type, assignment]
+
     else:
         raise NotImplementedError(f"Unsupported File type: {type}")
 

src/humanloop/evals/types.py

@@ -23,9 +23,7 @@
     UpdateDatesetAction as UpdateDatasetAction,
 )  # TODO: fix original type typo
 
-EvaluatorDict = Union[
-    CodeEvaluatorDict, LLMEvaluatorDict, HumanEvaluatorDict, ExternalEvaluator
-]
+EvaluatorDict = Union[CodeEvaluatorDict, LLMEvaluatorDict, HumanEvaluatorDict, ExternalEvaluator]
 Version = Union[FlowDict, PromptDict, ToolDict, EvaluatorDict]
 FileType = Literal["flow", "prompt", "tool", "evaluator"]
 
@@ -42,7 +40,7 @@ class Identifiers(TypedDict):
 class File(Identifiers):
     """A File on Humanloop (Flow, Prompt, Tool, Evaluator)."""
 
-    type: Literal["flow", "prompt"]
+    type: Literal["flow", "prompt", "agent"]
     """The type of File this callable relates to on Humanloop."""
     version: NotRequired[Version]
     """The contents uniquely define the version of the File on Humanloop."""