added test

Author: Andrei Bratu

src/humanloop/evals/run.py

@@ -122,7 +122,6 @@
 T = TypeVar("T")  # Add TypeVar T definition
 
 
-
 def print_error(message: str) -> None:
     """Print a formatted error message to stdout."""
     sys.stdout.write(f"{RED}{message}{RESET}")
@@ -459,7 +458,7 @@ def _resolve_file(client: "BaseHumanloop", file_config: FileEvalConfig) -> tuple
     except ApiError:
         if not version or not path or file_id:
             raise HumanloopRuntimeError(
-                "File does not exist on Humanloop. Please provide a `file.path` and a version to create a new version.",
+                "File does not exist on Humanloop. Please provide a `file.path` and a `file.version` to create a new version.",
             )
         return _upsert_file(file_config=file_config, client=client), callable or None
 

tests/custom/integration/test_evals.py

@@ -457,3 +457,53 @@ async def my_async_flow(question: str) -> str:
     evaluation_id = evaluations_response.items[0].id
     runs_response = humanloop_client.evaluations.list_runs_for_evaluation(id=evaluation_id)
     assert runs_response.runs[0].status == "completed"
+
+
+async def test_eval_simple_async_callable(
+    get_humanloop_client: GetHumanloopClientFn,
+    eval_dataset: ResourceIdentifiers,
+    output_not_null_evaluator: ResourceIdentifiers,
+    sdk_test_dir: str,
+):
+    humanloop_client = get_humanloop_client()
+
+    flow_path = f"{sdk_test_dir}/Test Async Flow"
+
+    # GIVEN a simple async callable
+    async def my_async_callable(question: str) -> str:
+        return "It's complicated don't worry about it!"
+
+    # WHEN we run an evaluation with the async callable
+    humanloop_client.evaluations.run(  # type: ignore [attr-defined]
+        name="test_async_eval_run",
+        file={
+            "path": flow_path,
+            "type": "flow",
+            "version": {
+                "attributes": {
+                    "foo": "bar",
+                },
+            },
+            "callable": my_async_callable,
+        },
+        dataset={
+            "path": eval_dataset.file_path,
+        },
+        evaluators=[
+            {
+                "path": output_not_null_evaluator.file_path,
+            }
+        ],
+    )
+
+    # THEN the Flow is created
+    flow = humanloop_client.files.retrieve_by_path(path=flow_path)
+    assert flow is not None
+
+    # THEN the evaluation finishes successfully
+    evaluations_response = humanloop_client.evaluations.list(file_id=flow.id)
+    assert evaluations_response.items and len(evaluations_response.items) == 1
+    # THEN the evaluation is completed
+    evaluation_id = evaluations_response.items[0].id
+    runs_response = humanloop_client.evaluations.list_runs_for_evaluation(id=evaluation_id)
+    assert runs_response.runs[0].status == "completed"