test: add tests for CLI

Author: AlePouroullis

src/humanloop/sync/sync_client.py

@@ -7,7 +7,6 @@
 import time
 from humanloop.error import HumanloopRuntimeError
 import json
-import re
 
 if TYPE_CHECKING:
     from humanloop.base_client import BaseHumanloop
@@ -41,8 +40,13 @@ def format_api_error(error: Exception) -> str:
         # Get the detail from the body
         detail = body.get("detail", {})
 
-        # Prefer description, fall back to msg
-        return detail.get("description") or detail.get("msg") or error_msg
+        # Handle both string and dictionary types for detail
+        if isinstance(detail, str):
+            return detail
+        elif isinstance(detail, dict):
+            return detail.get("description") or detail.get("msg") or error_msg
+        else:
+            return error_msg
     except Exception as e:
         logger.debug(f"Failed to parse error message: {str(e)}")
         return error_msg

tests/custom/conftest.py

@@ -88,7 +88,6 @@ def get_humanloop_client() -> GetHumanloopClientFn:
     def _get_humanloop_client(use_local_files: bool = False) -> Humanloop:
         return Humanloop(
             api_key=os.getenv("HUMANLOOP_API_KEY"),
-            base_url="http://localhost:80/v5",
             use_local_files=use_local_files,
         )
 

tests/custom/integration/conftest.py

@@ -2,13 +2,15 @@
 from dataclasses import dataclass
 import os
 import time
-from typing import Any, ContextManager, Generator
+from typing import Any, ContextManager, Generator, List, Union
 import io
 from typing import TextIO
 import uuid
 import pytest
 import dotenv
-from tests.custom.types import GetHumanloopClientFn
+from humanloop import AgentResponse, PromptResponse
+from tests.custom.types import GetHumanloopClientFn, SyncableFile
+from click.testing import CliRunner
 
 
 @dataclass
@@ -177,3 +179,81 @@ def id_for_staging_environment(get_humanloop_client: GetHumanloopClientFn, eval_
         if environment.name == "staging":
             return environment.id
     pytest.fail("Staging environment not found")
+
+
+@pytest.fixture
+def syncable_files_fixture(
+    get_humanloop_client: GetHumanloopClientFn,
+    sdk_test_dir: str,
+) -> Generator[list[SyncableFile], None, None]:
+    """Creates a predefined structure of files in Humanloop for testing sync."""
+    files: List[SyncableFile] = [
+        SyncableFile(
+            path="prompts/gpt-4",
+            type="prompt",
+            model="gpt-4",
+        ),
+        SyncableFile(
+            path="prompts/gpt-4o",
+            type="prompt",
+            model="gpt-4o",
+        ),
+        SyncableFile(
+            path="prompts/nested/complex/gpt-4o",
+            type="prompt",
+            model="gpt-4o",
+        ),
+        SyncableFile(
+            path="agents/gpt-4",
+            type="agent",
+            model="gpt-4",
+        ),
+        SyncableFile(
+            path="agents/gpt-4o",
+            type="agent",
+            model="gpt-4o",
+        ),
+    ]
+
+    humanloop_client = get_humanloop_client()
+    created_files = []
+    for file in files:
+        full_path = f"{sdk_test_dir}/{file.path}"
+        response: Union[AgentResponse, PromptResponse]
+        if file.type == "prompt":
+            response = humanloop_client.prompts.upsert(
+                path=full_path,
+                model=file.model,
+            )
+        elif file.type == "agent":
+            response = humanloop_client.agents.upsert(
+                path=full_path,
+                model=file.model,
+            )
+        created_files.append(
+            SyncableFile(
+                path=full_path, type=file.type, model=file.model, id=response.id, version_id=response.version_id
+            )
+        )
+
+    yield created_files
+
+
+@pytest.fixture
+def cli_runner() -> CliRunner:
+    """GIVEN a CLI runner
+    THEN it should be configured to catch exceptions
+    """
+    return CliRunner(mix_stderr=False)
+
+
+@pytest.fixture
+def no_humanloop_api_key_in_env(monkeypatch):
+    """Fixture that removes HUMANLOOP_API_KEY from environment variables.
+
+    Use this fixture in tests that verify behavior when no API key is available
+    in the environment (but could still be loaded from .env files).
+    """
+    # Remove API key from environment
+    monkeypatch.delenv("HUMANLOOP_API_KEY", raising=False)
+    yield

tests/custom/integration/test_sync.py

@@ -1,79 +1,11 @@
-from typing import Generator, List, NamedTuple, Union
+from typing import List, Union
 from pathlib import Path
 import pytest
-from humanloop import FileType, AgentResponse, PromptResponse
+from humanloop import AgentResponse, PromptResponse
 from humanloop.prompts.client import PromptsClient
 from humanloop.agents.client import AgentsClient
 from humanloop.error import HumanloopRuntimeError
-from tests.custom.types import GetHumanloopClientFn
-
-
-class SyncableFile(NamedTuple):
-    path: str
-    type: FileType
-    model: str
-    id: str = ""
-    version_id: str = ""
-
-
-@pytest.fixture
-def syncable_files_fixture(
-    get_humanloop_client: GetHumanloopClientFn,
-    sdk_test_dir: str,
-) -> Generator[list[SyncableFile], None, None]:
-    """Creates a predefined structure of files in Humanloop for testing sync."""
-    files: List[SyncableFile] = [
-        SyncableFile(
-            path="prompts/gpt-4",
-            type="prompt",
-            model="gpt-4",
-        ),
-        SyncableFile(
-            path="prompts/gpt-4o",
-            type="prompt",
-            model="gpt-4o",
-        ),
-        SyncableFile(
-            path="prompts/nested/complex/gpt-4o",
-            type="prompt",
-            model="gpt-4o",
-        ),
-        SyncableFile(
-            path="agents/gpt-4",
-            type="agent",
-            model="gpt-4",
-        ),
-        SyncableFile(
-            path="agents/gpt-4o",
-            type="agent",
-            model="gpt-4o",
-        ),
-    ]
-
-    humanloop_client = get_humanloop_client()
-    created_files = []
-    for file in files:
-        full_path = f"{sdk_test_dir}/{file.path}"
-        response: Union[AgentResponse, PromptResponse]
-        if file.type == "prompt":
-            response = humanloop_client.prompts.upsert(
-                path=full_path,
-                model=file.model,
-            )
-        elif file.type == "agent":
-            response = humanloop_client.agents.upsert(
-                path=full_path,
-                model=file.model,
-            )
-        created_files.append(
-            SyncableFile(
-                path=full_path, type=file.type, model=file.model, id=response.id, version_id=response.version_id
-            )
-        )
-
-    humanloop_client.pull()
-
-    yield created_files
+from tests.custom.types import GetHumanloopClientFn, SyncableFile
 
 
 @pytest.fixture
@@ -96,7 +28,7 @@ def test_pull_basic(
     humanloop_client = get_humanloop_client()
 
     # WHEN running the sync
-    successful_files = humanloop_client.pull()
+    humanloop_client.pull()
 
     # THEN our local filesystem should mirror the remote filesystem in the HL Workspace
     for file in syncable_files_fixture: