Fixes ENG-1417: TS SDK Utilities

.fernignore

@@ -1 +1,25 @@
 # Specify files that shouldn't be modified by Fern
+
+# Custom code
+
+src/otel
+src/eval_utils
+src/utilities
+src/index.ts
+src/humanloop.client.ts
+
+# Tests
+
+tests
+
+# CI Action
+
+.github/workflows/ci.yml
+
+# Prettier
+
+.prettierrc.yml
+
+# Package Scripts
+
+scripts

.github/workflows/ci.yml

@@ -3,55 +3,59 @@ name: ci
 on: [push]
 
 jobs:
-  compile:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Set up node
-        uses: actions/setup-node@v3
-
-      - name: Compile
-        run: yarn && yarn build
-
-  test:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Set up node
-        uses: actions/setup-node@v3
-
-      - name: Compile
-        run: yarn && yarn test    
-
-  publish:
-    needs: [ compile, test ]
-    if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      - name: Set up node
-        uses: actions/setup-node@v3
-      - name: Install dependencies
-        run: yarn install
-      - name: Build
-        run: yarn build
-
-      - name: Publish to npm
-        run: |
-          npm config set //registry.npmjs.org/:_authToken ${NPM_TOKEN}
-          if [[ ${GITHUB_REF} == *alpha* ]]; then
-            npm publish --access public --tag alpha
-          elif [[ ${GITHUB_REF} == *beta* ]]; then
-            npm publish --access public --tag beta
-          else
-            npm publish --access public
-          fi
-        env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+    compile:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout repo
+              uses: actions/checkout@v3
+
+            - name: Set up node
+              uses: actions/setup-node@v3
+
+            - name: Compile
+              run: yarn && yarn build
+
+    test:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout repo
+              uses: actions/checkout@v3
+
+            - name: Set up node
+              uses: actions/setup-node@v3
+
+            - name: Compile
+              run: yarn && yarn test
+              env:
+                  OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
+                  ANTHROPIC_KEY: ${{ secrets.ANTHROPIC_KEY }}
+                  COHERE_KEY: ${{ secrets.COHERE_KEY }}
+
+    publish:
+        needs: [compile, test]
+        if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout repo
+              uses: actions/checkout@v3
+            - name: Set up node
+              uses: actions/setup-node@v3
+            - name: Install dependencies
+              run: yarn install
+            - name: Build
+              run: yarn build
+
+            - name: Publish to npm
+              run: |
+                  npm config set //registry.npmjs.org/:_authToken ${NPM_TOKEN}
+                  if [[ ${GITHUB_REF} == *alpha* ]]; then
+                    npm publish --access public --tag alpha
+                  elif [[ ${GITHUB_REF} == *beta* ]]; then
+                    npm publish --access public --tag beta
+                  else
+                    npm publish --access public
+                  fi
+              env:
+                  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 .DS_Store
-/dist
+/dist
+.env

.prettierrc.yml

@@ -1,2 +1,10 @@
 tabWidth: 4
-printWidth: 120
+printWidth: 88
+semi: true
+importOrderSeparation: true
+importOrderSortSpecifiers: true
+importOrderGroupNamespaceSpecifiers: true
+importOrder: ["<THIRD_PARTY_MODULES>", "^[./]"]
+trailingComma: "all"
+plugins:
+    - "@trivago/prettier-plugin-sort-imports"

package.json

@@ -8,33 +8,51 @@
     "scripts": {
         "format": "prettier . --write --ignore-unknown",
         "build": "tsc",
-        "prepack": "cp -rv dist/. .",
-        "test": "jest"
+        "prepack": "yarn build && ./scripts/prepack.sh",
+        "clean": "./scripts/clean.sh",
+        "test": "jest --detectOpenHandles --forceExit"
     },
     "dependencies": {
-        "url-join": "4.0.1",
         "form-data": "^4.0.0",
+        "form-data-encoder": "^4.0.2",
         "formdata-node": "^6.0.3",
         "node-fetch": "2.7.0",
         "qs": "6.11.2",
         "readable-stream": "^4.5.2",
-        "form-data-encoder": "^4.0.2"
+        "ts-json-schema-generator": "^2.3.0",
+        "url-join": "4.0.1",
+        "@opentelemetry/api": "^1.9.0",
+        "@opentelemetry/auto-instrumentations-node": "^0.53.0",
+        "@opentelemetry/sdk-metrics": "^1.28.0",
+        "@opentelemetry/sdk-node": "^0.55.0",
+        "@opentelemetry/sdk-trace-node": "^1.28.0",
+        "@traceloop/ai-semantic-conventions": "^0.11.0",
+        "@traceloop/instrumentation-anthropic": "^0.11.1",
+        "@traceloop/instrumentation-cohere": "^0.11.1",
+        "@traceloop/instrumentation-openai": "^0.11.3",
+        "uuid": "^11.0.3"
     },
     "devDependencies": {
-        "@types/url-join": "4.0.1",
-        "@types/qs": "6.9.8",
+        "@types/jest": "29.5.5",
+        "@types/node": "17.0.33",
         "@types/node-fetch": "2.6.9",
+        "@types/qs": "6.9.8",
         "@types/readable-stream": "^4.0.15",
+        "@types/url-join": "4.0.1",
         "fetch-mock-jest": "^1.5.1",
-        "webpack": "^5.94.0",
-        "ts-loader": "^9.3.1",
         "jest": "29.7.0",
-        "@types/jest": "29.5.5",
-        "ts-jest": "29.1.1",
         "jest-environment-jsdom": "29.7.0",
-        "@types/node": "17.0.33",
-        "prettier": "2.7.1",
-        "typescript": "4.6.4"
+        "ts-jest": "29.1.1",
+        "ts-loader": "^9.3.1",
+        "typescript": "4.6.4",
+        "webpack": "^5.94.0",
+        "openai": "^4.74.0",
+        "@anthropic-ai/sdk": "^0.32.1",
+        "cohere-ai": "^7.15.0",
+        "dotenv": "^16.4.6",
+        "jsonschema": "^1.4.1",
+        "@trivago/prettier-plugin-sort-imports": "^5.2.0",
+        "prettier": "^3.4.2"
     },
     "browser": {
         "fs": false,

scripts/clean.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+cat .yalc.clean | xargs rm -rf;
+rm -f .yalc.clean;

scripts/prepack.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+if npm list --global | grep -q yalc; then
+    # Local dev environment
+    cp -rv dist/. . | awk '{print $3}' | tail -n +2 | cut -c 3- > .yalc.clean;
+    echo .yalc.clean >> .yalc.clean;
+else
+    # CI environment, releasing the package
+    cp -rv dist/. .;
+fi
+

src/eval_utils/context.ts

@@ -0,0 +1,22 @@
+export interface EvaluationContext {
+    /** Context Log to Humanloop.
+     * Per datapoint state that is set when an Evaluation is run.
+     */
+
+    /** Required for associating a Log with the Evaluation Run. */
+    source_datapoint_id: string;
+
+    /** Overloaded .log method call. */
+    upload_callback: (log: string) => void;
+
+    /** ID of the evaluated File. */
+    file_id: string;
+
+    /** Path of the evaluated File. */
+    path: string;
+
+    /** Required for associating a Log with the Evaluation Run. */
+    run_id: string;
+}
+
+export const EVALUATION_CONTEXT_VARIABLE_NAME = "__EVALUATION_CONTEXT";

src/eval_utils/index.ts

@@ -0,0 +1 @@
+export * from "./context";

src/humanloop.client.ts

@@ -0,0 +1,99 @@
+import { NodeTracerProvider, Tracer } from "@opentelemetry/sdk-trace-node";
+import { AnthropicInstrumentation } from "@traceloop/instrumentation-anthropic";
+import { CohereInstrumentation } from "@traceloop/instrumentation-cohere";
+import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
+
+import { HumanloopClient as BaseHumanloopClient } from "./Client";
+import { FlowKernelRequest } from "./api/types/FlowKernelRequest";
+import { ToolKernelRequest } from "./api/types/ToolKernelRequest";
+import { HumanloopSpanExporter } from "./otel/exporter";
+import { moduleIsInstalled } from "./otel/helpers";
+import { HumanloopSpanProcessor } from "./otel/processor";
+import { flowUtilityFactory } from "./utilities/flow";
+import { UtilityPromptKernel, promptUtilityFactory } from "./utilities/prompt";
+import { toolUtilityFactory } from "./utilities/tool";
+
+export class HumanloopClient extends BaseHumanloopClient {
+    protected readonly opentelemetryTracerProvider: NodeTracerProvider;
+    protected readonly opentelemetryTracer: Tracer;
+
+    constructor(_options: BaseHumanloopClient.Options) {
+        super(_options);
+
+        this.opentelemetryTracerProvider = new NodeTracerProvider({
+            spanProcessors: [
+                new HumanloopSpanProcessor(new HumanloopSpanExporter(this)),
+            ],
+        });
+
+        if (moduleIsInstalled("openai")) {
+            const openai = require("openai").default;
+            const instrumentor = new OpenAIInstrumentation({
+                enrichTokens: true,
+            });
+            instrumentor.manuallyInstrument(openai);
+            instrumentor.setTracerProvider(this.opentelemetryTracerProvider);
+            instrumentor.enable();
+        }
+
+        if (moduleIsInstalled("@anthropic-ai/sdk")) {
+            const anthropic = require("@anthropic-ai/sdk");
+            const instrumentor = new AnthropicInstrumentation();
+            instrumentor.manuallyInstrument(anthropic);
+            instrumentor.setTracerProvider(this.opentelemetryTracerProvider);
+            instrumentor.enable();
+        }
+
+        if (moduleIsInstalled("cohere-ai")) {
+            const cohere = require("cohere-ai");
+            const instrumentor = new CohereInstrumentation();
+            instrumentor.manuallyInstrument(cohere);
+            instrumentor.setTracerProvider(this.opentelemetryTracerProvider);
+            instrumentor.enable();
+        }
+
+        this.opentelemetryTracerProvider.register();
+
+        this.opentelemetryTracer =
+            this.opentelemetryTracerProvider.getTracer("humanloop.sdk");
+    }
+
+    public prompt<T extends (...args: any[]) => any>(promptUtilityArguments: {
+        callable: T;
+        promptKernel?: UtilityPromptKernel;
+        path?: string;
+    }) {
+        return promptUtilityFactory(
+            this.opentelemetryTracer,
+            promptUtilityArguments.callable,
+            promptUtilityArguments.promptKernel,
+            promptUtilityArguments.path,
+        );
+    }
+
+    public tool<T extends (...args: any[]) => any>(toolUtilityArguments: {
+        callable: T;
+        toolKernel: ToolKernelRequest;
+        path?: string;
+    }) {
+        return toolUtilityFactory(
+            this.opentelemetryTracer,
+            toolUtilityArguments.callable,
+            toolUtilityArguments.toolKernel,
+            toolUtilityArguments.path,
+        );
+    }
+
+    public flow<T extends (...args: any[]) => any>(flowUtilityArguments: {
+        callable: T;
+        flowKernel?: FlowKernelRequest;
+        path?: string;
+    }) {
+        return flowUtilityFactory(
+            this.opentelemetryTracer,
+            flowUtilityArguments.callable,
+            flowUtilityArguments.flowKernel,
+            flowUtilityArguments.path,
+        );
+    }
+}

src/index.ts

@@ -1,4 +1,4 @@
 export * as Humanloop from "./api";
-export { HumanloopClient } from "./Client";
+export { HumanloopClient } from "./humanloop.client";
 export { HumanloopEnvironment } from "./environments";
 export { HumanloopError, HumanloopTimeoutError } from "./errors";

src/otel/constants.ts

@@ -0,0 +1,10 @@
+// Attribute name prefix on Humanloop spans for file-related attributes + path
+export const HUMANLOOP_FILE_KEY = "humanloop.file";
+// Attribute name prefix on Humanloop spans for log-related attributes
+export const HUMANLOOP_LOG_KEY = "humanloop.log";
+export const HUMANLOOP_FILE_TYPE_KEY = "humanloop.file_type";
+export const HUMANLOOP_PATH_KEY = "humanloop.file.path";
+export const HUMANLOOP_PARENT_SPAN_CTX_KEY = "humanloop.context.parentSpanId";
+export const HUMANLOOP_TRACE_FLOW_CTX_KEY = "humanloop.context.traceFlow";
+
+export type AsyncFunction = (...args: unknown[]) => Promise<unknown>;