Fixes ENG-1430: Add eval run utility in TS

.fernignore

@@ -1 +1,27 @@
 # Specify files that shouldn't be modified by Fern
+
+# Custom code
+
+src/otel
+src/eval_utils
+src/utilities
+src/index.ts
+src/humanloop.client.ts
+
+# Tests
+
+tests
+
+# CI Action
+
+.github/workflows/ci.yml
+
+# Config files
+
+.prettierrc.yml
+babel.config.js
+jest.config.js
+
+# Package Scripts
+
+scripts

.github/workflows/ci.yml

@@ -3,55 +3,59 @@ name: ci
 on: [push]
 
 jobs:
-  compile:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Set up node
-        uses: actions/setup-node@v3
-
-      - name: Compile
-        run: yarn && yarn build
-
-  test:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Set up node
-        uses: actions/setup-node@v3
-
-      - name: Compile
-        run: yarn && yarn test    
-
-  publish:
-    needs: [ compile, test ]
-    if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      - name: Set up node
-        uses: actions/setup-node@v3
-      - name: Install dependencies
-        run: yarn install
-      - name: Build
-        run: yarn build
-
-      - name: Publish to npm
-        run: |
-          npm config set //registry.npmjs.org/:_authToken ${NPM_TOKEN}
-          if [[ ${GITHUB_REF} == *alpha* ]]; then
-            npm publish --access public --tag alpha
-          elif [[ ${GITHUB_REF} == *beta* ]]; then
-            npm publish --access public --tag beta
-          else
-            npm publish --access public
-          fi
-        env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+    compile:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout repo
+              uses: actions/checkout@v3
+
+            - name: Set up node
+              uses: actions/setup-node@v3
+
+            - name: Compile
+              run: yarn && yarn build
+
+    test:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout repo
+              uses: actions/checkout@v3
+
+            - name: Set up node
+              uses: actions/setup-node@v3
+
+            - name: Compile
+              run: yarn && yarn test
+              env:
+                  OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
+                  ANTHROPIC_KEY: ${{ secrets.ANTHROPIC_KEY }}
+                  COHERE_KEY: ${{ secrets.COHERE_KEY }}
+
+    publish:
+        needs: [compile, test]
+        if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout repo
+              uses: actions/checkout@v3
+            - name: Set up node
+              uses: actions/setup-node@v3
+            - name: Install dependencies
+              run: yarn install
+            - name: Build
+              run: yarn build
+
+            - name: Publish to npm
+              run: |
+                  npm config set //registry.npmjs.org/:_authToken ${NPM_TOKEN}
+                  if [[ ${GITHUB_REF} == *alpha* ]]; then
+                    npm publish --access public --tag alpha
+                  elif [[ ${GITHUB_REF} == *beta* ]]; then
+                    npm publish --access public --tag beta
+                  else
+                    npm publish --access public
+                  fi
+              env:
+                  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 .DS_Store
-/dist
+/dist
+.env

.prettierignore

@@ -0,0 +1,17 @@
+# Ignore Fern auto-generated files
+
+src/Client.ts
+src/api
+src/core
+src/errors
+src/serialization
+src/environment.ts
+src/version.ts
+
+tests/custom.test.ts
+tests/unit
+
+jest.config.js
+package.json
+yarn.lock
+tsconfig.json

.prettierrc.yml

@@ -1,2 +1,10 @@
 tabWidth: 4
-printWidth: 120
+printWidth: 88
+semi: true
+importOrderSeparation: true
+importOrderSortSpecifiers: true
+importOrderGroupNamespaceSpecifiers: true
+importOrder: ["<THIRD_PARTY_MODULES>", "^[./]"]
+trailingComma: "all"
+plugins:
+    - "@trivago/prettier-plugin-sort-imports"

babel.config.js

@@ -0,0 +1,11 @@
+module.exports = {
+  presets: [
+      [
+        "@babel/preset-env",
+        { targets: { node: "current" } }
+      ],
+  ],
+  plugins: [
+    "@babel/plugin-transform-modules-commonjs"
+  ]
+};

jest.config.js

@@ -2,4 +2,12 @@
 module.exports = {
     preset: "ts-jest",
     testEnvironment: "node",
+    // If Jest complains about an unknown symbol when running tests, you're dealing with dependency
+    // written in ES module instead of CJS format. Add the dependency in the exclusive regex group below.
+    // All modules NOT matching the pattern (thus exclusive grouping) will be passed to babel for
+    // transpilation before tests are ran.
+    transformIgnorePatterns: ["<rootDir>/node_modules/(?!p-map/)"],
+    transform: {
+        "\\.js$": "babel-jest"
+    }
 };

package.json

@@ -8,33 +8,62 @@
     "scripts": {
         "format": "prettier . --write --ignore-unknown",
         "build": "tsc",
-        "prepack": "cp -rv dist/. .",
-        "test": "jest"
+        "prepack": "yarn build && ./scripts/prepack.sh",
+        "clean": "./scripts/clean.sh",
+        "test": "jest --detectOpenHandles --forceExit"
     },
     "dependencies": {
-        "url-join": "4.0.1",
+        "@opentelemetry/api": "^1.9.0",
+        "@opentelemetry/auto-instrumentations-node": "^0.53.0",
+        "@opentelemetry/sdk-metrics": "^1.28.0",
+        "@opentelemetry/sdk-node": "^0.55.0",
+        "@opentelemetry/sdk-trace-node": "^1.28.0",
+        "@traceloop/ai-semantic-conventions": "^0.11.0",
+        "@traceloop/instrumentation-anthropic": "^0.11.1",
+        "@traceloop/instrumentation-cohere": "^0.11.1",
+        "@traceloop/instrumentation-openai": "^0.11.3",
+        "cli-progress": "^3.12.0",
         "form-data": "^4.0.0",
+        "form-data-encoder": "^4.0.2",
         "formdata-node": "^6.0.3",
+        "lodash": "^4.17.21",
+        "nanoid": "^5.0.9",
         "node-fetch": "2.7.0",
+        "p-map": "^7.0.3",
         "qs": "6.11.2",
         "readable-stream": "^4.5.2",
-        "form-data-encoder": "^4.0.2"
+        "stable-hash": "^0.0.4",
+        "ts-json-schema-generator": "^2.3.0",
+        "url-join": "4.0.1",
+        "uuid": "^11.0.3"
     },
     "devDependencies": {
-        "@types/url-join": "4.0.1",
-        "@types/qs": "6.9.8",
+        "@anthropic-ai/sdk": "^0.32.1",
+        "@babel/core": "^7.26.0",
+        "@babel/plugin-transform-modules-commonjs": "^7.26.3",
+        "@babel/preset-env": "^7.26.0",
+        "@trivago/prettier-plugin-sort-imports": "^5.2.0",
+        "@types/cli-progress": "^3.11.6",
+        "@types/jest": "29.5.5",
+        "@types/lodash": "4.17.5",
+        "@types/node": "17.0.33",
         "@types/node-fetch": "2.6.9",
+        "@types/qs": "6.9.8",
         "@types/readable-stream": "^4.0.15",
+        "@types/url-join": "4.0.1",
+        "babel-jest": "^29.7.0",
+        "cohere-ai": "^7.15.0",
+        "dotenv": "^16.4.6",
         "fetch-mock-jest": "^1.5.1",
-        "webpack": "^5.94.0",
-        "ts-loader": "^9.3.1",
         "jest": "29.7.0",
-        "@types/jest": "29.5.5",
-        "ts-jest": "29.1.1",
         "jest-environment-jsdom": "29.7.0",
-        "@types/node": "17.0.33",
-        "prettier": "2.7.1",
-        "typescript": "4.6.4"
+        "jsonschema": "^1.4.1",
+        "openai": "^4.74.0",
+        "prettier": "^3.4.2",
+        "ts-jest": "29.1.1",
+        "ts-loader": "^9.3.1",
+        "typescript": "4.6.4",
+        "webpack": "^5.94.0"
     },
     "browser": {
         "fs": false,

scripts/clean.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+cat .yalc.clean | xargs rm -rf;
+rm -f .yalc.clean;

scripts/prepack.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+if npm list --global | grep -q yalc; then
+    # Local dev environment
+    cp -rv dist/. . | awk '{print $3}' | tail -n +2 | cut -c 3- > .yalc.clean;
+    echo .yalc.clean >> .yalc.clean;
+else
+    # CI environment, releasing the package
+    cp -rv dist/. .;
+fi
+

src/eval_utils/context.ts

@@ -0,0 +1,97 @@
+import hash from "stable-hash";
+
+import { FlowLogRequest, PromptLogRequest } from "../api";
+import { DatapointResponse } from "../api";
+import { Humanloop } from "../index";
+import { Version } from "./types";
+
+type EvaluationContextState = {
+    fileId?: string;
+    path?: string;
+    uploadCallback: (logId: string, datapoint: DatapointResponse) => void;
+    evaluatedVersion?: Version;
+};
+
+type EvaluationContextKey = {
+    inputs: Record<string, unknown> | undefined;
+    messages: Humanloop.ChatMessage[] | undefined;
+};
+
+type EvaluationContextValue = {
+    runId: string;
+    sourceDatapointId: string;
+    uploadCallback: (logId: string) => void;
+};
+
+class EvaluationContext {
+    private state?: EvaluationContextState;
+    private static instance: EvaluationContext;
+    private inputMappings: Map<string, EvaluationContextValue[]> = new Map();
+
+    private constructor() {}
+
+    public static getInstance(): EvaluationContext {
+        if (!EvaluationContext.instance) {
+            EvaluationContext.instance = new EvaluationContext();
+        }
+        return EvaluationContext.instance;
+    }
+
+    public setState(state: EvaluationContextState): void {
+        this.state = state;
+    }
+
+    public getState(): Omit<EvaluationContextState, "uploadCallback"> | undefined {
+        return this.state === undefined
+            ? this.state
+            : {
+                  fileId: this.state.fileId,
+                  path: this.state.path,
+                  evaluatedVersion: this.state.evaluatedVersion,
+              };
+    }
+
+    public addDatapoint(datapoint: DatapointResponse, runId: string): void {
+        if (this.state === undefined) {
+            throw new Error("EvaluationContext state is not set");
+        }
+        const key = hash({ inputs: datapoint.inputs, messages: datapoint.messages });
+
+        if (!this.inputMappings.has(key)) {
+            this.inputMappings.set(key, []);
+        }
+        this.inputMappings.get(key)!.push({
+            runId,
+            sourceDatapointId: datapoint.id,
+            uploadCallback: (logId: string) =>
+                this.state!.uploadCallback(logId, datapoint),
+        });
+    }
+
+    public getDatapoint(key: EvaluationContextKey): EvaluationContextValue {
+        if (key.inputs !== undefined && "inputs" in key.inputs) {
+            key = { ...key, inputs: key.inputs.inputs as Record<string, unknown> };
+        }
+        const mappings = this.inputMappings.get(hash(key));
+        if (!mappings || mappings.length === 0) {
+            throw new Error(
+                `No input mappings found for: ${JSON.stringify(key)}. Try using peekDatapoint() first.`,
+            );
+        }
+        return mappings.pop()!;
+    }
+
+    public peekDatapoint(key: EvaluationContextKey): boolean {
+        const mappings = this.inputMappings.get(hash(key));
+        return mappings !== undefined && mappings.length > 0;
+    }
+
+    public isEvaluatedFile(args: FlowLogRequest | PromptLogRequest) {
+        return (
+            this.state &&
+            (this.state.fileId === args.id || this.state.path === args.path)
+        );
+    }
+}
+
+export const evaluationContext = EvaluationContext.getInstance();

src/eval_utils/index.ts

@@ -0,0 +1 @@
+export * from "./context";