Skip to content

Commit 392886c

Browse files
author
Andrei Bratu
authored
Merge pull request #19 from humanloop/decorators-fixes-latest
* Better typing DX on decorated functions and eval.run callable * Flow decorator picks up on logging via SDK functions * More graceful error messages in eval.run()
2 parents ff9442b + ea16dfe commit 392886c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2595
-1609
lines changed

‎.fernignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ tests/unit/fetcher/stream-wrappers/webpack.test.ts
2525
.prettierrc.yml
2626
.prettierignore
2727
.gitignore
28-
babel.config.js
2928

3029
# Package Scripts
3130

‎babel.config.js‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎package.json‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,7 @@
5353
"jsonschema": "^1.4.1",
5454
"@types/cli-progress": "^3.11.6",
5555
"@types/lodash": "4.14.74",
56-
"@trivago/prettier-plugin-sort-imports": "^4.3.0",
57-
"babel-jest": "^29.7.0",
58-
"@babel/core": "^7.26.0",
59-
"@babel/preset-env": "^7.26.0"
56+
"@trivago/prettier-plugin-sort-imports": "^4.3.0"
6057
},
6158
"browser": {
6259
"fs": false,

‎src/context.ts‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
import * as contextApi from "@opentelemetry/api";
2+
3+
import { HumanloopRuntimeError } from "./error";
4+
import {
5+
HUMANLOOP_CONTEXT_DECORATOR,
6+
HUMANLOOP_CONTEXT_EVALUATION,
7+
HUMANLOOP_CONTEXT_TRACE_ID,
8+
} from "./otel/constants";
9+
10+
export function getTraceId(): string | undefined {
11+
const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_TRACE_ID);
12+
const value = contextApi.context.active().getValue(key);
13+
return (value || undefined) as string | undefined;
14+
}
15+
16+
export function setTraceId(flowLogId: string): contextApi.Context {
17+
const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_TRACE_ID);
18+
return contextApi.context.active().setValue(key, flowLogId);
19+
}
20+
21+
export type DecoratorContext = {
22+
path: string;
23+
type: "prompt" | "tool" | "flow";
24+
version: Record<string, unknown>;
25+
};
26+
27+
export function setDecoratorContext(
28+
decoratorContext: DecoratorContext,
29+
): contextApi.Context {
30+
const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_DECORATOR);
31+
return contextApi.context.active().setValue(key, decoratorContext);
32+
}
33+
34+
export function getDecoratorContext(): DecoratorContext | undefined {
35+
const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_DECORATOR);
36+
return (contextApi.context.active().getValue(key) || undefined) as
37+
| DecoratorContext
38+
| undefined;
39+
}
40+
41+
export class EvaluationContext {
42+
public sourceDatapointId: string;
43+
public runId: string;
44+
public fileId: string;
45+
public path: string;
46+
private _logged: boolean;
47+
private _callback: (log_id: string) => Promise<void>;
48+
49+
constructor({
50+
sourceDatapointId,
51+
runId,
52+
evalCallback,
53+
fileId,
54+
path,
55+
}: {
56+
sourceDatapointId: string;
57+
runId: string;
58+
evalCallback: (log_id: string) => Promise<void>;
59+
fileId: string;
60+
path: string;
61+
}) {
62+
this.sourceDatapointId = sourceDatapointId;
63+
this.runId = runId;
64+
this._callback = evalCallback;
65+
this.fileId = fileId;
66+
this.path = path;
67+
this._logged = false;
68+
}
69+
70+
public get logged(): boolean {
71+
return this._logged;
72+
}
73+
74+
public logArgsWithContext({
75+
logArgs,
76+
forOtel,
77+
path,
78+
fileId,
79+
}: {
80+
logArgs: Record<string, any>;
81+
forOtel: boolean;
82+
path?: string;
83+
fileId?: string;
84+
}): [Record<string, any>, ((log_id: string) => Promise<void>) | null] {
85+
if (path === undefined && fileId === undefined) {
86+
throw new HumanloopRuntimeError(
87+
"Internal error: Evaluation context called without providing a path or file_id",
88+
);
89+
}
90+
91+
if (this._logged) {
92+
return [logArgs, null];
93+
}
94+
95+
if (this.path !== undefined && this.path === path) {
96+
this._logged = true;
97+
return [
98+
forOtel
99+
? {
100+
...logArgs,
101+
source_datapoint_id: this.sourceDatapointId,
102+
run_id: this.runId,
103+
}
104+
: {
105+
...logArgs,
106+
sourceDatapointId: this.sourceDatapointId,
107+
runId: this.runId,
108+
},
109+
this._callback,
110+
];
111+
} else if (this.fileId !== undefined && this.fileId === fileId) {
112+
this._logged = true;
113+
return [
114+
forOtel
115+
? {
116+
...logArgs,
117+
sourceDatapointId: this.sourceDatapointId,
118+
runId: this.runId,
119+
}
120+
: {
121+
...logArgs,
122+
sourceDatapointId: this.sourceDatapointId,
123+
runId: this.runId,
124+
},
125+
this._callback,
126+
];
127+
} else {
128+
return [logArgs, null];
129+
}
130+
}
131+
}
132+
133+
// ... existing code ...
134+
135+
export function setEvaluationContext(
136+
evaluationContext: EvaluationContext,
137+
): contextApi.Context {
138+
const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_EVALUATION);
139+
return contextApi.context.active().setValue(key, evaluationContext);
140+
}
141+
142+
export function getEvaluationContext(): EvaluationContext | undefined {
143+
const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_EVALUATION);
144+
return (contextApi.context.active().getValue(key) || undefined) as
145+
| EvaluationContext
146+
| undefined;
147+
}

‎src/decorators/flow.ts‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
import * as contextApi from "@opentelemetry/api";
2+
import { ReadableSpan, Tracer } from "@opentelemetry/sdk-trace-node";
3+
4+
import { HumanloopClient } from "../Client";
5+
import { ChatMessage, FlowLogRequest, FlowLogResponse } from "../api";
6+
import { getTraceId, setDecoratorContext, setTraceId } from "../context";
7+
import { jsonifyIfNotString, writeToOpenTelemetrySpan } from "../otel";
8+
import {
9+
HUMANLOOP_FILE_TYPE_KEY,
10+
HUMANLOOP_FLOW_SPAN_NAME,
11+
HUMANLOOP_LOG_KEY,
12+
HUMANLOOP_PATH_KEY,
13+
} from "../otel/constants";
14+
15+
export function flowUtilityFactory<I, O>(
16+
client: HumanloopClient,
17+
opentelemetryTracer: Tracer,
18+
callable: (
19+
args: I extends Record<string, unknown> & {
20+
messages?: ChatMessage[];
21+
}
22+
? I
23+
: never,
24+
) => O,
25+
path: string,
26+
attributes?: Record<string, unknown>,
27+
): (args: I) => Promise<O | undefined> & {
28+
file: {
29+
type: string;
30+
version: { attributes?: Record<string, unknown> };
31+
callable: (args: I) => Promise<O | undefined>;
32+
};
33+
} {
34+
const flowKernel = { attributes: attributes || {} };
35+
const fileType = "flow";
36+
37+
const wrappedFunction = async (
38+
inputs: I extends Record<string, unknown> & {
39+
messages?: ChatMessage[];
40+
}
41+
? I
42+
: never,
43+
) => {
44+
return contextApi.context.with(
45+
setDecoratorContext({
46+
path: path,
47+
type: fileType,
48+
version: flowKernel,
49+
}),
50+
async () => {
51+
return opentelemetryTracer.startActiveSpan(
52+
HUMANLOOP_FLOW_SPAN_NAME,
53+
async (span) => {
54+
span.setAttribute(HUMANLOOP_PATH_KEY, path);
55+
span.setAttribute(HUMANLOOP_FILE_TYPE_KEY, fileType);
56+
const traceId = getTraceId();
57+
58+
const logInputs = { ...inputs } as Record<string, unknown>;
59+
const logMessages = logInputs.messages as
60+
| ChatMessage[]
61+
| undefined;
62+
delete logInputs.messages;
63+
64+
const initLogInputs: FlowLogRequest = {
65+
inputs: logInputs,
66+
messages: logMessages,
67+
traceParentId: traceId,
68+
};
69+
70+
const flowLogResponse: FlowLogResponse =
71+
// @ts-ignore
72+
await client.flows._log({
73+
path,
74+
flow: flowKernel,
75+
logStatus: "incomplete",
76+
...initLogInputs,
77+
});
78+
79+
return await contextApi.context.with(
80+
setTraceId(flowLogResponse.id),
81+
async () => {
82+
let logOutput: string | undefined;
83+
let outputMessage: ChatMessage | undefined;
84+
let logError: string | undefined;
85+
let funcOutput: O | undefined;
86+
87+
try {
88+
funcOutput = await callable(inputs);
89+
if (
90+
// @ts-ignore
91+
funcOutput instanceof Object &&
92+
"role" in funcOutput &&
93+
"content" in funcOutput
94+
) {
95+
outputMessage =
96+
funcOutput as unknown as ChatMessage;
97+
logOutput = undefined;
98+
} else {
99+
logOutput = jsonifyIfNotString(
100+
callable,
101+
funcOutput,
102+
);
103+
outputMessage = undefined;
104+
}
105+
logError = undefined;
106+
} catch (err: any) {
107+
console.error(
108+
`Error calling ${callable.name}:`,
109+
err,
110+
);
111+
logOutput = undefined;
112+
outputMessage = undefined;
113+
logError = err.message || String(err);
114+
funcOutput = undefined as unknown as O;
115+
}
116+
117+
const updatedFlowLog = {
118+
log_status: "complete",
119+
output: logOutput,
120+
error: logError,
121+
output_message: outputMessage,
122+
id: flowLogResponse.id,
123+
};
124+
125+
writeToOpenTelemetrySpan(
126+
span as unknown as ReadableSpan,
127+
// @ts-ignore
128+
updatedFlowLog,
129+
HUMANLOOP_LOG_KEY,
130+
);
131+
132+
span.end();
133+
return funcOutput;
134+
},
135+
);
136+
},
137+
);
138+
},
139+
);
140+
};
141+
142+
// @ts-ignore
143+
return Object.assign(wrappedFunction, {
144+
file: {
145+
path: path,
146+
type: fileType,
147+
version: flowKernel,
148+
callable: wrappedFunction,
149+
},
150+
});
151+
}

‎src/decorators/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
export { flowUtilityFactory } from "./flow";
2+
export { toolUtilityFactory } from "./tool";

‎src/decorators/prompt.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
import * as contextApi from "@opentelemetry/api";
2+
3+
import { setDecoratorContext } from "../evals";
4+
5+
export function promptDecoratorFactory<I, O>(path: string, callable: (inputs: I) => O) {
6+
const fileType = "prompt";
7+
8+
const wrappedFunction = (inputs: I) => {
9+
return contextApi.context.with(
10+
setDecoratorContext({
11+
path: path,
12+
type: fileType,
13+
version: {
14+
// TODO: Implement reverse lookup of template
15+
template: undefined,
16+
},
17+
}),
18+
async () => {
19+
return await callable(inputs);
20+
},
21+
);
22+
};
23+
24+
return Object.assign(wrappedFunction, {
25+
file: {
26+
path: path,
27+
type: fileType,
28+
version: {
29+
// TODO: Implement reverse lookup of template
30+
template: undefined,
31+
},
32+
callable: wrappedFunction,
33+
},
34+
});
35+
}

0 commit comments

Comments
 (0)