From 77dd5c9419afae61719bee6da8c435ebb8ea7249 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Sat, 20 Jan 2024 07:49:10 -0800 Subject: [PATCH 01/15] initial project setup --- .../.vscode/c_cpp_properties.json | 21 ++++++ .../binding.gyp | 20 ++++++ .../lib/binding.ts | 21 ++++++ .../package.json | 33 ++++++++++ .../src/llamacpp_bindings.cc | 66 +++++++++++++++++++ .../src/llamacpp_bindings.h | 15 +++++ .../test/test_binding.js | 21 ++++++ .../tsconfig.json | 13 ++++ 8 files changed, 210 insertions(+) create mode 100644 packages/@modelfusion-llamacpp-bindings/.vscode/c_cpp_properties.json create mode 100644 packages/@modelfusion-llamacpp-bindings/binding.gyp create mode 100644 packages/@modelfusion-llamacpp-bindings/lib/binding.ts create mode 100644 packages/@modelfusion-llamacpp-bindings/package.json create mode 100644 packages/@modelfusion-llamacpp-bindings/src/llamacpp_bindings.cc create mode 100644 packages/@modelfusion-llamacpp-bindings/src/llamacpp_bindings.h create mode 100644 packages/@modelfusion-llamacpp-bindings/test/test_binding.js create mode 100644 packages/@modelfusion-llamacpp-bindings/tsconfig.json diff --git a/packages/@modelfusion-llamacpp-bindings/.vscode/c_cpp_properties.json b/packages/@modelfusion-llamacpp-bindings/.vscode/c_cpp_properties.json new file mode 100644 index 00000000..9fea6a52 --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/.vscode/c_cpp_properties.json @@ -0,0 +1,21 @@ +{ + "configurations": [ + { + "name": "Mac", + "includePath": [ + "${workspaceFolder}/**", + "${workspaceFolder}/node_modules/node-addon-api", + "/usr/local/include/node" + ], + "defines": [], + "macFrameworkPath": [ + "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/System/Library/Frameworks" + ], + "compilerPath": "/usr/bin/clang", + "cStandard": "c17", + "cppStandard": "c++17", + "intelliSenseMode": "macos-clang-arm64" + } + ], + "version": 4 +} diff --git a/packages/@modelfusion-llamacpp-bindings/binding.gyp b/packages/@modelfusion-llamacpp-bindings/binding.gyp new file mode 100644 index 00000000..a29a8cc9 --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/binding.gyp @@ -0,0 +1,20 @@ +{ + 'targets': [ + { + 'target_name': 'llamacpp-bindings-native', + 'sources': [ 'src/llamacpp_bindings.cc' ], + 'include_dirs': ["_greeterName = info[0].As().Utf8Value(); +} + +Napi::Value llamacppBindings::Greet(const Napi::CallbackInfo &info) +{ + Napi::Env env = info.Env(); + + if (info.Length() < 1) + { + Napi::TypeError::New(env, "Wrong number of arguments") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + if (!info[0].IsString()) + { + Napi::TypeError::New(env, "You need to introduce yourself to greet") + .ThrowAsJavaScriptException(); + return env.Null(); + } + + Napi::String name = info[0].As(); + + printf("Hello %s\n", name.Utf8Value().c_str()); + printf("I am %s\n", this->_greeterName.c_str()); + + return Napi::String::New(env, this->_greeterName); +} + +Napi::Function llamacppBindings::GetClass(Napi::Env env) +{ + return DefineClass(env, "llamacppBindings", { + llamacppBindings::InstanceMethod("greet", &llamacppBindings::Greet), + }); +} + +Napi::Object Init(Napi::Env env, Napi::Object exports) +{ + Napi::String name = Napi::String::New(env, "llamacppBindings"); + exports.Set(name, llamacppBindings::GetClass(env)); + return exports; +} + +NODE_API_MODULE(addon, Init) diff --git a/packages/@modelfusion-llamacpp-bindings/src/llamacpp_bindings.h b/packages/@modelfusion-llamacpp-bindings/src/llamacpp_bindings.h new file mode 100644 index 00000000..f8c2003d --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/src/llamacpp_bindings.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +class llamacppBindings : public Napi::ObjectWrap +{ +public: + llamacppBindings(const Napi::CallbackInfo &); + Napi::Value Greet(const Napi::CallbackInfo &); + + static Napi::Function GetClass(Napi::Env); + +private: + std::string _greeterName; +}; diff --git a/packages/@modelfusion-llamacpp-bindings/test/test_binding.js b/packages/@modelfusion-llamacpp-bindings/test/test_binding.js new file mode 100644 index 00000000..ed08ff62 --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/test/test_binding.js @@ -0,0 +1,21 @@ +const llamacppBindings = require("../dist/binding.js"); +const assert = require("assert"); + +assert(llamacppBindings, "The expected module is undefined"); + +function testBasic() +{ + const instance = new llamacppBindings("mr-yeoman"); + assert(instance.greet, "The expected method is not defined"); + assert.strictEqual(instance.greet("kermit"), "mr-yeoman", "Unexpected value returned"); +} + +function testInvalidParams() +{ + const instance = new llamacppBindings(); +} + +assert.doesNotThrow(testBasic, undefined, "testBasic threw an expection"); +assert.throws(testInvalidParams, undefined, "testInvalidParams didn't throw"); + +console.log("Tests passed- everything looks OK!"); \ No newline at end of file diff --git a/packages/@modelfusion-llamacpp-bindings/tsconfig.json b/packages/@modelfusion-llamacpp-bindings/tsconfig.json new file mode 100644 index 00000000..af079489 --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "declaration": true, + "lib": ["esnext"], + "module": "commonjs", + "outDir": "./dist", + "rootDir": "./lib", + "strict": true, + "target": "es6", + "types": ["node"], + "typeRoots": ["node_modules/@types"] + } +} From 968908f75a8c8b6b35532080a33d896941e53b28 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Sun, 21 Jan 2024 09:48:48 -0800 Subject: [PATCH 02/15] restructure folders for llamacpp dependencies --- .gitignore | 1 + .../package.json | 2 ++ .../{lib => src}/binding.ts | 0 .../tsconfig.json | 5 +++-- .../update_llamacpp.sh | 19 +++++++++++++++++++ 5 files changed, 25 insertions(+), 2 deletions(-) rename packages/@modelfusion-llamacpp-bindings/{lib => src}/binding.ts (100%) create mode 100755 packages/@modelfusion-llamacpp-bindings/update_llamacpp.sh diff --git a/.gitignore b/.gitignore index a2098a4e..ec702d90 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ examples/*/build/** packages/*/build/** packages/*/coverage/** packages/*/dist/** +packages/*/lib/** tools/*/build/** tools/*/coverage/** tools/*/dist/** \ No newline at end of file diff --git a/packages/@modelfusion-llamacpp-bindings/package.json b/packages/@modelfusion-llamacpp-bindings/package.json index a85e1b3e..b149cc17 100644 --- a/packages/@modelfusion-llamacpp-bindings/package.json +++ b/packages/@modelfusion-llamacpp-bindings/package.json @@ -4,6 +4,8 @@ "node-addon-api": "^7.0.0" }, "scripts": { + "build": "node-gyp rebuild && tsc", + "clean": "node-gyp clean && tsc --build --clean", "pretest": "tsc", "test": "node --napi-modules ./test/test_binding.js" }, diff --git a/packages/@modelfusion-llamacpp-bindings/lib/binding.ts b/packages/@modelfusion-llamacpp-bindings/src/binding.ts similarity index 100% rename from packages/@modelfusion-llamacpp-bindings/lib/binding.ts rename to packages/@modelfusion-llamacpp-bindings/src/binding.ts diff --git a/packages/@modelfusion-llamacpp-bindings/tsconfig.json b/packages/@modelfusion-llamacpp-bindings/tsconfig.json index af079489..be45c8eb 100644 --- a/packages/@modelfusion-llamacpp-bindings/tsconfig.json +++ b/packages/@modelfusion-llamacpp-bindings/tsconfig.json @@ -4,10 +4,11 @@ "lib": ["esnext"], "module": "commonjs", "outDir": "./dist", - "rootDir": "./lib", + "rootDir": "./src", "strict": true, "target": "es6", "types": ["node"], "typeRoots": ["node_modules/@types"] - } + }, + "include": ["src/**/*.ts"] } diff --git a/packages/@modelfusion-llamacpp-bindings/update_llamacpp.sh b/packages/@modelfusion-llamacpp-bindings/update_llamacpp.sh new file mode 100755 index 00000000..0b94ce58 --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/update_llamacpp.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Define the repository URL +REPO_URL="https://github.com/ggerganov/llama.cpp.git" + +# Define the directory where the repository will be cloned +DIR="lib/llamacpp" + +# If the directory already exists, pull the latest changes +if [ -d "$DIR" ]; then + echo "Directory $DIR exists." + echo "Pulling latest changes..." + cd "$DIR" && git pull origin master +else + # If the directory does not exist, clone the repository + echo "Directory $DIR does not exist." + echo "Cloning repository..." + git clone "$REPO_URL" "$DIR" +fi \ No newline at end of file From d9c17d740c223b8c0f1674df7ed674b19cc28cd2 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Mon, 22 Jan 2024 15:40:15 -0800 Subject: [PATCH 03/15] integrate llama.cpp with cmake-js --- .../CMakeLists.txt | 42 +++++++++++++++++++ .../binding.gyp | 26 +++++------- .../package.json | 1 + .../src/binding.ts | 5 +++ .../src/llamacpp_bindings.cc | 7 ++++ .../test/test_binding.js | 2 + 6 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 packages/@modelfusion-llamacpp-bindings/CMakeLists.txt diff --git a/packages/@modelfusion-llamacpp-bindings/CMakeLists.txt b/packages/@modelfusion-llamacpp-bindings/CMakeLists.txt new file mode 100644 index 00000000..5a0c6643 --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/CMakeLists.txt @@ -0,0 +1,42 @@ +cmake_minimum_required(VERSION 3.13) + +project("llamacpp-bindings-native" C CXX) + +if (MSVC) + # add_compile_options(/EHsc) +else() + add_compile_options(-fexceptions) +endif() + +add_definitions(-DNAPI_VERSION=7) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +execute_process(COMMAND node -p "require('node-addon-api').include.slice(1,-1)" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE NODE_ADDON_API_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE) + +include_directories(${NODE_ADDON_API_DIR} ${CMAKE_JS_INC}) + +add_subdirectory("lib/llamacpp") +include_directories("lib/llamacpp") +include_directories("./lib/llamacpp/common") + +file(GLOB SOURCE_FILES "src/llamacpp_bindings.cc") + +add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC}) +set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node") +target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB}) +target_link_libraries(${PROJECT_NAME} "llama") +target_link_libraries(${PROJECT_NAME} "common") + +# copy ggml-metal.metal to release folder +add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/lib/llamacpp/ggml-metal.metal + ${CMAKE_CURRENT_BINARY_DIR}/Release/ggml-metal.metal) + +if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET) + # Generate node.lib + execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS}) +endif() diff --git a/packages/@modelfusion-llamacpp-bindings/binding.gyp b/packages/@modelfusion-llamacpp-bindings/binding.gyp index a29a8cc9..db9c5eeb 100644 --- a/packages/@modelfusion-llamacpp-bindings/binding.gyp +++ b/packages/@modelfusion-llamacpp-bindings/binding.gyp @@ -1,20 +1,16 @@ { 'targets': [ - { - 'target_name': 'llamacpp-bindings-native', - 'sources': [ 'src/llamacpp_bindings.cc' ], - 'include_dirs': [" using namespace Napi; @@ -56,10 +57,16 @@ Napi::Function llamacppBindings::GetClass(Napi::Env env) }); } +Napi::Value systemInfo(const Napi::CallbackInfo &info) +{ + return Napi::String::From(info.Env(), llama_print_system_info()); +} + Napi::Object Init(Napi::Env env, Napi::Object exports) { Napi::String name = Napi::String::New(env, "llamacppBindings"); exports.Set(name, llamacppBindings::GetClass(env)); + exports.Set(Napi::String::New(env, "systemInfo"), Napi::Function::New(env, systemInfo)); return exports; } diff --git a/packages/@modelfusion-llamacpp-bindings/test/test_binding.js b/packages/@modelfusion-llamacpp-bindings/test/test_binding.js index ed08ff62..5a8e5b9f 100644 --- a/packages/@modelfusion-llamacpp-bindings/test/test_binding.js +++ b/packages/@modelfusion-llamacpp-bindings/test/test_binding.js @@ -2,12 +2,14 @@ const llamacppBindings = require("../dist/binding.js"); const assert = require("assert"); assert(llamacppBindings, "The expected module is undefined"); +assert(llamacppBindings.getSystemInfo, "The expected method is undefined"); function testBasic() { const instance = new llamacppBindings("mr-yeoman"); assert(instance.greet, "The expected method is not defined"); assert.strictEqual(instance.greet("kermit"), "mr-yeoman", "Unexpected value returned"); + assert(llamacppBindings.getSystemInfo().length > 2, "System info is too short"); } function testInvalidParams() From 5ef0a0824f6a73b99368ec93f9d41b9b9b77f8ea Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Mon, 22 Jan 2024 15:47:07 -0800 Subject: [PATCH 04/15] convert tests to typescript --- packages/@modelfusion-llamacpp-bindings/binding.gyp | 2 +- packages/@modelfusion-llamacpp-bindings/package.json | 2 +- .../{test/test_binding.js => src/binding.test.ts} | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) rename packages/@modelfusion-llamacpp-bindings/{test/test_binding.js => src/binding.test.ts} (77%) diff --git a/packages/@modelfusion-llamacpp-bindings/binding.gyp b/packages/@modelfusion-llamacpp-bindings/binding.gyp index db9c5eeb..1b07f601 100644 --- a/packages/@modelfusion-llamacpp-bindings/binding.gyp +++ b/packages/@modelfusion-llamacpp-bindings/binding.gyp @@ -1,5 +1,5 @@ { - 'targets': [ + "targets": [ { "target_name": "llamacpp-bindings-native", "type": "none", diff --git a/packages/@modelfusion-llamacpp-bindings/package.json b/packages/@modelfusion-llamacpp-bindings/package.json index eaaba19c..cdfe72e0 100644 --- a/packages/@modelfusion-llamacpp-bindings/package.json +++ b/packages/@modelfusion-llamacpp-bindings/package.json @@ -7,7 +7,7 @@ "build": "node-gyp rebuild && tsc", "clean": "node-gyp clean && tsc --build --clean", "pretest": "tsc", - "test": "node --napi-modules ./test/test_binding.js" + "test": "node --napi-modules ./dist/binding.test.js" }, "gypfile": true, "name": "@modelfusion/llamacpp-bindings", diff --git a/packages/@modelfusion-llamacpp-bindings/test/test_binding.js b/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts similarity index 77% rename from packages/@modelfusion-llamacpp-bindings/test/test_binding.js rename to packages/@modelfusion-llamacpp-bindings/src/binding.test.ts index 5a8e5b9f..17a65773 100644 --- a/packages/@modelfusion-llamacpp-bindings/test/test_binding.js +++ b/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts @@ -1,5 +1,5 @@ const llamacppBindings = require("../dist/binding.js"); -const assert = require("assert"); +import * as assert from "assert"; assert(llamacppBindings, "The expected module is undefined"); assert(llamacppBindings.getSystemInfo, "The expected method is undefined"); @@ -17,7 +17,7 @@ function testInvalidParams() const instance = new llamacppBindings(); } -assert.doesNotThrow(testBasic, undefined, "testBasic threw an expection"); -assert.throws(testInvalidParams, undefined, "testInvalidParams didn't throw"); +assert.doesNotThrow(testBasic, "testBasic threw an expection"); +assert.throws(testInvalidParams, "testInvalidParams didn't throw"); console.log("Tests passed- everything looks OK!"); \ No newline at end of file From 9b8c74ab793a2d50a1182e712c1195c20415aed9 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Tue, 23 Jan 2024 21:00:58 -0600 Subject: [PATCH 05/15] initial provider implementation --- .../LlamaCppBindingsCompletionModel.ts | 255 ++++++++++++++++++ .../LlamaCppBindingsFacade.ts | 10 + .../model-provider/llamacpp-bindings/index.ts | 1 + 3 files changed, 266 insertions(+) create mode 100644 packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsCompletionModel.ts create mode 100644 packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsFacade.ts create mode 100644 packages/modelfusion/src/model-provider/llamacpp-bindings/index.ts diff --git a/packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsCompletionModel.ts b/packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsCompletionModel.ts new file mode 100644 index 00000000..86d66e0d --- /dev/null +++ b/packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsCompletionModel.ts @@ -0,0 +1,255 @@ +import { AbstractModel } from "model-function/AbstractModel"; +import { + TextGenerationBaseModel, + TextGenerationModel, + TextGenerationModelSettings, +} from "../../model-function/generate-text/TextGenerationModel.js"; +import { TextGenerationPromptTemplateProvider } from "../../model-function/generate-text/prompt-template/PromptTemplateProvider.js"; +import { + InstructionPrompt, + ChatPrompt, + BasicTokenizer, + FullTokenizer, + TextGenerationResult, +} from "index.js"; + +export interface LlamaCppBindingsCompletionPrompt { + /** + * Text prompt. Images can be included through references such as `[img-ID]`, e.g. `[img-1]`. + */ + text: string; + + /** + * Maps image id to image base data. + */ + images?: Record; +} + +export interface LlamaCppBindingsCompletionModelSettings< + CONTEXT_WINDOW_SIZE extends number | undefined, +> extends TextGenerationModelSettings { + /** + * Specify the context window size of the model that you have loaded in your + * Llama.cpp server. + */ + contextWindowSize?: CONTEXT_WINDOW_SIZE; + + /** + * Adjust the randomness of the generated text (default: 0.8). + */ + temperature?: number; + + /** + * Limit the next token selection to the K most probable tokens (default: 40). + */ + topK?: number; + + /** + * Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P (default: 0.95). + */ + topP?: number; + + /** + * The minimum probability for a token to be considered, relative to the probability of the most likely token (default: 0.05). + */ + minP?: number; + + /** + * Specify the number of tokens from the prompt to retain when the context size is exceeded + * and tokens need to be discarded. By default, this value is set to 0 (meaning no tokens + * are kept). Use -1 to retain all tokens from the prompt. + */ + nKeep?: number; + + /** + * Enable tail free sampling with parameter z (default: 1.0, 1.0 = disabled). + */ + tfsZ?: number; + + /** + * Enable locally typical sampling with parameter p (default: 1.0, 1.0 = disabled). + */ + typicalP?: number; + + /** + * Control the repetition of token sequences in the generated text (default: 1.1). + */ + repeatPenalty?: number; + + /** + * Last n tokens to consider for penalizing repetition (default: 64, 0 = disabled, -1 = ctx-size). + */ + repeatLastN?: number; + + /** + * Penalize newline tokens when applying the repeat penalty (default: true). + */ + penalizeNl?: boolean; + + /** + * Repeat alpha presence penalty (default: 0.0, 0.0 = disabled). + */ + presencePenalty?: number; + + /** + * Repeat alpha frequency penalty (default: 0.0, 0.0 = disabled). + */ + frequencyPenalty?: number; + + /** + * This will replace the prompt for the purpose of the penalty evaluation. + * Can be either null, a string or an array of numbers representing tokens + * (default: null = use the original prompt). + */ + penaltyPrompt?: string | number[]; + + /** + * Enable Mirostat sampling, controlling perplexity during text generation + * (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0). + */ + mirostat?: number; + + /** + * Set the Mirostat target entropy, parameter tau (default: 5.0). + */ + mirostatTau?: number; + + /** + * Set the Mirostat learning rate, parameter eta (default: 0.1). + */ + mirostatEta?: number; + + /** + * Set grammar for grammar-based sampling (default: no grammar) + * + * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md + */ + grammar?: string; + + /** + * Set the random number generator (RNG) seed + * (default: -1, -1 = random seed). + */ + seed?: number; + + /** + * Ignore end of stream token and continue generating (default: false). + */ + ignoreEos?: boolean; + + /** + * Modify the likelihood of a token appearing in the generated text completion. + * For example, use "logit_bias": [[15043,1.0]] to increase the likelihood of the token + * 'Hello', or "logit_bias": [[15043,-1.0]] to decrease its likelihood. + * Setting the value to false, "logit_bias": [[15043,false]] ensures that the token Hello is + * never produced (default: []). + */ + logitBias?: Array<[number, number | false]>; + + /** + * If greater than 0, the response also contains the probabilities of top N tokens + * for each generated token (default: 0) + */ + nProbs?: number; + + /** + * Save the prompt and generation for avoid reprocess entire prompt if a part of this isn't change (default: false) + */ + cachePrompt?: boolean; + + /** + * Assign the completion task to an specific slot. + * If is -1 the task will be assigned to a Idle slot (default: -1) + */ + slotId?: number; + + /** + * Prompt template provider that is used when calling `.withTextPrompt()`, `withInstructionPrompt()` or `withChatPrompt()`. + */ + promptTemplate?: TextGenerationPromptTemplateProvider; +} + +export class LlamaCppBindingsCompletionModel< + CONTEXT_WINDOW_SIZE extends number | undefined, + > + extends AbstractModel< + LlamaCppBindingsCompletionModelSettings + > + implements + TextGenerationBaseModel< + LlamaCppBindingsCompletionPrompt, + LlamaCppBindingsCompletionModelSettings + > +{ + constructor( + settings: LlamaCppBindingsCompletionModelSettings = {} + ) { + super({ settings }); + } + withTextPrompt(): TextGenerationModel< + string, + LlamaCppBindingsCompletionModelSettings + > { + throw new Error("Method not implemented."); + } + withInstructionPrompt(): TextGenerationModel< + InstructionPrompt, + LlamaCppBindingsCompletionModelSettings + > { + throw new Error("Method not implemented."); + } + withChatPrompt(): TextGenerationModel< + ChatPrompt, + LlamaCppBindingsCompletionModelSettings + > { + throw new Error("Method not implemented."); + } + withPromptTemplate(): TextGenerationModel< + INPUT_PROMPT, + LlamaCppBindingsCompletionModelSettings + > { + throw new Error("Method not implemented."); + } + contextWindowSize: number | undefined; + tokenizer: BasicTokenizer | FullTokenizer | undefined; + countPromptTokens: + | ((prompt: LlamaCppBindingsCompletionPrompt) => PromiseLike) + | undefined; + doGenerateTexts(): PromiseLike<{ + rawResponse: unknown; + textGenerationResults: TextGenerationResult[]; + usage?: + | { promptTokens: number; completionTokens: number; totalTokens: number } + | undefined; + }> { + throw new Error("Method not implemented."); + } + restoreGeneratedTexts(): { + rawResponse: unknown; + textGenerationResults: TextGenerationResult[]; + usage?: + | { promptTokens: number; completionTokens: number; totalTokens: number } + | undefined; + } { + throw new Error("Method not implemented."); + } + withJsonOutput(): this { + throw new Error("Method not implemented."); + } + + readonly provider = "llamacpp"; + + get modelName() { + return null; + } + + get settingsForEvent(): Partial< + LlamaCppBindingsCompletionModelSettings + > { + return {}; + } + + withSettings(): this { + return this; + } +} diff --git a/packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsFacade.ts b/packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsFacade.ts new file mode 100644 index 00000000..6bef7ce0 --- /dev/null +++ b/packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsFacade.ts @@ -0,0 +1,10 @@ +import { + LlamaCppBindingsCompletionModel, + LlamaCppBindingsCompletionModelSettings, +} from "./LlamaCppBindingsCompletionModel.js"; + +export function CompletionTextGenerator( + settings: LlamaCppBindingsCompletionModelSettings = {} +) { + return new LlamaCppBindingsCompletionModel(settings); +} diff --git a/packages/modelfusion/src/model-provider/llamacpp-bindings/index.ts b/packages/modelfusion/src/model-provider/llamacpp-bindings/index.ts new file mode 100644 index 00000000..44299480 --- /dev/null +++ b/packages/modelfusion/src/model-provider/llamacpp-bindings/index.ts @@ -0,0 +1 @@ +export * as llamacppbindings from "./LlamaCppBindingsFacade.js"; From 908cbc5ef968745c1ccdb1a97d01de4bc7ab875d Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Wed, 24 Jan 2024 07:38:23 -0600 Subject: [PATCH 06/15] move provider implementation into package --- .gitignore | 3 +- .vscode/settings.json | 3 +- .../CMakeLists.txt | 10 +- .../package.json | 7 +- .../src}/LlamaCppBindingsCompletionModel.ts | 8 +- .../src}/LlamaCppBindingsFacade.ts | 0 .../src}/index.ts | 0 .../tsconfig.json | 5 +- ...pdate_llamacpp.sh => update_submodules.sh} | 0 pnpm-lock.yaml | 195 +++++++++++++----- 10 files changed, 167 insertions(+), 64 deletions(-) rename packages/{modelfusion/src/model-provider/llamacpp-bindings => @modelfusion-llamacpp-bindings/src}/LlamaCppBindingsCompletionModel.ts (95%) rename packages/{modelfusion/src/model-provider/llamacpp-bindings => @modelfusion-llamacpp-bindings/src}/LlamaCppBindingsFacade.ts (100%) rename packages/{modelfusion/src/model-provider/llamacpp-bindings => @modelfusion-llamacpp-bindings/src}/index.ts (100%) rename packages/@modelfusion-llamacpp-bindings/{update_llamacpp.sh => update_submodules.sh} (100%) diff --git a/.gitignore b/.gitignore index ec702d90..d5587b37 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,7 @@ runs node_modules package-lock.json -examples/*/build/** -packages/*/build/** +build packages/*/coverage/** packages/*/dist/** packages/*/lib/** diff --git a/.vscode/settings.json b/.vscode/settings.json index 381b93c0..4cd8eba1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -36,5 +36,6 @@ "upsert", "whispercpp" ], - "typescript.tsdk": "node_modules/typescript/lib" + "typescript.tsdk": "node_modules/typescript/lib", + "cmake.sourceDirectory": "packages/@modelfusion-llamacpp-bindings" } diff --git a/packages/@modelfusion-llamacpp-bindings/CMakeLists.txt b/packages/@modelfusion-llamacpp-bindings/CMakeLists.txt index 5a0c6643..a164b58e 100644 --- a/packages/@modelfusion-llamacpp-bindings/CMakeLists.txt +++ b/packages/@modelfusion-llamacpp-bindings/CMakeLists.txt @@ -18,9 +18,13 @@ execute_process(COMMAND node -p "require('node-addon-api').include.slice(1,-1)" include_directories(${NODE_ADDON_API_DIR} ${CMAKE_JS_INC}) -add_subdirectory("lib/llamacpp") -include_directories("lib/llamacpp") -include_directories("./lib/llamacpp/common") +if(EXISTS ${CMAKE_SOURCE_DIR}/lib/llamacpp) + add_subdirectory(${CMAKE_SOURCE_DIR}/lib/llamacpp) + include_directories(${CMAKE_SOURCE_DIR}/lib/llamacpp) + include_directories(${CMAKE_SOURCE_DIR}/lib/llamacpp/common) +else() + message(FATAL_ERROR "Directory 'lib/llamacpp' does not exist. Run ./update_submodules.sh") +endif() file(GLOB SOURCE_FILES "src/llamacpp_bindings.cc") diff --git a/packages/@modelfusion-llamacpp-bindings/package.json b/packages/@modelfusion-llamacpp-bindings/package.json index cdfe72e0..4acd837a 100644 --- a/packages/@modelfusion-llamacpp-bindings/package.json +++ b/packages/@modelfusion-llamacpp-bindings/package.json @@ -4,6 +4,7 @@ "node-addon-api": "^7.0.0" }, "scripts": { + "setup": "./update_submodules.sh", "build": "node-gyp rebuild && tsc", "clean": "node-gyp clean && tsc --build --clean", "pretest": "tsc", @@ -31,6 +32,10 @@ "devDependencies": { "@types/node": "^20.11.3", "cmake-js": "^7.3.0", - "typescript": "^5.3.3" + "typescript": "^5.3.3", + "modelfusion": "0.131.0" + }, + "peerDependencies": { + "modelfusion": ">=0.131.0" } } diff --git a/packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsCompletionModel.ts b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts similarity index 95% rename from packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsCompletionModel.ts rename to packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts index 86d66e0d..e2c2d9c3 100644 --- a/packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsCompletionModel.ts +++ b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts @@ -1,17 +1,15 @@ -import { AbstractModel } from "model-function/AbstractModel"; import { TextGenerationBaseModel, TextGenerationModel, TextGenerationModelSettings, -} from "../../model-function/generate-text/TextGenerationModel.js"; -import { TextGenerationPromptTemplateProvider } from "../../model-function/generate-text/prompt-template/PromptTemplateProvider.js"; -import { + TextGenerationPromptTemplateProvider, InstructionPrompt, ChatPrompt, BasicTokenizer, FullTokenizer, TextGenerationResult, -} from "index.js"; +} from "modelfusion"; +import { AbstractModel } from "modelfusion/model-function/AbstractModel"; export interface LlamaCppBindingsCompletionPrompt { /** diff --git a/packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsFacade.ts b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsFacade.ts similarity index 100% rename from packages/modelfusion/src/model-provider/llamacpp-bindings/LlamaCppBindingsFacade.ts rename to packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsFacade.ts diff --git a/packages/modelfusion/src/model-provider/llamacpp-bindings/index.ts b/packages/@modelfusion-llamacpp-bindings/src/index.ts similarity index 100% rename from packages/modelfusion/src/model-provider/llamacpp-bindings/index.ts rename to packages/@modelfusion-llamacpp-bindings/src/index.ts diff --git a/packages/@modelfusion-llamacpp-bindings/tsconfig.json b/packages/@modelfusion-llamacpp-bindings/tsconfig.json index be45c8eb..055ed86a 100644 --- a/packages/@modelfusion-llamacpp-bindings/tsconfig.json +++ b/packages/@modelfusion-llamacpp-bindings/tsconfig.json @@ -5,10 +5,11 @@ "module": "commonjs", "outDir": "./dist", "rootDir": "./src", + "skipLibCheck": true, "strict": true, "target": "es6", "types": ["node"], - "typeRoots": ["node_modules/@types"] + "typeRoots": ["node_modules/@types"], }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts"], } diff --git a/packages/@modelfusion-llamacpp-bindings/update_llamacpp.sh b/packages/@modelfusion-llamacpp-bindings/update_submodules.sh similarity index 100% rename from packages/@modelfusion-llamacpp-bindings/update_llamacpp.sh rename to packages/@modelfusion-llamacpp-bindings/update_submodules.sh diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e65b741a..5e776c51 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -497,6 +497,25 @@ importers: specifier: 20.2.3 version: 20.2.3 + packages/@modelfusion-llamacpp-bindings: + dependencies: + node-addon-api: + specifier: ^7.0.0 + version: 7.1.0 + devDependencies: + '@types/node': + specifier: ^20.11.3 + version: 20.11.6 + cmake-js: + specifier: ^7.3.0 + version: 7.3.0 + modelfusion: + specifier: 0.131.0 + version: link:../modelfusion/dist + typescript: + specifier: ^5.3.3 + version: 5.3.3 + packages/@modelfusion-pinecone: devDependencies: '@pinecone-database/pinecone': @@ -3869,7 +3888,7 @@ packages: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 20.10.5 + '@types/node': 20.11.6 '@types/yargs': 17.0.32 chalk: 4.1.2 dev: false @@ -4869,26 +4888,26 @@ packages: resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} dependencies: '@types/connect': 3.4.38 - '@types/node': 20.10.5 + '@types/node': 20.11.6 dev: false /@types/bonjour@3.5.13: resolution: {integrity: sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 dev: false /@types/connect-history-api-fallback@1.5.4: resolution: {integrity: sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==} dependencies: '@types/express-serve-static-core': 4.17.41 - '@types/node': 20.10.5 + '@types/node': 20.11.6 dev: false /@types/connect@3.4.38: resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 dev: false /@types/cookie@0.6.0: @@ -4937,7 +4956,7 @@ packages: /@types/express-serve-static-core@4.17.41: resolution: {integrity: sha512-OaJ7XLaelTgrvlZD8/aa0vvvxZdUmlCn6MtWeB7TkiKW70BQLc9XEPpDLPdbo52ZhXUCrznlWdCHWxJWtdyajA==} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 '@types/qs': 6.9.10 '@types/range-parser': 1.2.7 '@types/send': 0.17.4 @@ -4989,7 +5008,7 @@ packages: /@types/http-proxy@1.17.14: resolution: {integrity: sha512-SSrD0c1OQzlFX7pGu1eXxSEjemej64aaNPRhhVYUGqXh0BtldAAx37MG8btcumvpgKyZp1F5Gn3JkktdxiFv6w==} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 dev: false /@types/istanbul-lib-coverage@2.0.6: @@ -5050,7 +5069,7 @@ packages: /@types/node-forge@1.3.10: resolution: {integrity: sha512-y6PJDYN4xYBxwd22l+OVH35N+1fCYWiuC3aiP2SlXVE6Lo7SS+rSx9r89hLxrP4pn6n1lBGhHJ12pj3F3Mpttw==} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 /@types/node@17.0.45: resolution: {integrity: sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==} @@ -5064,6 +5083,12 @@ packages: resolution: {integrity: sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==} dependencies: undici-types: 5.26.5 + dev: true + + /@types/node@20.11.6: + resolution: {integrity: sha512-+EOokTnksGVgip2PbYbr3xnR7kZigh4LbybAfBAw5BpnQ+FqBYUsvCEjYd70IXKlbohQ64mzEYmMtlWUY8q//Q==} + dependencies: + undici-types: 5.26.5 /@types/node@20.2.3: resolution: {integrity: sha512-pg9d0yC4rVNWQzX8U7xb4olIOFuuVL9za3bzMT2pu2SU0SNEi66i2qrvhE2qt0HvkhuCaWJu7pLNOt/Pj8BIrw==} @@ -5145,7 +5170,7 @@ packages: /@types/sax@1.2.7: resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 dev: false /@types/scheduler@0.16.8: @@ -5159,7 +5184,7 @@ packages: resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==} dependencies: '@types/mime': 1.3.5 - '@types/node': 20.10.5 + '@types/node': 20.11.6 dev: false /@types/serve-index@1.9.4: @@ -5173,13 +5198,13 @@ packages: dependencies: '@types/http-errors': 2.0.4 '@types/mime': 3.0.4 - '@types/node': 20.10.5 + '@types/node': 20.11.6 dev: false /@types/sockjs@0.3.36: resolution: {integrity: sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 dev: false /@types/statuses@2.0.4: @@ -5199,7 +5224,7 @@ packages: /@types/ws@8.5.10: resolution: {integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 /@types/yargs-parser@21.0.3: resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} @@ -5877,8 +5902,6 @@ packages: /aproba@2.0.0: resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==} requiresBuild: true - dev: false - optional: true /archy@1.0.0: resolution: {integrity: sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw==} @@ -5893,6 +5916,14 @@ packages: dev: false optional: true + /are-we-there-yet@3.0.1: + resolution: {integrity: sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==} + engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + dependencies: + delegates: 1.0.0 + readable-stream: 3.6.2 + dev: true + /arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} @@ -5939,7 +5970,6 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: false /at-least-node@1.0.0: resolution: {integrity: sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==} @@ -5974,6 +6004,16 @@ packages: transitivePeerDependencies: - supports-color + /axios@1.6.5(debug@4.3.4): + resolution: {integrity: sha512-Ii012v05KEVuUoFWmMW/UQv9aRIc3ZwkWDcM+h5Il8izZCtRVpDUfwpoFf7eOtajT3QiGR4yDUx7lPqHJULgbg==} + dependencies: + follow-redirects: 1.15.5(debug@4.3.4) + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + dev: true + /axobject-query@3.2.1: resolution: {integrity: sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==} dependencies: @@ -6422,8 +6462,6 @@ packages: resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} engines: {node: '>=10'} requiresBuild: true - dev: false - optional: true /chrome-trace-event@1.0.3: resolution: {integrity: sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==} @@ -6547,6 +6585,28 @@ packages: engines: {node: '>=6'} dev: false + /cmake-js@7.3.0: + resolution: {integrity: sha512-dXs2zq9WxrV87bpJ+WbnGKv8WUBXDw8blNiwNHoRe/it+ptscxhQHKB1SJXa1w+kocLMeP28Tk4/eTCezg4o+w==} + engines: {node: '>= 14.15.0'} + hasBin: true + dependencies: + axios: 1.6.5(debug@4.3.4) + debug: 4.3.4 + fs-extra: 11.2.0 + lodash.isplainobject: 4.0.6 + memory-stream: 1.0.0 + node-api-headers: 1.1.0 + npmlog: 6.0.2 + rc: 1.2.8 + semver: 7.5.4 + tar: 6.2.0 + url-join: 4.0.1 + which: 2.0.2 + yargs: 17.7.2 + transitivePeerDependencies: + - supports-color + dev: true + /code-red@1.0.4: resolution: {integrity: sha512-7qJWqItLA8/VPVlKJlFXU+NBlo/qyfs39aJcuMT/2ere32ZqvF5OSxgdM5xOfJJ7O429gg2HM47y8v9P+9wrNw==} dependencies: @@ -6581,8 +6641,6 @@ packages: resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} hasBin: true requiresBuild: true - dev: false - optional: true /colord@2.9.3: resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==} @@ -6601,7 +6659,6 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 - dev: false /comma-separated-tokens@1.0.8: resolution: {integrity: sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==} @@ -6704,8 +6761,6 @@ packages: /console-control-strings@1.1.0: resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} requiresBuild: true - dev: false - optional: true /content-disposition@0.5.2: resolution: {integrity: sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==} @@ -7516,13 +7571,10 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - dev: false /delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} requiresBuild: true - dev: false - optional: true /depd@1.1.2: resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} @@ -8084,7 +8136,7 @@ packages: resolution: {integrity: sha512-EzV94NYKoO09GLXGjXj9JIlXijVck4ONSr5wiCWDvhsvj5jxSrzTmRU/9C1DyB6uToszLs8aifA6NQ7lEQdvFw==} engines: {node: '>= 0.8'} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 require-like: 0.1.2 dev: false @@ -8450,6 +8502,18 @@ packages: optional: true dev: false + /follow-redirects@1.15.5(debug@4.3.4): + resolution: {integrity: sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dependencies: + debug: 4.3.4 + dev: true + /foreground-child@3.1.1: resolution: {integrity: sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==} engines: {node: '>=14'} @@ -8501,7 +8565,6 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 - dev: false /format@0.2.2: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} @@ -8534,7 +8597,6 @@ packages: graceful-fs: 4.2.11 jsonfile: 6.1.0 universalify: 2.0.1 - dev: false /fs-extra@9.1.0: resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==} @@ -8552,8 +8614,6 @@ packages: requiresBuild: true dependencies: minipass: 3.3.6 - dev: false - optional: true /fs-monkey@1.0.5: resolution: {integrity: sha512-8uMbBjrhzW76TYgEV27Y5E//W2f/lTFmx78P2w19FZSxarhI/798APGQyuGCwmkNxgwGRhrLfvWyLBvNtuOmew==} @@ -8589,6 +8649,20 @@ packages: dev: false optional: true + /gauge@4.0.4: + resolution: {integrity: sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==} + engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + dependencies: + aproba: 2.0.0 + color-support: 1.1.3 + console-control-strings: 1.1.0 + has-unicode: 2.0.1 + signal-exit: 3.0.7 + string-width: 4.2.3 + strip-ansi: 6.0.1 + wide-align: 1.1.5 + dev: true + /gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -8853,8 +8927,6 @@ packages: /has-unicode@2.0.1: resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==} requiresBuild: true - dev: false - optional: true /has-yarn@3.0.0: resolution: {integrity: sha512-IrsVwUHhEULx3R8f/aA8AHuEzAorplsab/v8HBzEiIukwq5i/EC+xmOW+HfP1OaDP+2JkgT1yILHN2O3UFIbcA==} @@ -9663,7 +9735,7 @@ packages: engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: '@jest/types': 29.6.3 - '@types/node': 20.10.5 + '@types/node': 20.11.6 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -9674,7 +9746,7 @@ packages: resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} engines: {node: '>= 10.13.0'} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -9682,7 +9754,7 @@ packages: resolution: {integrity: sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: - '@types/node': 20.10.5 + '@types/node': 20.11.6 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -9813,7 +9885,6 @@ packages: universalify: 2.0.1 optionalDependencies: graceful-fs: 4.2.11 - dev: false /keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -9985,6 +10056,10 @@ packages: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} dev: false + /lodash.isplainobject@4.0.6: + resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} + dev: true + /lodash.memoize@4.1.2: resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==} dev: false @@ -10511,6 +10586,12 @@ packages: fs-monkey: 1.0.5 dev: false + /memory-stream@1.0.0: + resolution: {integrity: sha512-Wm13VcsPIMdG96dzILfij09PvuS3APtcKNh7M28FsCA/w6+1mjR7hhPmfFNoilX9xU7wTdhsH5lJAm6XNzdtww==} + dependencies: + readable-stream: 3.6.2 + dev: true + /merge-descriptors@1.0.1: resolution: {integrity: sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==} dev: false @@ -11252,15 +11333,11 @@ packages: requiresBuild: true dependencies: yallist: 4.0.0 - dev: false - optional: true /minipass@5.0.0: resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} engines: {node: '>=8'} requiresBuild: true - dev: false - optional: true /minipass@7.0.4: resolution: {integrity: sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==} @@ -11274,8 +11351,6 @@ packages: dependencies: minipass: 3.3.6 yallist: 4.0.0 - dev: false - optional: true /mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} @@ -11518,6 +11593,15 @@ packages: dependencies: semver: 7.5.4 + /node-addon-api@7.1.0: + resolution: {integrity: sha512-mNcltoe1R8o7STTegSOHdnJNN7s5EUvhoS7ShnTHDyOSd+8H+UdWODq6qSv67PjC8Zc5JRT8+oLAMCr0SIXw7g==} + engines: {node: ^16 || ^18 || >= 20} + dev: false + + /node-api-headers@1.1.0: + resolution: {integrity: sha512-ucQW+SbYCUPfprvmzBsnjT034IGRB2XK8rRc78BgjNKhTdFKgAwAmgW704bKIBmcYW48it0Gkjpkd39Azrwquw==} + dev: true + /node-emoji@2.1.3: resolution: {integrity: sha512-E2WEOVsgs7O16zsURJ/eH8BqhF029wGpEOnv7Urwdo2wmQanOACwJQh0devF9D9RhoZru0+9JXIS0dBXIAz+lA==} engines: {node: '>=18'} @@ -11610,6 +11694,16 @@ packages: dev: false optional: true + /npmlog@6.0.2: + resolution: {integrity: sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==} + engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + dependencies: + are-we-there-yet: 3.0.1 + console-control-strings: 1.1.0 + gauge: 4.0.4 + set-blocking: 2.0.0 + dev: true + /nprogress@0.2.0: resolution: {integrity: sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==} dev: false @@ -12668,6 +12762,10 @@ packages: forwarded: 0.2.0 ipaddr.js: 1.9.1 + /proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: true + /psl@1.9.0: resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} dev: false @@ -13625,8 +13723,6 @@ packages: /set-blocking@2.0.0: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} requiresBuild: true - dev: false - optional: true /set-cookie-parser@2.6.0: resolution: {integrity: sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==} @@ -14309,8 +14405,6 @@ packages: minizlib: 2.1.2 mkdirp: 1.0.4 yallist: 4.0.0 - dev: false - optional: true /terser-webpack-plugin@5.3.9(webpack@5.89.0): resolution: {integrity: sha512-ZuXsqE07EcggTWQjXUj+Aot/OMcD0bMKGgF63f7UxYcu5/AJF53aIpK1YoP5xR9l6s/Hy2b+t1AM0bLNPRuhwA==} @@ -14827,7 +14921,6 @@ packages: /universalify@2.0.1: resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} engines: {node: '>= 10.0.0'} - dev: false /unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} @@ -14874,6 +14967,10 @@ packages: dependencies: punycode: 2.3.1 + /url-join@4.0.1: + resolution: {integrity: sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==} + dev: true + /url-loader@4.1.1(file-loader@6.2.0)(webpack@5.89.0): resolution: {integrity: sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA==} engines: {node: '>= 10.13.0'} @@ -15439,8 +15536,6 @@ packages: requiresBuild: true dependencies: string-width: 4.2.3 - dev: false - optional: true /widest-line@4.0.1: resolution: {integrity: sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==} From 4d5e00e5f18e4102b99552cc3b17dae879c7329f Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Wed, 24 Jan 2024 21:25:18 -0600 Subject: [PATCH 07/15] use LlamaCppCompletionPrompt / Settings from modelfusion for consistency --- .../src/LlamaCppBindingsCompletionModel.ts | 184 ++---------------- .../src/LlamaCppBindingsFacade.ts | 8 +- 2 files changed, 16 insertions(+), 176 deletions(-) diff --git a/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts index e2c2d9c3..eb94705b 100644 --- a/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts +++ b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts @@ -1,217 +1,59 @@ import { TextGenerationBaseModel, TextGenerationModel, - TextGenerationModelSettings, - TextGenerationPromptTemplateProvider, InstructionPrompt, ChatPrompt, BasicTokenizer, FullTokenizer, TextGenerationResult, + LlamaCppCompletionModelSettings, + LlamaCppCompletionPrompt, } from "modelfusion"; -import { AbstractModel } from "modelfusion/model-function/AbstractModel"; - -export interface LlamaCppBindingsCompletionPrompt { - /** - * Text prompt. Images can be included through references such as `[img-ID]`, e.g. `[img-1]`. - */ - text: string; - - /** - * Maps image id to image base data. - */ - images?: Record; -} - -export interface LlamaCppBindingsCompletionModelSettings< - CONTEXT_WINDOW_SIZE extends number | undefined, -> extends TextGenerationModelSettings { - /** - * Specify the context window size of the model that you have loaded in your - * Llama.cpp server. - */ - contextWindowSize?: CONTEXT_WINDOW_SIZE; - - /** - * Adjust the randomness of the generated text (default: 0.8). - */ - temperature?: number; - - /** - * Limit the next token selection to the K most probable tokens (default: 40). - */ - topK?: number; - - /** - * Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P (default: 0.95). - */ - topP?: number; - - /** - * The minimum probability for a token to be considered, relative to the probability of the most likely token (default: 0.05). - */ - minP?: number; - - /** - * Specify the number of tokens from the prompt to retain when the context size is exceeded - * and tokens need to be discarded. By default, this value is set to 0 (meaning no tokens - * are kept). Use -1 to retain all tokens from the prompt. - */ - nKeep?: number; - - /** - * Enable tail free sampling with parameter z (default: 1.0, 1.0 = disabled). - */ - tfsZ?: number; - - /** - * Enable locally typical sampling with parameter p (default: 1.0, 1.0 = disabled). - */ - typicalP?: number; - - /** - * Control the repetition of token sequences in the generated text (default: 1.1). - */ - repeatPenalty?: number; - - /** - * Last n tokens to consider for penalizing repetition (default: 64, 0 = disabled, -1 = ctx-size). - */ - repeatLastN?: number; - - /** - * Penalize newline tokens when applying the repeat penalty (default: true). - */ - penalizeNl?: boolean; - - /** - * Repeat alpha presence penalty (default: 0.0, 0.0 = disabled). - */ - presencePenalty?: number; - - /** - * Repeat alpha frequency penalty (default: 0.0, 0.0 = disabled). - */ - frequencyPenalty?: number; - - /** - * This will replace the prompt for the purpose of the penalty evaluation. - * Can be either null, a string or an array of numbers representing tokens - * (default: null = use the original prompt). - */ - penaltyPrompt?: string | number[]; - - /** - * Enable Mirostat sampling, controlling perplexity during text generation - * (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0). - */ - mirostat?: number; - - /** - * Set the Mirostat target entropy, parameter tau (default: 5.0). - */ - mirostatTau?: number; - - /** - * Set the Mirostat learning rate, parameter eta (default: 0.1). - */ - mirostatEta?: number; - - /** - * Set grammar for grammar-based sampling (default: no grammar) - * - * @see https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md - */ - grammar?: string; - - /** - * Set the random number generator (RNG) seed - * (default: -1, -1 = random seed). - */ - seed?: number; - - /** - * Ignore end of stream token and continue generating (default: false). - */ - ignoreEos?: boolean; - - /** - * Modify the likelihood of a token appearing in the generated text completion. - * For example, use "logit_bias": [[15043,1.0]] to increase the likelihood of the token - * 'Hello', or "logit_bias": [[15043,-1.0]] to decrease its likelihood. - * Setting the value to false, "logit_bias": [[15043,false]] ensures that the token Hello is - * never produced (default: []). - */ - logitBias?: Array<[number, number | false]>; - - /** - * If greater than 0, the response also contains the probabilities of top N tokens - * for each generated token (default: 0) - */ - nProbs?: number; - - /** - * Save the prompt and generation for avoid reprocess entire prompt if a part of this isn't change (default: false) - */ - cachePrompt?: boolean; - - /** - * Assign the completion task to an specific slot. - * If is -1 the task will be assigned to a Idle slot (default: -1) - */ - slotId?: number; - - /** - * Prompt template provider that is used when calling `.withTextPrompt()`, `withInstructionPrompt()` or `withChatPrompt()`. - */ - promptTemplate?: TextGenerationPromptTemplateProvider; -} +import { AbstractModel } from "modelfusion/internal"; export class LlamaCppBindingsCompletionModel< CONTEXT_WINDOW_SIZE extends number | undefined, > - extends AbstractModel< - LlamaCppBindingsCompletionModelSettings - > + extends AbstractModel> implements TextGenerationBaseModel< - LlamaCppBindingsCompletionPrompt, - LlamaCppBindingsCompletionModelSettings + LlamaCppCompletionPrompt, + LlamaCppCompletionModelSettings > { constructor( - settings: LlamaCppBindingsCompletionModelSettings = {} + settings: LlamaCppCompletionModelSettings = {} ) { super({ settings }); } withTextPrompt(): TextGenerationModel< string, - LlamaCppBindingsCompletionModelSettings + LlamaCppCompletionModelSettings > { throw new Error("Method not implemented."); } withInstructionPrompt(): TextGenerationModel< InstructionPrompt, - LlamaCppBindingsCompletionModelSettings + LlamaCppCompletionModelSettings > { throw new Error("Method not implemented."); } withChatPrompt(): TextGenerationModel< ChatPrompt, - LlamaCppBindingsCompletionModelSettings + LlamaCppCompletionModelSettings > { throw new Error("Method not implemented."); } withPromptTemplate(): TextGenerationModel< INPUT_PROMPT, - LlamaCppBindingsCompletionModelSettings + LlamaCppCompletionModelSettings > { throw new Error("Method not implemented."); } contextWindowSize: number | undefined; tokenizer: BasicTokenizer | FullTokenizer | undefined; countPromptTokens: - | ((prompt: LlamaCppBindingsCompletionPrompt) => PromiseLike) + | ((prompt: LlamaCppCompletionPrompt) => PromiseLike) | undefined; doGenerateTexts(): PromiseLike<{ rawResponse: unknown; @@ -242,7 +84,7 @@ export class LlamaCppBindingsCompletionModel< } get settingsForEvent(): Partial< - LlamaCppBindingsCompletionModelSettings + LlamaCppCompletionModelSettings > { return {}; } diff --git a/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsFacade.ts b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsFacade.ts index 6bef7ce0..91267411 100644 --- a/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsFacade.ts +++ b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsFacade.ts @@ -1,10 +1,8 @@ -import { - LlamaCppBindingsCompletionModel, - LlamaCppBindingsCompletionModelSettings, -} from "./LlamaCppBindingsCompletionModel.js"; +import { LlamaCppCompletionModelSettings } from "modelfusion"; +import { LlamaCppBindingsCompletionModel } from "./LlamaCppBindingsCompletionModel.js"; export function CompletionTextGenerator( - settings: LlamaCppBindingsCompletionModelSettings = {} + settings: LlamaCppCompletionModelSettings = {} ) { return new LlamaCppBindingsCompletionModel(settings); } From 124e041059c8008fca7315565fadc5e9e19afbf9 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Thu, 25 Jan 2024 06:48:42 -0600 Subject: [PATCH 08/15] loading native node module with es6 enabled --- .../package.json | 1 + .../src/binding.test.ts | 36 +++++++++---------- .../src/binding.ts | 22 ++++++------ .../tsconfig.json | 6 ++-- 4 files changed, 35 insertions(+), 30 deletions(-) diff --git a/packages/@modelfusion-llamacpp-bindings/package.json b/packages/@modelfusion-llamacpp-bindings/package.json index 4acd837a..78e688ce 100644 --- a/packages/@modelfusion-llamacpp-bindings/package.json +++ b/packages/@modelfusion-llamacpp-bindings/package.json @@ -18,6 +18,7 @@ "type": "git", "url": "git+https://github.com/lgrammel/modelfusion.git" }, + "type": "module", "keywords": [ "modelfusion", "llamacpp", diff --git a/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts b/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts index 17a65773..0c39fb19 100644 --- a/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts +++ b/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts @@ -1,23 +1,23 @@ -const llamacppBindings = require("../dist/binding.js"); -import * as assert from "assert"; +import { LlamaCppBindings } from "./binding.js"; +import assert from "node:assert"; -assert(llamacppBindings, "The expected module is undefined"); -assert(llamacppBindings.getSystemInfo, "The expected method is undefined"); +assert(LlamaCppBindings, "The expected module is undefined"); +assert(LlamaCppBindings.getSystemInfo, "The expected method is undefined"); -function testBasic() -{ - const instance = new llamacppBindings("mr-yeoman"); - assert(instance.greet, "The expected method is not defined"); - assert.strictEqual(instance.greet("kermit"), "mr-yeoman", "Unexpected value returned"); - assert(llamacppBindings.getSystemInfo().length > 2, "System info is too short"); +function testBasic() { + const instance = new LlamaCppBindings("mr-yeoman"); + assert(instance.greet, "The expected method is not defined"); + assert.strictEqual( + instance.greet("kermit"), + "mr-yeoman", + "Unexpected value returned" + ); + assert( + LlamaCppBindings.getSystemInfo().then((info) => info.length > 2), + "System info is too short" + ); } -function testInvalidParams() -{ - const instance = new llamacppBindings(); -} - -assert.doesNotThrow(testBasic, "testBasic threw an expection"); -assert.throws(testInvalidParams, "testInvalidParams didn't throw"); +assert.doesNotThrow(testBasic, "testBasic threw an exception"); -console.log("Tests passed- everything looks OK!"); \ No newline at end of file +console.log("Tests passed- everything looks OK!"); diff --git a/packages/@modelfusion-llamacpp-bindings/src/binding.ts b/packages/@modelfusion-llamacpp-bindings/src/binding.ts index 7fbadc6a..f33698c5 100644 --- a/packages/@modelfusion-llamacpp-bindings/src/binding.ts +++ b/packages/@modelfusion-llamacpp-bindings/src/binding.ts @@ -1,12 +1,19 @@ -// eslint-disable-next-line @typescript-eslint/no-var-requires -const addon = require("../build/Release/llamacpp-bindings-native"); +// Import the native addon +import { createRequire } from "module"; +const require = createRequire(import.meta.url); +const addon = require("../build/Release/llamacpp-bindings-native.node"); -interface IllamacppBindingsNative { +// Correct the spelling in the interface name +export interface ILlamaCppBindingsNative { greet(strName: string): string; } -class llamacppBindings { +export class LlamaCppBindings { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private _addonInstance: ILlamaCppBindingsNative; + constructor(name: string) { + // Use the static _addon member to create a new instance this._addonInstance = new addon.llamacppBindings(name); } @@ -14,13 +21,8 @@ class llamacppBindings { return this._addonInstance.greet(strName); } - // private members - private _addonInstance: IllamacppBindingsNative; - // static members - static getSystemInfo(): string { + static async getSystemInfo(): Promise { return addon.systemInfo(); } } - -export = llamacppBindings; diff --git a/packages/@modelfusion-llamacpp-bindings/tsconfig.json b/packages/@modelfusion-llamacpp-bindings/tsconfig.json index 055ed86a..e8caff85 100644 --- a/packages/@modelfusion-llamacpp-bindings/tsconfig.json +++ b/packages/@modelfusion-llamacpp-bindings/tsconfig.json @@ -2,14 +2,16 @@ "compilerOptions": { "declaration": true, "lib": ["esnext"], - "module": "commonjs", + "module": "ES2020", + "moduleResolution": "node", "outDir": "./dist", "rootDir": "./src", "skipLibCheck": true, "strict": true, - "target": "es6", + "target": "ES2020", "types": ["node"], "typeRoots": ["node_modules/@types"], + "allowSyntheticDefaultImports": true, }, "include": ["src/**/*.ts"], } From 3f42f2f7e2cf105ba62058f7d686541863e5c648 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Thu, 25 Jan 2024 07:10:06 -0600 Subject: [PATCH 09/15] now using vitest for automated tests --- .../package.json | 2 +- .../src/binding.test.ts | 35 +++++++------------ 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/packages/@modelfusion-llamacpp-bindings/package.json b/packages/@modelfusion-llamacpp-bindings/package.json index 78e688ce..bbcb77f6 100644 --- a/packages/@modelfusion-llamacpp-bindings/package.json +++ b/packages/@modelfusion-llamacpp-bindings/package.json @@ -8,7 +8,7 @@ "build": "node-gyp rebuild && tsc", "clean": "node-gyp clean && tsc --build --clean", "pretest": "tsc", - "test": "node --napi-modules ./dist/binding.test.js" + "test": "vitest --run src" }, "gypfile": true, "name": "@modelfusion/llamacpp-bindings", diff --git a/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts b/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts index 0c39fb19..1f40fe68 100644 --- a/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts +++ b/packages/@modelfusion-llamacpp-bindings/src/binding.test.ts @@ -1,23 +1,14 @@ import { LlamaCppBindings } from "./binding.js"; -import assert from "node:assert"; - -assert(LlamaCppBindings, "The expected module is undefined"); -assert(LlamaCppBindings.getSystemInfo, "The expected method is undefined"); - -function testBasic() { - const instance = new LlamaCppBindings("mr-yeoman"); - assert(instance.greet, "The expected method is not defined"); - assert.strictEqual( - instance.greet("kermit"), - "mr-yeoman", - "Unexpected value returned" - ); - assert( - LlamaCppBindings.getSystemInfo().then((info) => info.length > 2), - "System info is too short" - ); -} - -assert.doesNotThrow(testBasic, "testBasic threw an exception"); - -console.log("Tests passed- everything looks OK!"); +import { test, expect } from "vitest"; + +test("testModule", () => { + expect(LlamaCppBindings).toBeDefined(); + expect(LlamaCppBindings.getSystemInfo).toBeDefined(); +}); + +test("testBasic", async () => { + const instance = new LlamaCppBindings("mistral"); + expect(instance.greet).toBeDefined(); + expect(instance.greet("kermit")).toBe("mistral"); + await expect(LlamaCppBindings.getSystemInfo()).resolves.toContain(" | "); +}); From 791aea19761cd6299eb7ff350f22253e0b0bb7d0 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Thu, 25 Jan 2024 07:16:31 -0600 Subject: [PATCH 10/15] put helper scripts into the bin --- .../@modelfusion-llamacpp-bindings/{ => bin}/update_submodules.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/@modelfusion-llamacpp-bindings/{ => bin}/update_submodules.sh (100%) diff --git a/packages/@modelfusion-llamacpp-bindings/update_submodules.sh b/packages/@modelfusion-llamacpp-bindings/bin/update_submodules.sh similarity index 100% rename from packages/@modelfusion-llamacpp-bindings/update_submodules.sh rename to packages/@modelfusion-llamacpp-bindings/bin/update_submodules.sh From 77c712588da4deba6ff245a8fbcc56e4b1198505 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Thu, 25 Jan 2024 07:44:38 -0600 Subject: [PATCH 11/15] revert build ignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d5587b37..ec702d90 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,8 @@ runs node_modules package-lock.json -build +examples/*/build/** +packages/*/build/** packages/*/coverage/** packages/*/dist/** packages/*/lib/** From a0702abdeecfd65fe850d41539c6f90cf804dacb Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Thu, 25 Jan 2024 08:07:16 -0600 Subject: [PATCH 12/15] fix ci workflow --- .github/workflows/continuous-integration.yml | 1 + package.json | 1 + packages/@modelfusion-llamacpp-bindings/package.json | 2 +- turbo.json | 5 +++++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index d9286f4b..31bf0146 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -29,5 +29,6 @@ jobs: cache: "pnpm" - run: pnpm install --frozen-lockfile + - run: pnpm build:setup - run: pnpm build - run: pnpm test diff --git a/package.json b/package.json index 42c42a40..56dc1b18 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "private": true, "scripts": { "build": "turbo build", + "build:setup": "turbo setup", "clean": "rimraf .turbo node_modules/.cache/turbo && pnpm --filter \"./packages/**\" clean && pnpm --filter \"./examples/**\" clean", "lint": "turbo lint", "setup": "husky install", diff --git a/packages/@modelfusion-llamacpp-bindings/package.json b/packages/@modelfusion-llamacpp-bindings/package.json index bbcb77f6..ca21aa85 100644 --- a/packages/@modelfusion-llamacpp-bindings/package.json +++ b/packages/@modelfusion-llamacpp-bindings/package.json @@ -4,7 +4,7 @@ "node-addon-api": "^7.0.0" }, "scripts": { - "setup": "./update_submodules.sh", + "setup": "./bin/update_submodules.sh", "build": "node-gyp rebuild && tsc", "clean": "node-gyp clean && tsc --build --clean", "pretest": "tsc", diff --git a/turbo.json b/turbo.json index 7942284c..f7366337 100644 --- a/turbo.json +++ b/turbo.json @@ -1,6 +1,11 @@ { "$schema": "https://turbo.build/schema.json", "pipeline": { + "setup": { + "dependsOn": ["^setup"], + "env": [], + "outputs": [] + }, "build": { "dependsOn": ["^build"], "env": [], From 928fa8d0ce713da803232d31d751b7c66eac23f4 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Sat, 27 Jan 2024 08:48:17 -0500 Subject: [PATCH 13/15] update lock --- pnpm-lock.yaml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad485d4f..fd3b2d9e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -511,7 +511,7 @@ importers: version: 7.3.0 modelfusion: specifier: 0.131.0 - version: link:../modelfusion/dist + version: 0.131.0 typescript: specifier: ^5.3.3 version: 5.3.3 @@ -8164,7 +8164,6 @@ packages: /eventsource-parser@1.1.1: resolution: {integrity: sha512-3Ej2iLj6ZnX+5CMxqyUb8syl9yVZwcwm8IIMrOJlF7I51zxOOrRlU3zxSb/6hFbl03ts1ZxHAGJdWLZOLyKG7w==} engines: {node: '>=14.18'} - dev: false /execa@5.1.1: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} @@ -9782,7 +9781,6 @@ packages: resolution: {integrity: sha512-biba8u/clw7iesNEWLOLwrNGoBP2lA+hTaBLs/D45pJdUPFXyxD6nhcDVtADChghv4GgyAiMKYMiRx7x6h7Biw==} dependencies: base64-js: 1.5.1 - dev: false /js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -11391,6 +11389,22 @@ packages: - utf-8-validate dev: false + /modelfusion@0.131.0: + resolution: {integrity: sha512-sMYIslWoaOj4SGzwbSexz0w2w0K4xdy0Mxs79xGCAJUgiVS+U4w6eb18291PGMR0whmbxohqmtsaZ+lJuZJE9g==} + engines: {node: '>=18'} + dependencies: + eventsource-parser: 1.1.1 + js-tiktoken: 1.0.7 + nanoid: 3.3.6 + secure-json-parse: 2.7.0 + ws: 8.14.2 + zod: 3.22.4 + zod-to-json-schema: 3.22.3(zod@3.22.4) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: true + /mri@1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} @@ -15769,7 +15783,6 @@ packages: zod: ^3.22.4 dependencies: zod: 3.22.4 - dev: false /zod@3.21.4: resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==} From 3a29f238a4249a0fd56be57c3e3f7f77c80428c8 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Sun, 28 Jan 2024 15:39:20 -0600 Subject: [PATCH 14/15] use LLamaCppCompletionModel as a base class --- .../package.json | 14 ++- .../src/LlamaCppBindingsCompletionModel.ts | 93 +------------------ 2 files changed, 18 insertions(+), 89 deletions(-) diff --git a/packages/@modelfusion-llamacpp-bindings/package.json b/packages/@modelfusion-llamacpp-bindings/package.json index ca21aa85..081318f8 100644 --- a/packages/@modelfusion-llamacpp-bindings/package.json +++ b/packages/@modelfusion-llamacpp-bindings/package.json @@ -1,5 +1,4 @@ { - "main": "lib/binding.js", "dependencies": { "node-addon-api": "^7.0.0" }, @@ -19,6 +18,19 @@ "url": "git+https://github.com/lgrammel/modelfusion.git" }, "type": "module", + "engines": { + "node": ">=18" + }, + "publishConfig": { + "directory": "dist", + "linkDirectory": true + }, + "exports": { + ".": { + "types": "./index.d.ts", + "import": "./index.js" + } + }, "keywords": [ "modelfusion", "llamacpp", diff --git a/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts index eb94705b..339128c9 100644 --- a/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts +++ b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts @@ -1,95 +1,12 @@ import { - TextGenerationBaseModel, - TextGenerationModel, - InstructionPrompt, - ChatPrompt, - BasicTokenizer, - FullTokenizer, - TextGenerationResult, LlamaCppCompletionModelSettings, - LlamaCppCompletionPrompt, + LlamaCppCompletionModel, } from "modelfusion"; -import { AbstractModel } from "modelfusion/internal"; export class LlamaCppBindingsCompletionModel< - CONTEXT_WINDOW_SIZE extends number | undefined, - > - extends AbstractModel> - implements - TextGenerationBaseModel< - LlamaCppCompletionPrompt, - LlamaCppCompletionModelSettings - > -{ - constructor( - settings: LlamaCppCompletionModelSettings = {} - ) { - super({ settings }); - } - withTextPrompt(): TextGenerationModel< - string, - LlamaCppCompletionModelSettings - > { - throw new Error("Method not implemented."); - } - withInstructionPrompt(): TextGenerationModel< - InstructionPrompt, - LlamaCppCompletionModelSettings - > { - throw new Error("Method not implemented."); - } - withChatPrompt(): TextGenerationModel< - ChatPrompt, - LlamaCppCompletionModelSettings - > { - throw new Error("Method not implemented."); - } - withPromptTemplate(): TextGenerationModel< - INPUT_PROMPT, - LlamaCppCompletionModelSettings - > { - throw new Error("Method not implemented."); - } - contextWindowSize: number | undefined; - tokenizer: BasicTokenizer | FullTokenizer | undefined; - countPromptTokens: - | ((prompt: LlamaCppCompletionPrompt) => PromiseLike) - | undefined; - doGenerateTexts(): PromiseLike<{ - rawResponse: unknown; - textGenerationResults: TextGenerationResult[]; - usage?: - | { promptTokens: number; completionTokens: number; totalTokens: number } - | undefined; - }> { - throw new Error("Method not implemented."); - } - restoreGeneratedTexts(): { - rawResponse: unknown; - textGenerationResults: TextGenerationResult[]; - usage?: - | { promptTokens: number; completionTokens: number; totalTokens: number } - | undefined; - } { - throw new Error("Method not implemented."); - } - withJsonOutput(): this { - throw new Error("Method not implemented."); - } - - readonly provider = "llamacpp"; - - get modelName() { - return null; - } - - get settingsForEvent(): Partial< - LlamaCppCompletionModelSettings - > { - return {}; - } - - withSettings(): this { - return this; + CONTEXT_WINDOW_SIZE extends number | undefined, +> extends LlamaCppCompletionModel { + constructor(settings: LlamaCppCompletionModelSettings) { + super(settings); } } From e309265290e5b689f8902e283758ec6bf8f884d6 Mon Sep 17 00:00:00 2001 From: Nick Nance Date: Tue, 6 Feb 2024 07:29:29 -0600 Subject: [PATCH 15/15] initial shell of completion model --- .../package.json | 11 +- .../src/LlamaCppBindingsCompletionModel.ts | 227 +++++++++++++++++- .../src/base/AbstractModel.ts | 27 +++ .../src/base/LlamaCppPrompt.ts | 83 +++++++ .../src/base/convertJsonSchemaToGBNF.ts | 135 +++++++++++ .../src/binding.ts | 7 +- pnpm-lock.yaml | 17 ++ 7 files changed, 491 insertions(+), 16 deletions(-) create mode 100644 packages/@modelfusion-llamacpp-bindings/src/base/AbstractModel.ts create mode 100644 packages/@modelfusion-llamacpp-bindings/src/base/LlamaCppPrompt.ts create mode 100644 packages/@modelfusion-llamacpp-bindings/src/base/convertJsonSchemaToGBNF.ts diff --git a/packages/@modelfusion-llamacpp-bindings/package.json b/packages/@modelfusion-llamacpp-bindings/package.json index 081318f8..5f181b69 100644 --- a/packages/@modelfusion-llamacpp-bindings/package.json +++ b/packages/@modelfusion-llamacpp-bindings/package.json @@ -21,16 +21,7 @@ "engines": { "node": ">=18" }, - "publishConfig": { - "directory": "dist", - "linkDirectory": true - }, - "exports": { - ".": { - "types": "./index.d.ts", - "import": "./index.js" - } - }, + "main": "./dist/index.js", "keywords": [ "modelfusion", "llamacpp", diff --git a/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts index 339128c9..01a5b769 100644 --- a/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts +++ b/packages/@modelfusion-llamacpp-bindings/src/LlamaCppBindingsCompletionModel.ts @@ -1,12 +1,229 @@ import { LlamaCppCompletionModelSettings, - LlamaCppCompletionModel, + LlamaCppCompletionPrompt, + FunctionCallOptions, + LlamaCppTokenizer, + TextStreamingBaseModel, + BasicTokenizer, + ChatPrompt, + Delta, + FullTokenizer, + InstructionPrompt, + JsonSchemaProducer, + Schema, + TextGenerationPromptTemplate, + TextGenerationResult, + TextStreamingModel, + FlexibleObjectFromTextPromptTemplate, + ObjectFromTextPromptTemplate, + ObjectFromTextStreamingModel, + PromptTemplateTextStreamingModel, + TextGenerationPromptTemplateProvider, + textGenerationModelProperties, } from "modelfusion"; +import { Text } from "./base/LlamaCppPrompt"; +import { convertJsonSchemaToGBNF } from "./base/convertJsonSchemaToGBNF"; +import { AbstractModel } from "./base/AbstractModel"; +import { LlamaCppBindings } from "./binding"; + export class LlamaCppBindingsCompletionModel< - CONTEXT_WINDOW_SIZE extends number | undefined, -> extends LlamaCppCompletionModel { - constructor(settings: LlamaCppCompletionModelSettings) { - super(settings); + CONTEXT_WINDOW_SIZE extends number | undefined, + > + extends AbstractModel> + implements + TextStreamingBaseModel< + LlamaCppCompletionPrompt, + LlamaCppCompletionModelSettings + > +{ + constructor( + settings: LlamaCppCompletionModelSettings = {} + ) { + super({ settings }); + this.tokenizer = new LlamaCppTokenizer(this.settings.api); + } + + readonly provider = "llamacppbindgins"; + contextWindowSize: number | undefined; + tokenizer: BasicTokenizer | FullTokenizer | undefined; + + get modelName() { + return null; + } + + get settingsForEvent(): Partial< + LlamaCppCompletionModelSettings + > { + const eventSettingProperties: Array = [ + ...textGenerationModelProperties, + "contextWindowSize", + "temperature", + "topK", + "topP", + "minP", + "nKeep", + "tfsZ", + "typicalP", + "repeatPenalty", + "repeatLastN", + "penalizeNl", + "presencePenalty", + "frequencyPenalty", + "penaltyPrompt", + "mirostat", + "mirostatTau", + "mirostatEta", + "grammar", + "seed", + "ignoreEos", + "logitBias", + "nProbs", + "cachePrompt", + "slotId", + ] satisfies (keyof LlamaCppCompletionModelSettings)[]; + + return Object.fromEntries( + Object.entries(this.settings).filter(([key]) => + eventSettingProperties.includes(key) + ) + ); + } + + private get promptTemplateProvider(): TextGenerationPromptTemplateProvider { + return this.settings.promptTemplate ?? Text; + } + + withTextPrompt(): TextStreamingModel< + string, + LlamaCppCompletionModelSettings + > { + throw new Error("Method not implemented."); + } + + withInstructionPrompt(): TextStreamingModel< + InstructionPrompt, + LlamaCppCompletionModelSettings + > { + return this.withPromptTemplate(this.promptTemplateProvider.instruction()); + } + + withChatPrompt(): PromptTemplateTextStreamingModel< + ChatPrompt, + LlamaCppCompletionPrompt, + LlamaCppCompletionModelSettings, + this + > { + return this.withPromptTemplate(this.promptTemplateProvider.chat()); + } + + /** + * Maps the prompt for the full Llama.cpp prompt template (incl. image support). + */ + withPromptTemplate( + promptTemplate: TextGenerationPromptTemplate< + INPUT_PROMPT, + LlamaCppCompletionPrompt + > + ): PromptTemplateTextStreamingModel< + INPUT_PROMPT, + LlamaCppCompletionPrompt, + LlamaCppCompletionModelSettings, + this + > { + return new PromptTemplateTextStreamingModel({ + model: this.withSettings({ + stopSequences: [ + ...(this.settings.stopSequences ?? []), + ...promptTemplate.stopSequences, + ], + }), + promptTemplate, + }); + } + + withSettings( + additionalSettings: Partial< + LlamaCppCompletionModelSettings + > + ) { + return new LlamaCppBindingsCompletionModel( + Object.assign({}, this.settings, additionalSettings) + ) as this; + } + + doStreamText( + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: LlamaCppCompletionPrompt, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + options?: FunctionCallOptions | undefined + ): PromiseLike>> { + throw new Error("Method not implemented."); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + extractTextDelta(delta: unknown): string | undefined { + throw new Error("Method not implemented."); + } + + countPromptTokens: + | ((prompt: LlamaCppCompletionPrompt) => PromiseLike) + | undefined; + + async doGenerateTexts( + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: LlamaCppCompletionPrompt, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + options?: FunctionCallOptions | undefined + ): Promise<{ + rawResponse: unknown; + textGenerationResults: TextGenerationResult[]; + usage?: + | { promptTokens: number; completionTokens: number; totalTokens: number } + | undefined; + }> { + const info = await LlamaCppBindings.getSystemInfo(); + console.log(info); + throw new Error("Method not implemented."); + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + restoreGeneratedTexts(rawResponse: unknown): { + rawResponse: unknown; + textGenerationResults: TextGenerationResult[]; + usage?: + | { promptTokens: number; completionTokens: number; totalTokens: number } + | undefined; + } { + throw new Error("Method not implemented."); + } + + asObjectGenerationModel( + promptTemplate: + | ObjectFromTextPromptTemplate + | FlexibleObjectFromTextPromptTemplate + ) { + return "adaptModel" in promptTemplate + ? new ObjectFromTextStreamingModel({ + model: promptTemplate.adaptModel(this), + template: promptTemplate, + }) + : new ObjectFromTextStreamingModel({ + model: this as TextStreamingModel, + template: promptTemplate, + }); + } + + withJsonOutput(schema: Schema & JsonSchemaProducer): this { + // don't override the grammar if it's already set (to allow user to override) + if (this.settings.grammar != null) { + return this; + } + + const grammar = convertJsonSchemaToGBNF(schema.getJsonSchema()); + + return this.withSettings({ + grammar: grammar, + }); } } diff --git a/packages/@modelfusion-llamacpp-bindings/src/base/AbstractModel.ts b/packages/@modelfusion-llamacpp-bindings/src/base/AbstractModel.ts new file mode 100644 index 00000000..cfcb1a25 --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/src/base/AbstractModel.ts @@ -0,0 +1,27 @@ +import { ModelInformation } from "modelfusion"; +import { Model, ModelSettings } from "modelfusion"; + +export abstract class AbstractModel + implements Model +{ + readonly settings: SETTINGS; + + constructor({ settings }: { settings: SETTINGS }) { + this.settings = settings; + } + + abstract readonly provider: string; + abstract readonly modelName: string | null; + + // implemented as a separate accessor to remove all other properties from the model + get modelInformation(): ModelInformation { + return { + provider: this.provider, + modelName: this.modelName, + }; + } + + abstract get settingsForEvent(): Partial; + + abstract withSettings(additionalSettings: Partial): this; +} diff --git a/packages/@modelfusion-llamacpp-bindings/src/base/LlamaCppPrompt.ts b/packages/@modelfusion-llamacpp-bindings/src/base/LlamaCppPrompt.ts new file mode 100644 index 00000000..8e7ce526 --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/src/base/LlamaCppPrompt.ts @@ -0,0 +1,83 @@ +import { + LlamaCppCompletionPrompt, + TextGenerationPromptTemplate, + TextGenerationPromptTemplateProvider, + MistralInstructPrompt, + ChatMLPrompt, + Llama2Prompt, + NeuralChatPrompt, + AlpacaPrompt, + SynthiaPrompt, + VicunaPrompt, + TextPrompt, +} from "modelfusion"; + +export function asLlamaCppPromptTemplate( + promptTemplate: TextGenerationPromptTemplate +): TextGenerationPromptTemplate { + return { + format: (prompt) => ({ + text: promptTemplate.format(prompt), + }), + stopSequences: promptTemplate.stopSequences, + }; +} + +export function asLlamaCppTextPromptTemplateProvider( + promptTemplateProvider: TextGenerationPromptTemplateProvider +): TextGenerationPromptTemplateProvider { + return { + text: () => asLlamaCppPromptTemplate(promptTemplateProvider.text()), + + instruction: () => + asLlamaCppPromptTemplate(promptTemplateProvider.instruction()), + + chat: () => asLlamaCppPromptTemplate(promptTemplateProvider.chat()), + }; +} + +export const Text = asLlamaCppTextPromptTemplateProvider(TextPrompt); + +/** + * Formats text, instruction or chat prompts as a Mistral instruct prompt. + * + * Note that Mistral does not support system prompts. We emulate them. + * + * Text prompt: + * ``` + * [INST] { instruction } [/INST] + * ``` + * + * Instruction prompt when system prompt is set: + * ``` + * [INST] ${ system prompt } [/INST] [INST] ${instruction} [/INST] ${ response prefix } + * ``` + * + * Instruction prompt template when there is no system prompt: + * ``` + * [INST] ${ instruction } [/INST] ${ response prefix } + * ``` + * + * Chat prompt when system prompt is set: + * ``` + * [INST] ${ system prompt } [/INST] [INST] ${ user msg 1 } [/INST] ${ model response 1 } [INST] ${ user msg 2 } [/INST] ${ model response 2 } [INST] ${ user msg 3 } [/INST] + * ``` + * + * Chat prompt when there is no system prompt: + * ``` + * [INST] ${ user msg 1 } [/INST] ${ model response 1 } [INST] ${ user msg 2 } [/INST] ${ model response 2 } [INST] ${ user msg 3 } [/INST] + * ``` + * + * @see https://docs.mistral.ai/models/#chat-template + */ +export const Mistral = asLlamaCppTextPromptTemplateProvider( + MistralInstructPrompt +); + +export const ChatML = asLlamaCppTextPromptTemplateProvider(ChatMLPrompt); +export const Llama2 = asLlamaCppTextPromptTemplateProvider(Llama2Prompt); +export const NeuralChat = + asLlamaCppTextPromptTemplateProvider(NeuralChatPrompt); +export const Alpaca = asLlamaCppTextPromptTemplateProvider(AlpacaPrompt); +export const Synthia = asLlamaCppTextPromptTemplateProvider(SynthiaPrompt); +export const Vicuna = asLlamaCppTextPromptTemplateProvider(VicunaPrompt); diff --git a/packages/@modelfusion-llamacpp-bindings/src/base/convertJsonSchemaToGBNF.ts b/packages/@modelfusion-llamacpp-bindings/src/base/convertJsonSchemaToGBNF.ts new file mode 100644 index 00000000..fb8431c9 --- /dev/null +++ b/packages/@modelfusion-llamacpp-bindings/src/base/convertJsonSchemaToGBNF.ts @@ -0,0 +1,135 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +/** + * Convert JSON Schema to a GBNF grammar. + * + * This is a modified version of + * https://github.com/ggerganov/llama.cpp/blob/master/examples/server/public/json-schema-to-grammar.mjs + */ +export function convertJsonSchemaToGBNF(schema: unknown): string { + const rules = new RuleMap(); + + rules.add("space", SPACE_RULE); + + visit(schema, undefined, rules); + + return rules.toGBNF(); +} +const SPACE_RULE = '" "?'; + +const PRIMITIVE_RULES = { + boolean: '("true" | "false") space', + number: + '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space', + integer: '("-"? ([0-9] | [1-9] [0-9]*)) space', + string: ` "\\"" ( [^"\\\\] | "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) )* "\\"" space`, + null: '"null" space', +} as Record; + +class RuleMap { + readonly rules = new Map(); + + add(name: string, rule: string): string { + const escapedName = this.escapeRuleName(name, rule); + this.rules.set(escapedName, rule); + return escapedName; + } + + /** + * Replace invalid characters in rule name with hyphens. + * Disambiguate the name if it already exists. + */ + private escapeRuleName(name: string, rule: string) { + const baseName = name.replace(/[^\dA-Za-z-]+/g, "-"); + + if (!this.rules.has(baseName) || this.rules.get(baseName) === rule) { + return baseName; + } + + let i = 0; + while (this.rules.has(`${baseName}${i}`)) { + if (this.rules.get(`${baseName}${i}`) === rule) { + return `${baseName}${i}`; + } + + i++; + } + + return `${baseName}${i}`; + } + + toGBNF() { + return Array.from(this.rules) + .map(([name, rule]) => `${name} ::= ${rule}`) + .join("\n"); + } +} + +const GRAMMAR_LITERAL_ESCAPES = { + "\r": "\\r", + "\n": "\\n", + '"': '\\"', +} as Record; + +function formatLiteral(literal: string) { + const escaped = JSON.stringify(literal).replace( + /[\n\r"]/g, + (m) => GRAMMAR_LITERAL_ESCAPES[m] + ); + + return `"${escaped}"`; +} + +function visit(schema: any, name: string | undefined, rules: RuleMap): string { + const schemaType = schema.type; + const ruleName = name || "root"; + + if (schema.oneOf || schema.anyOf) { + const rule = (schema.oneOf || schema.anyOf) + .map((altSchema: any, i: number) => + visit(altSchema, `${name}${name ? "-" : ""}${i}`, rules) + ) + .join(" | "); + + return rules.add(ruleName, rule); + } else if ("const" in schema) { + return rules.add(ruleName, formatLiteral(schema.const)); + } else if ("enum" in schema) { + const rule = schema.enum.map(formatLiteral).join(" | "); + return rules.add(ruleName, rule); + } else if (schemaType === "object" && "properties" in schema) { + const propPairs = Object.entries(schema.properties); + + let rule = '"{" space'; + propPairs.forEach(([propName, propSchema], i) => { + const propRuleName = visit( + propSchema, + `${name ?? ""}${name ? "-" : ""}${propName}`, + rules + ); + if (i > 0) { + rule += ' "," space'; + } + rule += ` ${formatLiteral(propName)} space ":" space ${propRuleName}`; + }); + rule += ' "}" space'; + + return rules.add(ruleName, rule); + } else if (schemaType === "array" && "items" in schema) { + const itemRuleName = visit( + schema.items, + `${name ?? ""}${name ? "-" : ""}item`, + rules + ); + const rule = `"[" space (${itemRuleName} ("," space ${itemRuleName})*)? "]" space`; + return rules.add(ruleName, rule); + } else { + if (!PRIMITIVE_RULES[schemaType]) { + throw new Error(`Unrecognized schema: ${JSON.stringify(schema)}`); + } + return rules.add( + ruleName === "root" ? "root" : schemaType, + PRIMITIVE_RULES[schemaType] + ); + } +} diff --git a/packages/@modelfusion-llamacpp-bindings/src/binding.ts b/packages/@modelfusion-llamacpp-bindings/src/binding.ts index f33698c5..61375c73 100644 --- a/packages/@modelfusion-llamacpp-bindings/src/binding.ts +++ b/packages/@modelfusion-llamacpp-bindings/src/binding.ts @@ -8,7 +8,12 @@ export interface ILlamaCppBindingsNative { greet(strName: string): string; } -export class LlamaCppBindings { +export interface ILlamaCppBindingsNativeConstructor { + new (name: string): ILlamaCppBindingsNative; + getSystemInfo(): Promise; +} + +export class LlamaCppBindings implements ILlamaCppBindingsNative { // eslint-disable-next-line @typescript-eslint/no-explicit-any private _addonInstance: ILlamaCppBindingsNative; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0bbc0f9a..343112bc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -518,6 +518,7 @@ importers: typescript: specifier: ^5.3.3 version: 5.3.3 + publishDirectory: dist packages/@modelfusion-pinecone: devDependencies: @@ -11365,6 +11366,22 @@ packages: obliterator: 2.0.4 dev: false + /modelfusion@0.131.0: + resolution: {integrity: sha512-sMYIslWoaOj4SGzwbSexz0w2w0K4xdy0Mxs79xGCAJUgiVS+U4w6eb18291PGMR0whmbxohqmtsaZ+lJuZJE9g==} + engines: {node: '>=18'} + dependencies: + eventsource-parser: 1.1.1 + js-tiktoken: 1.0.7 + nanoid: 3.3.6 + secure-json-parse: 2.7.0 + ws: 8.14.2 + zod: 3.22.4 + zod-to-json-schema: 3.22.3(zod@3.22.4) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: true + /mri@1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'}