diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c2ddf5c..d72541a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -41,7 +41,7 @@ These instructions define the conventions for ElBruno's .NET NuGet library repos ## Project Conventions ### Library Projects -- Target `net8.0;net10.0` (multi-target: LTS + latest) +- Target `net10.0` only (single target). .NET 8 support was dropped ahead of its end of support. - Package naming: `{Owner}.{ProjectName}.{Feature}` (e.g., `ElBruno.QRCodeGenerator.Svg`) - Each packable project must include the NuGet icon: ```xml @@ -52,14 +52,14 @@ These instructions define the conventions for ElBruno's .NET NuGet library repos - Enable deterministic CI builds: `true` ### Test Projects -- Target `net8.0` only (single target) +- Target `net10.0` only (single target) - Framework: xUnit with coverlet.collector - Naming: `{LibraryProject}.Tests` - Location: `src/tests/{LibraryProject}.Tests/` - Set `false` and `true` ### Tool Projects (dotnet global tools) -- Target `net8.0` only (single target β€” CI runners may not have preview SDKs) +- Target `net10.0` only (single target) - Set `true` and `{toolname}` ### Sample Projects @@ -111,7 +111,7 @@ Use `"rollForward": "latestMajor"` for flexibility across dev machines and CI: ```json { "sdk": { - "version": "8.0.0", + "version": "10.0.0", "rollForward": "latestMajor" } } @@ -154,10 +154,9 @@ Thumbs.db - **Triggers:** push to `main`, PR to `main` - **Runner:** `ubuntu-latest` -- **SDK:** `dotnet-version: 8.0.x` +- **SDK:** `dotnet-version: 10.0.x` - **Commands use solution-level operations** (not per-project) -- **Use `-p:TargetFrameworks=net8.0`** for restore and build (CI runners may not have preview SDKs) -- **Use `--framework net8.0`** for test +- Projects single-target `net10.0`, so no per-framework overrides are needed ```yaml name: CI Build @@ -176,13 +175,13 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.x + dotnet-version: 10.0.x - name: Restore - run: dotnet restore {ProjectName}.slnx -p:TargetFrameworks=net8.0 + run: dotnet restore {ProjectName}.slnx - name: Build - run: dotnet build {ProjectName}.slnx --no-restore -p:TargetFrameworks=net8.0 + run: dotnet build {ProjectName}.slnx --no-restore - name: Test - run: dotnet test {ProjectName}.slnx --no-build --framework net8.0 + run: dotnet test {ProjectName}.slnx --no-build ``` --- @@ -225,21 +224,21 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.x + dotnet-version: 10.0.x - name: Determine version id: version run: | # Extract from release tag, input, or csproj # Validate: ^[0-9]+\.[0-9]+\.[0-9]+ - name: Restore - run: dotnet restore {Solution}.slnx -p:TargetFrameworks=net8.0 + run: dotnet restore {Solution}.slnx - name: Build - run: dotnet build {Solution}.slnx -c Release --no-restore -p:TargetFrameworks=net8.0 -p:Version=${{ steps.version.outputs.version }} + run: dotnet build {Solution}.slnx -c Release --no-restore -p:Version=${{ steps.version.outputs.version }} - name: Test - run: dotnet test {Solution}.slnx -c Release --no-build --framework net8.0 + run: dotnet test {Solution}.slnx -c Release --no-build - name: Pack run: | - dotnet pack src/{Package}/{Package}.csproj -c Release --no-build -p:TargetFrameworks=net8.0 -p:Version=${{ steps.version.outputs.version }} --output ./nupkgs + dotnet pack src/{Package}/{Package}.csproj -c Release --no-build -p:Version=${{ steps.version.outputs.version }} --output ./nupkgs # Repeat for each packable project - name: NuGet login (OIDC trusted publishing) uses: NuGet/login@v1 @@ -292,7 +291,7 @@ Follow this exact order: - Test project naming: `{LibraryProject}.Tests` - Tests live in `src/tests/{TestProject}/` - All tests must pass before publishing (enforced in both CI and publish workflows) -- Test projects target `net8.0` only +- Test projects target `net10.0` only --- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05d1d79..d647b8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,18 +18,16 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: | - 8.0.x - 10.0.x + dotnet-version: 10.0.x - name: Restore - run: dotnet restore ElBruno.LocalLLMs.slnx -p:TargetFramework=net8.0 -p:TargetFrameworks=net8.0 -p:NoBuild=false + run: dotnet restore ElBruno.LocalLLMs.slnx -p:NoBuild=false - name: Build - run: dotnet build ElBruno.LocalLLMs.slnx --no-restore -p:TargetFramework=net8.0 -p:TargetFrameworks=net8.0 -p:NoBuild=false + run: dotnet build ElBruno.LocalLLMs.slnx --no-restore -p:NoBuild=false - name: Test - run: dotnet test ElBruno.LocalLLMs.slnx --no-build --framework net8.0 --verbosity normal --logger "trx;LogFileName=test-results.trx" + run: dotnet test ElBruno.LocalLLMs.slnx --no-build --verbosity normal --logger "trx;LogFileName=test-results.trx" - name: Upload test results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/publish-bitnet-native.yml b/.github/workflows/publish-bitnet-native.yml index 7599101..0961f9a 100644 --- a/.github/workflows/publish-bitnet-native.yml +++ b/.github/workflows/publish-bitnet-native.yml @@ -30,7 +30,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.x + dotnet-version: 10.0.x - name: Determine version id: version diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 712838d..8abe957 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,9 +23,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: | - 8.0.x - 10.0.x + dotnet-version: 10.0.x - name: Determine version id: version @@ -54,12 +52,15 @@ jobs: dotnet build src/ElBruno.LocalLLMs.Rag/ElBruno.LocalLLMs.Rag.csproj -c Release --no-restore -p:Version=${{ steps.version.outputs.version }} dotnet build src/ElBruno.LocalLLMs.BitNet/ElBruno.LocalLLMs.BitNet.csproj -c Release --no-restore -p:Version=${{ steps.version.outputs.version }} dotnet build src/ElBruno.LocalLLMs.BlazorComponents/ElBruno.LocalLLMs.BlazorComponents.csproj -c Release --no-restore -p:Version=${{ steps.version.outputs.version }} + dotnet build src/ElBruno.LocalLLMs.Decisions/ElBruno.LocalLLMs.Decisions.csproj -c Release --no-restore -p:Version=${{ steps.version.outputs.version }} - name: Unit Tests run: | - dotnet test src/tests/ElBruno.LocalLLMs.Tests/ElBruno.LocalLLMs.Tests.csproj -c Release --framework net8.0 -p:Version=${{ steps.version.outputs.version }} --verbosity normal - dotnet test src/tests/ElBruno.LocalLLMs.Rag.Tests/ElBruno.LocalLLMs.Rag.Tests.csproj -c Release --framework net8.0 -p:Version=${{ steps.version.outputs.version }} --verbosity normal - dotnet test src/tests/ElBruno.LocalLLMs.BitNet.Tests/ElBruno.LocalLLMs.BitNet.Tests.csproj -c Release --framework net8.0 -p:Version=${{ steps.version.outputs.version }} --verbosity normal + dotnet test src/tests/ElBruno.LocalLLMs.Tests/ElBruno.LocalLLMs.Tests.csproj -c Release -p:Version=${{ steps.version.outputs.version }} --verbosity normal + dotnet test src/tests/ElBruno.LocalLLMs.Rag.Tests/ElBruno.LocalLLMs.Rag.Tests.csproj -c Release -p:Version=${{ steps.version.outputs.version }} --verbosity normal + dotnet test src/tests/ElBruno.LocalLLMs.BitNet.Tests/ElBruno.LocalLLMs.BitNet.Tests.csproj -c Release -p:Version=${{ steps.version.outputs.version }} --verbosity normal + dotnet test src/tests/ElBruno.LocalLLMs.Decisions.Tests/ElBruno.LocalLLMs.Decisions.Tests.csproj -c Release -p:Version=${{ steps.version.outputs.version }} --verbosity normal + dotnet test src/tests/ElBruno.LocalLLMs.BlazorComponents.Tests/ElBruno.LocalLLMs.BlazorComponents.Tests.csproj -c Release -p:Version=${{ steps.version.outputs.version }} --verbosity normal - name: Validate README What's New policy run: | @@ -83,6 +84,7 @@ jobs: dotnet pack src/ElBruno.LocalLLMs.Rag/ElBruno.LocalLLMs.Rag.csproj -c Release --no-build -p:Version=${{ steps.version.outputs.version }} -o artifacts dotnet pack src/ElBruno.LocalLLMs.BitNet/ElBruno.LocalLLMs.BitNet.csproj -c Release --no-build -p:Version=${{ steps.version.outputs.version }} -o artifacts dotnet pack src/ElBruno.LocalLLMs.BlazorComponents/ElBruno.LocalLLMs.BlazorComponents.csproj -c Release --no-build -p:Version=${{ steps.version.outputs.version }} -o artifacts + dotnet pack src/ElBruno.LocalLLMs.Decisions/ElBruno.LocalLLMs.Decisions.csproj -c Release --no-build -p:Version=${{ steps.version.outputs.version }} -o artifacts - name: Validate packed assembly versions shell: pwsh run: ./scripts/Validate-PackageAssemblyVersions.ps1 -PackageDirectory ./artifacts diff --git a/.github/workflows/squad-ci.yml b/.github/workflows/squad-ci.yml index 42a433b..8ae30df 100644 --- a/.github/workflows/squad-ci.yml +++ b/.github/workflows/squad-ci.yml @@ -1,9 +1,12 @@ name: Squad CI -# dotnet project β€” configure build/test commands below + +# Real build and test for the dev-first branches. +# PRs targeting main are covered by ci.yml, so main is deliberately absent +# here to avoid running the same suite twice on the same commit. on: pull_request: - branches: [dev, preview, main, insider] + branches: [dev, preview, insider] types: [opened, synchronize, reopened] push: branches: [dev, insider] @@ -17,12 +20,25 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Build and test - run: | - # TODO: Add your dotnet build/test commands here - # Go: go test ./... - # Python: pip install -r requirements.txt && pytest - # .NET: dotnet test - # Java (Maven): mvn test - # Java (Gradle): ./gradlew test - echo "No build commands configured β€” update squad-ci.yml" + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + + # -p:NoBuild=false is required: the BitNet native projects default it to + # true, which otherwise makes restore and build silently skip them. + - name: Restore + run: dotnet restore ElBruno.LocalLLMs.slnx -p:NoBuild=false + + - name: Build + run: dotnet build ElBruno.LocalLLMs.slnx --no-restore -p:NoBuild=false + + - name: Test + run: dotnet test ElBruno.LocalLLMs.slnx --no-build --verbosity normal --logger "trx;LogFileName=test-results.trx" + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results + path: "**/test-results.trx" diff --git a/Directory.Build.props b/Directory.Build.props index 9f97ccf..8bf1b7a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -23,12 +23,13 @@ Copyright Β© Bruno Capuano $([System.DateTime]::Now.Year) MIT nuget_logo.png - 0.21.0 + 0.22.0 true + Or '$(MSBuildProjectName)' == 'ElBruno.LocalLLMs.BlazorComponents' + Or '$(MSBuildProjectName)' == 'ElBruno.LocalLLMs.Decisions'">true diff --git a/ElBruno.LocalLLMs.slnx b/ElBruno.LocalLLMs.slnx index 737b6fe..f720c32 100644 --- a/ElBruno.LocalLLMs.slnx +++ b/ElBruno.LocalLLMs.slnx @@ -2,6 +2,7 @@ + @@ -14,6 +15,7 @@ + @@ -34,6 +36,8 @@ + + diff --git a/README.md b/README.md index bc2463f..ef56ed9 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Build Status](https://github.com/elbruno/ElBruno.LocalLLMs/actions/workflows/ci.yml/badge.svg)](https://github.com/elbruno/ElBruno.LocalLLMs/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](LICENSE) [![HuggingFace](https://img.shields.io/badge/πŸ€—_HuggingFace-ONNX_Models-orange?style=flat-square)](https://huggingface.co/elbruno) -[![.NET](https://img.shields.io/badge/.NET-8.0%20|%2010.0-512BD4?style=flat-square&logo=dotnet)](https://dotnet.microsoft.com/) +[![.NET](https://img.shields.io/badge/.NET-10.0-512BD4?style=flat-square&logo=dotnet)](https://dotnet.microsoft.com/) [![GitHub stars](https://img.shields.io/github/stars/elbruno/ElBruno.LocalLLMs?style=social)](https://github.com/elbruno/ElBruno.LocalLLMs) [![Twitter Follow](https://img.shields.io/twitter/follow/elbruno?style=social)](https://twitter.com/elbruno) @@ -17,15 +17,16 @@ Run local LLMs in .NET through `IChatClient` β€” the same interface you'd use fo > The last 5 notable additions to the library. Updated with each NuGet release. +- 🎯 **`ElBruno.LocalLLMs.Decisions`** β€” new package for local **System One** decision models. Typed `choice`, `score` and yes/no answers with full probability distributions in a single forward pass (~600 ms on CPU), running [Laya](https://github.com/NandhaKishorM/laya) **fully in-process on ONNX Runtime** β€” no server, no Python. Built for routing, triage, moderation and intent detection, where a chat model is slow and overkill. Call `services.AddLocalDecisions()` and inject `IDecisionClient`. See the [Local Decisions Guide](docs/decisions-guide.md) and the [LocalDecisions sample](src/samples/LocalDecisions/). +- ⬆️ **.NET 10 only** β€” every project now single-targets `net10.0`. .NET 8 reaches end of support in November 2026, so multi-targeting was dropped ahead of it. **Breaking:** consumers still on .NET 8 must stay on `v0.21.0` or upgrade to .NET 10. - 🧩 **`ElBruno.LocalLLMs.BlazorComponents`** β€” new Razor Class Library with 7 ready-to-use Blazor components: `ModelStatusCard` (download progress bar + actions), `ModelGallery` (filterable grid), `ModelSelector` (two-way-bindable dropdown), `ChatBox` (streaming token display), `EnvironmentDashboard` (CPU/CUDA/DirectML badges), `LocalLLMHealthBadge` (nav-bar status dot), and `RagPlayground`. Call `services.AddLocalLLMsBlazorComponents()` to register. See the [Blazor Components Guide](docs/blazor-components.md) and the [BlazorDemo sample](src/samples/BlazorDemo/). - 🧠 **GPT-OSS 20B support** β€” OpenAI's open-weight MoE model (Apache-2.0) now runs locally via the official `onnxruntime/gpt-oss-20b-onnx` artifacts. Adds the **Harmony** prompt format, channel-aware output filtering (chain-of-thought is stripped, never shown to users), Harmony tool calling, and a `ReasoningEffort` option. Two model IDs: `gpt-oss-20b` (CPU INT4) and `gpt-oss-20b-cuda`. See the [GptOssChat sample](src/samples/GptOssChat/). Also fixes a token-duplication bug that repeated the final token of every generation. -- πŸ” **`v0.21.0`** β€” Clean re-publish after `v0.20.12` failed to propagate on NuGet.org; carries forward the issue #49 assembly-version fix and issue #51 vision-probe hardening. - πŸš€ **`v0.20.12`** β€” Corrects sibling-package assembly versions, hardens vision token probing against model context limits, and verifies Fara smart image resizing for screenshot workflows. -- ⬆️ **`v0.20.9`** β€” Upgraded `onnxruntime-genai` to **0.15.1** and `Microsoft.Extensions.AI.Abstractions` to **10.8.3** across all projects. No API changes. ## Features - 🧩 **Blazor components** β€” `ModelStatusCard`, `ChatBox`, `ModelGallery`, `ModelSelector`, `EnvironmentDashboard`, `LocalLLMHealthBadge`, `RagPlayground` via `ElBruno.LocalLLMs.BlazorComponents` ([guide](docs/blazor-components.md)) +- 🎯 **Local decision models** β€” typed choice/score/yes-no answers with probabilities in one forward pass via `ElBruno.LocalLLMs.Decisions` ([guide](docs/decisions-guide.md)) - πŸ”Œ **`IChatClient` implementation** β€” seamless integration with [Microsoft.Extensions.AI](https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai) - πŸ“¦ **Automatic model download** β€” models are fetched from HuggingFace on first use - πŸš€ **Zero friction** β€” works out of the box with sensible defaults (Phi-3.5 mini) @@ -47,6 +48,7 @@ Run local LLMs in .NET through `IChatClient` β€” the same interface you'd use fo | `ElBruno.LocalLLMs.Rag` | [![NuGet](https://img.shields.io/nuget/v/ElBruno.LocalLLMs.Rag.svg?style=flat-square)](https://www.nuget.org/packages/ElBruno.LocalLLMs.Rag) | [![Downloads](https://img.shields.io/nuget/dt/ElBruno.LocalLLMs.Rag.svg?style=flat-square)](https://www.nuget.org/packages/ElBruno.LocalLLMs.Rag) | RAG pipeline β€” document chunking, indexing, retrieval | | `ElBruno.LocalLLMs.BitNet` | [![NuGet](https://img.shields.io/nuget/v/ElBruno.LocalLLMs.BitNet.svg?style=flat-square)](https://www.nuget.org/packages/ElBruno.LocalLLMs.BitNet) | [![Downloads](https://img.shields.io/nuget/dt/ElBruno.LocalLLMs.BitNet.svg?style=flat-square)](https://www.nuget.org/packages/ElBruno.LocalLLMs.BitNet) | BitNet 1.58-bit models via bitnet.cpp + IChatClient | | `ElBruno.LocalLLMs.BlazorComponents` | [![NuGet](https://img.shields.io/nuget/v/ElBruno.LocalLLMs.BlazorComponents.svg?style=flat-square)](https://www.nuget.org/packages/ElBruno.LocalLLMs.BlazorComponents) | [![Downloads](https://img.shields.io/nuget/dt/ElBruno.LocalLLMs.BlazorComponents.svg?style=flat-square)](https://www.nuget.org/packages/ElBruno.LocalLLMs.BlazorComponents) | Blazor components β€” ModelStatusCard, ChatBox, ModelGallery, and more | +| `ElBruno.LocalLLMs.Decisions` | [![NuGet](https://img.shields.io/nuget/v/ElBruno.LocalLLMs.Decisions.svg?style=flat-square)](https://www.nuget.org/packages/ElBruno.LocalLLMs.Decisions) | [![Downloads](https://img.shields.io/nuget/dt/ElBruno.LocalLLMs.Decisions.svg?style=flat-square)](https://www.nuget.org/packages/ElBruno.LocalLLMs.Decisions) | System One decision models β€” typed choice, score and yes/no answers | ## Installation @@ -253,6 +255,53 @@ var options = new LocalLLMsOptions See [docs/observability.md](docs/observability.md) for the lifecycle event contract, metric names, and Aspire wiring notes, and [docs/cancellation.md](docs/cancellation.md) for voice barge-in cancellation behavior. +## Local Decisions + +Sometimes you don't need prose β€” you need a decision. `ElBruno.LocalLLMs.Decisions` runs a +**System One** model that returns a *typed* answer with its full probability distribution in a +single forward pass, fast enough to call on every request. + +```bash +dotnet add package ElBruno.LocalLLMs.Decisions +``` + +The model runs **in-process on ONNX Runtime** β€” no server to start and no Python. The weights are +downloaded from [`elbruno/laya-onnx`](https://huggingface.co/elbruno/laya-onnx) on first use +(about 800 MB) and cached afterwards. + +Ask several questions at once β€” they share one forward pass, so four cost about what one costs: + +```csharp +using var client = new LayaOnnxDecisionClient(); + +DecisionResult result = await client.EvaluateAsync( + new DecisionRequest("My invoice charged me twice and I want my money back.") + .Choose("team", new Dictionary + { + ["billing"] = "Payment, invoice and refund problems", + ["technical"] = "Bugs, outages and API errors", + ["sales"] = "Pricing, upgrades and new purchases" + }) + .Score("urgency", new[] { "no rush", "normal", "urgent", "critical" }) + .Ask("refund", "The customer is asking for a refund.")); + +ChoiceResult team = result.Choice("team"); + +// Gate on the probability so ambiguous tickets reach a human +// instead of being confidently misrouted. +string route = team.ChoiceOrNull(0.6) ?? "human-review"; + +Console.WriteLine(route); // billing +Console.WriteLine(result.Score("urgency").Score); // 1.67 of 3 +Console.WriteLine(result.Probability("refund").IsTrue); // True +``` + +> ⚠️ Laya's public checkpoints report **uncalibrated confidence**. Fit `DecisionThreshold` against +> your own labelled examples before relying on a boolean verdict. Answers whose temperature bucket +> the checkpoint got wrong are flagged with `CalibrationClamped` β€” see the +> [Local Decisions Guide](docs/decisions-guide.md#clamped-calibration-buckets) and the +> [DecisionCalibration sample](src/samples/DecisionCalibration/). + ## Cache Management Inspect and manage the local model cache programmatically: @@ -380,12 +429,14 @@ See the [Supported Models Guide](docs/supported-models.md) for detailed model ca | [GptOssChat](src/samples/GptOssChat) | GPT-OSS 20B chat, streaming, reasoning effort, and tool calling | | [MagenticUIServer](src/samples/MagenticUIServer) | ASP.NET Core + SignalR multi-agent server (FileSurfer, WebFetcher, Coder) | | [ConsoleAppDemo](src/samples/ConsoleAppDemo) | Interactive console application | +| [LocalDecisions](src/samples/LocalDecisions) | Support-ticket triage with a local System One decision model | +| [DecisionCalibration](src/samples/DecisionCalibration) | Calibration clamping and fp16 batch variance, made visible | > 🌐 **Reference App:** [ElBruno.MagenticUI](https://github.com/elbruno/ElBruno.MagenticUI) β€” full Blazor Server port of [microsoft/magentic-ui](https://github.com/microsoft/magentic-ui) running locally with this library. ## Requirements -- .NET 8.0 or .NET 10.0 +- .NET 10.0 - CPU (default), NVIDIA GPU (CUDA), or Windows GPU (DirectML) - ~2-8 GB disk space per model (depending on size and quantization) @@ -396,13 +447,13 @@ git clone https://github.com/elbruno/ElBruno.LocalLLMs.git cd ElBruno.LocalLLMs dotnet restore ElBruno.LocalLLMs.slnx dotnet build ElBruno.LocalLLMs.slnx -dotnet test ElBruno.LocalLLMs.slnx --framework net8.0 +dotnet test ElBruno.LocalLLMs.slnx ``` **Run integration tests** (downloads real models β€” requires internet): ```bash -RUN_INTEGRATION_TESTS=true dotnet test ElBruno.LocalLLMs.slnx --framework net8.0 +RUN_INTEGRATION_TESTS=true dotnet test ElBruno.LocalLLMs.slnx ``` Integration tests validate the full lifecycle (download β†’ infer β†’ cache hit β†’ delete) for all 35 supported models. See [docs/tests/README.md](docs/tests/README.md) for details. @@ -414,6 +465,8 @@ Integration tests validate the full lifecycle (download β†’ infer β†’ cache hit - [BitNet Guide](docs/bitnet-guide.md) β€” setup and usage of 1.58-bit BitNet models - [Cancellation Guide](docs/cancellation.md) β€” streaming cancellation semantics and voice barge-in usage - [Observability](docs/observability.md) β€” OpenTelemetry, lifecycle events, metrics, Aspire wiring +- [Blazor Components Guide](docs/blazor-components.md) β€” the 7 Razor components and how to wire them +- [Local Decisions Guide](docs/decisions-guide.md) β€” typed decision models, calibration caveats, thresholds - [Architecture](docs/architecture.md) β€” design decisions and internal structure - [Samples Guide](docs/samples.md) β€” walkthrough of each sample application - [Benchmarks](docs/benchmarks.md) β€” how to run and interpret performance benchmarks diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 01d79f6..c8a028f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,9 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- -## [Unreleased] +## [0.22.0] - 2026-09-30 + +### Changed +- **BREAKING: .NET 10 is now the only supported target framework.** Every library, test, sample, and benchmark project single-targets `net10.0`; the `net8.0;net10.0` multi-targeting was removed ahead of .NET 8's November 2026 end of support. `global.json` now pins SDK `10.0.0`, and the CI/publish workflows install `10.0.x` only and no longer pass per-framework overrides. Consumers still on .NET 8 must remain on `v0.21.0` or upgrade to .NET 10. ### Added +- **`ElBruno.LocalLLMs.Decisions`** β€” new package for local System One decision models, running [Laya](https://github.com/NandhaKishorM/laya) (Apache-2.0) **fully in-process on ONNX Runtime**: no server, no Python, and no network call at inference time. Provides `IDecisionClient` with `ChooseAsync` (typed label + full distribution), `ScoreAsync` (fractional expected position across a 2–10 level rubric), `AskAsync` (proposition probability), and `EvaluateAsync` for batching several questions into a **single forward pass**. Includes `DecisionRequest`/`DecisionResult` builders, `ChoiceResult.ChoiceOrNull(threshold)` for confidence-gated routing, `ProbabilityResult.AtThreshold` for re-thresholding without re-running inference, and `AddLocalDecisions()` DI registration. `LayaOnnxDecisionClient` ports Laya's sequence construction, ByteLevel BPE tokenizer, fitted temperature scaling, and decode math to managed .NET, and reproduces the reference checkpoint's published probabilities exactly. Model weights are downloaded from HuggingFace on first use and cached; the default source is [`elbruno/laya-onnx`](https://huggingface.co/elbruno/laya-onnx), an unmodified mirror of the `inferenceprince/laya-onnx` export verified byte-identical to it, and `DecisionOptions.ModelRepository` and `ModelPath` make the source fully configurable. Laya's `choice:11+` temperature of `0.1006` is clamped into its own valid `0.5`–`5.0` range, and every answer carries a `CalibrationClamped` flag reporting whether its own temperature bucket was clamped. See the [Local Decisions Guide](decisions-guide.md). +- **`LocalDecisions` sample** β€” support-ticket triage that routes, scores urgency, and detects refund/anger signals for three tickets in one request each. +- **`DecisionCalibration` sample** β€” makes the checkpoint's two numerical caveats visible: the clamped `choice:11+` calibration bucket (including the fact that clamping bounds the damage but still leaves saturated confidence) and fp16 batch variance between a question asked alone and the same question asked inside a batch. - **GPT-OSS 20B support** β€” OpenAI's Apache-2.0 open-weight MoE model (21B total / 3.6B active), using the official `onnxruntime/gpt-oss-20b-onnx` artifacts. Two registry entries: `gpt-oss-20b` (CPU INT4) and `gpt-oss-20b-cuda` (CUDA INT4). Requires no ONNX Runtime GenAI upgrade β€” gpt-oss support landed in the already-referenced 0.15.x line. - **`ChatTemplateFormat.Harmony`** and `HarmonyFormatter` β€” the OpenAI Harmony prompt format, implemented against the `chat_template.jinja` shipped in the model repository. Renders the caller's system prompt as a `developer` message and tool definitions as a TypeScript-style `namespace functions { ... }` block. - **Harmony channel filtering** β€” GPT-OSS splits output into `analysis` (chain-of-thought), `commentary` (tool calls), and `final` channels. The library now surfaces only the `final` channel in both buffered and streaming responses; chain-of-thought is never exposed to callers, per the GPT-OSS model card. The streaming filter is resilient to control markers split across token boundaries. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 6843dfc..f77152e 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -8,7 +8,7 @@ Thank you for your interest in contributing! This guide explains how to build, t ### Prerequisites -- **.NET 8.0 or 10.0** β€” [Download](https://dotnet.microsoft.com/en-us/download) +- **.NET 10.0** β€” [Download](https://dotnet.microsoft.com/en-us/download) - **Git** β€” for version control - **Python 3.10+** β€” only if converting models to ONNX diff --git a/docs/architecture.md b/docs/architecture.md index 65dd210..7eebfb7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -582,7 +582,7 @@ public class MyService(IChatClient chatClient) { ... } ```xml - net8.0;net10.0 + net10.0 true ElBruno.LocalLLMs Local LLM chat completions using Microsoft.Extensions.AI and ONNX Runtime GenAI. IChatClient implementation for running LLMs locally. @@ -924,4 +924,4 @@ ElBruno.LocalLLMs/ | **`ModelDefinition` as record** | Immutable, data-first. Adding a model = adding a record to `KnownModels`. | | **Chat templates as internal strategy pattern** | Users pick a model; the template is resolved automatically. No template API in public surface. | | **`ElBruno.HuggingFace.Downloader` for downloads** | Proven in both reference repos. Consistent download/cache behavior. | -| **`net8.0;net10.0` multi-targeting** | Matches reference repos. .NET 8 LTS + .NET 10 current. | +| **`net10.0` single-targeting** | .NET 8 support was dropped ahead of its end of support, so every project targets .NET 10 only. | diff --git a/docs/benchmarks.md b/docs/benchmarks.md index a4ba938..6b18712 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -97,7 +97,7 @@ The benchmarks use these BenchmarkDotNet settings: - **`[MemoryDiagnoser]`** β€” Tracks GC and memory allocation - **`[SimpleJob(warmupCount: 3, iterationCount: 10)]`** β€” 3 warmup iterations, 10 measured iterations (fast runs for development) -- **Target framework:** `net8.0` +- **Target framework:** `net10.0` ## Adding New Benchmarks diff --git a/docs/blazor-components.md b/docs/blazor-components.md index fdaacbf..ad776ee 100644 --- a/docs/blazor-components.md +++ b/docs/blazor-components.md @@ -319,6 +319,6 @@ Your app must also call `AddLocalLLMs()` (or `AddLocalVisionLLM()`) to register ## Target frameworks -`ElBruno.LocalLLMs.BlazorComponents` targets **net8.0** only (Blazor Server and +`ElBruno.LocalLLMs.BlazorComponents` targets **net10.0** only (Blazor Server and Blazor WASM both require ASP.NET Core). It is not compatible with console or non-web projects. diff --git a/docs/decisions-guide.md b/docs/decisions-guide.md new file mode 100644 index 0000000..fa3744f --- /dev/null +++ b/docs/decisions-guide.md @@ -0,0 +1,311 @@ +# Local Decisions Guide + +`ElBruno.LocalLLMs.Decisions` runs a **System One** decision model on your machine. Instead of +generating text, it returns a *typed* answer β€” a choice, a score, or a probability β€” together with +the full distribution behind it, in a single forward pass. + +That makes it fast enough to sit on a request path and cheap enough to call on every message, +which is exactly where a chat model is the wrong tool. + +## Decision model vs. chat model + +| | Chat model (`IChatClient`) | Decision model (`IDecisionClient`) | +|---|---|---| +| Output | free text, token by token | one typed answer + probabilities | +| Passes | autoregressive, one token at a time | a single forward pass | +| Typical latency | seconds | tens to hundreds of milliseconds | +| Failure mode | plausible prose that is wrong | a flat distribution you can detect | +| Good for | writing, summarising, reasoning | routing, triage, moderation, intent | + +The two compose well: use a decision model to decide *whether* and *where*, then hand the work to a +chat model when you actually need prose. + +## Running the model + +There is nothing to install and nothing to start. The package runs +[Laya](https://github.com/NandhaKishorM/laya) (Apache-2.0) **in-process on ONNX Runtime** β€” no +Python, no server, no network call at inference time. + +The model weights are downloaded from HuggingFace the first time a client is used (about 800 MB) +and cached under `%LOCALAPPDATA%\ElBruno\LocalLLMs\decisions`, so the first call is slow and the +rest are not. + +> Laya's authors publish PyTorch weights only, so every ONNX export of Laya is community-produced. +> The default, [`elbruno/laya-onnx`](https://huggingface.co/elbruno/laya-onnx), mirrors the +> [`inferenceprince/laya-onnx`](https://huggingface.co/inferenceprince/laya-onnx) export unmodified +> and was verified to reproduce the numbers that export's model card reports. Point at a different +> repository if you would rather pin your own: +> +> ```csharp +> options.ModelRepository = "your-org/your-laya-export"; +> ``` + +To avoid the download entirely, point at a directory you have already populated with the ONNX +graph, the checkpoint config and the tokenizer: + +```csharp +options.ModelPath = @"D:\models\laya"; +``` + +## Installation + +```bash +dotnet add package ElBruno.LocalLLMs.Decisions +``` + +## Quick start + +```csharp +using ElBruno.LocalLLMs.Decisions; + +using var client = new LayaOnnxDecisionClient(); + +ChoiceResult team = await client.ChooseAsync( + "My invoice charged me twice and I want my money back.", + new Dictionary + { + ["billing"] = "Payment, invoice and refund problems", + ["technical"] = "Bugs, outages and API errors", + ["sales"] = "Pricing, upgrades and new purchases" + }, + "Which team should handle this support ticket?"); + +Console.WriteLine(team.Choice); // billing +Console.WriteLine(team.Probability); // 0.9824 +``` + +## Ask several questions at once + +Every question in a `DecisionRequest` is answered in the **same forward pass**, so four questions +cost roughly what one costs. Prefer batching over sequential calls. + +```csharp +DecisionResult result = await client.EvaluateAsync( + new DecisionRequest(ticket) + .Choose("team", teams, "Which team should handle this?") + .Score("urgency", new[] { "no rush", "normal", "urgent", "critical" }) + .Ask("refund", "The customer is asking for a refund.") + .Ask("angry", "The customer is angry or frustrated.")); + +string route = result.Choice("team").Choice; +double urgency = result.Score("urgency").Score; +bool refund = result.Probability("refund").IsTrue; +``` + +Read answers back with the same name you used to add the question. Reading a name that was not +asked throws `KeyNotFoundException`; reading it as the wrong type throws `InvalidOperationException`. + +## The three question types + +### `Choose` β€” pick one label + +Returns the selected label plus a probability for every option. + +**Option descriptions matter.** The model reads the option text, so a bare `"billing"` is weaker +than `"billing"` described as `"payment, invoice and refund problems"`. There is an overload that +takes just labels, but prefer the dictionary when accuracy matters. + +### `Score` β€” rate against a rubric + +Takes 2–10 ordered levels and returns a **fractional expected position**, not an index. A score of +`1.67` across four levels sits between "normal" and "urgent" and means the model is genuinely +between them. + +Each level should be self-contained, because the model sees the level text rather than its +position. `"urgent, answer within an hour"` works better than `"3"`. + +Use `Score` for a continuous reading and `MostLikelyLevel` when you need a single bucket β€” they can +differ when the distribution is skewed or bimodal. + +### `Ask` β€” probability of a proposition + +Returns a probability rather than a boolean, deliberately. `IsTrue` applies +`DecisionOptions.DecisionThreshold` (default `0.5`), and `AtThreshold(x)` re-applies a different +threshold without calling the model again. + +Phrase the proposition so that "true" is unambiguous: `"The customer is asking for a refund."` +rather than `"Is this about refunds or something else?"`. + +## Act on probabilities, not just labels + +The highest-probability label is always returned, even when the model is torn. Gate on the +probability so ambiguous inputs fall through to a human instead of being confidently misrouted: + +```csharp +string route = team.ChoiceOrNull(0.6) ?? "human-review"; +``` + +## Calibration β€” read this before you trust a threshold + +**Laya's public checkpoints are not reliably calibrated, and this is the single biggest risk in +using them.** Concretely, observed on the English checkpoint: + +- Some checkpoints ship temperature values outside the valid range. The English checkpoint's + `choice:11+` bucket is `0.1006`, which sharpens the logits roughly tenfold. This package clamps + such values into Laya's own `0.5`–`5.0` range and reports it per answer via + `CalibrationClamped` β€” but **clamping bounds the damage, it does not fix the calibration**. + See [Clamped calibration buckets](#clamped-calibration-buckets) below for what that leaves you with. +- `Confidence` can be low on an answer whose distribution is actually sharp, and vice versa. +- The English checkpoint stays confident on non-Latin scripts while being wrong; use a + multilingual export if your input is not English. +- Reported accuracy on some held-out tasks (moderation, for example) is close to chance. + +None of that makes the model useless β€” routing and triage worked well in practice β€” but it does +mean: + +1. **Fit `DecisionThreshold` on your own labelled examples.** The 0.5 default is a starting point, + not a decision boundary. +2. **Prefer the probability over `IsTrue`** so you can re-tune later without re-running inference. +3. **Measure before you ship.** Treat the model as a component you evaluate, not an oracle. + +### Clamped calibration buckets + +Laya fits a separate temperature for each `(question type, option count)` bucket, keyed as +`choice:2`, `choice:3-5`, `choice:6-10`, `choice:11+` and the `score:` / `noul:` equivalents. The +English checkpoint ships `choice:11+ = 0.1006`, far below Laya's own `TEMP_MIN` of `0.5`. Dividing +the logits by it multiplies them roughly tenfold before the softmax. + +This package clamps that value to `0.5` and sets `CalibrationClamped` on the affected answer: + +```csharp +var result = await client.ChooseAsync(ticket, twelveDepartments, "Route this ticket"); + +if (result.CalibrationClamped) +{ + // The ranking is still meaningful. The confidence number is not. + logger.LogWarning("Confidence for {Choice} is not a fitted calibration.", result.Choice); +} +``` + +**Be clear about what the clamp does and does not buy you.** A temperature below `1.0` still +sharpens the distribution β€” `0.5` doubles the logits. Clamping only prevents the far more extreme +tenfold sharpening. Running the [DecisionCalibration sample](../src/samples/DecisionCalibration/) +against the same ticket shows what survives: + +| Options | Bucket | Choice | Reported confidence | `CalibrationClamped` | +|---|---|---|---|---| +| 5 | `choice:3-5` | `billing` | 86.9% | `false` | +| 12 | `choice:11+` | `billing` | **100.0%** | `true` | + +Both pick the right department. But the 12-option answer reports **saturated confidence** even +after clamping, because `0.5` still over-sharpens. So: + +- **Use the ranking, not the number**, for choices with 11 or more options. `Choice` and the + relative ordering of `Probabilities` are trustworthy; the magnitude is not. +- **`ChoiceOrNull(threshold)` is close to useless in this bucket** β€” a saturated probability clears + every threshold you would plausibly set, so it will never abstain. +- **Prefer fewer options.** Splitting one 12-way choice into two smaller ones keeps you inside a + bucket whose fitted temperature the checkpoint actually got right. +- If you need calibrated confidence over many options, fit your own temperature on labelled data + and apply it to the probabilities yourself. + +### fp16 batch variance + +The published checkpoint stores fp16 weights. Asking a question on its own and asking the same +question inside a larger batch produce slightly different probabilities, because the batched matrix +multiplications accumulate in a different order. Measured with the same sample: + +| Label | Alone | Batched with 3 others | Delta | +|---|---|---|---| +| `billing` | 0.93738423 | 0.93739642 | 1.22e-05 | +| `sales` | 0.03319702 | 0.03318152 | 1.55e-05 | +| `technical` | 0.02941875 | 0.02942205 | 3.31e-06 | + +Deltas land between roughly `1e-6` and `1e-4` depending on the input and batch size. This is +accumulation noise, **not** a sign that markers are misaligned β€” the answer and the ordering are +unchanged. + +It is far too small to flip a routing decision, but large enough to matter if you: + +- **assert exact equality** in tests β€” compare with a tolerance (`1e-3` is comfortable); +- **cache or deduplicate on the probability value** β€” key on the label instead; +- **persist probabilities and diff them across runs** β€” expect churn in the low decimals; +- **hash a result for change detection** β€” round first, or hash only the choice. + +Batching is still the right default: four questions in one pass cost about what one costs. Just +treat the probabilities as approximate, which the calibration caveats above already require. + +## Dependency injection + +```csharp +builder.Services.AddLocalDecisions(options => +{ + options.ModelRepository = "elbruno/laya-onnx"; + options.DecisionThreshold = 0.7; +}); +``` + +Then inject `IDecisionClient`. The client is thread-safe and registered as a singleton, because +loading the model is expensive and the loaded session is safe to share. Registration does not +touch the network β€” the model is loaded lazily on the first call. + +## Options + +| Option | Default | Notes | +|---|---|---| +| `ModelRepository` | `elbruno/laya-onnx` | The HuggingFace repository to download from. Mirrors the community `inferenceprince/laya-onnx` export unmodified. | +| `ModelPath` | `null` | A local directory holding the model files. When set, nothing is downloaded. | +| `CacheDirectory` | `%LOCALAPPDATA%\ElBruno\LocalLLMs\decisions` | Where downloads are cached. | +| `IntraOpNumThreads` | `null` | Threads used within a single ONNX operator. Worth pinning to a small number under concurrency, since the default fills every core for one call. | +| `DecisionThreshold` | `0.5` | Applied by `ProbabilityResult.IsTrue`. | + +## Testing + +`IDecisionClient` is a plain interface, so unit tests can implement it directly with whatever +distributions the test needs β€” no model download and no ONNX session: + +```csharp +internal sealed class StubDecisionClient : IDecisionClient { /* ... */ } +``` + +## Error handling + +`DecisionException` is thrown when the model cannot be downloaded or loaded, when the model +directory is incomplete, or when a question does not fit the model's token window. The message +says which of those happened and what to do about it. + +```csharp +try +{ + var result = await client.EvaluateAsync(request); +} +catch (DecisionException ex) +{ + // Download failed, model directory incomplete, or too many options to score. +} +``` + +## Performance + +Measured on CPU with the English checkpoint, 4 questions per request: + +| | Latency | +|---|---| +| First ever call (download, ~800 MB) | minutes | +| First call after that (session load) | a few seconds | +| Warm, 4 questions, CPU | ~600 ms | + +Latency scales with input length far more than with question count, which is why batching is close +to free. + +## Sample + +[`src/samples/LocalDecisions`](../src/samples/LocalDecisions/) triages three support tickets and +prints the routing decision, urgency score and both probabilities for each. + +```bash +dotnet run --project src/samples/LocalDecisions + +# or against a model directory you already have, skipping the download +dotnet run --project src/samples/LocalDecisions -- D:\models\laya +``` + +## Limitations + +- **The ONNX export is community-produced.** Laya's authors ship PyTorch weights only. The default + export was verified against its own model card, but it is not first-party. +- **Confidence is not reliably calibrated.** See the section above; this is the real constraint. +- **English checkpoint is English-only in practice.** It stays confident on non-Latin scripts while + being wrong. Point `ModelRepository` at a multilingual export if your input is not English. +- **The first run downloads about 800 MB.** Pre-populate `ModelPath` in environments where that is + not acceptable. diff --git a/docs/getting-started.md b/docs/getting-started.md index 5c67679..3f12b91 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -8,7 +8,7 @@ Welcome! This guide will help you set up and run local LLMs in your .NET applica Before you start, ensure you have: -- **.NET 8.0 or .NET 10.0** β€” [Download](https://dotnet.microsoft.com/en-us/download) +- **.NET 10.0** β€” [Download](https://dotnet.microsoft.com/en-us/download) - **~2-8 GB disk space** β€” to cache downloaded models - **A compatible processor** β€” CPU (default), NVIDIA GPU (CUDA 11.8+), or Windows GPU (DirectML) diff --git a/docs/magentic-ui-dotnet.md b/docs/magentic-ui-dotnet.md index 87db3c0..f066fa4 100644 --- a/docs/magentic-ui-dotnet.md +++ b/docs/magentic-ui-dotnet.md @@ -74,7 +74,7 @@ dotnet restore src/samples/MagenticUIServer/MagenticUIServer.Agents/MagenticUISe dotnet build src/samples/MagenticUIServer/MagenticUIServer.Agents/MagenticUIServer.Agents.csproj # Run all Phase 3 tests -dotnet test src/tests/MagenticUIServer.Agents.Tests/MagenticUIServer.Agents.Tests.csproj --framework net8.0 +dotnet test src/tests/MagenticUIServer.Agents.Tests/MagenticUIServer.Agents.Tests.csproj # Build the web host dotnet build src/samples/MagenticUIServer/MagenticUIServer/MagenticUIServer.csproj diff --git a/docs/publishing.md b/docs/publishing.md index b24b3a8..bf84639 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -77,7 +77,7 @@ Before triggering publish: 1. `` matches the release you intend to ship. 2. `docs/CHANGELOG.md` contains the same version. 3. README and any feature docs reflect the shipped behavior. -4. `dotnet test ElBruno.LocalLLMs.slnx --framework net8.0` passes locally. +4. `dotnet test ElBruno.LocalLLMs.slnx` passes locally. 5. `dotnet pack src/ElBruno.LocalLLMs/ElBruno.LocalLLMs.csproj -c Release` succeeds locally. --- @@ -96,7 +96,7 @@ Before triggering publish: β”‚ publish.yml β”‚ β”‚ β”‚ β”‚ 1. Checkout code β”‚ - β”‚ 2. Setup .NET 8 + 10 β”‚ + β”‚ 2. Setup .NET 10 β”‚ β”‚ 3. Determine version (tag/input/ β”‚ β”‚ csproj) β”‚ β”‚ 4. Restore β†’ Build β†’ Test β†’ Pack β”‚ diff --git a/docs/samples.md b/docs/samples.md index 0dec9e7..80bf44d 100644 --- a/docs/samples.md +++ b/docs/samples.md @@ -4,7 +4,7 @@ Runnable examples demonstrating how to use `ElBruno.LocalLLMs` in different scen ## Prerequisites -- .NET 8.0+ SDK +- .NET 10.0 SDK - ~2-4 GB free disk space (models are downloaded on first run) - CPU is sufficient; GPU (CUDA/DirectML) is optional @@ -456,6 +456,116 @@ API reference and configuration options. --- +## LocalDecisions β€” Support-Ticket Triage with a Decision Model + +Unlike every other sample here, this one does not use a chat model. It uses a +**System One decision model**, which returns a typed answer with probabilities +in a single forward pass instead of generating text. + +The model runs in-process on ONNX Runtime β€” nothing to install and nothing to +start. The weights are downloaded from HuggingFace on first run (about 800 MB) +and cached afterwards: + +```bash +dotnet run --project src/samples/LocalDecisions + +# or against a model directory you already have, skipping the download +dotnet run --project src/samples/LocalDecisions -- D:\models\laya +``` + +It asks **four questions per ticket in one request** β€” which team should own it, +how urgent it is, whether a refund is being requested, and whether the customer +is angry: + +```csharp +DecisionResult result = await client.EvaluateAsync( + new DecisionRequest(ticket) + .Choose("team", teams, "Which team should handle this support ticket?") + .Score("urgency", urgencyLevels, "How urgent is this ticket?") + .Ask("refund", "The customer is asking for a refund.") + .Ask("angry", "The customer is angry or frustrated.")); +``` + +Output: + +```text +My invoice charged me twice for the same subscription month and I want my m… + + route : billing (billing @ 98.4%) + urgency : urgent, answer within an hour + score 1.67 of 3 + refund : yes (p = 0.940) + angry : yes (p = 0.768) + + distribution: billing 98% sales 1% technical 1% + 244 input tokens, 664 ms, model 'elbruno/laya-onnx' +``` + +Note the routing line: the sample calls `team.ChoiceOrNull(0.6)` and falls back +to `human-review`, so a ticket the model is unsure about reaches a person rather +than being confidently misrouted. + +See the [Local Decisions Guide](decisions-guide.md) for the question types and, +importantly, the calibration caveats before you pick a threshold. + +--- + +## DecisionCalibration β€” Seeing the Checkpoint's Two Numerical Caveats + +The Laya checkpoint has two behaviours that are easy to trip over and hard to +notice, because neither one produces an error. This sample makes both visible: + +```bash +dotnet run --project src/samples/DecisionCalibration + +# or against a model directory you already have +dotnet run --project src/samples/DecisionCalibration -- D:\models\laya +``` + +**1. A clamped calibration bucket.** Laya fits a temperature per option-count +bucket, and the English checkpoint ships `choice:11+ = 0.1006` β€” far below its +own minimum of `0.5`. The sample routes the same ticket twice, once with 5 +options and once with 12: + +```text + 5 options (choice:3-5) + choice : billing + probability : 86.9% + CalibrationClamped : False + + 12 options (choice:11+) + choice : billing + probability : 100.0% + CalibrationClamped : True +``` + +Both answers are right, and `CalibrationClamped` tells you which one you can +trust the *number* from. Note that the clamped answer still reports saturated +confidence: clamping to `0.5` bounds the damage but does not fix the +calibration, because any temperature below `1.0` still sharpens. + +**2. fp16 batch variance.** The same question answered alone and answered inside +a batch returns slightly different probabilities, because fp16 accumulates in a +different order: + +```text + label alone batched delta + billing 0.93738423 0.93739642 1.22E-005 + sales 0.03319702 0.03318152 1.55E-005 + technical 0.02941875 0.02942205 3.31E-006 + + same choice : True (billing) + max delta : 1.55E-005 +``` + +Far too small to change a decision, but large enough to break an exact-equality +assertion or a cache keyed on the probability. Compare with a tolerance. + +See the [Local Decisions Guide](decisions-guide.md#clamped-calibration-buckets) +for what to do about both. + +--- + ## Next Steps - πŸ“– [Getting Started](getting-started.md) β€” full setup guide with GPU configuration @@ -463,3 +573,4 @@ API reference and configuration options. - πŸ“Š [Benchmarks](benchmarks.md) β€” measure performance on your hardware - πŸ—οΈ [Architecture](architecture.md) β€” understand the internal design - 🧩 [Blazor Components](blazor-components.md) β€” ready-to-use UI components for Blazor apps +- 🎯 [Local Decisions](decisions-guide.md) β€” typed decision models for routing and triage diff --git a/docs/tests/README.md b/docs/tests/README.md index 1e042e3..69119f6 100644 --- a/docs/tests/README.md +++ b/docs/tests/README.md @@ -45,18 +45,18 @@ Integration tests require: ```powershell # Run all integration tests (downloads models as needed) $env:RUN_INTEGRATION_TESTS = "true" -dotnet test src/tests/ElBruno.LocalLLMs.IntegrationTests --framework net8.0 +dotnet test src/tests/ElBruno.LocalLLMs.IntegrationTests # Run only lifecycle tests for text models -dotnet test src/tests/ElBruno.LocalLLMs.IntegrationTests --framework net8.0 ` +dotnet test src/tests/ElBruno.LocalLLMs.IntegrationTests ` --filter "FullyQualifiedName~ModelLifecycleTests" # Run only tool-calling lifecycle tests -dotnet test src/tests/ElBruno.LocalLLMs.IntegrationTests --framework net8.0 ` +dotnet test src/tests/ElBruno.LocalLLMs.IntegrationTests ` --filter "FullyQualifiedName~ToolCallingLifecycleTests" # Run only vision lifecycle tests -dotnet test src/tests/ElBruno.LocalLLMs.IntegrationTests --framework net8.0 ` +dotnet test src/tests/ElBruno.LocalLLMs.IntegrationTests ` --filter "FullyQualifiedName~VisionLifecycleTests" ``` diff --git a/global.json b/global.json index 8eb62be..8f11e95 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.0", + "version": "10.0.0", "rollForward": "latestMajor" } } diff --git a/scripts/README.md b/scripts/README.md index 28e0378..dccc8e6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -25,7 +25,7 @@ bash scripts/run-tests.sh | `-SkipBuild` / `-NoBuild` | β€” | Skip `dotnet build` | | `-SkipUnitTests` | β€” | Skip unit test project | | `-SkipIntegrationTests` | β€” | Skip integration tests (build + unit only) | -| `-Framework` | `net8.0` | Target framework | +| `-Framework` | `net10.0` | Target framework | | `-HfToken ` | β€” | Sets `HF_TOKEN` for private HuggingFace repos | | `-Filter ` | β€” | xUnit `--filter` expression (integration tests only) | @@ -36,7 +36,7 @@ bash scripts/run-tests.sh | `--skip-build` / `-B` / `--no-build` | β€” | Skip build | | `--skip-unit-tests` / `-U` | β€” | Skip unit tests | | `--skip-integration-tests` / `-I` | β€” | Skip integration tests | -| `--framework ` | `net8.0` | Target framework | +| `--framework ` | `net10.0` | Target framework | | `--hf-token ` | β€” | Sets `HF_TOKEN` | | `--filter ` | β€” | xUnit filter expression | diff --git a/scripts/run-tests.ps1 b/scripts/run-tests.ps1 index c6a475b..fd6f523 100644 --- a/scripts/run-tests.ps1 +++ b/scripts/run-tests.ps1 @@ -29,7 +29,7 @@ Skip the integration test project run (useful for fast CI pre-commit checks). .PARAMETER Framework - Target framework passed to dotnet build and dotnet test. Defaults to 'net8.0'. + Target framework passed to dotnet build and dotnet test. Defaults to 'net10.0'. .PARAMETER HfToken HuggingFace token for private model repositories. Sets the HF_TOKEN environment variable @@ -82,7 +82,7 @@ param( [switch]$NoBuild, [switch]$SkipUnitTests, [switch]$SkipIntegrationTests, - [string]$Framework = 'net8.0', + [string]$Framework = 'net10.0', [string]$HfToken, [string]$Filter, [string]$LogFile # Optional explicit log path. Auto-generates docs/tests/YYYY-MM-DD-HH-run.log when omitted. diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index a143c80..0eb363b 100644 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -38,7 +38,7 @@ fi SKIP_BUILD=false SKIP_UNIT=false SKIP_INTEGRATION=false -FRAMEWORK="net8.0" +FRAMEWORK="net10.0" HF_TOKEN_VALUE="" FILTER="" @@ -54,7 +54,7 @@ Options: --no-build Alias for --skip-build --skip-unit-tests, -U Skip unit tests --skip-integration-tests, -I Skip integration tests - --framework Target framework (default: net8.0) + --framework Target framework (default: net10.0) --hf-token Set HF_TOKEN for private HuggingFace repos --filter xUnit filter string for integration tests --help, -h Show this help message diff --git a/src/ElBruno.LocalLLMs.BitNet/ElBruno.LocalLLMs.BitNet.csproj b/src/ElBruno.LocalLLMs.BitNet/ElBruno.LocalLLMs.BitNet.csproj index 5b18b77..0eb59f4 100644 --- a/src/ElBruno.LocalLLMs.BitNet/ElBruno.LocalLLMs.BitNet.csproj +++ b/src/ElBruno.LocalLLMs.BitNet/ElBruno.LocalLLMs.BitNet.csproj @@ -1,6 +1,6 @@ - net8.0;net10.0 + net10.0 true ElBruno.LocalLLMs.BitNet BitNet 1.58-bit LLM inference using bitnet.cpp. IChatClient implementation for Microsoft.Extensions.AI. diff --git a/src/ElBruno.LocalLLMs.BlazorComponents/ElBruno.LocalLLMs.BlazorComponents.csproj b/src/ElBruno.LocalLLMs.BlazorComponents/ElBruno.LocalLLMs.BlazorComponents.csproj index 973190f..7896faa 100644 --- a/src/ElBruno.LocalLLMs.BlazorComponents/ElBruno.LocalLLMs.BlazorComponents.csproj +++ b/src/ElBruno.LocalLLMs.BlazorComponents/ElBruno.LocalLLMs.BlazorComponents.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 enable enable true diff --git a/src/ElBruno.LocalLLMs.Decisions/DecisionClientExtensions.cs b/src/ElBruno.LocalLLMs.Decisions/DecisionClientExtensions.cs new file mode 100644 index 0000000..67c23f0 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/DecisionClientExtensions.cs @@ -0,0 +1,61 @@ +namespace ElBruno.LocalLLMs.Decisions; + +/// +/// Convenience overloads for . +/// +public static class DecisionClientExtensions +{ + /// + /// Selects one of for the supplied text, without option descriptions. + /// + /// The decision client. + /// The text to reason about. + /// The option labels. Descriptions are omitted, so the labels must be self-explanatory. + /// Optional natural-language guidance, such as a question to answer. + /// A token to cancel the request. + /// The selected label and the full distribution over options. + /// + /// Descriptions matter more than they look: the model reads the option text, so + /// "billing" alone is weaker than "billing" described as + /// "payment, invoice and refund problems". Prefer the dictionary overload when accuracy matters. + /// + public static Task ChooseAsync( + this IDecisionClient client, + string state, + IEnumerable options, + string? instructions = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(client); + ArgumentNullException.ThrowIfNull(options); + + var map = options.ToDictionary(option => option, _ => (string?)null, StringComparer.Ordinal); + return client.ChooseAsync(state, map, instructions, cancellationToken); + } + + /// + /// Assesses a proposition and returns only the boolean verdict, using the client's configured threshold. + /// + /// The decision client. + /// The text to reason about. + /// The statement to assess. + /// A token to cancel the request. + /// when the probability meets the configured threshold. + /// + /// Discarding the probability discards the information you need to tune the threshold later. + /// Prefer unless you are certain a boolean is enough. + /// + public static async Task IsTrueAsync( + this IDecisionClient client, + string state, + string proposition, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(client); + ProbabilityResult result = await client + .AskAsync(state, proposition, cancellationToken) + .ConfigureAwait(false); + + return result.IsTrue; + } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/DecisionException.cs b/src/ElBruno.LocalLLMs.Decisions/DecisionException.cs new file mode 100644 index 0000000..88b50b1 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/DecisionException.cs @@ -0,0 +1,16 @@ +namespace ElBruno.LocalLLMs.Decisions; + +/// +/// Thrown when a local decision model cannot be reached or returns something unusable. +/// +public sealed class DecisionException : Exception +{ + /// Creates an exception with a message. + /// The error description. + public DecisionException(string message) : base(message) { } + + /// Creates an exception with a message and underlying cause. + /// The error description. + /// The underlying failure. + public DecisionException(string message, Exception innerException) : base(message, innerException) { } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/DecisionOptions.cs b/src/ElBruno.LocalLLMs.Decisions/DecisionOptions.cs new file mode 100644 index 0000000..d4a86db --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/DecisionOptions.cs @@ -0,0 +1,105 @@ +namespace ElBruno.LocalLLMs.Decisions; + +/// +/// Configuration for a local decision client. The model runs in-process through ONNX Runtime, +/// so there is no server to start and no Python to install. +/// +public sealed class DecisionOptions +{ + /// + /// Gets or sets the HuggingFace repository the model is downloaded from. + /// Defaults to an ONNX export of Laya's English checkpoint. + /// + /// + /// Laya's authors publish PyTorch weights only, so every ONNX export of Laya is + /// community-produced. The default mirrors the inferenceprince/laya-onnx export + /// unmodified, and was verified to reproduce the numbers that export's model card reports. + /// + public string ModelRepository { get; set; } = "elbruno/laya-onnx"; + + /// + /// Gets or sets a local directory holding the model files. When set, nothing is downloaded + /// and is ignored. + /// + /// + /// The directory must contain the ONNX graph, the checkpoint config carrying the fitted + /// temperatures, and the tokenizer definition. + /// + public string? ModelPath { get; set; } + + /// + /// Gets or sets where downloaded models are cached. Defaults to a folder under the user's + /// local application data. + /// + public string? CacheDirectory { get; set; } + + /// + /// Gets or sets the number of threads ONNX Runtime uses within a single operator. + /// Leave null to let the runtime decide. + /// + /// + /// Worth pinning to a small number when many requests are served concurrently, because the + /// default fills every core for one call and the batches here are short. + /// + public int? IntraOpNumThreads { get; set; } + + /// + /// Gets or sets the probability at or above which reports true. + /// Defaults to 0.5. + /// + /// + /// This is a convenience default, not a calibrated boundary. Laya's public checkpoints are + /// over-confident, and one shipped temperature is invalid enough that this package clamps it. + /// Fit this threshold against your own labelled examples before depending on the boolean. + /// + public double DecisionThreshold { get; set; } = 0.5; + + /// + /// Gets the repository files that must be present for the model to run. + /// + internal IReadOnlyList ModelFiles { get; } = + [ + "model.onnx", + "rl_agent_config.json", + "tokenizer/tokenizer.json", + ]; + + /// + /// Gets repository files that are downloaded when present. The weights of an externally + /// stored graph live here, as do the alternative layouts other exports use. + /// + internal IReadOnlyList OptionalModelFiles { get; } = + [ + "model.onnx.data", + "model.onnx_data", + "config.json", + "tokenizer.json", + "tokenizer/tokenizer_config.json", + ]; + + internal string ResolveCacheDirectory() => + CacheDirectory ?? Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "ElBruno", "LocalLLMs", "decisions"); + + internal void Validate() + { + if (string.IsNullOrWhiteSpace(ModelPath) && string.IsNullOrWhiteSpace(ModelRepository)) + { + throw new InvalidOperationException( + "DecisionOptions needs either a ModelRepository to download from or a ModelPath to load from."); + } + + if (IntraOpNumThreads is <= 0) + { + throw new InvalidOperationException( + $"DecisionOptions.IntraOpNumThreads must be greater than zero, but was {IntraOpNumThreads}."); + } + + if (DecisionThreshold is < 0 or > 1 || double.IsNaN(DecisionThreshold)) + { + throw new InvalidOperationException( + $"DecisionOptions.DecisionThreshold must be between 0 and 1, but was {DecisionThreshold}."); + } + } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/DecisionRequest.cs b/src/ElBruno.LocalLLMs.Decisions/DecisionRequest.cs new file mode 100644 index 0000000..099fe68 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/DecisionRequest.cs @@ -0,0 +1,172 @@ +using System.Collections.ObjectModel; + +namespace ElBruno.LocalLLMs.Decisions; + +/// +/// The kind of answer a question expects. +/// +public enum DecisionQuestionType +{ + /// Selects one caller-defined label from a set. + Choice, + + /// Scores against an ordered rubric. + Score, + + /// Assesses the probability of a proposition being true. + Probability +} + +/// +/// A single named question within a . +/// +public sealed class DecisionQuestion +{ + internal DecisionQuestion( + DecisionQuestionType type, + string? instructions, + IReadOnlyDictionary? options, + IReadOnlyList? levels) + { + Type = type; + Instructions = instructions; + Options = options; + Levels = levels; + } + + /// Gets the answer kind this question expects. + public DecisionQuestionType Type { get; } + + /// Gets the natural-language instructions, or null when omitted. + public string? Instructions { get; } + + /// Gets the option labels and descriptions for a choice question, otherwise null. + public IReadOnlyDictionary? Options { get; } + + /// Gets the ordered rubric levels for a score question, otherwise null. + public IReadOnlyList? Levels { get; } +} + +/// +/// A batch of independent questions evaluated against one shared piece of text. +/// +/// +/// Every question in a request is answered in a single forward pass, so asking five questions +/// together costs roughly the same as asking one. Prefer batching over sequential calls. +/// Instances are immutable; each Choose, Score or Ask returns a new request. +/// +public sealed class DecisionRequest +{ + private readonly Dictionary _questions; + + /// + /// Creates a request for the supplied text with no questions attached. + /// + /// The text to reason about. Must not be null or whitespace. + public DecisionRequest(string state) + { + ArgumentException.ThrowIfNullOrWhiteSpace(state); + + State = state; + _questions = new Dictionary(StringComparer.Ordinal); + Questions = new ReadOnlyDictionary(_questions); + } + + private DecisionRequest(string state, Dictionary questions) + { + State = state; + _questions = questions; + Questions = new ReadOnlyDictionary(questions); + } + + /// Gets the shared text all questions are evaluated against. + public string State { get; } + + /// Gets the questions, keyed by the names supplied when they were added. + public IReadOnlyDictionary Questions { get; } + + /// + /// Adds a choice question that selects one of . + /// + /// The result key used to read the answer back. Must be unique within the request. + /// Option labels mapped to optional descriptions. Between 1 and 255 entries. + /// Optional natural-language guidance, such as a question to answer. + /// A new request including this question. + public DecisionRequest Choose( + string name, + IReadOnlyDictionary options, + string? instructions = null) + { + ArgumentNullException.ThrowIfNull(options); + if (options.Count is < 1 or > 255) + { + throw new ArgumentOutOfRangeException(nameof(options), "A choice question requires 1-255 options."); + } + + var copy = new Dictionary(options, StringComparer.Ordinal); + return Add(name, new DecisionQuestion(DecisionQuestionType.Choice, instructions, copy, null)); + } + + /// + /// Adds a choice question whose options carry no descriptions. + /// + /// The result key used to read the answer back. + /// The option labels, in any order. + /// Optional natural-language guidance. + /// A new request including this question. + public DecisionRequest Choose(string name, IEnumerable options, string? instructions = null) + { + ArgumentNullException.ThrowIfNull(options); + var map = options.ToDictionary(option => option, _ => (string?)null, StringComparer.Ordinal); + return Choose(name, map, instructions); + } + + /// + /// Adds a score question against an ordered rubric. + /// + /// The result key used to read the answer back. + /// + /// Between 2 and 10 rubric levels, ordered from lowest to highest. Each level should be + /// self-contained, because the model sees the level text rather than its position. + /// + /// Optional natural-language guidance. + /// A new request including this question. + public DecisionRequest Score(string name, IReadOnlyList levels, string? instructions = null) + { + ArgumentNullException.ThrowIfNull(levels); + if (levels.Count is < 2 or > 10) + { + throw new ArgumentOutOfRangeException(nameof(levels), "A score question requires 2-10 ordered levels."); + } + + return Add(name, new DecisionQuestion(DecisionQuestionType.Score, instructions, null, levels.ToArray())); + } + + /// + /// Adds a yes/no proposition answered with a probability. + /// + /// The result key used to read the answer back. + /// The statement to assess, phrased so that "true" is unambiguous. + /// A new request including this question. + public DecisionRequest Ask(string name, string proposition) + { + ArgumentException.ThrowIfNullOrWhiteSpace(proposition); + return Add(name, new DecisionQuestion(DecisionQuestionType.Probability, proposition, null, null)); + } + + private DecisionRequest Add(string name, DecisionQuestion question) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + if (_questions.ContainsKey(name)) + { + throw new ArgumentException($"A question named '{name}' was already added.", nameof(name)); + } + + var questions = new Dictionary(_questions, StringComparer.Ordinal) + { + [name] = question + }; + + return new DecisionRequest(State, questions); + } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/DecisionResult.cs b/src/ElBruno.LocalLLMs.Decisions/DecisionResult.cs new file mode 100644 index 0000000..1d27cd2 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/DecisionResult.cs @@ -0,0 +1,82 @@ +namespace ElBruno.LocalLLMs.Decisions; + +/// +/// The answers to a , read back by question name. +/// +public sealed class DecisionResult +{ + private readonly IReadOnlyDictionary _answers; + + internal DecisionResult( + string model, + long? inputTokens, + TimeSpan duration, + IReadOnlyDictionary answers) + { + Model = model; + InputTokens = inputTokens; + Duration = duration; + _answers = answers; + } + + /// Gets the model this result came from: the HuggingFace repository it was + /// downloaded from, or the local path it was loaded from. + public string Model { get; } + + /// + /// Gets the number of input tokens the whole batch was encoded into, or null when + /// that is not known. + /// + public long? InputTokens { get; } + + /// Gets the wall-clock time taken by the call, including tokenization. + public TimeSpan Duration { get; } + + /// Gets the question names present in this result. + public IReadOnlyCollection Names => (IReadOnlyCollection)_answers.Keys; + + /// + /// Reads a choice answer. + /// + /// The question name supplied to . + /// The choice outcome. + /// No question with this name was answered. + /// The named question was not a choice question. + public ChoiceResult Choice(string name) => Get(name, "choice"); + + /// + /// Reads a score answer. + /// + /// The question name supplied to . + /// The score outcome. + /// No question with this name was answered. + /// The named question was not a score question. + public ScoreResult Score(string name) => Get(name, "score"); + + /// + /// Reads a yes/no probability answer. + /// + /// The question name supplied to . + /// The probability outcome. + /// No question with this name was answered. + /// The named question was not a proposition. + public ProbabilityResult Probability(string name) => Get(name, "probability"); + + private T Get(string name, string expected) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + if (!_answers.TryGetValue(name, out object? answer)) + { + throw new KeyNotFoundException( + $"No answer named '{name}'. Available answers: {string.Join(", ", _answers.Keys)}."); + } + + if (answer is not T typed) + { + throw new InvalidOperationException( + $"Answer '{name}' is not a {expected} answer; the model returned {answer.GetType().Name}."); + } + + return typed; + } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/DecisionResults.cs b/src/ElBruno.LocalLLMs.Decisions/DecisionResults.cs new file mode 100644 index 0000000..27fe43e --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/DecisionResults.cs @@ -0,0 +1,106 @@ +namespace ElBruno.LocalLLMs.Decisions; + +/// +/// The outcome of a choice question: one selected label plus the full probability distribution. +/// +/// The highest-probability option label. Always a key of . +/// Probability assigned to every option label, keyed exactly as supplied. +/// +/// The model's self-reported confidence. Treat as uncalibrated unless you have validated it +/// on your own labelled data; see . +/// +/// +/// when the checkpoint's fitted temperature for this question's bucket was +/// outside its own valid range and had to be clamped, so and +/// are not the checkpoint's fitted calibration. See +/// the guide. +/// +public sealed record ChoiceResult( + string Choice, + IReadOnlyDictionary Probabilities, + double Confidence, + bool CalibrationClamped = false) +{ + /// + /// Gets the probability assigned to the selected . + /// + public double Probability => Probabilities[Choice]; + + /// + /// Returns the selected choice only when its probability meets , + /// and null otherwise. Use this to route ambiguous inputs to a fallback or to a larger model. + /// + /// The inclusive probability floor, between 0 and 1. + /// The selected label, or null when the distribution is too flat to act on. + public string? ChoiceOrNull(double minimumProbability) => + Probability >= minimumProbability ? Choice : null; +} + +/// +/// The outcome of a score question against an ordered rubric. +/// +/// +/// The expected value over the rubric levels, between 0 and Legend.Count - 1. +/// This is a continuous number, not a level index; a score of 1.6 sits between levels 1 and 2. +/// +/// Probability of each zero-based rubric level. +/// The rubric levels in their supplied order, echoed by the model. +/// The model's self-reported confidence. Treat as uncalibrated until validated. +/// +/// when the checkpoint's fitted temperature for this question's bucket was +/// outside its own valid range and had to be clamped. +/// +public sealed record ScoreResult( + double Score, + IReadOnlyDictionary Probabilities, + IReadOnlyList Legend, + double Confidence, + bool CalibrationClamped = false) +{ + /// + /// Gets the zero-based index of the single most likely rubric level. + /// This can differ from rounding when the distribution is skewed or bimodal. + /// + public int MostLikelyLevel => Probabilities.OrderByDescending(p => p.Value).First().Key; + + /// + /// Gets the rubric text of . + /// + public string MostLikelyLabel => Legend[MostLikelyLevel]; + + /// + /// Gets the probability of . + /// + public double Probability => Probabilities[MostLikelyLevel]; +} + +/// +/// The outcome of a yes/no proposition, expressed as a probability rather than a boolean. +/// +/// The probability that the proposition is true, between 0 and 1. +/// The threshold applied by . +/// +/// when the checkpoint's fitted temperature for this question's bucket was +/// outside its own valid range and had to be clamped. +/// +public readonly record struct ProbabilityResult( + double Probability, + double Threshold, + bool CalibrationClamped = false) +{ + /// + /// Gets a value indicating whether meets . + /// + /// + /// The default threshold of 0.5 is a starting point, not a calibrated decision boundary. + /// Fit it against your own labelled examples before relying on it in production. + /// + public bool IsTrue => Probability >= Threshold; + + /// + /// Applies a different threshold to the same probability without re-running the model. + /// + /// The inclusive probability floor, between 0 and 1. + /// when the probability meets the supplied threshold. + public bool AtThreshold(double threshold) => Probability >= threshold; +} diff --git a/src/ElBruno.LocalLLMs.Decisions/DecisionServiceExtensions.cs b/src/ElBruno.LocalLLMs.Decisions/DecisionServiceExtensions.cs new file mode 100644 index 0000000..e8799da --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/DecisionServiceExtensions.cs @@ -0,0 +1,46 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace ElBruno.LocalLLMs.Decisions; + +/// +/// Extension methods for registering local decision services in dependency injection. +/// +public static class DecisionServiceExtensions +{ + /// + /// Registers an that runs a Laya decision model in-process + /// through ONNX Runtime. + /// + /// The service collection. + /// Optional action to configure the model source and threshold. + /// The service collection for chaining. + /// + /// The client is registered as a singleton because loading the model is expensive and the + /// loaded session is safe to share. The model is downloaded lazily on first use, so + /// registration does not block startup. + /// + /// + /// + /// builder.Services.AddLocalDecisions(options => + /// { + /// options.ModelRepository = "elbruno/laya-onnx"; + /// options.DecisionThreshold = 0.7; + /// }); + /// + /// + public static IServiceCollection AddLocalDecisions( + this IServiceCollection services, + Action? configureOptions = null) + { + ArgumentNullException.ThrowIfNull(services); + + var options = new DecisionOptions(); + configureOptions?.Invoke(options); + options.Validate(); + + services.AddSingleton(options); + services.AddSingleton(_ => new LayaOnnxDecisionClient(options)); + + return services; + } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/ElBruno.LocalLLMs.Decisions.csproj b/src/ElBruno.LocalLLMs.Decisions/ElBruno.LocalLLMs.Decisions.csproj new file mode 100644 index 0000000..aa91778 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/ElBruno.LocalLLMs.Decisions.csproj @@ -0,0 +1,31 @@ + + + net10.0 + true + ElBruno.LocalLLMs.Decisions + Local System One decision models for ElBruno.LocalLLMs. Typed choice, score and yes/no decisions with probabilities in a single forward pass, running fully in-process on ONNX Runtime with no server and no Python. + https://github.com/elbruno/ElBruno.LocalLLMs + README.md + decisions;classification;laya;onnx;system-one;local-llm;routing;triage + true + true + snupkg + true + + + + + + + + + + + + + + + + + + diff --git a/src/ElBruno.LocalLLMs.Decisions/IDecisionClient.cs b/src/ElBruno.LocalLLMs.Decisions/IDecisionClient.cs new file mode 100644 index 0000000..b7cc5c1 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/IDecisionClient.cs @@ -0,0 +1,72 @@ +namespace ElBruno.LocalLLMs.Decisions; + +/// +/// Makes typed decisions about text using a local System One model. +/// +/// +/// +/// A decision model is not a chat model. It does not generate text; it produces a typed answer +/// with a probability distribution in a single forward pass, which makes it fast enough to sit on +/// a request path and cheap enough to call on every message. +/// +/// +/// Use it for routing, triage, moderation gates, intent detection and similar classification work, +/// and hand the result to a generative model when you actually need prose. +/// +/// +public interface IDecisionClient +{ + /// + /// Selects one of for the supplied text. + /// + /// The text to reason about. + /// Option labels mapped to optional descriptions. + /// Optional natural-language guidance, such as a question to answer. + /// A token to cancel the request. + /// The selected label and the full distribution over options. + Task ChooseAsync( + string state, + IReadOnlyDictionary options, + string? instructions = null, + CancellationToken cancellationToken = default); + + /// + /// Scores the supplied text against an ordered rubric. + /// + /// The text to reason about. + /// Between 2 and 10 rubric levels, ordered from lowest to highest. + /// Optional natural-language guidance. + /// A token to cancel the request. + /// The expected score and the distribution over rubric levels. + Task ScoreAsync( + string state, + IReadOnlyList levels, + string? instructions = null, + CancellationToken cancellationToken = default); + + /// + /// Assesses how likely a proposition is to be true of the supplied text. + /// + /// The text to reason about. + /// The statement to assess, phrased so that "true" is unambiguous. + /// A token to cancel the request. + /// The probability that the proposition holds. + Task AskAsync( + string state, + string proposition, + CancellationToken cancellationToken = default); + + /// + /// Answers several independent questions about one piece of text in a single forward pass. + /// + /// The questions to answer and the text they share. + /// A token to cancel the request. + /// All answers, read back by question name. + /// + /// Batching is close to free: five questions cost roughly what one costs. Prefer this over + /// issuing several single-question calls. + /// + Task EvaluateAsync( + DecisionRequest request, + CancellationToken cancellationToken = default); +} diff --git a/src/ElBruno.LocalLLMs.Decisions/Internal/ByteLevelPreTokenizer.cs b/src/ElBruno.LocalLLMs.Decisions/Internal/ByteLevelPreTokenizer.cs new file mode 100644 index 0000000..9bb7d0b --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/Internal/ByteLevelPreTokenizer.cs @@ -0,0 +1,40 @@ +using System.Text.RegularExpressions; +using Microsoft.ML.Tokenizers; + +namespace ElBruno.LocalLLMs.Decisions.Internal; + +/// +/// Splits text the way HuggingFace's ByteLevel pre-tokenizer does, which is what Laya's +/// checkpoints were tokenized with. +/// +/// +/// The pattern keeps a leading space attached to the word that follows it. That detail is not +/// cosmetic: "world" and " world" are different tokens in a byte-level vocabulary, so dropping +/// the space shifts every id after it and produces a different answer from the model. +/// +internal sealed partial class ByteLevelPreTokenizer : PreTokenizer +{ + internal static ByteLevelPreTokenizer Instance { get; } = new(); + + [GeneratedRegex(@"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+")] + private static partial Regex SplitPattern(); + + public override IEnumerable<(int Offset, int Length)> PreTokenize(string? text) + { + if (string.IsNullOrEmpty(text)) + { + yield break; + } + + foreach (Match match in SplitPattern().Matches(text)) + { + if (match.Length > 0) + { + yield return (match.Index, match.Length); + } + } + } + + public override IEnumerable<(int Offset, int Length)> PreTokenize(ReadOnlySpan text) + => PreTokenize(text.ToString()); +} diff --git a/src/ElBruno.LocalLLMs.Decisions/Internal/DecisionMath.cs b/src/ElBruno.LocalLLMs.Decisions/Internal/DecisionMath.cs new file mode 100644 index 0000000..215217a --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/Internal/DecisionMath.cs @@ -0,0 +1,135 @@ +namespace ElBruno.LocalLLMs.Decisions.Internal; + +/// +/// The numeric steps that turn raw marker logits into the answers Laya reports. +/// +/// +/// These run outside the ONNX graph because the option count varies per call. Keeping them here, +/// rather than baking them into the export, is also what lets the temperature be corrected +/// without re-exporting the model. +/// +internal static class DecisionMath +{ + /// + /// Divides logits by a temperature and normalizes them into a probability distribution. + /// + /// The raw marker logits, one per option. + /// The fitted temperature for this question kind and size. + /// Probabilities summing to 1. + public static double[] Softmax(ReadOnlySpan logits, double temperature) + { + var probabilities = new double[logits.Length]; + if (logits.Length == 0) + { + return probabilities; + } + + var maximum = double.NegativeInfinity; + for (var i = 0; i < logits.Length; i++) + { + probabilities[i] = logits[i] / temperature; + if (probabilities[i] > maximum) + { + maximum = probabilities[i]; + } + } + + var total = 0.0; + for (var i = 0; i < probabilities.Length; i++) + { + // Subtract the maximum first: the exported logits can exceed 80 once divided by a + // temperature below 1, which overflows to infinity without this shift. + probabilities[i] = Math.Exp(probabilities[i] - maximum); + total += probabilities[i]; + } + + if (total <= 0 || double.IsNaN(total) || double.IsInfinity(total)) + { + Array.Fill(probabilities, 1.0 / probabilities.Length); + return probabilities; + } + + for (var i = 0; i < probabilities.Length; i++) + { + probabilities[i] /= total; + } + + return probabilities; + } + + /// + /// Returns the probability mass on the answer being reported, which is max(p). + /// + /// The distribution over options. + /// A value between 0 and 1. + /// + /// This is the quantity temperature scaling fits, so it is the one to gate on. It still only + /// carries its calibration guarantee on a checkpoint whose temperatures were fitted and + /// validated for your option counts. + /// + public static double AnswerConfidence(ReadOnlySpan probabilities) + { + if (probabilities.Length < 1) + { + return 1.0; + } + + var maximum = 0.0; + foreach (var probability in probabilities) + { + if (probability > maximum) + { + maximum = probability; + } + } + + return Math.Clamp(maximum, 0.0, 1.0); + } + + /// + /// Returns normalized Shannon entropy confidence, 1 - H(p) / log(k). + /// + /// The distribution over options. + /// A value between 0 and 1. + /// + /// This measures how concentrated the whole distribution is. It is not what temperature + /// scaling fits, so it must not be compared against the same threshold as + /// . + /// + public static double EntropyConfidence(ReadOnlySpan probabilities) + { + if (probabilities.Length < 2) + { + return 1.0; + } + + var entropy = 0.0; + foreach (var probability in probabilities) + { + entropy -= probability * Math.Log(Math.Clamp(probability, 1e-12, 1.0)); + } + + return Math.Clamp(1.0 - (entropy / Math.Log(probabilities.Length)), 0.0, 1.0); + } + + /// + /// Returns the expected rubric level under the distribution. + /// + /// The distribution over ordered levels. + /// A continuous value between 0 and probabilities.Length - 1. + /// + /// This is an expectation, not an argmax, so a bimodal distribution over levels 0 and 2 + /// reports 1 β€” a level the model never actually favoured. Read + /// when that distinction matters. + /// + public static double ExpectedScore(ReadOnlySpan probabilities) + { + var score = 0.0; + for (var i = 0; i < probabilities.Length; i++) + { + score += i * probabilities[i]; + } + + return score; + } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/Internal/LayaModel.cs b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaModel.cs new file mode 100644 index 0000000..2b03314 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaModel.cs @@ -0,0 +1,134 @@ +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; + +namespace ElBruno.LocalLLMs.Decisions.Internal; + +/// +/// Wraps the exported Laya graph and runs a batch of questions through it in one forward pass. +/// +/// +/// The graph takes input_ids, attention_mask, marker_pos, marker_mask +/// and qtype, and returns raw logits plus act_logits. Rows are independent, +/// so several questions about the same text share a single pass. +/// +internal sealed class LayaModel : IDisposable +{ + private readonly InferenceSession _session; + private readonly int _padTokenId; + + private LayaModel(InferenceSession session, int padTokenId) + { + _session = session; + _padTokenId = padTokenId; + } + + /// + /// Opens an ONNX session over the exported graph. + /// + /// Full path to the .onnx file. + /// The token id used to square off a batch. + /// Optional thread count for the CPU execution provider. + /// A ready session. + public static LayaModel Load(string modelPath, int padTokenId, int? intraOpNumThreads) + { + var options = new SessionOptions + { + GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL, + }; + + if (intraOpNumThreads is > 0) + { + options.IntraOpNumThreads = intraOpNumThreads.Value; + } + + try + { + return new LayaModel(new InferenceSession(modelPath, options), padTokenId); + } + catch (Exception ex) when (ex is OnnxRuntimeException or FileNotFoundException) + { + options.Dispose(); + throw new DecisionException( + $"The decision model at '{modelPath}' could not be loaded. " + + "Check that the download completed and that any companion '.onnx.data' file sits beside it.", + ex); + } + } + + /// + /// Runs every sequence in one pass and returns the marker logits for each. + /// + /// The tokenized questions. + /// One logit array per sequence, trimmed to that sequence's option count. + public float[][] Run(IReadOnlyList sequences) + { + var batch = sequences.Count; + var length = sequences.Max(sequence => sequence.TokenIds.Count); + var markers = sequences.Max(sequence => sequence.MarkerPositions.Count); + + var inputIds = new long[batch * length]; + var attentionMask = new long[batch * length]; + var markerPositions = new long[batch * markers]; + var markerMask = new bool[batch * markers]; + var questionTypes = new long[batch]; + + // Padded rows must still be valid indices: the head gathers marker positions before it + // masks them, so an out-of-range position would fault rather than be ignored. + Array.Fill(inputIds, _padTokenId); + + for (var row = 0; row < batch; row++) + { + var sequence = sequences[row]; + var tokenOffset = row * length; + for (var i = 0; i < sequence.TokenIds.Count; i++) + { + inputIds[tokenOffset + i] = sequence.TokenIds[i]; + attentionMask[tokenOffset + i] = 1; + } + + var markerOffset = row * markers; + for (var i = 0; i < sequence.MarkerPositions.Count; i++) + { + markerPositions[markerOffset + i] = sequence.MarkerPositions[i]; + markerMask[markerOffset + i] = true; + } + + questionTypes[row] = sequence.QuestionType; + } + + var inputs = new List + { + NamedOnnxValue.CreateFromTensor("input_ids", + new DenseTensor(new Memory(inputIds), new[] { batch, length })), + NamedOnnxValue.CreateFromTensor("attention_mask", + new DenseTensor(new Memory(attentionMask), new[] { batch, length })), + NamedOnnxValue.CreateFromTensor("marker_pos", + new DenseTensor(new Memory(markerPositions), new[] { batch, markers })), + NamedOnnxValue.CreateFromTensor("marker_mask", + new DenseTensor(new Memory(markerMask), new[] { batch, markers })), + NamedOnnxValue.CreateFromTensor("qtype", + new DenseTensor(new Memory(questionTypes), new[] { batch })), + }; + + using var outputs = _session.Run(inputs); + var logits = outputs.First(output => output.Name == "logits").AsTensor(); + + var results = new float[batch][]; + for (var row = 0; row < batch; row++) + { + var count = sequences[row].MarkerPositions.Count; + var row_ = new float[count]; + for (var i = 0; i < count; i++) + { + row_[i] = logits[row, i]; + } + + results[row] = row_; + } + + return results; + } + + /// + public void Dispose() => _session.Dispose(); +} diff --git a/src/ElBruno.LocalLLMs.Decisions/Internal/LayaModelFiles.cs b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaModelFiles.cs new file mode 100644 index 0000000..cef74f7 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaModelFiles.cs @@ -0,0 +1,161 @@ +using ElBruno.HuggingFace; + +namespace ElBruno.LocalLLMs.Decisions.Internal; + +/// +/// Resolves the four files a decision model needs, downloading them from HuggingFace on first use. +/// +/// The ONNX graph. +/// The checkpoint settings, including fitted temperatures. +/// The tokenizer definition. +internal sealed record LayaModelFiles(string ModelPath, string ConfigPath, string TokenizerPath) +{ + private static readonly SemaphoreSlim s_downloadLock = new(1, 1); + + /// + /// Ensures the model files exist locally and returns their paths. + /// + /// The configured model source and cache location. + /// A token to cancel the download. + /// The resolved file paths. + public static async Task EnsureAsync( + DecisionOptions options, + CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(options.ModelPath)) + { + return Locate(options.ModelPath!, mustExist: true); + } + + var directory = Path.Combine(options.ResolveCacheDirectory(), Sanitize(options.ModelRepository)); + var candidate = TryLocate(directory); + if (candidate is not null) + { + return candidate; + } + + await s_downloadLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + // Another caller may have completed the download while this one waited. + candidate = TryLocate(directory); + if (candidate is not null) + { + return candidate; + } + + Directory.CreateDirectory(directory); + + var downloader = new HuggingFaceDownloader(); + await downloader.DownloadFilesAsync( + new DownloadRequest + { + RepoId = options.ModelRepository, + LocalDirectory = directory, + RequiredFiles = options.ModelFiles.ToArray(), + OptionalFiles = options.OptionalModelFiles.ToArray(), + }, + cancellationToken).ConfigureAwait(false); + + return Locate(directory, mustExist: true); + } + catch (Exception ex) when (ex is not OperationCanceledException and not DecisionException) + { + throw new DecisionException( + $"The decision model '{options.ModelRepository}' could not be downloaded. " + + "Check network access, or set DecisionOptions.ModelPath to a directory you have " + + "already populated.", + ex); + } + finally + { + s_downloadLock.Release(); + } + } + + private static LayaModelFiles? TryLocate(string directory) + { + if (!Directory.Exists(directory)) + { + return null; + } + + try + { + return Locate(directory, mustExist: false); + } + catch (DecisionException) + { + return null; + } + } + + private static LayaModelFiles Locate(string directory, bool mustExist) + { + if (!Directory.Exists(directory)) + { + throw new DecisionException($"The model directory '{directory}' does not exist."); + } + + var model = FindFirst(directory, "model.onnx", "laya.onnx", "onnx/model.onnx"); + var config = FindFirst(directory, "rl_agent_config.json", "config.json"); + var tokenizer = FindFirst(directory, "tokenizer/tokenizer.json", "tokenizer.json"); + + if (model is null || config is null || tokenizer is null) + { + var missing = new List(); + if (model is null) missing.Add("the ONNX graph (model.onnx)"); + if (config is null) missing.Add("the checkpoint config (rl_agent_config.json or config.json)"); + if (tokenizer is null) missing.Add("the tokenizer (tokenizer.json)"); + + throw new DecisionException( + $"The model directory '{directory}' is missing {string.Join(", ", missing)}." + + (mustExist ? " The download may have been interrupted; delete the directory and retry." : string.Empty)); + } + + // A graph that uses external data is unusable without it, and the failure surfaces much + // later as an opaque ONNX Runtime error, so check for it while the context is still clear. + var externalData = model + ".data"; + if (mustExist && new FileInfo(model).Length < 50L * 1024 * 1024 && !File.Exists(externalData)) + { + var siblings = Directory.EnumerateFiles(Path.GetDirectoryName(model)!, "*.onnx*data*").ToArray(); + if (siblings.Length == 0) + { + throw new DecisionException( + $"The ONNX graph at '{model}' is too small to contain weights and no companion " + + "'.onnx.data' file sits beside it. The download was probably interrupted; " + + "delete the directory and retry."); + } + } + + return new LayaModelFiles(model, config, tokenizer); + } + + private static string? FindFirst(string directory, params string[] relativePaths) + { + foreach (var relative in relativePaths) + { + var full = Path.Combine(directory, relative.Replace('/', Path.DirectorySeparatorChar)); + if (File.Exists(full)) + { + return full; + } + } + + return null; + } + + private static string Sanitize(string repository) + { + var characters = repository.ToCharArray(); + for (var i = 0; i < characters.Length; i++) + { + if (Array.IndexOf(Path.GetInvalidFileNameChars(), characters[i]) >= 0) + { + characters[i] = '_'; + } + } + + return new string(characters); + } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/Internal/LayaRuntimeConfig.cs b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaRuntimeConfig.cs new file mode 100644 index 0000000..a6e1ba9 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaRuntimeConfig.cs @@ -0,0 +1,198 @@ +using System.Globalization; +using System.Text.Json; + +namespace ElBruno.LocalLLMs.Decisions.Internal; + +/// +/// The checkpoint settings Laya keeps outside the ONNX graph: the token budgets that shape the +/// input sequence, and the fitted temperatures applied to the logits afterwards. +/// +internal sealed class LayaRuntimeConfig +{ + /// The lowest temperature that will be applied, matching Laya's TEMP_MIN. + public const double MinimumTemperature = 0.5; + + /// The highest temperature that will be applied, matching Laya's TEMP_MAX. + public const double MaximumTemperature = 5.0; + + /// The per-option token cap Laya applies when rendering options. + public const int MaxOptionTokens = 48; + + private readonly double[] _temperatureByType; + private readonly bool[] _clampedByType; + private readonly Dictionary _temperatureByOptions; + private readonly HashSet _clampedByOptions; + + private LayaRuntimeConfig( + int maxLength, + int headMaxLength, + double[] temperatureByType, + bool[] clampedByType, + Dictionary temperatureByOptions, + HashSet clampedByOptions, + bool temperaturesWereClamped) + { + MaxLength = maxLength; + HeadMaxLength = headMaxLength; + _temperatureByType = temperatureByType; + _clampedByType = clampedByType; + _temperatureByOptions = temperatureByOptions; + _clampedByOptions = clampedByOptions; + TemperaturesWereClamped = temperaturesWereClamped; + } + + /// Gets the total token budget for one sequence. + public int MaxLength { get; } + + /// Gets the token budget shared by the instructions and the rendered options. + public int HeadMaxLength { get; } + + /// + /// Gets a value indicating whether the checkpoint shipped a temperature outside + /// .. that had to be clamped. + /// + /// + /// Laya's public English checkpoint ships choice:11+ = 0.1006, which sharpens the + /// logits roughly tenfold and turns a 24% top probability into a reported 99%. Clamping keeps + /// the reported confidence honest, at the cost of diverging from the raw checkpoint value. + /// + public bool TemperaturesWereClamped { get; } + + /// + /// Reads rl_agent_config.json from a checkpoint directory. + /// + /// Full path to the configuration file. + /// The parsed configuration. + public static LayaRuntimeConfig Load(string configPath) + { + using var document = JsonDocument.Parse(File.ReadAllText(configPath)); + var root = document.RootElement; + + // The onnx-community exports nest the same values under a "laya" object; the + // convaiinnovations-derived exports put them at the root. + var source = root.TryGetProperty("laya", out var nested) ? nested : root; + + var maxLength = source.TryGetProperty("max_len", out var maxLen) ? maxLen.GetInt32() : 512; + var headMaxLength = source.TryGetProperty("head_max_len", out var headMax) ? headMax.GetInt32() : 192; + + var clamped = false; + var byType = new double[3]; + var clampedByType = new bool[3]; + Array.Fill(byType, 1.0); + + if (source.TryGetProperty("temperature", out var temperatures) + && temperatures.ValueKind == JsonValueKind.Array) + { + var index = 0; + foreach (var value in temperatures.EnumerateArray()) + { + if (index >= byType.Length) + { + break; + } + + (byType[index], clampedByType[index]) = ClampTemperature(value); + clamped |= clampedByType[index]; + index++; + } + } + + var byOptions = new Dictionary(StringComparer.Ordinal); + var clampedByOptions = new HashSet(StringComparer.Ordinal); + if (source.TryGetProperty("temperature_by_options", out var buckets) + && buckets.ValueKind == JsonValueKind.Object) + { + foreach (var bucket in buckets.EnumerateObject()) + { + var (value, wasClamped) = ClampTemperature(bucket.Value); + byOptions[bucket.Name] = value; + if (wasClamped) + { + clampedByOptions.Add(bucket.Name); + clamped = true; + } + } + } + + return new LayaRuntimeConfig( + maxLength, + headMaxLength, + byType, + clampedByType, + byOptions, + clampedByOptions, + clamped); + } + + /// + /// Returns the temperature to divide the logits by for a question of this kind and size. + /// + /// The question type index: 0 choice, 1 score, 2 noul. + /// The number of options the caller supplied. + /// A temperature within the permitted range. + public double TemperatureFor(int questionType, int optionCount) + { + var bucket = TemperatureBucket(questionType, optionCount); + if (_temperatureByOptions.TryGetValue(bucket, out var fitted)) + { + return fitted; + } + + return questionType >= 0 && questionType < _temperatureByType.Length + ? _temperatureByType[questionType] + : 1.0; + } + + /// + /// Returns whether the temperature used for a question of this kind and size was clamped, + /// meaning the checkpoint shipped a value outside its own valid range for that bucket. + /// + /// The question type index: 0 choice, 1 score, 2 noul. + /// The number of options the caller supplied. + /// when the applied temperature is not the fitted one. + public bool WasClampedFor(int questionType, int optionCount) + { + var bucket = TemperatureBucket(questionType, optionCount); + if (_temperatureByOptions.ContainsKey(bucket)) + { + return _clampedByOptions.Contains(bucket); + } + + return questionType >= 0 + && questionType < _clampedByType.Length + && _clampedByType[questionType]; + } + + private static string TemperatureBucket(int questionType, int optionCount) + { + var size = optionCount <= 2 ? "2" + : optionCount <= 5 ? "3-5" + : optionCount <= 10 ? "6-10" + : "11+"; + + var name = questionType switch + { + 0 => "choice", + 1 => "score", + _ => "noul", + }; + + return string.Create(CultureInfo.InvariantCulture, $"{name}:{size}"); + } + + private static (double Value, bool Clamped) ClampTemperature(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Number || !element.TryGetDouble(out var value)) + { + return (1.0, true); + } + + if (double.IsNaN(value) || double.IsInfinity(value)) + { + return (1.0, true); + } + + var bounded = Math.Clamp(value, MinimumTemperature, MaximumTemperature); + return (bounded, bounded != value); + } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/Internal/LayaSequenceBuilder.cs b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaSequenceBuilder.cs new file mode 100644 index 0000000..2d770c2 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaSequenceBuilder.cs @@ -0,0 +1,206 @@ +using System.Globalization; + +namespace ElBruno.LocalLLMs.Decisions.Internal; + +/// +/// One tokenized question, ready to be batched into a forward pass. +/// +/// The caller's question name, used to key the answer. +/// The question this sequence was built from. +/// The full input sequence. +/// The index of each option's [MASK] marker within . +/// The option labels in marker order, as the caller supplied them. +/// The type index the model expects: 0 choice, 1 score, 2 noul. +internal sealed record LayaSequence( + string Name, + DecisionQuestion Question, + IReadOnlyList TokenIds, + IReadOnlyList MarkerPositions, + IReadOnlyList Labels, + int QuestionType); + +/// +/// Builds the input sequence Laya's checkpoints were trained on. +/// +/// +/// +/// The format is [CLS] {type} question: {instructions} [SEP] [MASK] opt0 [MASK] opt1 … [SEP] state [SEP]. +/// Every option is rendered as ordinary text preceded by a [MASK] marker, and the model +/// emits one logit per marker. That is why arbitrary caller-supplied labels work: nothing in the +/// graph is indexed by label, so the option set can change per call. +/// +/// +/// The budgets are applied in Laya's order β€” options first, then instructions, then whatever +/// room is left goes to the state. Changing that order changes the token ids and therefore the +/// answer. +/// +/// +internal sealed class LayaSequenceBuilder(LayaTokenizer tokenizer, LayaRuntimeConfig config) +{ + /// + /// Builds one sequence per question, reusing a single tokenization of the shared state. + /// + /// The questions and the text they share. + /// One sequence per question, in request order. + public IReadOnlyList Build(DecisionRequest request) + { + // Laya tokenizes the state once and reuses it for every question in the batch. + var stateIds = tokenizer.Encode(StripMaskToken(request.State)); + + var sequences = new List(request.Questions.Count); + foreach (var (name, question) in request.Questions) + { + sequences.Add(BuildOne(name, question, stateIds)); + } + + return sequences; + } + + private LayaSequence BuildOne(string name, DecisionQuestion question, IReadOnlyList stateIds) + { + var questionType = ToQuestionType(question.Type); + var typeName = questionType switch + { + 0 => "choice", + 1 => "score", + _ => "noul", + }; + + var (labels, optionTexts) = RenderOptions(question); + + var instructions = StripMaskToken(question.Instructions ?? string.Empty); + var headIds = tokenizer.Encode( + string.Create(CultureInfo.InvariantCulture, $"{typeName} question: {instructions}")); + + var optionIds = new List>(optionTexts.Count); + foreach (var optionText in optionTexts) + { + // Truncate at the tokenizer rather than after the fact, so a very long description + // costs the same as a short one. + var tokens = tokenizer.EncodeTruncated( + " " + StripMaskToken(optionText), + LayaRuntimeConfig.MaxOptionTokens); + + var option = new List(tokens.Count + 1) { tokenizer.MaskTokenId }; + option.AddRange(tokens); + optionIds.Add(option); + } + + var optionBudget = config.HeadMaxLength - optionIds.Sum(option => option.Count); + if (optionBudget < 16) + { + // Shrink every option evenly rather than dropping the ones at the end, so each option + // keeps a marker and the answer still covers the full label set. + var perOption = Math.Max(4, (config.HeadMaxLength - 16) / Math.Max(1, optionIds.Count)); + for (var i = 0; i < optionIds.Count; i++) + { + if (optionIds[i].Count > perOption) + { + optionIds[i] = optionIds[i].GetRange(0, perOption); + } + } + + optionBudget = config.HeadMaxLength - optionIds.Sum(option => option.Count); + } + + var headBudget = Math.Max(8, optionBudget); + if (headIds.Count > headBudget) + { + headIds = headIds.Take(headBudget).ToArray(); + } + + var ids = new List(config.MaxLength) { tokenizer.ClsTokenId }; + ids.AddRange(headIds); + ids.Add(tokenizer.SepTokenId); + + var markers = new List(optionIds.Count); + foreach (var option in optionIds) + { + markers.Add(ids.Count); + ids.AddRange(option); + } + + ids.Add(tokenizer.SepTokenId); + + var room = Math.Max(0, config.MaxLength - ids.Count - 1); + var used = Math.Min(room, stateIds.Count); + for (var i = 0; i < used; i++) + { + ids.Add(stateIds[i]); + } + + ids.Add(tokenizer.SepTokenId); + + if (ids.Count > config.MaxLength) + { + ids = ids.GetRange(0, config.MaxLength); + } + + var keptMarkers = markers.Where(marker => marker < config.MaxLength).ToArray(); + if (keptMarkers.Length != labels.Count) + { + throw new DecisionException( + $"Question '{name}' does not fit the model's {config.MaxLength}-token window: " + + $"{labels.Count} options were supplied but only {keptMarkers.Length} could be scored. " + + "Use fewer or shorter options."); + } + + return new LayaSequence(name, question, ids, keptMarkers, labels, questionType); + } + + private static (IReadOnlyList Labels, IReadOnlyList Texts) RenderOptions(DecisionQuestion question) + { + switch (question.Type) + { + case DecisionQuestionType.Choice: + { + var options = question.Options + ?? throw new DecisionException("A choice question must define its options."); + + var labels = new List(options.Count); + var texts = new List(options.Count); + foreach (var (label, description) in options) + { + labels.Add(label); + texts.Add(string.IsNullOrEmpty(description) ? label : $"{label}: {description}"); + } + + return (labels, texts); + } + + case DecisionQuestionType.Score: + { + var levels = question.Levels + ?? throw new DecisionException("A score question must define its rubric levels."); + + var labels = new List(levels.Count); + var texts = new List(levels.Count); + for (var i = 0; i < levels.Count; i++) + { + labels.Add(levels[i]); + texts.Add(string.Create(CultureInfo.InvariantCulture, $"level {i}: {levels[i]}")); + } + + return (labels, texts); + } + + default: + // Noul options are fixed and always in this order, so index 1 is always "true". + return ( + ["false", "true"], + ["false: no, the statement does not hold", "true: yes, the statement holds"]); + } + } + + private static int ToQuestionType(DecisionQuestionType type) => type switch + { + DecisionQuestionType.Choice => 0, + DecisionQuestionType.Score => 1, + _ => 2, + }; + + private static string StripMaskToken(string text) => + text.Contains(LayaTokenizer.MaskToken, StringComparison.Ordinal) + ? text.Replace(LayaTokenizer.MaskToken, " ", StringComparison.Ordinal) + : text; +} diff --git a/src/ElBruno.LocalLLMs.Decisions/Internal/LayaTokenizer.cs b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaTokenizer.cs new file mode 100644 index 0000000..e6bd14b --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/Internal/LayaTokenizer.cs @@ -0,0 +1,188 @@ +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using Microsoft.ML.Tokenizers; + +namespace ElBruno.LocalLLMs.Decisions.Internal; + +/// +/// A token that HuggingFace keeps out of the BPE merge process and matches against the raw text. +/// +/// The literal text that identifies the token. +/// The vocabulary id emitted when the content matches. +/// Whether whitespace immediately before the content is absorbed into the token. +/// Whether whitespace immediately after the content is absorbed into the token. +internal sealed record LayaAddedToken(string Content, int Id, bool LStrip, bool RStrip); + +/// +/// Reproduces the HuggingFace fast tokenizer that Laya's English checkpoint ships, reading the +/// checkpoint's own tokenizer.json so the vocabulary and merges are never duplicated here. +/// +/// +/// +/// Token ids must match the Python reference exactly. The model scores the hidden state at each +/// [MASK] marker position, so a single extra or missing token moves every marker after it +/// and the answer silently changes rather than failing. +/// +/// +/// Three pieces have to line up: NFC normalization, ByteLevel pre-tokenization, and the added +/// vocabulary, which is matched against the text before BPE runs. +/// +/// +internal sealed class LayaTokenizer +{ + private readonly BpeTokenizer _bpe; + private readonly Regex _addedPattern; + private readonly Dictionary _idsByContent; + + private LayaTokenizer(BpeTokenizer bpe, IReadOnlyList addedTokens) + { + _bpe = bpe; + _idsByContent = addedTokens + .GroupBy(token => token.Content, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.First().Id, StringComparer.Ordinal); + _addedPattern = BuildAddedPattern(addedTokens); + + ClsTokenId = RequireToken("[CLS]"); + SepTokenId = RequireToken("[SEP]"); + MaskTokenId = RequireToken("[MASK]"); + PadTokenId = RequireToken("[PAD]"); + } + + /// Gets the id of the [CLS] token that opens every sequence. + public int ClsTokenId { get; } + + /// Gets the id of the [SEP] token that separates the sequence sections. + public int SepTokenId { get; } + + /// Gets the id of the [MASK] token that marks a scorable option. + public int MaskTokenId { get; } + + /// Gets the id of the [PAD] token used to square off a batch. + public int PadTokenId { get; } + + /// Gets the literal [MASK] text, which is stripped from caller-supplied text. + public const string MaskToken = "[MASK]"; + + /// + /// Loads a tokenizer from a checkpoint's tokenizer.json. + /// + /// Full path to the tokenizer definition. + /// A tokenizer that reproduces the checkpoint's Python behaviour. + public static LayaTokenizer Load(string tokenizerJsonPath) + { + // Read as text rather than bytes so a byte-order mark, which some tooling adds when + // re-saving a tokenizer, does not fail the parse. + using var document = JsonDocument.Parse(File.ReadAllText(tokenizerJsonPath)); + var root = document.RootElement; + var model = root.GetProperty("model"); + + var vocabulary = model.GetProperty("vocab") + .EnumerateObject() + .Select(entry => new KeyValuePair(entry.Name, entry.Value.GetInt32())) + .ToList(); + + var merges = model.GetProperty("merges") + .EnumerateArray() + .Select(pair => pair[0].GetString() + " " + pair[1].GetString()) + .ToList(); + + var addedTokens = root.GetProperty("added_tokens") + .EnumerateArray() + .Select(token => new LayaAddedToken( + token.GetProperty("content").GetString()!, + token.GetProperty("id").GetInt32(), + token.GetProperty("lstrip").GetBoolean(), + token.GetProperty("rstrip").GetBoolean())) + .ToList(); + + var bpe = BpeTokenizer.Create(new BpeOptions(vocabulary) + { + Merges = merges, + ByteLevel = true, + PreTokenizer = ByteLevelPreTokenizer.Instance, + UnknownToken = null, + }); + + return new LayaTokenizer(bpe, addedTokens); + } + + /// + /// Encodes text to token ids, without adding the surrounding special tokens. + /// + /// The text to encode. + /// The token ids, in order. + public IReadOnlyList Encode(string text) + { + if (string.IsNullOrEmpty(text)) + { + return []; + } + + var normalized = text.Normalize(NormalizationForm.FormC); + var ids = new List(); + var cursor = 0; + + foreach (Match match in _addedPattern.Matches(normalized)) + { + if (match.Index > cursor) + { + ids.AddRange(_bpe.EncodeToIds(normalized[cursor..match.Index])); + } + + ids.Add(_idsByContent[match.Groups["content"].Value]); + cursor = match.Index + match.Length; + } + + if (cursor < normalized.Length) + { + ids.AddRange(_bpe.EncodeToIds(normalized[cursor..])); + } + + return ids; + } + + /// + /// Encodes text and keeps at most tokens from the start. + /// + /// The text to encode. + /// The inclusive token ceiling. + /// The truncated token ids. + public IReadOnlyList EncodeTruncated(string text, int maxTokens) + { + var ids = Encode(text); + return ids.Count <= maxTokens ? ids : ids.Take(maxTokens).ToArray(); + } + + private int RequireToken(string content) => + _idsByContent.TryGetValue(content, out var id) + ? id + : throw new DecisionException( + $"The tokenizer is missing the '{content}' token, so it cannot be a Laya checkpoint. " + + "Check that the model directory holds the tokenizer that shipped with the model."); + + private static Regex BuildAddedPattern(IReadOnlyList addedTokens) + { + // Longest content first: the added vocabulary contains overlapping whitespace runs, and + // regex alternation is first-match-wins rather than longest-match-wins. + var alternatives = addedTokens + .OrderByDescending(token => token.Content.Length) + .Select(token => + { + var pattern = "(?" + Regex.Escape(token.Content) + ")"; + if (token.LStrip) + { + pattern = @"\s*" + pattern; + } + + if (token.RStrip) + { + pattern += @"\s*"; + } + + return pattern; + }); + + return new Regex(string.Join("|", alternatives), RegexOptions.Compiled | RegexOptions.CultureInvariant); + } +} diff --git a/src/ElBruno.LocalLLMs.Decisions/LayaOnnxDecisionClient.cs b/src/ElBruno.LocalLLMs.Decisions/LayaOnnxDecisionClient.cs new file mode 100644 index 0000000..0eaf7d3 --- /dev/null +++ b/src/ElBruno.LocalLLMs.Decisions/LayaOnnxDecisionClient.cs @@ -0,0 +1,232 @@ +using System.Diagnostics; +using ElBruno.LocalLLMs.Decisions.Internal; + +namespace ElBruno.LocalLLMs.Decisions; + +/// +/// An that runs a Laya decision model in-process through ONNX Runtime. +/// +/// +/// +/// Nothing leaves the machine and there is no server to start. The model files are downloaded +/// from HuggingFace the first time the client is used and cached afterwards, so the first call +/// is slow and the rest are not. +/// +/// +/// A single instance is safe to share and expensive to create, so register it as a singleton. +/// +/// +public sealed class LayaOnnxDecisionClient : IDecisionClient, IDisposable +{ + private readonly DecisionOptions _options; + private readonly SemaphoreSlim _initializationLock = new(1, 1); + private Runtime? _runtime; + private bool _disposed; + + /// + /// Creates a client using the supplied configuration. The model is not loaded until the + /// first call. + /// + /// The model source and decoding settings. + public LayaOnnxDecisionClient(DecisionOptions? options = null) + { + _options = options ?? new DecisionOptions(); + _options.Validate(); + } + + /// + public async Task ChooseAsync( + string state, + IReadOnlyDictionary options, + string? instructions = null, + CancellationToken cancellationToken = default) + { + var result = await EvaluateAsync( + new DecisionRequest(state).Choose("choice", options, instructions), + cancellationToken).ConfigureAwait(false); + + return result.Choice("choice"); + } + + /// + public async Task ScoreAsync( + string state, + IReadOnlyList levels, + string? instructions = null, + CancellationToken cancellationToken = default) + { + var result = await EvaluateAsync( + new DecisionRequest(state).Score("score", levels, instructions), + cancellationToken).ConfigureAwait(false); + + return result.Score("score"); + } + + /// + public async Task AskAsync( + string state, + string proposition, + CancellationToken cancellationToken = default) + { + var result = await EvaluateAsync( + new DecisionRequest(state).Ask("ask", proposition), + cancellationToken).ConfigureAwait(false); + + return result.Probability("ask"); + } + + /// + public async Task EvaluateAsync( + DecisionRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + ObjectDisposedException.ThrowIf(_disposed, this); + + if (request.Questions.Count == 0) + { + throw new ArgumentException("A decision request must contain at least one question.", nameof(request)); + } + + var runtime = await EnsureRuntimeAsync(cancellationToken).ConfigureAwait(false); + var started = Stopwatch.GetTimestamp(); + + var sequences = runtime.Builder.Build(request); + + cancellationToken.ThrowIfCancellationRequested(); + + // ONNX Runtime work is synchronous and CPU-bound, so move it off the calling thread + // rather than blocking a request thread for the duration of the forward pass. + var logits = await Task.Run(() => runtime.Model.Run(sequences), cancellationToken) + .ConfigureAwait(false); + + var answers = new Dictionary(sequences.Count, StringComparer.Ordinal); + var tokens = 0L; + + for (var i = 0; i < sequences.Count; i++) + { + var sequence = sequences[i]; + tokens += sequence.TokenIds.Count; + + var temperature = runtime.Config.TemperatureFor(sequence.QuestionType, sequence.Labels.Count); + var clamped = runtime.Config.WasClampedFor(sequence.QuestionType, sequence.Labels.Count); + var probabilities = DecisionMath.Softmax(logits[i], temperature); + + answers[sequence.Name] = Decode(sequence, probabilities, clamped); + } + + return new DecisionResult( + runtime.ModelIdentifier, + tokens, + Stopwatch.GetElapsedTime(started), + answers); + } + + private object Decode(LayaSequence sequence, double[] probabilities, bool calibrationClamped) + { + switch (sequence.Question.Type) + { + case DecisionQuestionType.Choice: + { + var map = new Dictionary(sequence.Labels.Count, StringComparer.Ordinal); + var best = 0; + for (var i = 0; i < sequence.Labels.Count; i++) + { + map[sequence.Labels[i]] = probabilities[i]; + if (probabilities[i] > probabilities[best]) + { + best = i; + } + } + + return new ChoiceResult( + sequence.Labels[best], + map, + DecisionMath.AnswerConfidence(probabilities), + calibrationClamped); + } + + case DecisionQuestionType.Score: + { + var map = new Dictionary(sequence.Labels.Count); + for (var i = 0; i < sequence.Labels.Count; i++) + { + map[i] = probabilities[i]; + } + + return new ScoreResult( + DecisionMath.ExpectedScore(probabilities), + map, + sequence.Labels, + DecisionMath.AnswerConfidence(probabilities), + calibrationClamped); + } + + default: + // Noul options are rendered false-then-true, so index 1 is the probability of true. + return new ProbabilityResult(probabilities[1], _options.DecisionThreshold, calibrationClamped); + } + } + + private async Task EnsureRuntimeAsync(CancellationToken cancellationToken) + { + if (_runtime is not null) + { + return _runtime; + } + + await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_runtime is not null) + { + return _runtime; + } + + var files = await LayaModelFiles.EnsureAsync(_options, cancellationToken).ConfigureAwait(false); + + var tokenizer = LayaTokenizer.Load(files.TokenizerPath); + var config = LayaRuntimeConfig.Load(files.ConfigPath); + var model = LayaModel.Load(files.ModelPath, tokenizer.PadTokenId, _options.IntraOpNumThreads); + + var identifier = string.IsNullOrWhiteSpace(_options.ModelPath) + ? _options.ModelRepository + : files.ModelPath; + + _runtime = new Runtime( + tokenizer, + config, + model, + new LayaSequenceBuilder(tokenizer, config), + identifier); + + return _runtime; + } + finally + { + _initializationLock.Release(); + } + } + + /// + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _runtime?.Model.Dispose(); + _runtime = null; + _initializationLock.Dispose(); + } + + private sealed record Runtime( + LayaTokenizer Tokenizer, + LayaRuntimeConfig Config, + LayaModel Model, + LayaSequenceBuilder Builder, + string ModelIdentifier); +} diff --git a/src/ElBruno.LocalLLMs.Rag/ElBruno.LocalLLMs.Rag.csproj b/src/ElBruno.LocalLLMs.Rag/ElBruno.LocalLLMs.Rag.csproj index c0ee306..151b836 100644 --- a/src/ElBruno.LocalLLMs.Rag/ElBruno.LocalLLMs.Rag.csproj +++ b/src/ElBruno.LocalLLMs.Rag/ElBruno.LocalLLMs.Rag.csproj @@ -1,6 +1,6 @@ - net8.0;net10.0 + net10.0 true ElBruno.LocalLLMs.Rag RAG (Retrieval-Augmented Generation) pipeline for ElBruno.LocalLLMs. Provides document chunking, embedding storage, and semantic search. diff --git a/src/ElBruno.LocalLLMs/ElBruno.LocalLLMs.csproj b/src/ElBruno.LocalLLMs/ElBruno.LocalLLMs.csproj index 2b729b6..6472420 100644 --- a/src/ElBruno.LocalLLMs/ElBruno.LocalLLMs.csproj +++ b/src/ElBruno.LocalLLMs/ElBruno.LocalLLMs.csproj @@ -1,6 +1,6 @@ - net8.0;net10.0 + net10.0 true ElBruno.LocalLLMs Local LLM chat completions using Microsoft.Extensions.AI and ONNX Runtime GenAI. IChatClient implementation for running LLMs locally. diff --git a/src/benchmarks/ElBruno.LocalLLMs.Benchmarks/ElBruno.LocalLLMs.Benchmarks.csproj b/src/benchmarks/ElBruno.LocalLLMs.Benchmarks/ElBruno.LocalLLMs.Benchmarks.csproj index 7c2916a..8c1d066 100644 --- a/src/benchmarks/ElBruno.LocalLLMs.Benchmarks/ElBruno.LocalLLMs.Benchmarks.csproj +++ b/src/benchmarks/ElBruno.LocalLLMs.Benchmarks/ElBruno.LocalLLMs.Benchmarks.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net10.0 false diff --git a/src/samples/BlazorDemo/BlazorDemo.csproj b/src/samples/BlazorDemo/BlazorDemo.csproj index 99b475a..63b8a32 100644 --- a/src/samples/BlazorDemo/BlazorDemo.csproj +++ b/src/samples/BlazorDemo/BlazorDemo.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 enable enable false diff --git a/src/samples/ConsoleAppDemo/ConsoleAppDemo.csproj b/src/samples/ConsoleAppDemo/ConsoleAppDemo.csproj index b42e5ac..a656560 100644 --- a/src/samples/ConsoleAppDemo/ConsoleAppDemo.csproj +++ b/src/samples/ConsoleAppDemo/ConsoleAppDemo.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/samples/DecisionCalibration/DecisionCalibration.csproj b/src/samples/DecisionCalibration/DecisionCalibration.csproj new file mode 100644 index 0000000..ede0f72 --- /dev/null +++ b/src/samples/DecisionCalibration/DecisionCalibration.csproj @@ -0,0 +1,11 @@ + + + net10.0 + Exe + false + + + + + + diff --git a/src/samples/DecisionCalibration/Program.cs b/src/samples/DecisionCalibration/Program.cs new file mode 100644 index 0000000..6b773a8 --- /dev/null +++ b/src/samples/DecisionCalibration/Program.cs @@ -0,0 +1,140 @@ +using ElBruno.LocalLLMs.Decisions; + +// Demonstrates the two numerical caveats of the Laya ONNX checkpoint that are easy to +// trip over and hard to notice: a clamped calibration bucket, and fp16 batch variance. +// +// Usage: dotnet run [path-to-local-model-directory] +// With no argument the model is downloaded from HuggingFace on first use (~800 MB). + +var options = new DecisionOptions(); +if (args.Length > 0) +{ + options.ModelPath = args[0]; +} + +using var client = new LayaOnnxDecisionClient(options); + +Console.WriteLine($"model: {(args.Length > 0 ? args[0] : options.ModelRepository)}"); +Console.WriteLine("loading (first run downloads the checkpoint)...\n"); + +await ShowCalibrationClampAsync(client); +await ShowBatchVarianceAsync(client); + +// --------------------------------------------------------------------------------------- +// 1. Calibration clamp +// --------------------------------------------------------------------------------------- +// Laya's English checkpoint fits a temperature per (question type, option count) bucket. +// The choice:11+ bucket ships as 0.1006, far below Laya's own TEMP_MIN of 0.5. Dividing +// the logits by it sharpens the distribution roughly tenfold, which would report a +// genuinely uncertain answer as near-certain. The client clamps it back to 0.5 and sets +// CalibrationClamped so you can tell the difference. +static async Task ShowCalibrationClampAsync(IDecisionClient client) +{ + Console.WriteLine("=== 1. Calibration clamp (choice:11+) ===\n"); + + const string ticket = "The invoice I received this morning is for the wrong amount."; + + // Five options land in the choice:3-5 bucket, which ships a valid fitted temperature. + var small = await client.ChooseAsync( + ticket, + BuildDepartments(5), + "Route this ticket to the right team"); + + // Twelve options land in choice:11+, whose fitted temperature is out of range. + var large = await client.ChooseAsync( + ticket, + BuildDepartments(12), + "Route this ticket to the right team"); + + Report("5 options (choice:3-5)", small); + Report("12 options (choice:11+)", large); + + Console.WriteLine( + large.CalibrationClamped + ? " The 12-option answer used a clamped temperature. Both answers are correct, but\n" + + " note the confidence: clamping to 0.5 bounds the damage, it does not fix the\n" + + " calibration - a temperature below 1.0 still sharpens, so the number saturates.\n" + + " Use the ranking for 11+ options; ChoiceOrNull will never abstain here.\n" + : " This checkpoint ships a valid choice:11+ temperature; nothing was clamped.\n"); + + static void Report(string title, ChoiceResult result) + { + Console.WriteLine($" {title}"); + Console.WriteLine($" choice : {result.Choice}"); + Console.WriteLine($" probability : {result.Probability:P1}"); + Console.WriteLine($" confidence : {result.Confidence:F4}"); + Console.WriteLine($" CalibrationClamped : {result.CalibrationClamped}"); + Console.WriteLine(); + } + + static Dictionary BuildDepartments(int count) + { + // "billing" stays first so both sizes answer the same underlying question. + string[] all = + [ + "billing", "technical", "sales", "shipping", "returns", + "accounts", "security", "legal", "partnerships", "press", + "careers", "feedback", + ]; + + var map = new Dictionary(StringComparer.Ordinal); + for (var i = 0; i < count && i < all.Length; i++) + { + map[all[i]] = null; + } + + return map; + } +} + +// --------------------------------------------------------------------------------------- +// 2. fp16 batch variance +// --------------------------------------------------------------------------------------- +// The published checkpoint stores fp16 weights. Running a question alone and running the +// same question alongside others produce slightly different numbers, because the batched +// matrix multiplications accumulate in a different order. The difference is around 1e-4: +// far too small to flip a routing decision, but large enough to break an exact-equality +// assertion or a cache keyed on the probability value. +static async Task ShowBatchVarianceAsync(IDecisionClient client) +{ + Console.WriteLine("=== 2. fp16 batch variance ===\n"); + + const string ticket = "I was charged twice for my subscription this month and want a refund."; + var departments = new Dictionary(StringComparer.Ordinal) + { + ["billing"] = null, + ["technical"] = null, + ["sales"] = null, + }; + + // Alone: a batch of one. + var alone = await client.ChooseAsync(ticket, departments, "Route this ticket to the right team"); + + // Batched: the identical question, sharing a forward pass with three others. + var batchedResult = await client.EvaluateAsync( + new DecisionRequest(ticket) + .Choose("route", departments, "Route this ticket to the right team") + .Score("urgency", ["not urgent", "normal", "urgent", "critical"], "How urgent is this?") + .Ask("refund", "the customer is asking for a refund") + .Ask("angry", "the customer is angry")); + + var batched = batchedResult.Choice("route"); + + Console.WriteLine($" {"label",-12} {"alone",12} {"batched",12} delta"); + var worst = 0.0; + foreach (var pair in alone.Probabilities.OrderBy(p => p.Key, StringComparer.Ordinal)) + { + var other = batched.Probabilities[pair.Key]; + var delta = Math.Abs(pair.Value - other); + worst = Math.Max(worst, delta); + Console.WriteLine($" {pair.Key,-12} {pair.Value,12:F8} {other,12:F8} {delta:E2}"); + } + + Console.WriteLine(); + Console.WriteLine($" same choice : {alone.Choice == batched.Choice} ({alone.Choice})"); + Console.WriteLine($" max delta : {worst:E2}"); + Console.WriteLine($" batch cost : {batchedResult.Duration.TotalMilliseconds:N0} ms for 4 questions"); + Console.WriteLine(); + Console.WriteLine(" Compare probabilities with a tolerance rather than ==, and do not persist"); + Console.WriteLine(" them as if they were stable identifiers. The ranking is what is reliable."); +} diff --git a/src/samples/DependencyInjection/DependencyInjection.csproj b/src/samples/DependencyInjection/DependencyInjection.csproj index 981637f..1b8fee9 100644 --- a/src/samples/DependencyInjection/DependencyInjection.csproj +++ b/src/samples/DependencyInjection/DependencyInjection.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/samples/FaraVisionAgent/FaraVisionAgent.csproj b/src/samples/FaraVisionAgent/FaraVisionAgent.csproj index 0270321..c4f435b 100644 --- a/src/samples/FaraVisionAgent/FaraVisionAgent.csproj +++ b/src/samples/FaraVisionAgent/FaraVisionAgent.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 false diff --git a/src/samples/FineTunedToolCalling/FineTunedToolCalling.csproj b/src/samples/FineTunedToolCalling/FineTunedToolCalling.csproj index b42e5ac..a656560 100644 --- a/src/samples/FineTunedToolCalling/FineTunedToolCalling.csproj +++ b/src/samples/FineTunedToolCalling/FineTunedToolCalling.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/samples/GptOssChat/GptOssChat.csproj b/src/samples/GptOssChat/GptOssChat.csproj index 4560d76..d6b34e7 100644 --- a/src/samples/GptOssChat/GptOssChat.csproj +++ b/src/samples/GptOssChat/GptOssChat.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/samples/HelloChat/HelloChat.csproj b/src/samples/HelloChat/HelloChat.csproj index b42e5ac..a656560 100644 --- a/src/samples/HelloChat/HelloChat.csproj +++ b/src/samples/HelloChat/HelloChat.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/samples/LocalDecisions/LocalDecisions.csproj b/src/samples/LocalDecisions/LocalDecisions.csproj new file mode 100644 index 0000000..ede0f72 --- /dev/null +++ b/src/samples/LocalDecisions/LocalDecisions.csproj @@ -0,0 +1,11 @@ + + + net10.0 + Exe + false + + + + + + diff --git a/src/samples/LocalDecisions/Program.cs b/src/samples/LocalDecisions/Program.cs new file mode 100644 index 0000000..9fd3af6 --- /dev/null +++ b/src/samples/LocalDecisions/Program.cs @@ -0,0 +1,97 @@ +using ElBruno.LocalLLMs.Decisions; + +// Triages a support ticket with a local System One decision model. +// +// Nothing to install and nothing to start: the model runs in-process on ONNX +// Runtime. It is downloaded from HuggingFace on first run (about 800 MB) and +// cached afterwards, so the first run is slow and the rest are not. +// +// Pass a local model directory as the first argument to skip the download. +// +// Every question below is answered in ONE forward pass, so asking four costs +// about what asking one costs. + +using var client = new LayaOnnxDecisionClient(new DecisionOptions +{ + ModelPath = args.Length > 0 ? args[0] : null, + DecisionThreshold = 0.5 +}); + +string[] tickets = +{ + "My invoice charged me twice for the same subscription month and I want my money back.", + "The export endpoint returns a 500 every time I pass more than 50 ids. Here is the stack trace.", + "Hi! Just wondering what the difference is between the Pro and Team plans before we upgrade." +}; + +var teams = new Dictionary +{ + ["billing"] = "Payment, invoice and refund problems", + ["technical"] = "Bugs, outages, API errors and anything with a stack trace", + ["sales"] = "Pricing, plan comparisons, upgrades and new purchases" +}; + +string[] urgencyLevels = +{ + "no rush, answer whenever", + "normal, answer within a day", + "urgent, answer within an hour", + "critical, the customer is blocked right now" +}; + +foreach (string ticket in tickets) +{ + Console.WriteLine(new string('-', 78)); + Console.WriteLine(Truncate(ticket, 76)); + Console.WriteLine(); + + DecisionResult result; + try + { + result = await client.EvaluateAsync( + new DecisionRequest(ticket) + .Choose("team", teams, "Which team should handle this support ticket?") + .Score("urgency", urgencyLevels, "How urgent is this ticket?") + .Ask("refund", "The customer is asking for a refund.") + .Ask("angry", "The customer is angry or frustrated.")); + } + catch (DecisionException ex) + { + Console.WriteLine($" {ex.Message}"); + return 1; + } + + ChoiceResult team = result.Choice("team"); + ScoreResult urgency = result.Score("urgency"); + ProbabilityResult refund = result.Probability("refund"); + ProbabilityResult angry = result.Probability("angry"); + + // Routing on the probability rather than the label lets ambiguous tickets + // fall through to a human instead of being confidently misrouted. + string route = team.ChoiceOrNull(0.6) ?? "human-review"; + + Console.WriteLine($" route : {route,-14} ({team.Choice} @ {team.Probability:P1})"); + Console.WriteLine($" urgency : {urgency.MostLikelyLabel}"); + Console.WriteLine($" score {urgency.Score:F2} of {urgency.Legend.Count - 1}"); + Console.WriteLine($" refund : {YesNo(refund)}"); + Console.WriteLine($" angry : {YesNo(angry)}"); + Console.WriteLine(); + Console.WriteLine($" distribution: {string.Join(" ", team.Probabilities + .OrderByDescending(p => p.Value) + .Select(p => $"{p.Key} {p.Value:P0}"))}"); + Console.WriteLine($" {result.InputTokens} input tokens, {result.Duration.TotalMilliseconds:F0} ms, model '{result.Model}'"); +} + +Console.WriteLine(new string('-', 78)); +Console.WriteLine(); +Console.WriteLine("Note: confidence values from Laya's public checkpoints are not reliably"); +Console.WriteLine("calibrated. Validate any threshold against your own labelled tickets"); +Console.WriteLine("before trusting it in production."); + +return 0; + +static string YesNo(ProbabilityResult result) => + $"{(result.IsTrue ? "yes" : "no"),-4} (p = {result.Probability:F3})"; + +static string Truncate(string value, int length) => + value.Length <= length ? value : string.Concat(value.AsSpan(0, length - 1), "\u2026"); diff --git a/src/samples/MagenticBrainAgent/MagenticBrainAgent.csproj b/src/samples/MagenticBrainAgent/MagenticBrainAgent.csproj index b42e5ac..a656560 100644 --- a/src/samples/MagenticBrainAgent/MagenticBrainAgent.csproj +++ b/src/samples/MagenticBrainAgent/MagenticBrainAgent.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/samples/MagenticUIServer/MagenticUIServer.Agents/MagenticUIServer.Agents.csproj b/src/samples/MagenticUIServer/MagenticUIServer.Agents/MagenticUIServer.Agents.csproj index aad886a..877a7a3 100644 --- a/src/samples/MagenticUIServer/MagenticUIServer.Agents/MagenticUIServer.Agents.csproj +++ b/src/samples/MagenticUIServer/MagenticUIServer.Agents/MagenticUIServer.Agents.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 enable enable false diff --git a/src/samples/MagenticUIServer/MagenticUIServer/MagenticUIServer.csproj b/src/samples/MagenticUIServer/MagenticUIServer/MagenticUIServer.csproj index 9dc7401..f3dcfa8 100644 --- a/src/samples/MagenticUIServer/MagenticUIServer/MagenticUIServer.csproj +++ b/src/samples/MagenticUIServer/MagenticUIServer/MagenticUIServer.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 enable enable false diff --git a/src/samples/MultiModelChat/MultiModelChat.csproj b/src/samples/MultiModelChat/MultiModelChat.csproj index b42e5ac..a656560 100644 --- a/src/samples/MultiModelChat/MultiModelChat.csproj +++ b/src/samples/MultiModelChat/MultiModelChat.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/samples/OpenAiServer/OpenAiServer.csproj b/src/samples/OpenAiServer/OpenAiServer.csproj index 71fd264..c14e6f4 100644 --- a/src/samples/OpenAiServer/OpenAiServer.csproj +++ b/src/samples/OpenAiServer/OpenAiServer.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/samples/RagChatbot/RagChatbot.csproj b/src/samples/RagChatbot/RagChatbot.csproj index 04d6ea5..437ac4c 100644 --- a/src/samples/RagChatbot/RagChatbot.csproj +++ b/src/samples/RagChatbot/RagChatbot.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 Exe false diff --git a/src/samples/StreamingChat/StreamingChat.csproj b/src/samples/StreamingChat/StreamingChat.csproj index b42e5ac..a656560 100644 --- a/src/samples/StreamingChat/StreamingChat.csproj +++ b/src/samples/StreamingChat/StreamingChat.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/samples/ToolCallingAgent/ToolCallingAgent.csproj b/src/samples/ToolCallingAgent/ToolCallingAgent.csproj index b42e5ac..a656560 100644 --- a/src/samples/ToolCallingAgent/ToolCallingAgent.csproj +++ b/src/samples/ToolCallingAgent/ToolCallingAgent.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net10.0 diff --git a/src/tests/ElBruno.LocalLLMs.BitNet.Tests/ElBruno.LocalLLMs.BitNet.Tests.csproj b/src/tests/ElBruno.LocalLLMs.BitNet.Tests/ElBruno.LocalLLMs.BitNet.Tests.csproj index 4c2cf7c..cb484e1 100644 --- a/src/tests/ElBruno.LocalLLMs.BitNet.Tests/ElBruno.LocalLLMs.BitNet.Tests.csproj +++ b/src/tests/ElBruno.LocalLLMs.BitNet.Tests/ElBruno.LocalLLMs.BitNet.Tests.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 false true diff --git a/src/tests/ElBruno.LocalLLMs.BlazorComponents.Tests/ElBruno.LocalLLMs.BlazorComponents.Tests.csproj b/src/tests/ElBruno.LocalLLMs.BlazorComponents.Tests/ElBruno.LocalLLMs.BlazorComponents.Tests.csproj index 5975722..cfc5f23 100644 --- a/src/tests/ElBruno.LocalLLMs.BlazorComponents.Tests/ElBruno.LocalLLMs.BlazorComponents.Tests.csproj +++ b/src/tests/ElBruno.LocalLLMs.BlazorComponents.Tests/ElBruno.LocalLLMs.BlazorComponents.Tests.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 false true diff --git a/src/tests/ElBruno.LocalLLMs.Decisions.Tests/DecisionMathTests.cs b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/DecisionMathTests.cs new file mode 100644 index 0000000..e077b02 --- /dev/null +++ b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/DecisionMathTests.cs @@ -0,0 +1,103 @@ +using ElBruno.LocalLLMs.Decisions.Internal; +using Xunit; + +namespace ElBruno.LocalLLMs.Decisions.Tests; + +public class DecisionMathTests +{ + [Fact] + public void Softmax_ProducesDistributionSummingToOne() + { + double[] probabilities = DecisionMath.Softmax([1.0f, 2.0f, 3.0f], 1.0); + + Assert.Equal(1.0, probabilities.Sum(), 10); + Assert.All(probabilities, p => Assert.InRange(p, 0.0, 1.0)); + } + + [Fact] + public void Softmax_MatchesLayaPublishedExample() + { + // The logits and temperature published on the English checkpoint's model card, which + // reports 0.9696 / 0.0152 / 0.0152 for billing, technical and sales. + double[] probabilities = DecisionMath.Softmax([4.2919f, -3.0214f, -3.0209f], 1.760152); + + Assert.Equal(0.9696, probabilities[0], 4); + Assert.Equal(0.0152, probabilities[1], 4); + Assert.Equal(0.0152, probabilities[2], 4); + } + + [Fact] + public void Softmax_LowerTemperatureSharpensDistribution() + { + double[] warm = DecisionMath.Softmax([2.0f, 1.0f], 2.0); + double[] cold = DecisionMath.Softmax([2.0f, 1.0f], 0.5); + + Assert.True(cold[0] > warm[0]); + } + + [Fact] + public void Softmax_SurvivesLogitsThatWouldOverflowWithoutShifting() + { + // A large logit divided by a small temperature overflows exp() unless the maximum is + // subtracted first, which would produce NaN rather than a distribution. + double[] probabilities = DecisionMath.Softmax([900.0f, 1.0f], 0.5); + + Assert.Equal(1.0, probabilities[0], 10); + Assert.Equal(0.0, probabilities[1], 10); + } + + [Fact] + public void Softmax_FallsBackToUniformWhenNormalizationFails() + { + double[] probabilities = DecisionMath.Softmax([float.NaN, float.NaN], 1.0); + + Assert.Equal(0.5, probabilities[0]); + Assert.Equal(0.5, probabilities[1]); + } + + [Fact] + public void AnswerConfidence_ReturnsLargestProbability() + { + Assert.Equal(0.7, DecisionMath.AnswerConfidence([0.7, 0.2, 0.1]), 10); + } + + [Fact] + public void EntropyConfidence_IsZeroForUniformAndOneForCertain() + { + Assert.Equal(0.0, DecisionMath.EntropyConfidence([0.25, 0.25, 0.25, 0.25]), 6); + Assert.Equal(1.0, DecisionMath.EntropyConfidence([1.0, 0.0, 0.0, 0.0]), 6); + } + + [Fact] + public void EntropyConfidence_ReturnsOneForSingleOption() + { + Assert.Equal(1.0, DecisionMath.EntropyConfidence([1.0])); + } + + [Fact] + public void EntropyConfidence_DiffersFromAnswerConfidence() + { + // The two numbers measure different things, so they must not be compared against the + // same threshold. + double[] probabilities = [0.6, 0.2, 0.2]; + + Assert.NotEqual( + DecisionMath.AnswerConfidence(probabilities), + DecisionMath.EntropyConfidence(probabilities), + 2); + } + + [Fact] + public void ExpectedScore_WeightsLevelsByProbability() + { + Assert.Equal(1.5, DecisionMath.ExpectedScore([0.0, 0.5, 0.5]), 10); + } + + [Fact] + public void ExpectedScore_CanLandOnALevelTheModelNeverFavoured() + { + // A bimodal distribution over levels 0 and 2 reports level 1, which is exactly why + // ScoreResult also exposes MostLikelyLevel. + Assert.Equal(1.0, DecisionMath.ExpectedScore([0.5, 0.0, 0.5]), 10); + } +} diff --git a/src/tests/ElBruno.LocalLLMs.Decisions.Tests/DecisionOptionsTests.cs b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/DecisionOptionsTests.cs new file mode 100644 index 0000000..dbecedb --- /dev/null +++ b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/DecisionOptionsTests.cs @@ -0,0 +1,118 @@ +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace ElBruno.LocalLLMs.Decisions.Tests; + +public class DecisionOptionsTests +{ + [Fact] + public void Defaults_RunTheModelLocallyFromAKnownRepository() + { + var options = new DecisionOptions(); + + Assert.Equal("elbruno/laya-onnx", options.ModelRepository); + Assert.Null(options.ModelPath); + Assert.Null(options.CacheDirectory); + Assert.Null(options.IntraOpNumThreads); + Assert.Equal(0.5, options.DecisionThreshold); + } + + [Fact] + public void Validate_AcceptsTheDefaults() + { + new DecisionOptions().Validate(); + } + + [Fact] + public void Validate_AcceptsALocalModelPathWithoutARepository() + { + var options = new DecisionOptions { ModelRepository = string.Empty, ModelPath = @"C:\models\laya" }; + + options.Validate(); + } + + [Fact] + public void Validate_RejectsHavingNeitherASourceNorAPath() + { + var options = new DecisionOptions { ModelRepository = " " }; + + var exception = Assert.Throws(options.Validate); + Assert.Contains("ModelPath", exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(-0.1)] + [InlineData(1.1)] + [InlineData(double.NaN)] + public void Validate_RejectsThresholdOutsideUnitInterval(double threshold) + { + var options = new DecisionOptions { DecisionThreshold = threshold }; + + Assert.Throws(options.Validate); + } + + [Theory] + [InlineData(0)] + [InlineData(-4)] + public void Validate_RejectsNonPositiveThreadCounts(int threads) + { + var options = new DecisionOptions { IntraOpNumThreads = threads }; + + Assert.Throws(options.Validate); + } + + [Fact] + public void ResolveCacheDirectory_PrefersAnExplicitDirectory() + { + var options = new DecisionOptions { CacheDirectory = @"D:\cache" }; + + Assert.Equal(@"D:\cache", options.ResolveCacheDirectory()); + } + + [Fact] + public void ResolveCacheDirectory_FallsBackToLocalApplicationData() + { + var resolved = new DecisionOptions().ResolveCacheDirectory(); + + Assert.Contains("ElBruno", resolved, StringComparison.Ordinal); + Assert.Contains("decisions", resolved, StringComparison.Ordinal); + } + + [Fact] + public void AddLocalDecisions_RegistersASingletonClient() + { + var services = new ServiceCollection(); + + services.AddLocalDecisions(options => options.DecisionThreshold = 0.75); + + using ServiceProvider provider = services.BuildServiceProvider(); + var first = provider.GetRequiredService(); + var second = provider.GetRequiredService(); + + Assert.Same(first, second); + Assert.IsType(first); + Assert.Equal(0.75, provider.GetRequiredService().DecisionThreshold); + } + + [Fact] + public void AddLocalDecisions_DoesNotLoadTheModelDuringRegistration() + { + // Registration must stay cheap: the model is several hundred megabytes and is fetched + // lazily on first use, so adding the service cannot block startup or touch the network. + var services = new ServiceCollection(); + services.AddLocalDecisions(options => options.ModelPath = @"C:\does\not\exist"); + + using ServiceProvider provider = services.BuildServiceProvider(); + + Assert.NotNull(provider.GetRequiredService()); + } + + [Fact] + public void AddLocalDecisions_RejectsInvalidOptionsEagerly() + { + var services = new ServiceCollection(); + + Assert.Throws(() => + services.AddLocalDecisions(options => options.DecisionThreshold = 2.0)); + } +} diff --git a/src/tests/ElBruno.LocalLLMs.Decisions.Tests/DecisionRequestTests.cs b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/DecisionRequestTests.cs new file mode 100644 index 0000000..9fa7e9e --- /dev/null +++ b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/DecisionRequestTests.cs @@ -0,0 +1,107 @@ +using Xunit; + +namespace ElBruno.LocalLLMs.Decisions.Tests; + +public class DecisionRequestTests +{ + [Fact] + public void Constructor_RejectsEmptyState() + { + Assert.Throws(() => new DecisionRequest(" ")); + Assert.Throws(() => new DecisionRequest(string.Empty)); + Assert.Throws(() => new DecisionRequest(null!)); + } + + [Fact] + public void Builders_AreImmutable() + { + var original = new DecisionRequest("text"); + DecisionRequest withQuestion = original.Ask("q", "proposition"); + + Assert.Empty(original.Questions); + Assert.Single(withQuestion.Questions); + } + + [Fact] + public void DuplicateQuestionName_IsRejected() + { + DecisionRequest request = new DecisionRequest("text").Ask("q", "proposition"); + + Assert.Throws(() => request.Ask("q", "another")); + Assert.Throws(() => request.Choose("q", new[] { "a", "b" })); + } + + [Fact] + public void Choose_RequiresAtLeastOneOption() + { + var request = new DecisionRequest("text"); + + Assert.Throws( + () => request.Choose("q", Array.Empty())); + } + + [Fact] + public void Choose_RejectsMoreThan255Options() + { + var request = new DecisionRequest("text"); + string[] options = Enumerable.Range(0, 256).Select(i => $"option{i}").ToArray(); + + Assert.Throws(() => request.Choose("q", options)); + } + + [Theory] + [InlineData(1)] + [InlineData(11)] + public void Score_RejectsRubricOutsideTwoToTen(int levelCount) + { + var request = new DecisionRequest("text"); + string[] levels = Enumerable.Range(0, levelCount).Select(i => $"level{i}").ToArray(); + + Assert.Throws(() => request.Score("q", levels)); + } + + [Theory] + [InlineData(2)] + [InlineData(10)] + public void Score_AcceptsRubricBoundaries(int levelCount) + { + var request = new DecisionRequest("text"); + string[] levels = Enumerable.Range(0, levelCount).Select(i => $"level{i}").ToArray(); + + DecisionRequest result = request.Score("q", levels); + + Assert.Equal(levelCount, result.Questions["q"].Levels!.Count); + } + + [Fact] + public void Ask_RejectsEmptyProposition() + { + var request = new DecisionRequest("text"); + + Assert.Throws(() => request.Ask("q", " ")); + } + + [Fact] + public void Questions_RecordTheirType() + { + DecisionRequest request = new DecisionRequest("text") + .Choose("a", new[] { "x", "y" }) + .Score("b", new[] { "low", "high" }) + .Ask("c", "proposition"); + + Assert.Equal(DecisionQuestionType.Choice, request.Questions["a"].Type); + Assert.Equal(DecisionQuestionType.Score, request.Questions["b"].Type); + Assert.Equal(DecisionQuestionType.Probability, request.Questions["c"].Type); + } + + [Fact] + public void Choose_SnapshotsCallerDictionary() + { + var options = new Dictionary { ["a"] = "first" }; + DecisionRequest request = new DecisionRequest("text").Choose("q", options); + + options["b"] = "added later"; + + Assert.Single(request.Questions["q"].Options!); + } +} diff --git a/src/tests/ElBruno.LocalLLMs.Decisions.Tests/ElBruno.LocalLLMs.Decisions.Tests.csproj b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/ElBruno.LocalLLMs.Decisions.Tests.csproj new file mode 100644 index 0000000..b3ae8a2 --- /dev/null +++ b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/ElBruno.LocalLLMs.Decisions.Tests.csproj @@ -0,0 +1,19 @@ + + + net10.0 + false + true + + + + + + + + + + + + + + diff --git a/src/tests/ElBruno.LocalLLMs.Decisions.Tests/LayaRuntimeConfigTests.cs b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/LayaRuntimeConfigTests.cs new file mode 100644 index 0000000..ae5958e --- /dev/null +++ b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/LayaRuntimeConfigTests.cs @@ -0,0 +1,197 @@ +using ElBruno.LocalLLMs.Decisions.Internal; +using Xunit; + +namespace ElBruno.LocalLLMs.Decisions.Tests; + +public class LayaRuntimeConfigTests : IDisposable +{ + private readonly string _directory = + Directory.CreateTempSubdirectory("laya-config-tests").FullName; + + private string WriteConfig(string json) + { + var path = Path.Combine(_directory, $"{Guid.NewGuid():N}.json"); + File.WriteAllText(path, json); + return path; + } + + [Fact] + public void Load_ReadsRootLevelLayout() + { + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + """ + { + "max_len": 512, + "head_max_len": 192, + "temperature": [1.6369, 1.2514, 1.9834], + "temperature_by_options": { "choice:3-5": 1.7602 } + } + """)); + + Assert.Equal(512, config.MaxLength); + Assert.Equal(192, config.HeadMaxLength); + Assert.Equal(1.7602, config.TemperatureFor(0, 3), 4); + Assert.False(config.TemperaturesWereClamped); + } + + [Fact] + public void Load_ReadsNestedLayaLayout() + { + // The onnx-community exports nest the same values under a "laya" key. + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + """ + { "model_type": "laya", "laya": { "max_len": 256, "head_max_len": 96 } } + """)); + + Assert.Equal(256, config.MaxLength); + Assert.Equal(96, config.HeadMaxLength); + } + + [Fact] + public void Load_FallsBackToDefaultBudgetsWhenAbsent() + { + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig("{}")); + + Assert.Equal(512, config.MaxLength); + Assert.Equal(192, config.HeadMaxLength); + } + + [Fact] + public void Load_DefaultsToNeutralTemperatureWhenNoneAreFitted() + { + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig("{}")); + + Assert.Equal(1.0, config.TemperatureFor(0, 3)); + Assert.Equal(1.0, config.TemperatureFor(1, 5)); + Assert.Equal(1.0, config.TemperatureFor(2, 2)); + } + + [Theory] + [InlineData(2, "choice:2")] + [InlineData(3, "choice:3-5")] + [InlineData(5, "choice:3-5")] + [InlineData(6, "choice:6-10")] + [InlineData(10, "choice:6-10")] + [InlineData(11, "choice:11+")] + [InlineData(40, "choice:11+")] + public void TemperatureFor_SelectsTheBucketMatchingTheOptionCount(int optionCount, string bucket) + { + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + $$""" + { + "temperature": [9.9, 9.9, 9.9], + "temperature_by_options": { "{{bucket}}": 2.5 } + } + """)); + + Assert.Equal(2.5, config.TemperatureFor(0, optionCount)); + } + + [Fact] + public void TemperatureFor_FallsBackToTheTypeTemperatureWhenNoBucketMatches() + { + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + """ + { "temperature": [1.6369, 1.2514, 1.9834], "temperature_by_options": { "choice:2": 1.9 } } + """)); + + Assert.Equal(1.6369, config.TemperatureFor(0, 7), 4); + Assert.Equal(1.2514, config.TemperatureFor(1, 4), 4); + Assert.Equal(1.9834, config.TemperatureFor(2, 2), 4); + } + + [Fact] + public void Load_ClampsTheInvalidTemperatureShippedByThePublicEnglishCheckpoint() + { + // choice:11+ ships as 0.1006, which sharpens the logits roughly tenfold and turns a 24% + // top probability into a reported 99%. + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + """ + { "temperature_by_options": { "choice:11+": 0.1006 } } + """)); + + Assert.Equal(LayaRuntimeConfig.MinimumTemperature, config.TemperatureFor(0, 12)); + Assert.True(config.TemperaturesWereClamped); + } + + [Fact] + public void WasClampedFor_IsTrueOnlyForTheBucketThatWasClamped() + { + // The public English checkpoint ships a valid choice:3-5 and an invalid choice:11+, + // so the flag has to distinguish them rather than condemn the whole checkpoint. + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + """ + { + "temperature": [1.6369, 1.2514, 1.9834], + "temperature_by_options": { "choice:3-5": 1.7602, "choice:11+": 0.1006 } + } + """)); + + Assert.False(config.WasClampedFor(0, 3)); + Assert.True(config.WasClampedFor(0, 12)); + Assert.True(config.TemperaturesWereClamped); + } + + [Fact] + public void WasClampedFor_FallsBackToTheTypeTemperatureWhenNoBucketMatches() + { + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + """ + { "temperature": [0.01, 1.2514, 1.9834] } + """)); + + // No bucket matches, so the clamped type temperature is what actually gets applied. + Assert.True(config.WasClampedFor(0, 7)); + Assert.False(config.WasClampedFor(1, 7)); + } + + [Fact] + public void WasClampedFor_IsFalseWhenEveryTemperatureIsValid() + { + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + """ + { "temperature": [1.6369, 1.2514, 1.9834], "temperature_by_options": { "choice:2": 1.9 } } + """)); + + Assert.False(config.WasClampedFor(0, 2)); + Assert.False(config.WasClampedFor(0, 40)); + Assert.False(config.TemperaturesWereClamped); + } + + [Fact] + public void Load_ClampsTemperaturesAboveTheMaximum() + { + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + """ + { "temperature_by_options": { "choice:2": 99.0 } } + """)); + + Assert.Equal(LayaRuntimeConfig.MaximumTemperature, config.TemperatureFor(0, 2)); + Assert.True(config.TemperaturesWereClamped); + } + + [Fact] + public void Load_TreatsNonNumericTemperaturesAsNeutral() + { + LayaRuntimeConfig config = LayaRuntimeConfig.Load(WriteConfig( + """ + { "temperature_by_options": { "choice:2": null } } + """)); + + Assert.Equal(1.0, config.TemperatureFor(0, 2)); + Assert.True(config.TemperaturesWereClamped); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + try + { + Directory.Delete(_directory, recursive: true); + } + catch (IOException) + { + // A leftover temp directory is not worth failing a test run over. + } + } +} diff --git a/src/tests/ElBruno.LocalLLMs.Decisions.Tests/LayaSequenceBuilderTests.cs b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/LayaSequenceBuilderTests.cs new file mode 100644 index 0000000..1f7ed04 --- /dev/null +++ b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/LayaSequenceBuilderTests.cs @@ -0,0 +1,202 @@ +using ElBruno.LocalLLMs.Decisions.Internal; +using Xunit; + +namespace ElBruno.LocalLLMs.Decisions.Tests; + +public class LayaSequenceBuilderTests : IDisposable +{ + private readonly string _directory = + Directory.CreateTempSubdirectory("laya-sequence-tests").FullName; + + private readonly LayaTokenizer _tokenizer; + + public LayaSequenceBuilderTests() + { + _tokenizer = SyntheticTokenizer.Create(_directory); + } + + private LayaSequenceBuilder Builder(int maxLength = 512, int headMaxLength = 192) + { + var path = Path.Combine(_directory, $"{Guid.NewGuid():N}.json"); + File.WriteAllText(path, $$"""{"max_len": {{maxLength}}, "head_max_len": {{headMaxLength}}}"""); + return new LayaSequenceBuilder(_tokenizer, LayaRuntimeConfig.Load(path)); + } + + [Fact] + public void Build_ProducesOneSequencePerQuestion() + { + var request = new DecisionRequest("a support ticket") + .Choose("route", ["billing", "technical"]) + .Score("urgency", ["low", "high"]) + .Ask("angry", "the customer is angry"); + + IReadOnlyList sequences = Builder().Build(request); + + Assert.Equal(3, sequences.Count); + Assert.Equal(["route", "urgency", "angry"], sequences.Select(s => s.Name)); + } + + [Fact] + public void Build_AssignsTheQuestionTypeTheModelExpects() + { + var request = new DecisionRequest("text") + .Choose("a", ["x", "y"]) + .Score("b", ["low", "high"]) + .Ask("c", "something holds"); + + IReadOnlyList sequences = Builder().Build(request); + + Assert.Equal(0, sequences[0].QuestionType); + Assert.Equal(1, sequences[1].QuestionType); + Assert.Equal(2, sequences[2].QuestionType); + } + + [Fact] + public void Build_WrapsTheSequenceInTheExpectedSpecialTokens() + { + LayaSequence sequence = Builder().Build( + new DecisionRequest("text").Choose("route", ["billing", "technical"]))[0]; + + Assert.Equal(SyntheticTokenizer.ClsId, sequence.TokenIds[0]); + Assert.Equal(SyntheticTokenizer.SepId, sequence.TokenIds[^1]); + } + + [Fact] + public void Build_PlacesOneMaskMarkerPerOption() + { + LayaSequence sequence = Builder().Build( + new DecisionRequest("text").Choose("route", ["billing", "technical", "sales"]))[0]; + + Assert.Equal(3, sequence.MarkerPositions.Count); + Assert.All( + sequence.MarkerPositions, + position => Assert.Equal(SyntheticTokenizer.MaskId, sequence.TokenIds[position])); + } + + [Fact] + public void Build_KeepsMarkersInTheOrderTheLabelsWereSupplied() + { + LayaSequence sequence = Builder().Build( + new DecisionRequest("text").Choose("route", ["billing", "technical", "sales"]))[0]; + + Assert.Equal(["billing", "technical", "sales"], sequence.Labels); + Assert.Equal( + sequence.MarkerPositions.OrderBy(position => position), + sequence.MarkerPositions); + } + + [Fact] + public void Build_AlwaysRendersPropositionsAsFalseThenTrue() + { + // The decoder reads index 1 as the probability of true, so this order is load-bearing. + LayaSequence sequence = Builder().Build( + new DecisionRequest("text").Ask("angry", "the customer is angry"))[0]; + + Assert.Equal(["false", "true"], sequence.Labels); + } + + [Fact] + public void Build_StripsLiteralMaskTokensFromCallerText() + { + // A [MASK] in caller text would create a phantom marker the head would try to score. + LayaSequence sequence = Builder().Build( + new DecisionRequest("please [MASK] this").Choose("route", ["a", "b"]))[0]; + + Assert.Equal(2, sequence.TokenIds.Count(id => id == SyntheticTokenizer.MaskId)); + } + + [Fact] + public void Build_StripsLiteralMaskTokensFromInstructionsAndOptions() + { + LayaSequence sequence = Builder().Build( + new DecisionRequest("text").Choose( + "route", + new Dictionary { ["a"] = "[MASK] description", ["b"] = null }, + "where should [MASK] go?"))[0]; + + Assert.Equal(2, sequence.TokenIds.Count(id => id == SyntheticTokenizer.MaskId)); + } + + [Fact] + public void Build_NeverExceedsTheConfiguredWindow() + { + var state = string.Join(' ', Enumerable.Repeat("some fairly long supporting text", 400)); + + LayaSequence sequence = Builder(maxLength: 256).Build( + new DecisionRequest(state).Choose("route", ["billing", "technical"]))[0]; + + Assert.True(sequence.TokenIds.Count <= 256); + Assert.All(sequence.MarkerPositions, position => Assert.True(position < 256)); + } + + [Fact] + public void Build_KeepsEveryOptionScorableWhenOptionsExceedTheirBudget() + { + // Long descriptions are shrunk evenly rather than dropped, so every label keeps a marker. + var options = Enumerable.Range(0, 10).ToDictionary( + i => $"option{i}", + i => (string?)string.Join(' ', Enumerable.Repeat("a very long option description", 20))); + + LayaSequence sequence = Builder().Build( + new DecisionRequest("text").Choose("route", options))[0]; + + Assert.Equal(10, sequence.MarkerPositions.Count); + Assert.Equal(10, sequence.Labels.Count); + } + + [Fact] + public void Build_ThrowsWhenOptionsCannotAllBeScored() + { + var options = Enumerable.Range(0, 60).ToDictionary(i => $"option{i}", _ => (string?)null); + + var exception = Assert.Throws(() => + Builder(maxLength: 64, headMaxLength: 48).Build( + new DecisionRequest("text").Choose("route", options))); + + Assert.Contains("fewer or shorter options", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void Build_ReusesTheSameStateTokensForEveryQuestion() + { + var request = new DecisionRequest("a shared support ticket") + .Choose("a", ["x", "y"]) + .Choose("b", ["x", "y"]); + + IReadOnlyList sequences = Builder().Build(request); + + Assert.Equal(sequences[0].TokenIds, sequences[1].TokenIds); + } + + [Fact] + public void Build_DistinguishesOptionsThatCarryDescriptions() + { + LayaSequence bare = Builder().Build( + new DecisionRequest("text").Choose("route", ["billing", "technical"]))[0]; + + LayaSequence described = Builder().Build( + new DecisionRequest("text").Choose( + "route", + new Dictionary + { + ["billing"] = "invoices and payments", + ["technical"] = "bugs and outages", + }))[0]; + + Assert.True(described.TokenIds.Count > bare.TokenIds.Count); + Assert.Equal(bare.Labels, described.Labels); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + try + { + Directory.Delete(_directory, recursive: true); + } + catch (IOException) + { + // A leftover temp directory is not worth failing a test run over. + } + } +} diff --git a/src/tests/ElBruno.LocalLLMs.Decisions.Tests/LayaTokenizerTests.cs b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/LayaTokenizerTests.cs new file mode 100644 index 0000000..342a8af --- /dev/null +++ b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/LayaTokenizerTests.cs @@ -0,0 +1,118 @@ +using ElBruno.LocalLLMs.Decisions.Internal; +using Xunit; + +namespace ElBruno.LocalLLMs.Decisions.Tests; + +public class LayaTokenizerTests : IDisposable +{ + private readonly string _directory = + Directory.CreateTempSubdirectory("laya-tokenizer-tests").FullName; + + private readonly LayaTokenizer _tokenizer; + + public LayaTokenizerTests() + { + _tokenizer = SyntheticTokenizer.Create(_directory); + } + + [Fact] + public void Load_ExposesTheSpecialTokensTheSequenceFormatNeeds() + { + Assert.Equal(SyntheticTokenizer.ClsId, _tokenizer.ClsTokenId); + Assert.Equal(SyntheticTokenizer.SepId, _tokenizer.SepTokenId); + Assert.Equal(SyntheticTokenizer.MaskId, _tokenizer.MaskTokenId); + Assert.Equal(SyntheticTokenizer.PadId, _tokenizer.PadTokenId); + } + + [Fact] + public void Encode_ReturnsNothingForEmptyText() + { + Assert.Empty(_tokenizer.Encode(string.Empty)); + } + + [Fact] + public void Encode_IsStableForTheSameInput() + { + Assert.Equal(_tokenizer.Encode("hello world"), _tokenizer.Encode("hello world")); + } + + [Fact] + public void Encode_DistinguishesALeadingSpace() + { + // ByteLevel pre-tokenization keeps the space as part of the following token, which is the + // single most common way a hand-rolled port diverges from the Python reference. + Assert.NotEqual(_tokenizer.Encode("world"), _tokenizer.Encode(" world")); + } + + [Fact] + public void Encode_RecognisesAddedTokensAsSingleIds() + { + IReadOnlyList ids = _tokenizer.Encode("[MASK]"); + + Assert.Equal([SyntheticTokenizer.MaskId], ids); + } + + [Fact] + public void Encode_AbsorbsTheSpaceBeforeAnLStripAddedToken() + { + // [MASK] ships with lstrip=true, so the preceding space belongs to the token. + IReadOnlyList withSpace = _tokenizer.Encode("a [MASK]"); + IReadOnlyList withoutSpace = _tokenizer.Encode("a[MASK]"); + + Assert.Equal(withoutSpace, withSpace); + } + + [Fact] + public void Encode_NormalizesToCompositeForm() + { + // "Γ©" as a single code point and as "e" plus a combining accent must encode identically. + Assert.Equal(_tokenizer.Encode("caf\u00e9"), _tokenizer.Encode("cafe\u0301")); + } + + [Fact] + public void Encode_HandlesTextOutsideTheBasicLatinRange() + { + Assert.NotEmpty(_tokenizer.Encode("наши Π΄Π°Π½Π½Ρ‹Π΅ ζ—₯本θͺž πŸŽ‰")); + } + + [Fact] + public void EncodeTruncated_KeepsAtMostTheRequestedTokens() + { + IReadOnlyList ids = _tokenizer.EncodeTruncated("a fairly long piece of option text", 5); + + Assert.Equal(5, ids.Count); + Assert.Equal(_tokenizer.Encode("a fairly long piece of option text").Take(5), ids); + } + + [Fact] + public void EncodeTruncated_LeavesShortTextUntouched() + { + Assert.Equal(_tokenizer.Encode("short"), _tokenizer.EncodeTruncated("short", 64)); + } + + [Fact] + public void Load_RejectsATokenizerMissingLayasSpecialTokens() + { + var path = Path.Combine(_directory, "incomplete.json"); + File.WriteAllText(path, + """ + {"added_tokens":[],"model":{"type":"BPE","vocab":{"a":0},"merges":[]}} + """); + + var exception = Assert.Throws(() => LayaTokenizer.Load(path)); + Assert.Contains("[CLS]", exception.Message, StringComparison.Ordinal); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + try + { + Directory.Delete(_directory, recursive: true); + } + catch (IOException) + { + // A leftover temp directory is not worth failing a test run over. + } + } +} diff --git a/src/tests/ElBruno.LocalLLMs.Decisions.Tests/SyntheticTokenizer.cs b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/SyntheticTokenizer.cs new file mode 100644 index 0000000..6881710 --- /dev/null +++ b/src/tests/ElBruno.LocalLLMs.Decisions.Tests/SyntheticTokenizer.cs @@ -0,0 +1,116 @@ +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using ElBruno.LocalLLMs.Decisions.Internal; + +namespace ElBruno.LocalLLMs.Decisions.Tests; + +/// +/// Builds a minimal but genuine byte-level tokenizer.json so the sequence builder can be +/// tested without downloading a real 3 MB checkpoint tokenizer. +/// +/// +/// The vocabulary covers every byte-level character and defines no merges, so each byte becomes +/// one token. That makes token counts predictable while still exercising the real +/// code path, including byte-level mapping and the added vocabulary. +/// +internal static class SyntheticTokenizer +{ + public const int UnknownId = 256; + public const int ClsId = 257; + public const int SepId = 258; + public const int PadId = 259; + public const int MaskId = 260; + + /// + /// Writes a synthetic tokenizer definition and loads it. + /// + /// The directory to write the definition into. + /// A loaded tokenizer. + public static LayaTokenizer Create(string directory) + { + var path = Path.Combine(directory, $"tokenizer-{Guid.NewGuid():N}.json"); + File.WriteAllText(path, BuildJson(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + return LayaTokenizer.Load(path); + } + + private static string BuildJson() + { + var vocabulary = new JsonObject(); + var alphabet = ByteLevelAlphabet(); + for (var i = 0; i < alphabet.Length; i++) + { + vocabulary[alphabet[i].ToString()] = i; + } + + vocabulary["[UNK]"] = UnknownId; + vocabulary["[CLS]"] = ClsId; + vocabulary["[SEP]"] = SepId; + vocabulary["[PAD]"] = PadId; + vocabulary["[MASK]"] = MaskId; + + var added = new JsonArray(); + foreach (var (content, id, lstrip) in new[] + { + ("[UNK]", UnknownId, false), + ("[CLS]", ClsId, false), + ("[SEP]", SepId, false), + ("[PAD]", PadId, false), + // The real checkpoint marks [MASK] lstrip, so a preceding space is absorbed. + ("[MASK]", MaskId, true), + }) + { + added.Add(new JsonObject + { + ["id"] = id, + ["content"] = content, + ["lstrip"] = lstrip, + ["rstrip"] = false, + ["single_word"] = false, + ["normalized"] = false, + ["special"] = true, + }); + } + + var root = new JsonObject + { + ["version"] = "1.0", + ["added_tokens"] = added, + ["model"] = new JsonObject + { + ["type"] = "BPE", + ["vocab"] = vocabulary, + ["merges"] = new JsonArray(), + }, + }; + + return root.ToJsonString(new JsonSerializerOptions { WriteIndented = false }); + } + + private static char[] ByteLevelAlphabet() + { + var bytes = new List(); + for (var c = '!'; c <= '~'; c++) bytes.Add(c); + for (var c = '\u00a1'; c <= '\u00ac'; c++) bytes.Add(c); + for (var c = '\u00ae'; c <= '\u00ff'; c++) bytes.Add(c); + + var characters = new List(bytes); + var next = 0; + for (var b = 0; b < 256; b++) + { + if (!bytes.Contains(b)) + { + bytes.Add(b); + characters.Add(256 + next); + next++; + } + } + + // Order by byte value so the vocabulary ids line up with the bytes they encode. + return bytes + .Select((value, index) => (Byte: value, Character: characters[index])) + .OrderBy(pair => pair.Byte) + .Select(pair => (char)pair.Character) + .ToArray(); + } +} diff --git a/src/tests/ElBruno.LocalLLMs.FineTuneEval/ElBruno.LocalLLMs.FineTuneEval.csproj b/src/tests/ElBruno.LocalLLMs.FineTuneEval/ElBruno.LocalLLMs.FineTuneEval.csproj index 872bc77..3978e6a 100644 --- a/src/tests/ElBruno.LocalLLMs.FineTuneEval/ElBruno.LocalLLMs.FineTuneEval.csproj +++ b/src/tests/ElBruno.LocalLLMs.FineTuneEval/ElBruno.LocalLLMs.FineTuneEval.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 false true diff --git a/src/tests/ElBruno.LocalLLMs.IntegrationTests/ElBruno.LocalLLMs.IntegrationTests.csproj b/src/tests/ElBruno.LocalLLMs.IntegrationTests/ElBruno.LocalLLMs.IntegrationTests.csproj index dbe9f09..921da04 100644 --- a/src/tests/ElBruno.LocalLLMs.IntegrationTests/ElBruno.LocalLLMs.IntegrationTests.csproj +++ b/src/tests/ElBruno.LocalLLMs.IntegrationTests/ElBruno.LocalLLMs.IntegrationTests.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 false true diff --git a/src/tests/ElBruno.LocalLLMs.IntegrationTests/PlaceholderIntegrationTests.cs b/src/tests/ElBruno.LocalLLMs.IntegrationTests/PlaceholderIntegrationTests.cs index ac74b4b..173e49e 100644 --- a/src/tests/ElBruno.LocalLLMs.IntegrationTests/PlaceholderIntegrationTests.cs +++ b/src/tests/ElBruno.LocalLLMs.IntegrationTests/PlaceholderIntegrationTests.cs @@ -20,7 +20,7 @@ public void Pack_ShouldIncludeBuildTransitiveAssets() var process = Process.Start(new ProcessStartInfo { FileName = "dotnet", - Arguments = $"pack \"{csprojPath}\" -c Release -p:TargetFrameworks=net8.0 -o \"{packageOutDir}\"", + Arguments = $"pack \"{csprojPath}\" -c Release -o \"{packageOutDir}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false diff --git a/src/tests/ElBruno.LocalLLMs.Rag.Tests/ElBruno.LocalLLMs.Rag.Tests.csproj b/src/tests/ElBruno.LocalLLMs.Rag.Tests/ElBruno.LocalLLMs.Rag.Tests.csproj index 82a4628..a028ed1 100644 --- a/src/tests/ElBruno.LocalLLMs.Rag.Tests/ElBruno.LocalLLMs.Rag.Tests.csproj +++ b/src/tests/ElBruno.LocalLLMs.Rag.Tests/ElBruno.LocalLLMs.Rag.Tests.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 false true diff --git a/src/tests/ElBruno.LocalLLMs.Tests/ElBruno.LocalLLMs.Tests.csproj b/src/tests/ElBruno.LocalLLMs.Tests/ElBruno.LocalLLMs.Tests.csproj index f1ca859..20fd563 100644 --- a/src/tests/ElBruno.LocalLLMs.Tests/ElBruno.LocalLLMs.Tests.csproj +++ b/src/tests/ElBruno.LocalLLMs.Tests/ElBruno.LocalLLMs.Tests.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 false true diff --git a/src/tests/MagenticUIServer.Agents.Tests/MagenticUIServer.Agents.Tests.csproj b/src/tests/MagenticUIServer.Agents.Tests/MagenticUIServer.Agents.Tests.csproj index a15be35..6c0da1c 100644 --- a/src/tests/MagenticUIServer.Agents.Tests/MagenticUIServer.Agents.Tests.csproj +++ b/src/tests/MagenticUIServer.Agents.Tests/MagenticUIServer.Agents.Tests.csproj @@ -1,6 +1,6 @@ - net8.0 + net10.0 enable enable false