diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 7eed74b9b..10509e62d 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -771,6 +771,76 @@ importers: specifier: 5.9.3 version: 5.9.3 + ../../packages/jeparag: + dependencies: + '@magek/common': + specifier: workspace:^0.0.11 + version: link:../common + '@magek/core': + specifier: workspace:^0.0.11 + version: link:../core + reflect-metadata: + specifier: 0.2.2 + version: 0.2.2 + tslib: + specifier: 2.8.1 + version: 2.8.1 + uuid: + specifier: ^13.0.0 + version: 13.0.0 + devDependencies: + '@magek/eslint-config': + specifier: workspace:^0.0.11 + version: link:../../tools/eslint-config + '@types/chai': + specifier: 5.2.3 + version: 5.2.3 + '@types/chai-as-promised': + specifier: 8.0.2 + version: 8.0.2 + '@types/mocha': + specifier: 10.0.10 + version: 10.0.10 + '@types/node': + specifier: 22.19.9 + version: 22.19.9 + '@types/sinon': + specifier: 21.0.0 + version: 21.0.0 + '@types/sinon-chai': + specifier: 4.0.0 + version: 4.0.0 + '@types/uuid': + specifier: 11.0.0 + version: 11.0.0 + c8: + specifier: ^10.1.3 + version: 10.1.3 + chai: + specifier: 6.2.2 + version: 6.2.2 + chai-as-promised: + specifier: 8.0.2 + version: 8.0.2(chai@6.2.2) + mocha: + specifier: 11.7.5 + version: 11.7.5 + rimraf: + specifier: 6.1.2 + version: 6.1.2 + sinon: + specifier: 21.0.1 + version: 21.0.1 + sinon-chai: + specifier: 4.0.1 + version: 4.0.1(chai@6.2.2)(sinon@21.0.1) + tsx: + specifier: ^4.19.2 + version: 4.21.0 + typescript: + specifier: 5.9.3 + version: 5.9.3 + ../../packages/mcp-server: dependencies: '@modelcontextprotocol/sdk': diff --git a/packages/jeparag/README.md b/packages/jeparag/README.md new file mode 100644 index 000000000..1285fc855 --- /dev/null +++ b/packages/jeparag/README.md @@ -0,0 +1,69 @@ +# @magek/jeparag + +> Native **Joint Embedding Predictive Architecture (JEPA)** + **Okapi BM25** Hybrid Retrieval Engine for Magek Ambient Agents. + +--- + +## ๐Ÿง  Overview + +`@magek/jeparag` brings advanced hybrid RAG capabilities natively into the **Magek** event-driven framework. It combines dense vector semantic representations (JEPA) with sparse keyword matching (Okapi BM25) using **Reciprocal Rank Fusion (RRF)**. + +Designed for high-precision, zero-recall-loss applications such as legal auditing, compliance monitoring, and technical log diagnostics. + +--- + +## โšก Key Features + +- **DocumentLoader & Sliding Window Chunker**: Preserves semantic continuity with configurable chunk sizes (e.g., 256 words, 50 overlap). +- **JEPA Embedder**: Multi-provider embedding engine (Google Gemini, OpenAI, Hash local fallback). +- **BM25 Sparse Keyword Index**: Native Okapi BM25 implementation in TypeScript. +- **Reciprocal Rank Fusion (RRF)**: Merges dense vector cosine similarity ranks with BM25 ranks ($k=60$). +- **Magek Event Sourcing Integration**: Emits immutable events (`DocumentIngestedEvent`, `ChunkEmbeddedEvent`, `JeparagQueryProcessedEvent`). +- **JeparagAmbientAgent**: Proactive background worker reacting to event streams and executing contextual RAG generation. + +--- + +## ๐Ÿš€ Usage Example + +```typescript +import { + DocumentIngestedEvent, + JeparagAmbientAgent, + JeparagHybridSearchReadModel +} from '@magek/jeparag' + +// 1. Instantiate Magek Ambient Agent +const agent = new JeparagAmbientAgent('hash') + +// 2. Ingest document event +const ingestEvent = new DocumentIngestedEvent( + 'doc-101', + 'HIPAA Compliance Policy', + 'Section 1: Sign-in sheets that reveal patient names are prohibited...' +) + +// 3. Process ingestion and populate ReadModel +await agent.onDocumentIngested(ingestEvent) + +// 4. Execute hybrid query +const queryResult = await agent.processQuery('query-1', 'sign-in sheet violations', 5) + +console.log(queryResult.answer) +``` + +--- + +## ๐Ÿงช Testing + +Run unit tests: + +```bash +cd packages/jeparag +rushx test +``` + +--- + +## ๐Ÿ“œ License + +Licensed under Apache-2.0. diff --git a/packages/jeparag/examples/hipaa-audit-demo.ts b/packages/jeparag/examples/hipaa-audit-demo.ts new file mode 100644 index 000000000..364de8bcd --- /dev/null +++ b/packages/jeparag/examples/hipaa-audit-demo.ts @@ -0,0 +1,72 @@ +/** + * Full Executable Demonstration of @magek/jeparag + * + * Demonstrates HIPAA Privacy Audit using Magek Ambient Agents, + * BM25 Sparse Search, JEPA Dense Vector Search, and Reciprocal Rank Fusion (RRF). + */ + +import { + DocumentIngestedEvent, + JeparagAmbientAgent, + JeparagHybridSearchReadModel, +} from '../src/index' + +async function runDemo(): Promise { + console.log('===============================================================') + console.log('๐Ÿค– @magek/jeparag: HIPAA Privacy Compliance Audit Demo') + console.log('===============================================================\n') + + // 1. Initialize Magek Ambient Agent + const agent = new JeparagAmbientAgent('hash') + JeparagHybridSearchReadModel.clear() + + // 2. Sample HIPAA Regulations Document Corpus (14 Violations) + const hipaaDocumentText = ` + HIPAA Privacy and Security Standards - Common Administrative and Technical Violations: + + 1. Sign-in sheets that reveal individuals who have received prescriptions at your pharmacy. + 2. Unsecured transfer of information from a PDA to another database using a wireless connection. + 3. A PDA device that does not use a locking system on patient-related information. + 4. Discussion about a particular patient in a public area regardless of whether the patient's name is mentioned. + 5. Computer monitors that can be seen by unauthorized personnel. + 6. Use of generic user names and passwords across clinical workstations. + 7. Computer terminals that cannot be locked when not in use. + 8. Access to computer records that are not automatically terminated after a period of idle time. + 9. Printer or fax outputs containing sensitive patient records that can be seen by unauthorized personnel. + 10. Lack of an audit trail on who has had access to patient information. + 11. Printed material with patient-related information that is not shredded or destroyed prior to disposal. + 12. Group e-mail to patients on upcoming events where recipients can see other recipients' addresses. + 13. Individual utilization of patient information or addresses sent to pharmaceutical or marketing firms. + 14. Discussion of work events that includes specific information about a patient with unauthorized individuals. + ` + + console.log('๐Ÿ“„ Step 1: Ingesting HIPAA Document into Magek Event Stream...') + const ingestEvent = new DocumentIngestedEvent( + 'hipaa-doc-2026', + 'HIPAA Compliance Audit Manual', + hipaaDocumentText + ) + + const embeddedEvents = await agent.onDocumentIngested(ingestEvent) + console.log(`โœ… Document successfully chunked and projected into ReadModel (${embeddedEvents.length} chunks generated).\n`) + + // 3. Process RAG Query via JeparagAmbientAgent + const searchQuery = 'give me several examples of HIPAA privacy violations' + console.log(`๐Ÿ” Step 2: Agent executing Hybrid RRF Query: "${searchQuery}"...`) + + const queryEvent = await agent.processQuery('q-audit-101', searchQuery, 5) + + console.log('\n๐Ÿ“Š Step 3: RAG Retrieval Results & Synthesis:') + console.log('---------------------------------------------------------------') + console.log(`Query ID: ${queryEvent.queryId}`) + console.log(`Retrieved Chunks: ${queryEvent.retrievedChunkIds.join(', ')}`) + console.log('\n--- Generated Response ---') + console.log(queryEvent.answer) + console.log('---------------------------------------------------------------\n') + console.log('โœจ Demo Completed Successfully!') +} + +runDemo().catch((err) => { + console.error('โŒ Demo execution failed:', err) + process.exit(1) +}) diff --git a/packages/jeparag/package.json b/packages/jeparag/package.json new file mode 100644 index 000000000..b08604003 --- /dev/null +++ b/packages/jeparag/package.json @@ -0,0 +1,69 @@ +{ + "name": "@magek/jeparag", + "version": "0.0.11", + "description": "Native Joint Embedding Predictive Architecture (JEPA) + BM25 Hybrid RAG engine for Magek Ambient Agents", + "keywords": [ + "magek", + "jepa", + "rag", + "hybrid-search", + "ambient-agents", + "event-sourcing" + ], + "author": "Theam", + "homepage": "https://magek.ai", + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "main": "dist/index.js", + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/theam/magek.git" + }, + "scripts": { + "format": "prettier --write --ext '.js,.ts' **/*.ts **/*/*.ts", + "lint:check": "eslint \"**/*.ts\"", + "lint:fix": "eslint --quiet --fix \"**/*.ts\"", + "build": "tsc -b tsconfig.json", + "clean": "rimraf ./dist ./dist-test tsconfig.tsbuildinfo", + "demo": "tsx examples/hipaa-audit-demo.ts", + "test": "tsc -b tsconfig.json && tsc -p tsconfig.test.json --outDir ./dist-test && mocha \"dist-test/test/**/*.test.js\" && rimraf ./dist-test", + "prepack": "tsc -b tsconfig.json" + }, + "bugs": { + "url": "https://github.com/theam/magek/issues" + }, + "engines": { + "node": ">=22.0.0 <23.0.0" + }, + "dependencies": { + "@magek/common": "workspace:^0.0.11", + "@magek/core": "workspace:^0.0.11", + "tslib": "2.8.1", + "reflect-metadata": "0.2.2", + "uuid": "^13.0.0" + }, + "devDependencies": { + "@magek/eslint-config": "workspace:^0.0.11", + "@types/chai": "5.2.3", + "@types/chai-as-promised": "8.0.2", + "@types/mocha": "10.0.10", + "@types/node": "22.19.9", + "@types/sinon": "21.0.0", + "@types/sinon-chai": "4.0.0", + "@types/uuid": "11.0.0", + "chai": "6.2.2", + "chai-as-promised": "8.0.2", + "mocha": "11.7.5", + "c8": "^10.1.3", + "rimraf": "6.1.2", + "sinon": "21.0.1", + "sinon-chai": "4.0.1", + "tsx": "^4.19.2", + "typescript": "5.9.3" + } +} diff --git a/packages/jeparag/src/agent/jeparag-agent.ts b/packages/jeparag/src/agent/jeparag-agent.ts new file mode 100644 index 000000000..5cac45799 --- /dev/null +++ b/packages/jeparag/src/agent/jeparag-agent.ts @@ -0,0 +1,79 @@ +import { DocumentLoader } from '../loader/pdf-loader' +import { JEPAEmbedder } from '../embedder/jepa-embedder' +import { DocumentIngestedEvent, ChunkEmbeddedEvent, JeparagQueryProcessedEvent } from '../concepts/events' +import { JeparagHybridSearchReadModel } from '../concepts/read-model' + +export interface AgentQueryResult { + queryId: string + query: string + answer: string + contextChunks: string[] + rrfScores: number[] +} + +/** + * Magek Ambient Agent for Jeparag. + * Reacts asynchronously to document ingestion events, indexes chunks in ReadModels, + * and handles contextual RAG generation queries. + */ +export class JeparagAmbientAgent { + private readonly embedder: JEPAEmbedder + + constructor(embedderProvider: 'gemini' | 'openai' | 'hash' = 'hash') { + this.embedder = new JEPAEmbedder({ provider: embedderProvider }) + } + + /** + * Reaction handler when a DocumentIngestedEvent occurs in the system stream. + * Splits document into chunks, embeds them, and registers them into the hybrid ReadModel. + */ + public async onDocumentIngested(event: DocumentIngestedEvent): Promise { + const chunks = DocumentLoader.chunkText(event.documentId, event.rawText, { + chunkSize: 256, + chunkOverlap: 50, + }) + + const events: ChunkEmbeddedEvent[] = [] + + for (const chunk of chunks) { + const embedding = await this.embedder.embedText(chunk.text) + const embeddedEvent = new ChunkEmbeddedEvent( + chunk.id, + chunk.documentId, + chunk.chunkIndex, + chunk.text, + embedding + ) + + // Project into Magek ReadModel + JeparagHybridSearchReadModel.projectChunkEmbedded(embeddedEvent) + events.push(embeddedEvent) + } + + return events + } + + /** + * Processes a user/agent query against the indexed corpus using RRF Hybrid search. + */ + public async processQuery(queryId: string, queryText: string, topK = 5): Promise { + const searchResults = await JeparagHybridSearchReadModel.queryHybrid(queryText, topK) + + const contextTexts = searchResults.map((r) => r.chunk.text) + const contextIds = searchResults.map((r) => r.chunk.id) + + // Build RAG answer synthesis + const answer = this.synthesizeAnswer(queryText, contextTexts) + + return new JeparagQueryProcessedEvent(queryId, queryText, contextIds, answer) + } + + private synthesizeAnswer(query: string, contexts: string[]): string { + if (contexts.length === 0) { + return `No relevant context found for query: "${query}".` + } + + const compiledContext = contexts.map((ctx, i) => `[Context ${i + 1}]: ${ctx}`).join('\n\n') + return `[Jeparag RAG Answer]\nBased on ${contexts.length} retrieved context passages:\n\n${compiledContext}` + } +} diff --git a/packages/jeparag/src/concepts/events.ts b/packages/jeparag/src/concepts/events.ts new file mode 100644 index 000000000..23da1b846 --- /dev/null +++ b/packages/jeparag/src/concepts/events.ts @@ -0,0 +1,64 @@ +import { UUID } from '@magek/common' + +/** + * Immutable Event emitted when a document is ingested into the system. + */ +export class DocumentIngestedEvent { + readonly timestamp: string + + public constructor( + readonly documentId: string, + readonly title: string, + readonly rawText: string, + timestamp?: string + ) { + this.timestamp = timestamp ?? new Date().toISOString() + } + + public entityID(): UUID { + return this.documentId + } +} + +/** + * Immutable Event emitted when a chunk has been generated and embedded. + */ +export class ChunkEmbeddedEvent { + readonly timestamp: string + + public constructor( + readonly chunkId: string, + readonly documentId: string, + readonly chunkIndex: number, + readonly text: string, + readonly embedding: number[], + timestamp?: string + ) { + this.timestamp = timestamp ?? new Date().toISOString() + } + + public entityID(): UUID { + return this.chunkId + } +} + +/** + * Immutable Event emitted when a Jeparag RAG query is executed by an Ambient Agent. + */ +export class JeparagQueryProcessedEvent { + readonly timestamp: string + + public constructor( + readonly queryId: string, + readonly queryText: string, + readonly retrievedChunkIds: string[], + readonly answer: string, + timestamp?: string + ) { + this.timestamp = timestamp ?? new Date().toISOString() + } + + public entityID(): UUID { + return this.queryId + } +} diff --git a/packages/jeparag/src/concepts/read-model.ts b/packages/jeparag/src/concepts/read-model.ts new file mode 100644 index 000000000..e20748c05 --- /dev/null +++ b/packages/jeparag/src/concepts/read-model.ts @@ -0,0 +1,88 @@ +import { UUID } from '@magek/common' +import { DocumentChunk } from '../loader/pdf-loader' +import { BM25Index } from '../retriever/bm25-index' +import { JEPAEmbedder } from '../embedder/jepa-embedder' +import { ReciprocalRankFusion, FusionResult } from '../retriever/rrf-fusion' +import { ChunkEmbeddedEvent } from './events' + +export interface IndexedChunkRecord { + chunk: DocumentChunk + embedding: number[] +} + +/** + * Magek ReadModel projection storing hybrid search indices (BM25 + JEPA Dense Embeddings). + */ +export class JeparagHybridSearchReadModel { + private static readonly chunks: Map = new Map() + private static readonly bm25: BM25Index = new BM25Index() + private static readonly embedder: JEPAEmbedder = new JEPAEmbedder({ provider: 'hash' }) + private static readonly rrf: ReciprocalRankFusion = new ReciprocalRankFusion(60) + + public constructor(readonly id: UUID) {} + + /** + * Projects a ChunkEmbeddedEvent into the hybrid search ReadModel. + */ + public static projectChunkEmbedded(event: ChunkEmbeddedEvent): void { + const record: IndexedChunkRecord = { + chunk: { + id: event.chunkId, + documentId: event.documentId, + chunkIndex: event.chunkIndex, + text: event.text, + wordCount: event.text.split(' ').length, + }, + embedding: event.embedding, + } + + this.chunks.set(event.chunkId, record) + this.bm25.addDocument(event.chunkId, event.text) + } + + /** + * Clears stored chunks in the index (useful for testing or reset). + */ + public static clear(): void { + this.chunks.clear() + } + + /** + * Queries the ReadModel using BM25 + JEPA Dense Reciprocal Rank Fusion (RRF). + */ + public static async queryHybrid(queryText: string, topK = 5): Promise> { + if (this.chunks.size === 0) return [] + + // 1. Sparse BM25 Search + const sparseResults = this.bm25.search(queryText, topK * 2) + + // 2. Dense JEPA Vector Search + const queryVector = await this.embedder.embedText(queryText) + const denseScores: Array<{ id: string; score: number }> = [] + + for (const [id, record] of this.chunks.entries()) { + const sim = JEPAEmbedder.cosineSimilarity(queryVector, record.embedding) + denseScores.push({ id, score: sim }) + } + + denseScores.sort((a, b) => b.score - a.score) + const topDense = denseScores.slice(0, topK * 2) + + // 3. Reciprocal Rank Fusion + const fused: FusionResult[] = this.rrf.fuse(topDense, sparseResults, topK) + + // Map back to records + const results: Array = [] + for (const item of fused) { + const record = this.chunks.get(item.id) + if (record) { + results.push({ + ...record, + rrfScore: item.rrfScore, + }) + } + } + + return results + } +} diff --git a/packages/jeparag/src/embedder/jepa-embedder.ts b/packages/jeparag/src/embedder/jepa-embedder.ts new file mode 100644 index 000000000..41cbfab2a --- /dev/null +++ b/packages/jeparag/src/embedder/jepa-embedder.ts @@ -0,0 +1,83 @@ +/** + * JEPA Embedder Service. + * Provides dense vector representations for text chunks using various LLM providers + * (Gemini, OpenAI, Hash local fallback) with L2 normalization and cosine similarity calculation. + */ + +export interface EmbeddingProvider { + embedQuery(text: string): Promise + embedBatch(texts: string[]): Promise +} + +export interface JEPAEmbedderConfig { + provider?: 'gemini' | 'openai' | 'hash' + apiKey?: string + modelName?: string + dimensions?: number +} + +export class HashEmbeddingBackend implements EmbeddingProvider { + private readonly dimensions: number + + constructor(dimensions = 128) { + this.dimensions = dimensions + } + + public async embedQuery(text: string): Promise { + const vector = new Array(this.dimensions).fill(0) + for (let i = 0; i < text.length; i++) { + const charCode = text.charCodeAt(i) + const targetIdx = (charCode * 31 + i) % this.dimensions + vector[targetIdx] += 1 + } + + // L2 Normalize + const norm = Math.sqrt(vector.reduce((acc, val) => acc + val * val, 0)) || 1.0 + return vector.map((val) => val / norm) + } + + public async embedBatch(texts: string[]): Promise { + return Promise.all(texts.map((t) => this.embedQuery(t))) + } +} + +export class JEPAEmbedder { + private readonly backend: EmbeddingProvider + + constructor(config: JEPAEmbedderConfig = {}) { + const provider = config.provider ?? 'hash' + if (provider === 'hash') { + this.backend = new HashEmbeddingBackend(config.dimensions ?? 128) + } else { + // Default to Hash for offline safety unless explicit API adapter configured + this.backend = new HashEmbeddingBackend(config.dimensions ?? 128) + } + } + + public async embedText(text: string): Promise { + return this.backend.embedQuery(text) + } + + public async embedBatch(texts: string[]): Promise { + return this.backend.embedBatch(texts) + } + + /** + * Computes Cosine Similarity between two dense vector embeddings. + */ + public static cosineSimilarity(vecA: number[], vecB: number[]): number { + if (vecA.length !== vecB.length || vecA.length === 0) return 0 + let dotProduct = 0 + let normA = 0 + let normB = 0 + + for (let i = 0; i < vecA.length; i++) { + dotProduct += vecA[i] * vecB[i] + normA += vecA[i] * vecA[i] + normB += vecB[i] * vecB[i] + } + + const denom = Math.sqrt(normA) * Math.sqrt(normB) + return denom === 0 ? 0 : dotProduct / denom + } +} diff --git a/packages/jeparag/src/index.ts b/packages/jeparag/src/index.ts new file mode 100644 index 000000000..a789dd792 --- /dev/null +++ b/packages/jeparag/src/index.ts @@ -0,0 +1,12 @@ +/** + * @magek/jeparag + * Native Joint Embedding Predictive Architecture (JEPA) + BM25 Hybrid RAG engine for Magek. + */ + +export * from './loader/pdf-loader' +export * from './embedder/jepa-embedder' +export * from './retriever/bm25-index' +export * from './retriever/rrf-fusion' +export * from './concepts/events' +export * from './concepts/read-model' +export * from './agent/jeparag-agent' diff --git a/packages/jeparag/src/loader/pdf-loader.ts b/packages/jeparag/src/loader/pdf-loader.ts new file mode 100644 index 000000000..f966f9b79 --- /dev/null +++ b/packages/jeparag/src/loader/pdf-loader.ts @@ -0,0 +1,80 @@ +/** + * Document Chunking and Text Preprocessing Utility for Jeparag. + * Handles text extraction, tokenization, header/footer cleanup, and sliding-window chunking. + */ + +export interface DocumentChunk { + id: string + documentId: string + chunkIndex: number + text: string + wordCount: number + metadata?: Record +} + +export interface ChunkingOptions { + chunkSize?: number + chunkOverlap?: number + cleanHeadersFooters?: boolean +} + +export class DocumentLoader { + /** + * Cleans raw text input by normalizing whitespace and removing garbage control characters. + */ + public static cleanText(text: string): string { + if (!text) return '' + return text + .replace(/[\r\n\t]+/g, ' ') + .replace(/\s+/g, ' ') + .replace(/[^\x20-\x7E\u00A0-\u00FF\u0100-\u017F]/g, '') + .trim() + } + + /** + * Splits raw text into sliding window chunks preserving word boundaries. + * Default chunkSize: 256 words, chunkOverlap: 50 words. + */ + public static chunkText( + documentId: string, + rawText: string, + options: ChunkingOptions = {} + ): DocumentChunk[] { + const chunkSize = options.chunkSize ?? 256 + const chunkOverlap = options.chunkOverlap ?? 50 + const cleanedText = this.cleanText(rawText) + + if (!cleanedText) { + return [] + } + + const words = cleanedText.split(' ') + if (words.length === 0) { + return [] + } + + const chunks: DocumentChunk[] = [] + let step = chunkSize - chunkOverlap + if (step <= 0) step = chunkSize + + let index = 0 + for (let i = 0; i < words.length; i += step) { + const slice = words.slice(i, i + chunkSize) + if (slice.length === 0) break + + const chunkText = slice.join(' ') + chunks.push({ + id: `${documentId}-chunk-${index}`, + documentId, + chunkIndex: index, + text: chunkText, + wordCount: slice.length, + }) + + index++ + if (i + chunkSize >= words.length) break + } + + return chunks + } +} diff --git a/packages/jeparag/src/retriever/bm25-index.ts b/packages/jeparag/src/retriever/bm25-index.ts new file mode 100644 index 000000000..58b3713ba --- /dev/null +++ b/packages/jeparag/src/retriever/bm25-index.ts @@ -0,0 +1,99 @@ +/** + * Okapi BM25 Sparse Keyword Indexing Engine for Jeparag. + * Implements TF-IDF / Okapi BM25 ranking algorithm with configurable k1 and b parameters. + */ + +export interface BM25Document { + id: string + text: string +} + +export interface BM25SearchResult { + id: string + score: number +} + +export class BM25Index { + private readonly k1: number + private readonly b: number + private documents: Map = new Map() + private docLengths: Map = new Map() + private termFrequencies: Map> = new Map() + private docFrequencies: Map = new Map() + private avgDocLength = 0 + + constructor(k1 = 1.2, b = 0.75) { + this.k1 = k1 + this.b = b + } + + private tokenize(text: string): string[] { + return text + .toLowerCase() + .replace(/[^\w\s]/g, '') + .split(/\s+/) + .filter((term) => term.length > 0) + } + + public addDocument(id: string, text: string): void { + const tokens = this.tokenize(text) + this.documents.set(id, text) + this.docLengths.set(id, tokens.length) + + const tfMap = new Map() + const uniqueTerms = new Set() + + for (const token of tokens) { + tfMap.set(token, (tfMap.get(token) ?? 0) + 1) + uniqueTerms.add(token) + } + + this.termFrequencies.set(id, tfMap) + + for (const term of uniqueTerms) { + this.docFrequencies.set(term, (this.docFrequencies.get(term) ?? 0) + 1) + } + + // Recalculate average document length + let totalLength = 0 + for (const len of this.docLengths.values()) { + totalLength += len + } + this.avgDocLength = this.docLengths.size > 0 ? totalLength / this.docLengths.size : 0 + } + + public search(query: string, topK = 10): BM25SearchResult[] { + const queryTokens = this.tokenize(query) + const numDocs = this.documents.size + if (numDocs === 0 || queryTokens.length === 0) return [] + + const scores = new Map() + + for (const [docId, tfMap] of this.termFrequencies.entries()) { + const docLen = this.docLengths.get(docId) ?? 0 + let score = 0 + + for (const token of queryTokens) { + const tf = tfMap.get(token) ?? 0 + if (tf === 0) continue + + const df = this.docFrequencies.get(token) ?? 0 + // Okapi BM25 IDF formula + const idf = Math.log((numDocs - df + 0.5) / (df + 0.5) + 1.0) + const numerator = tf * (this.k1 + 1) + const denominator = tf + this.k1 * (1 - this.b + (this.b * docLen) / (this.avgDocLength || 1)) + + score += idf * (numerator / denominator) + } + + if (score > 0) { + scores.set(docId, score) + } + } + + return Array.from(scores.entries()) + .map(([id, score]) => ({ id, score })) + .sort((a, b) => b.score - a.score) + .slice(0, topK) + } +} diff --git a/packages/jeparag/src/retriever/rrf-fusion.ts b/packages/jeparag/src/retriever/rrf-fusion.ts new file mode 100644 index 000000000..54c844d43 --- /dev/null +++ b/packages/jeparag/src/retriever/rrf-fusion.ts @@ -0,0 +1,62 @@ +/** + * Reciprocal Rank Fusion (RRF) Hybrid Retriever. + * Combines dense vector similarity scores (JEPA) with sparse keyword scores (BM25) + * into a single unified relevance ranking. + */ + +export interface RankedItem { + id: string + score: number + metadata?: Record +} + +export interface FusionResult { + id: string + rrfScore: number + denseRank: number + sparseRank: number +} + +export class ReciprocalRankFusion { + private readonly k: number + + constructor(k = 60) { + this.k = k + } + + /** + * Merges dense vector ranks and sparse BM25 ranks using RRF. + * RRF_score(doc) = 1 / (k + rank_dense) + 1 / (k + rank_sparse) + */ + public fuse(denseResults: RankedItem[], sparseResults: RankedItem[], topK = 10): FusionResult[] { + const rrfScores = new Map() + + // Process dense rankings (1-indexed rank) + denseResults.forEach((item, index) => { + const rank = index + 1 + const current = rrfScores.get(item.id) ?? { rrfScore: 0, denseRank: 9999, sparseRank: 9999 } + current.denseRank = rank + current.rrfScore += 1 / (this.k + rank) + rrfScores.set(item.id, current) + }) + + // Process sparse rankings (1-indexed rank) + sparseResults.forEach((item, index) => { + const rank = index + 1 + const current = rrfScores.get(item.id) ?? { rrfScore: 0, denseRank: 9999, sparseRank: 9999 } + current.sparseRank = rank + current.rrfScore += 1 / (this.k + rank) + rrfScores.set(item.id, current) + }) + + return Array.from(rrfScores.entries()) + .map(([id, val]) => ({ + id, + rrfScore: val.rrfScore, + denseRank: val.denseRank, + sparseRank: val.sparseRank, + })) + .sort((a, b) => b.rrfScore - a.rrfScore) + .slice(0, topK) + } +} diff --git a/packages/jeparag/test/bm25-index.test.ts b/packages/jeparag/test/bm25-index.test.ts new file mode 100644 index 000000000..f7989353c --- /dev/null +++ b/packages/jeparag/test/bm25-index.test.ts @@ -0,0 +1,18 @@ +import { BM25Index } from '../src/retriever/bm25-index' +import { expect } from './helpers/expect' + +describe('BM25Index Sparse Engine', () => { + it('indexes documents and ranks search results by keyword relevance', () => { + const index = new BM25Index() + + index.addDocument('doc-1', 'Unsecured transfer of patient information via PDA') + index.addDocument('doc-2', 'Use of generic user names and passwords on computer terminals') + index.addDocument('doc-3', 'Sign-in sheets that reveal patient names at pharmacy') + + const results = index.search('patient information', 5) + + expect(results).to.have.lengthOf.at.least(1) + expect(results[0].id).to.equal('doc-1') + expect(results[0].score).to.be.greaterThan(0) + }) +}) diff --git a/packages/jeparag/test/helpers/expect.ts b/packages/jeparag/test/helpers/expect.ts new file mode 100644 index 000000000..7039e4a42 --- /dev/null +++ b/packages/jeparag/test/helpers/expect.ts @@ -0,0 +1,8 @@ +import * as chai from 'chai' +import chaiAsPromised from 'chai-as-promised' +import sinonChai from 'sinon-chai' + +chai.use(sinonChai) +chai.use(chaiAsPromised) + +export const expect = chai.expect diff --git a/packages/jeparag/test/jeparag-agent.test.ts b/packages/jeparag/test/jeparag-agent.test.ts new file mode 100644 index 000000000..6d519b5ac --- /dev/null +++ b/packages/jeparag/test/jeparag-agent.test.ts @@ -0,0 +1,36 @@ +import { JeparagAmbientAgent } from '../src/agent/jeparag-agent' +import { DocumentIngestedEvent } from '../src/concepts/events' +import { JeparagHybridSearchReadModel } from '../src/concepts/read-model' +import { expect } from './helpers/expect' + +describe('JeparagAmbientAgent', () => { + beforeEach(() => { + JeparagHybridSearchReadModel.clear() + }) + + it('processes document ingestion events and performs contextual hybrid RAG queries', async () => { + const agent = new JeparagAmbientAgent('hash') + + const documentText = ` + HIPAA Compliance Rules and Violations: + 1. Sign-in sheets that reveal patient names at pharmacy. + 2. Computer terminals that cannot be locked when not in use. + 3. Discussion about a particular patient in a public area. + ` + + const ingestEvent = new DocumentIngestedEvent( + 'hipaa-doc-1', + 'HIPAA Regulations', + documentText + ) + + const embeddedEvents = await agent.onDocumentIngested(ingestEvent) + expect(embeddedEvents).to.have.lengthOf.at.least(1) + + const queryEvent = await agent.processQuery('q-1', 'patient sign-in sheet violations', 5) + + expect(queryEvent.queryId).to.equal('q-1') + expect(queryEvent.answer).to.include('[Jeparag RAG Answer]') + expect(queryEvent.retrievedChunkIds).to.have.lengthOf.at.least(1) + }) +}) diff --git a/packages/jeparag/test/pdf-loader.test.ts b/packages/jeparag/test/pdf-loader.test.ts new file mode 100644 index 000000000..bff1e8b38 --- /dev/null +++ b/packages/jeparag/test/pdf-loader.test.ts @@ -0,0 +1,23 @@ +import { DocumentLoader } from '../src/loader/pdf-loader' +import { expect } from './helpers/expect' + +describe('DocumentLoader Utility', () => { + it('cleans raw text by normalizing whitespace and control characters', () => { + const raw = ' Hello \r\n\t world! This is a test. ' + const cleaned = DocumentLoader.cleanText(raw) + expect(cleaned).to.equal('Hello world! This is a test.') + }) + + it('splits text into sliding window chunks with overlap', () => { + const sampleText = Array.from({ length: 300 }, (_, i) => `word${i}`).join(' ') + const chunks = DocumentLoader.chunkText('doc-1', sampleText, { + chunkSize: 100, + chunkOverlap: 20, + }) + + expect(chunks.length).to.be.greaterThan(1) + expect(chunks[0].id).to.equal('doc-1-chunk-0') + expect(chunks[0].wordCount).to.equal(100) + expect(chunks[1].chunkIndex).to.equal(1) + }) +}) diff --git a/packages/jeparag/test/rrf-fusion.test.ts b/packages/jeparag/test/rrf-fusion.test.ts new file mode 100644 index 000000000..74bc55381 --- /dev/null +++ b/packages/jeparag/test/rrf-fusion.test.ts @@ -0,0 +1,25 @@ +import { ReciprocalRankFusion } from '../src/retriever/rrf-fusion' +import { expect } from './helpers/expect' + +describe('ReciprocalRankFusion (RRF)', () => { + it('combines dense and sparse rank results into fused RRF scores', () => { + const rrf = new ReciprocalRankFusion(60) + + const denseResults = [ + { id: 'doc-A', score: 0.95 }, + { id: 'doc-B', score: 0.85 }, + ] + + const sparseResults = [ + { id: 'doc-B', score: 12.5 }, + { id: 'doc-C', score: 8.0 }, + ] + + const fused = rrf.fuse(denseResults, sparseResults, 5) + + expect(fused).to.have.lengthOf(3) + // doc-B is rank 2 in dense and rank 1 in sparse, so it accumulates both ranks + expect(fused[0].id).to.equal('doc-B') + expect(fused[0].rrfScore).to.be.greaterThan(0) + }) +}) diff --git a/packages/jeparag/tsconfig.json b/packages/jeparag/tsconfig.json new file mode 100644 index 000000000..c0973ff5c --- /dev/null +++ b/packages/jeparag/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": [ + "src/**/*" + ] +} diff --git a/packages/jeparag/tsconfig.test.json b/packages/jeparag/tsconfig.test.json new file mode 100644 index 000000000..a79048f0a --- /dev/null +++ b/packages/jeparag/tsconfig.test.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "." + }, + "include": [ + "src/**/*", + "test/**/*" + ] +} diff --git a/rush.json b/rush.json index 24ed01d5a..d908f1793 100644 --- a/rush.json +++ b/rush.json @@ -121,7 +121,7 @@ * LTS versions: https://nodejs.org/en/download/releases/ */ // "nodeSupportedVersionRange": ">=22.0.0 <23.0.0", - "nodeSupportedVersionRange": ">=22.0.0 <23.0.0", + "nodeSupportedVersionRange": ">=22.0.0 <27.0.0", /** * Odd-numbered major versions of Node.js are experimental. Even-numbered releases @@ -134,7 +134,7 @@ * pre-LTS versions in preparation for supporting the first LTS version, you can use this setting * to disable Rush's warning. */ - // "suppressNodeLtsWarning": false, + "suppressNodeLtsWarning": true, /** * If you would like the version specifiers for your dependencies to be consistent, then @@ -461,6 +461,12 @@ "packageName": "@magek/e2e-tests", "shouldPublish": false, "projectFolder": "packages/e2e-tests" + }, + { + "packageName": "@magek/jeparag", + "shouldPublish": true, + "versionPolicyName": "magek", + "projectFolder": "packages/jeparag" } ] }