feat(search): add language parameter to search queries and update cur…

backend/scripts/search.ts

@@ -34,6 +34,8 @@ import {
  *   --fields <a,b,c>             (optional, default: all fields) The source fields to search on
  *                                (preferredLabel, description, altLabels, scopeNote).
  *   --limit <n>                  (optional, default: 10) The maximum number of hits per query.
+ *   --language <dbKeyName>       (optional, default: en) The language dbKeyName to restrict the search to.
+ *                                Must match the language the model was embedded in (e.g. "en", "fr").
  *
  * Example:
  *   yarn search --model 6123abc... --service 77bb8ff3-a6b0-460b-bcaa-00631a907852 \
@@ -105,6 +107,7 @@ interface IParsedArgs {
   queries: string[];
   fields: EmbeddableField[];
   limit: number;
+  language: string;
 }
 
 /**
@@ -161,7 +164,9 @@ export function parseArgs(argv: string[]): IParsedArgs {
     throw new Error(`Invalid --limit '${values.limit?.[0]}'. Must be a positive integer`);
   }
 
-  return { modelId, embeddingServiceId, collection, queries, fields: fields as EmbeddableField[], limit };
+  const language = values.language?.[0] ?? "en";
+
+  return { modelId, embeddingServiceId, collection, queries, fields: fields as EmbeddableField[], limit, language };
 }
 
 /**
@@ -181,6 +186,7 @@ async function runSearch(config: ICollectionConfig, args: IParsedArgs, query: st
     indexName: config.indexName,
     modelId: args.modelId,
     embeddingServiceId: args.embeddingServiceId,
+    language: args.language,
     queryVector,
     searchFields: args.fields,
     limit: args.limit,
@@ -216,7 +222,8 @@ async function main(): Promise<void> {
 
     console.info(
       `Searching '${args.collection}' embeddings of model ${args.modelId} ` +
-        `with embedding service ${args.embeddingServiceId} on fields [${args.fields.join(", ")}].`
+        `with embedding service ${args.embeddingServiceId} on fields [${args.fields.join(", ")}] ` +
+        `in language '${args.language}'.`
     );
 
     for (const query of args.queries) {

backend/src/esco/common/searchCondition.test.ts

@@ -24,13 +24,13 @@ describe("Test buildSearchCondition()", () => {
     });
   });
 
-  test("should match a translatable field on the path of the fall back language", () => {
+  test("should match a translatable field on the path of the fall back language when no language is given", () => {
     // GIVEN a search on a translatable field and on a monolingual field
     const givenSearch = { value: "cook", fields: ["preferredLabel", "code"] };
     // AND preferredLabel is translated, code is not
     const givenTranslatableFields = ["preferredLabel", "altLabels"];
 
-    // WHEN the condition is built
+    // WHEN the condition is built without an explicit language
     const actualCondition = buildSearchCondition(givenSearch, givenTranslatableFields);
 
     // THEN expect the translatable field to be matched on its fall back language path, the other one as it is stored
@@ -42,6 +42,26 @@ describe("Test buildSearchCondition()", () => {
     });
   });
 
+  test("should match a translatable field on the path of the given language", () => {
+    // GIVEN a search on a translatable field and on a monolingual field
+    const givenSearch = { value: "cuisinier", fields: ["preferredLabel", "code"] };
+    // AND preferredLabel is translated
+    const givenTranslatableFields = ["preferredLabel", "altLabels"];
+    // AND the requested language is French
+    const givenLanguage = "fr";
+
+    // WHEN the condition is built with the French language
+    const actualCondition = buildSearchCondition(givenSearch, givenTranslatableFields, givenLanguage);
+
+    // THEN expect the translatable field to be matched on its French path, not the fallback
+    expect(actualCondition).toEqual({
+      $or: [
+        { "preferredLabel.fr": { $regex: "cuisinier", $options: "i" } },
+        { code: { $regex: "cuisinier", $options: "i" } },
+      ],
+    });
+  });
+
   test("should match the value literally, so that a regular expression cannot be injected", () => {
     // GIVEN a search value that carries regular expression special characters
     const givenSearch = { value: "a.*(b)", fields: ["code"] };

backend/src/esco/common/searchCondition.ts

@@ -13,22 +13,25 @@ export interface ISearch {
  * Builds the $or condition that matches a search value on the requested fields.
  *
  * The value is matched literally (escaped) and case insensitively. A field that is translated is stored as a
- * localized sub document, so it is matched on the path of the fall back language, e.g. preferredLabel.en. $regex
- * matches an array field such as altLabels element wise, so array and scalar fields are handled uniformly.
+ * localized sub document, so it is matched on the path of the given language (e.g. preferredLabel.fr), falling
+ * back to the fallback language when none is supplied. $regex matches an array field such as altLabels element
+ * wise, so array and scalar fields are handled uniformly.
  *
  * @param search the value to match and the fields to match it on
  * @param translatableFields the fields that are translated, none by default
+ * @param language the language dbKeyName to match translatable fields in; defaults to the fallback language
  * @returns the $or condition, to be ANDed with the rest of the match stage
  */
 export function buildSearchCondition(
   search: ISearch,
-  translatableFields: readonly string[] = []
+  translatableFields: readonly string[] = [],
+  language?: string
 ): { $or: Record<string, unknown>[] } {
   const escapedValue = escapeRegExp(search.value);
-  const fallbackDbKeyName = getFallbackLanguageConfig().dbKeyName;
+  const dbKeyName = language ?? getFallbackLanguageConfig().dbKeyName;
   return {
     $or: search.fields.map((field) => {
-      const path = translatableFields.includes(field) ? `${field}.${fallbackDbKeyName}` : field;
+      const path = translatableFields.includes(field) ? `${field}.${dbKeyName}` : field;
       return { [path]: { $regex: escapedValue, $options: "i" } };
     }),
   };

backend/src/esco/common/searchCursor.test.ts

@@ -1,31 +1,120 @@
-import { decodeSearchCursor, encodeSearchCursor } from "./searchCursor";
+import LanguageAPISpecs from "api-specifications/language";
+import {
+  decodeSearchCursor,
+  encodeSearchCursor,
+  parseSearchCursor,
+  SearchCursorLanguageMismatchError,
+} from "./searchCursor";
+
+const FALLBACK_DB_KEY_NAME = LanguageAPISpecs.Constants.FALLBACK_LANGUAGE.dbKeyName;
 
 describe("search cursor", () => {
-  test.each([0, 5, 100, 1_000_000])("should round-trip the offset %s", (givenOffset) => {
-    // WHEN the offset is encoded and then decoded
-    const actual = decodeSearchCursor(encodeSearchCursor(givenOffset));
+  describe("encodeSearchCursor / decodeSearchCursor round-trip", () => {
+    test.each([
+      [0, "en"],
+      [5, "fr"],
+      [100, "de"],
+      [1_000_000, "en"],
+    ])("should round-trip offset %s and language '%s'", (givenOffset, givenLanguage) => {
+      // WHEN the offset and language are encoded and then decoded
+      const actual = decodeSearchCursor(encodeSearchCursor(givenOffset, givenLanguage), givenLanguage);
+
+      // THEN expect the original offset back
+      expect(actual).toBe(givenOffset);
+    });
+
+    test("should produce an opaque (base64) cursor string containing both offset and language", () => {
+      // GIVEN an offset and language
+      const givenOffset = 10;
+      const givenLanguage = "fr";
+
+      // WHEN the cursor is encoded
+      const actual = encodeSearchCursor(givenOffset, givenLanguage);
 
-    // THEN expect the original offset back
-    expect(actual).toBe(givenOffset);
+      // THEN expect a base64 string that decodes to a payload with both fields
+      expect(typeof actual).toBe("string");
+      expect(JSON.parse(Buffer.from(actual, "base64").toString("utf-8"))).toEqual({
+        offset: givenOffset,
+        language: givenLanguage,
+      });
+    });
   });
 
-  test("should produce an opaque (base64) cursor string", () => {
-    // WHEN an offset is encoded
-    const actual = encodeSearchCursor(10);
+  describe("decodeSearchCursor language mismatch", () => {
+    test("should throw SearchCursorLanguageMismatchError when the cursor language differs from the requested language", () => {
+      // GIVEN a cursor issued for French
+      const givenCursor = encodeSearchCursor(5, "fr");
+
+      // WHEN decoding it with English as the requested language
+      // THEN expect a language mismatch error
+      expect(() => decodeSearchCursor(givenCursor, "en")).toThrow(SearchCursorLanguageMismatchError);
+    });
+
+    test("should include the cursor and requested languages in the mismatch error message", () => {
+      // GIVEN a cursor issued for French
+      const givenCursor = encodeSearchCursor(5, "fr");
+
+      // WHEN decoding it with English as the requested language
+      let actualError: Error | undefined;
+      try {
+        decodeSearchCursor(givenCursor, "en");
+      } catch (e) {
+        actualError = e as Error;
+      }
+
+      // THEN expect the error message to name both languages
+      expect(actualError?.message).toContain("fr");
+      expect(actualError?.message).toContain("en");
+    });
+  });
+
+  describe("parseSearchCursor (structural validation only)", () => {
+    test("should parse a well-formed cursor without checking language", () => {
+      // GIVEN a cursor issued for French
+      const givenCursor = encodeSearchCursor(7, "fr");
+
+      // WHEN parsing it (no language to match)
+      const actual = parseSearchCursor(givenCursor);
+
+      // THEN expect the offset and language to be returned as-is
+      expect(actual).toEqual({ offset: 7, language: "fr" });
+    });
+
+    test.each([
+      ["malformed base64/JSON", "not-base64-json-!@#$%"],
+      ["a negative offset", Buffer.from(JSON.stringify({ offset: -1, language: "en" })).toString("base64")],
+      ["a non-integer offset", Buffer.from(JSON.stringify({ offset: 1.5, language: "en" })).toString("base64")],
+      ["a non-numeric offset", Buffer.from(JSON.stringify({ offset: "x", language: "en" })).toString("base64")],
+      ["a missing offset", Buffer.from(JSON.stringify({ language: "en" })).toString("base64")],
+      ["an empty language", Buffer.from(JSON.stringify({ offset: 0, language: "" })).toString("base64")],
+    ])("should throw when the cursor holds %s", (_description, givenCursor) => {
+      // WHEN parsing an invalid cursor THEN expect it to throw
+      expect(() => parseSearchCursor(givenCursor)).toThrow();
+    });
+
+    test("should fall back to the fallback language for a cursor with no language field (backward compat)", () => {
+      // GIVEN a pre-migration cursor that has no language field
+      const givenLegacyCursor = Buffer.from(JSON.stringify({ offset: 3 })).toString("base64");
+
+      // WHEN parsing it
+      const actual = parseSearchCursor(givenLegacyCursor);
 
-    // THEN expect a base64 string that decodes to the offset payload
-    expect(typeof actual).toBe("string");
-    expect(JSON.parse(Buffer.from(actual, "base64").toString("utf-8"))).toEqual({ offset: 10 });
+      // THEN expect it to succeed, with language set to the fallback
+      expect(actual.offset).toBe(3);
+      expect(actual.language).toBe(FALLBACK_DB_KEY_NAME);
+    });
   });
 
-  test.each([
-    ["malformed base64/JSON", "not-base64-json-!@#$%"],
-    ["a negative offset", Buffer.from(JSON.stringify({ offset: -1 })).toString("base64")],
-    ["a non-integer offset", Buffer.from(JSON.stringify({ offset: 1.5 })).toString("base64")],
-    ["a non-numeric offset", Buffer.from(JSON.stringify({ offset: "x" })).toString("base64")],
-    ["a missing offset", Buffer.from(JSON.stringify({})).toString("base64")],
-  ])("should throw when the cursor holds %s", (_description, givenCursor) => {
-    // WHEN decoding an invalid cursor THEN expect it to throw
-    expect(() => decodeSearchCursor(givenCursor)).toThrow();
+  describe("decodeSearchCursor structural validation", () => {
+    test.each([
+      ["malformed base64/JSON", "not-base64-json-!@#$%"],
+      ["a negative offset", Buffer.from(JSON.stringify({ offset: -1, language: "en" })).toString("base64")],
+      ["a non-integer offset", Buffer.from(JSON.stringify({ offset: 1.5, language: "en" })).toString("base64")],
+      ["a non-numeric offset", Buffer.from(JSON.stringify({ offset: "x", language: "en" })).toString("base64")],
+      ["a missing offset", Buffer.from(JSON.stringify({ language: "en" })).toString("base64")],
+    ])("should throw when the cursor holds %s", (_description, givenCursor) => {
+      // WHEN decoding an invalid cursor THEN expect it to throw
+      expect(() => decodeSearchCursor(givenCursor, "en")).toThrow();
+    });
   });
 });

backend/src/esco/common/searchCursor.ts

@@ -1,3 +1,5 @@
+import { getFallbackLanguageConfig } from "common/language/fallbackLanguage";
+
 /**
  * The cursor used to paginate a vector (embeddings) search.
  *
@@ -6,34 +8,82 @@
  * therefore paginated by rank offset: the cursor simply encodes how many of the ranked results have already been
  * returned. Because the query vector is deterministic, re-running the search and skipping `offset` results yields
  * the next page.
+ *
+ * The cursor also stores the language that was active when it was issued. If the client sends a cursor back with
+ * a different language the rank ordering is no longer valid (it was computed for the previous language), so the
+ * service should treat such a cursor as invalid.
  */
 interface SearchCursorPayload {
   offset: number;
+  language: string;
+}
+
+/**
+ * Thrown by {@link decodeSearchCursor} when the cursor was issued for a different language than the one now
+ * requested. The rank offset is valid but the result set would be from the wrong language, so callers should
+ * reject the cursor and ask the client to start from page 1 in the new language.
+ */
+export class SearchCursorLanguageMismatchError extends Error {
+  constructor(cursorLanguage: string, requestedLanguage: string) {
+    super(
+      `Search cursor was issued for language '${cursorLanguage}' but the current request uses '${requestedLanguage}'. ` +
+        `Start a new search without a cursor to use the requested language.`
+    );
+    this.name = "SearchCursorLanguageMismatchError";
+  }
 }
 
 /**
- * Encodes a vector-search pagination offset into an opaque base64 cursor string.
+ * Encodes a vector-search pagination offset and language into an opaque base64 cursor string.
  *
  * @param {number} offset - The number of ranked results already returned.
+ * @param {string} language - The language dbKeyName the search was executed in.
  * @return {string} - The base64 encoded cursor.
  */
-export function encodeSearchCursor(offset: number): string {
-  const payload: SearchCursorPayload = { offset };
+export function encodeSearchCursor(offset: number, language: string): string {
+  const payload: SearchCursorPayload = { offset, language };
   return Buffer.from(JSON.stringify(payload)).toString("base64");
 }
 
 /**
- * Decodes an opaque base64 vector-search cursor back into its pagination offset.
+ * Parses an opaque base64 vector-search cursor, validating its structure.
+ * Does NOT check the language — use this only for structural validation (e.g. to decide whether a cursor string
+ * looks like a search cursor at all). Use {@link decodeSearchCursor} when you also need to enforce language.
  *
  * @param {string} cursor - The base64 encoded cursor.
- * @return {number} - The decoded, non-negative integer offset.
- * @throws {Error} - If the cursor is malformed or does not hold a valid non-negative integer offset.
+ * @return {{ offset: number; language: string }} - The decoded payload.
+ * @throws {Error} - If the cursor is malformed.
  */
-export function decodeSearchCursor(cursor: string): number {
+export function parseSearchCursor(cursor: string): { offset: number; language: string } {
   const json = Buffer.from(cursor, "base64").toString("utf-8");
   const payload = JSON.parse(json) as SearchCursorPayload;
   if (typeof payload.offset !== "number" || !Number.isInteger(payload.offset) || payload.offset < 0) {
     throw new Error("Invalid search cursor: offset must be a non-negative integer");
   }
-  return payload.offset;
+  // Cursors issued before language was added to the payload have no language field.
+  // Treat a missing language as the fallback for one release cycle to avoid breaking
+  // in-progress pagination sessions on deploy. An explicitly empty string is still invalid.
+  if (typeof payload.language === "string" && payload.language.length === 0) {
+    throw new Error("Invalid search cursor: language must be a non-empty string");
+  }
+  const language = typeof payload.language === "string" ? payload.language : getFallbackLanguageConfig().dbKeyName;
+  return { offset: payload.offset, language };
+}
+
+/**
+ * Decodes an opaque base64 vector-search cursor back into its pagination offset, validating both structure and
+ * that the cursor language matches the currently requested language.
+ *
+ * @param {string} cursor - The base64 encoded cursor.
+ * @param {string} requestedLanguage - The language dbKeyName of the current request.
+ * @return {number} - The decoded, non-negative integer offset.
+ * @throws {SearchCursorLanguageMismatchError} - If the cursor was issued for a different language.
+ * @throws {Error} - If the cursor is malformed or does not hold a valid non-negative integer offset.
+ */
+export function decodeSearchCursor(cursor: string, requestedLanguage: string): number {
+  const { offset, language } = parseSearchCursor(cursor);
+  if (language !== requestedLanguage) {
+    throw new SearchCursorLanguageMismatchError(language, requestedLanguage);
+  }
+  return offset;
 }

backend/src/esco/occupationGroup/GET/index.ts

@@ -16,7 +16,7 @@ import { decodeCursor, encodeCursor, getOccupationGroupsPathParameters } from ".
 import { transformPaginated } from "./response";
 import { parseBooleanQueryParam } from "common/formatters/parseBooleanQueryParam";
 import { EmbeddableField } from "embeddings/service/types";
-import { decodeSearchCursor } from "esco/common/searchCursor";
+import { parseSearchCursor, SearchCursorLanguageMismatchError } from "esco/common/searchCursor";
 import { resolveLanguageFromModelResult } from "../_shared/resolveLanguageFromModelResult";
 
 /**
@@ -36,7 +36,7 @@ function isWellFormedSearchCursor(cursor: string): boolean {
   }
 
   try {
-    decodeSearchCursor(cursor);
+    parseSearchCursor(cursor);
     return true;
   } catch {
     return false;
@@ -303,6 +303,14 @@ export class OccupationGroupListController {
       );
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
     } catch (error: any) {
+      if (error instanceof SearchCursorLanguageMismatchError) {
+        return errorResponseGET(
+          StatusCodes.BAD_REQUEST,
+          OccupationGroupGETAPISpecs.Enums.Response.Status400.ErrorCodes.INVALID_NEXT_CURSOR_PARAMETER,
+          error.message,
+          ""
+        );
+      }
       console.error("Failed to retrieve occupation groups:", error);
       errorLoggerInstance.logError("Failed to retrieve the occupation groups from the DB", error.name);
       return errorResponseGET(