diff --git a/.gitignore b/.gitignore index a6de7ab12..525d84064 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,6 @@ integtest-results.xml test-results.xml .tox/ local_testing -.mcp.json \ No newline at end of file +.mcp.json +# Local feature-e2e planning scratch (never committed) +.feature-e2e/ diff --git a/TOOLS.md b/TOOLS.md index 1c4f87680..0053d1c9e 100644 --- a/TOOLS.md +++ b/TOOLS.md @@ -12,6 +12,7 @@ description, and a list of created table names. - [get_components](#get_components): Retrieves detailed information about one or more components by their IDs. - [get_config_examples](#get_config_examples): Retrieves sample configuration examples for a specific component. - [get_configs](#get_configs): Retrieves component configurations in the project with optional filtering. +- [get_shared_codes](#get_shared_codes): Discovers Keboola shared-code libraries and their reusable snippets in the current project. - [run_sync_action](#run_sync_action): Executes a synchronous action for a component configuration or a component row configuration. - [update_config](#update_config): Updates an existing root component configuration by modifying its parameters, storage mappings, name or description. - [update_config_row](#update_config_row): Updates an existing component configuration row by modifying its parameters, storage mappings, name, or description. @@ -101,6 +102,10 @@ Skipping these steps will cause a schema validation error. USAGE: - Use when you want to create a new row configuration for a specific component configuration. +SHARED CODE ROWS: +- For `keboola.shared-code` rows, set `row_id` to the Mustache placeholder key (e.g. `dumpfiles`) + and put the snippet body in `parameters` as `{"code_content": [""]}`. + WHEN NOT TO USE: - `keboola.orchestrator` / `keboola.flow` → use flows tools - `keboola.data-apps` → use data applications tools @@ -161,6 +166,11 @@ EXAMPLES: "type": "object" }, "type": "array" + }, + "row_id": { + "default": "", + "description": "Optional explicit row ID. When provided, becomes the row identifier in SAPI (forwarded as `rowId`). For `keboola.shared-code` rows this is the Mustache placeholder key used in transformation scripts (e.g. `dumpfiles` \u2192 referenced as `{{ dumpfiles }}`). Row IDs are case-sensitive. Leave empty to let SAPI auto-assign a numeric ID.", + "type": "string" } }, "required": [ @@ -195,6 +205,16 @@ Skipping these steps will cause a schema validation error. USAGE: - Use when you want to create a new root configuration for a specific component. +SHARED CODE: +- For `keboola.shared-code` parent libraries: pass `component_id="keboola.shared-code"`, + `parameters={"componentId": ""}`, AND `configuration_id="shared-codes."` + (e.g. `shared-codes.snowflake-transformation`). The platform stores `componentId` AT THE + CONFIGURATION ROOT for shared-code (not nested under `parameters`); this tool unwraps the + provided parameters dict accordingly. The conventional `configuration_id` is required — + auto-generated IDs are not recognised by the runtime expansion. +- For Python/R transformations that should reuse shared snippets, set `shared_code_id` and + `shared_code_row_ids` and embed `{{ rowId }}` Mustache placeholders in the component's script. + WHEN NOT TO USE: - `keboola.orchestrator` / `keboola.flow` → use flows tools - `keboola.data-apps` → use data applications tools @@ -288,6 +308,24 @@ EXAMPLES: }, "type": "array" }, + "shared_code_id": { + "default": "", + "description": "Optional. The configuration ID of the parent `keboola.shared-code` library this configuration references at the root level. Useful when creating Python (`keboola.python-transformation-v2`), R (`keboola.r-transformation-v2`) transformation configurations that need to reuse shared snippets. Must be paired with `shared_code_row_ids` and matching `{{ rowId }}` placeholders in the component's script. Leave empty when not using shared code.", + "type": "string" + }, + "shared_code_row_ids": { + "default": [], + "description": "Optional. The list of shared code row IDs (Mustache placeholder keys) referenced from the configuration. Each entry must exist as a row in the parent shared-code configuration and appear as `{{ rowId }}` in the component's script. Row IDs are case-sensitive.", + "items": { + "type": "string" + }, + "type": "array" + }, + "configuration_id": { + "default": "", + "description": "Optional explicit configuration ID. When non-empty, forwarded to SAPI as `configurationId`. REQUIRED for `keboola.shared-code` parent libraries \u2014 pass the conventional `shared-codes.` value (e.g. `shared-codes.snowflake-transformation`, `shared-codes.google-bigquery-transformation`, `shared-codes.python-transformation-v2`, `shared-codes.r-transformation-v2`). The UI and runtime expansion look up shared-code libraries by this exact ID. Leave empty to let SAPI auto-assign for any other component.", + "type": "string" + }, "variables": { "anyOf": [ { @@ -343,6 +381,11 @@ CONSIDERATIONS: - If there are 20 or more SQL transformations in the project, consider organizing them with a folder: existing folder names are surfaced in the response's change_summary — use one of them or create a new one. +SHARED CODE LINKAGE: +- To reuse snippets from the project's `keboola.shared-code` library, embed `{{ rowId }}` placeholders in the + script AND pass `shared_code_id` + `shared_code_row_ids`. Both must be set together; the placeholders alone + have no effect. Discover existing libraries with `get_shared_codes` before creating new ones. + USAGE: - Use when you want to create a new SQL transformation. @@ -442,6 +485,19 @@ EXAMPLES: "description": "Folder name to organize this transformation in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more transformations in the project. If there are 20 or more transformations, you should assign one of the existing folders or create a new one that clearly reflects the transformation purpose.", "type": "string" }, + "shared_code_id": { + "default": "", + "description": "Optional. The configuration ID of the parent `keboola.shared-code` library this transformation should reference (e.g. `shared-codes.snowflake-transformation`). When provided together with `shared_code_row_ids`, every `{{ rowId }}` Mustache placeholder used in the SQL script is expanded at runtime to the matching row's code. Discover available libraries via `get_shared_codes`. Leave empty when not using shared code.", + "type": "string" + }, + "shared_code_row_ids": { + "default": [], + "description": "Optional. The list of shared code row IDs (Mustache placeholder keys) referenced from the SQL script. Each entry must (a) exist as a row in the parent shared-code configuration and (b) appear as `{{ rowId }}` in at least one script block. Row IDs are case-sensitive.", + "items": { + "type": "string" + }, + "type": "array" + }, "variables": { "anyOf": [ { @@ -658,6 +714,52 @@ EXAMPLES: } ``` +--- + +## get_shared_codes +**Annotations**: `read-only` + +**Tags**: `components` + +**Description**: + +Discovers Keboola shared-code libraries and their reusable snippets in the current project. + +Shared code is a Keboola primitive for storing reusable SQL/Python/R snippets that transformations +reference via Mustache placeholders (`{{ rowId }}`). Each transformation backend can have a parent +`keboola.shared-code` configuration whose rows are the individual snippets. The `rowId` of each row +is the case-sensitive Mustache key used in transformation scripts; the `code_content` of the row is +the snippet body expanded at runtime. + +WHEN TO USE: +- Before writing or editing transformation code, check whether the project already maintains a + reusable snippet you can reference via `{{ rowId }}` instead of duplicating logic inline. +- When the user asks about existing shared code libraries or wants to inventory reusable snippets. + +RETURNS: +- A list of `SharedCodeConfig` entries, each with `config_id` (use as `shared_code_id` in + transformations), `transformation_component_id` (which backend the library belongs to), + and the `rows` available — each row's `row_id` is the Mustache placeholder key. + + +**Input JSON Schema**: +```json +{ + "additionalProperties": false, + "properties": { + "transformation_component_ids": { + "default": [], + "description": "Optional filter limiting results to shared-code libraries that belong to the given transformation components. Accepted values are `keboola.snowflake-transformation`, `keboola.google-bigquery-transformation`, `keboola.python-transformation-v2`, and `keboola.r-transformation-v2`. When empty, every shared-code library in the project is returned.", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" +} +``` + --- ## run_sync_action @@ -733,6 +835,7 @@ WHEN TO USE: - Modifying configuration parameters (credentials, settings, API keys, etc.) - Updating storage mappings (input/output tables or files) - Changing configuration name or description +- Adding/removing shared-code linkage on Python/R transformations (via `shared_code_id`) - Any combination of the above WHEN NOT TO USE: @@ -979,6 +1082,26 @@ WORKFLOW: "default": null, "description": "Folder name to organize this configuration in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more configurations in the project. If there are 20 or more configurations, you should assign one of the existing folders or create a new one that clearly reflects the configuration purpose." }, + "shared_code_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional. Updates the shared-code linkage on the configuration root. Non-empty string: sets `shared_code_id` (parent `keboola.shared-code` config ID) and replaces `shared_code_row_ids` with the value below. Empty string `\"\"`: clears the linkage (removes both root fields). `None` (default): leaves the existing linkage untouched. Use for Python/R transformations; SQL transformations use update_sql_transformation." + }, + "shared_code_row_ids": { + "default": [], + "description": "Optional. The list of shared code row IDs (Mustache placeholder keys). Only applied when `shared_code_id` is non-empty; ignored otherwise. Row IDs are case-sensitive.", + "items": { + "type": "string" + }, + "type": "array" + }, "variables": { "anyOf": [ { @@ -1629,6 +1752,19 @@ Example 4 - Update storage mappings: ], "type": "object" }, + "TfRemoveSharedCode": { + "description": "Remove the shared-code linkage from the transformation (clears both root fields).", + "properties": { + "op": { + "const": "remove_shared_code", + "type": "string" + } + }, + "required": [ + "op" + ], + "type": "object" + }, "TfRenameBlock": { "description": "Rename an existing block in the transformation.", "properties": { @@ -1708,6 +1844,32 @@ Example 4 - Update storage mappings: ], "type": "object" }, + "TfSetSharedCode": { + "description": "Link the transformation to shared code snippets at the configuration root.\n\nSets both `shared_code_id` (the parent `keboola.shared-code` configuration ID) and\n`shared_code_row_ids` (the list of Mustache placeholder keys referenced from the script).\nReplaces any existing shared-code linkage on the transformation.", + "properties": { + "op": { + "const": "set_shared_code", + "type": "string" + }, + "shared_code_id": { + "description": "The parent `keboola.shared-code` configuration ID", + "type": "string" + }, + "shared_code_row_ids": { + "description": "The list of shared code row IDs (Mustache keys) the script references", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "op", + "shared_code_id", + "shared_code_row_ids" + ], + "type": "object" + }, "TfStrReplace": { "description": "Replace a substring in SQL statements in the transformation.", "properties": { @@ -1821,9 +1983,11 @@ Example 4 - Update storage mappings: "add_script": "#/$defs/TfAddScript", "remove_block": "#/$defs/TfRemoveBlock", "remove_code": "#/$defs/TfRemoveCode", + "remove_shared_code": "#/$defs/TfRemoveSharedCode", "rename_block": "#/$defs/TfRenameBlock", "rename_code": "#/$defs/TfRenameCode", "set_code": "#/$defs/TfSetCode", + "set_shared_code": "#/$defs/TfSetSharedCode", "str_replace": "#/$defs/TfStrReplace" }, "propertyName": "op" @@ -1855,6 +2019,12 @@ Example 4 - Update storage mappings: }, { "$ref": "#/$defs/TfStrReplace" + }, + { + "$ref": "#/$defs/TfSetSharedCode" + }, + { + "$ref": "#/$defs/TfRemoveSharedCode" } ] }, diff --git a/feature_spec/shared_code_support/RFC.md b/feature_spec/shared_code_support/RFC.md new file mode 100644 index 000000000..1e1f632fe --- /dev/null +++ b/feature_spec/shared_code_support/RFC.md @@ -0,0 +1,518 @@ +# RFC: Shared Code Support + +Linear: [AI-1167](https://linear.app/keboola/issue/AI-1167/add-support-for-shared-codes-to-sql-transformation-tooling) + +> **Status:** shipped. Amended 2026-05-14 to reflect post-implementation deltas +> — the original spec captured the high-level shape correctly but missed several +> wire-format and runtime-substitution details that only surfaced during live +> testing on real Snowflake / BigQuery projects. Sections marked **[amended]** +> were rewritten after the corresponding commit; see *Implementation Deltas* at +> the bottom for the change log keyed to commit SHAs. + +## Problem + +The MCP server has no awareness of Keboola's shared code feature. As a result: + +- The LLM cannot discover what reusable snippets already exist in a project before writing + transformation code. +- The LLM cannot create shared code entries even when the user explicitly asks for reusable + code or when the same logic should span multiple transformations. +- Transformation tools (`create_sql_transformation`, `update_sql_transformation`, + `create_config`) have no parameters for `shared_code_id` / `shared_code_row_ids` — the + top-level config fields that wire a transformation to its shared code library — so there + is no path from "write shared code" to "reference it in a transformation". +- Mustache placeholders (`{{ rowId }}`) placed in transformation scripts have no effect + unless the linkage fields are also set on the configuration root. + +This gap is actively blocking customers (e.g. Apify, see SUPPORT-15438). + +## Background + +### Keboola Shared Code API [amended] + +Reference: https://developers.keboola.com/integrate/variables/#shared-code + +Shared code is stored under the `keboola.shared-code` component. There is one parent +**configuration** per transformation type. The platform expects a **flat configuration +body** for shared-code — `componentId` sits at the configuration root, *not* under a +`"parameters"` key: + +```json +{ "componentId": "keboola.snowflake-transformation" } +``` + +The conventional config ID is **required, not optional** — the UI and the runtime resolver +look up shared-code libraries by exact ID: + +| Transformation type | Required shared-code config ID | +|---|---| +| `keboola.snowflake-transformation` | `shared-codes.snowflake-transformation` | +| `keboola.google-bigquery-transformation` | `shared-codes.google-bigquery-transformation` | +| `keboola.python-transformation-v2` | `shared-codes.python-transformation-v2` | +| `keboola.r-transformation-v2` | `shared-codes.r-transformation-v2` | + +SAPI-auto-assigned UUIDs work as IDs but are invisible to the UI / runtime, so the tool +forwards `configurationId` explicitly and the system prompt instructs the LLM to pass the +conventional value. + +Each **row** of the config is one reusable snippet (also flat-bodied): + +- `rowId` — set explicitly at row-creation time; becomes the Mustache key (e.g. `dumpfiles`) +- `configuration.code_content` — array of code strings at the row configuration root, + e.g. `{"code_content": ["SELECT 1"]}` (not wrapped under `"parameters"`) + +Fetching a parent library's rows requires `?include=rows` on the Storage API detail call; +without it the rows array is omitted. + +### Using Shared Code in a Transformation [amended] + +A transformation references shared code via three coupled pieces: + +1. **Configuration root** — two top-level fields alongside `parameters` and `storage`: + `shared_code_id` (parent library ID) and `shared_code_row_ids` (list of row IDs). +2. **One marker code block per referenced row** — a dedicated code with name + `Shared Code (-)` and script `["{{ rowId }}"]`. The runtime + substitutes the placeholder *only* when it is the sole array element of a script; + inline `{{ rowId }}` inside a longer SQL string is **not** substituted. +3. **The user-authored code blocks** — these read the side-effects of the expanded + snippet (e.g. a session variable set, a temp table created, a UDF defined). + +```json +{ + "parameters": { + "blocks": [ + { + "name": "Blocks", + "codes": [ + { "name": "Shared Code (shared-codes.snowflake-transformation-dumpfiles)", + "script": ["{{ dumpfiles }}"] }, + { "name": "User code", "script": ["SELECT * FROM dumped_table"] } + ] + } + ] + }, + "storage": {}, + "shared_code_id": "shared-codes.snowflake-transformation", + "shared_code_row_ids": ["dumpfiles"] +} +``` + +The LLM/agent never authors the marker blocks by hand — `create_sql_transformation`, +`update_sql_transformation`, `create_config`, and `update_config` emit them automatically +whenever the target component is a transformation backend AND linkage fields are set. +See *Marker Handling* in §6. + +### Current Codebase State + +- Component constants in `src/keboola_mcp_server/tools/components/utils.py:64-67`: + `SNOWFLAKE_TRANSFORMATION_ID`, `BIGQUERY_TRANSFORMATION_ID`, + `PYTHON_TRANSFORMATION_ID`, `R_TRANSFORMATION_ID` +- `TransformationConfiguration` model (`model.py`) has no `shared_code_id` or + `shared_code_row_ids` fields — these are silently dropped during + deserialize/re-serialize cycles in `update_sql_transformation` +- `create_sql_transformation` (`tools.py:382`) and `update_sql_transformation` + (`tools.py:537`) have no parameters for shared code linkage +- `create_config` (`tools.py:1005`) builds `configuration_payload` with only `storage`, + `parameters`, and `processors` — no mechanism to inject root-level fields +- `update_config.parameter_updates` paths are relative to `parameters`, making it + impossible to reach `shared_code_id` at the config root via the existing tool +- `storage.py` already exposes `configuration_create` (`line 487`) and + `configuration_row_create` (`line 664`) — sufficient for all CRUD calls, but the + initial implementations were missing form-field forwarding **[amended]**: + - `configuration_create` did not forward `configurationId` — the SAPI auto-assigned + UUIDs are unusable as shared-code library IDs (added in 67b893c7) + - `configuration_row_create` did not forward `rowId` — the row ID is the Mustache + key, must be settable explicitly (added in 9af709fc) + - `configuration_detail` did not accept `include` — without `?include=rows` the + Storage API omits row data and `get_shared_codes` returns empty rows (added in + 30faec45) + +## Proposed Changes + +### 1. System Prompt (`project_system_prompt.md`) + +Add a **"Shared Code"** section immediately after the existing "Transformations" section. + +#### Discovery + +Before writing or editing transformation code, call `get_shared_codes` filtered to the +relevant transformation component. This surfaces snippets the project already maintains. +Use existing shared code via Mustache references rather than duplicating logic inline. + +#### When to Create Shared Code + +Create a new shared code entry when: +- The user explicitly asks for reusable / shared code. +- The same logic needs to appear in multiple transformations. + +Do **not** proactively convert every snippet to shared code — only do so when reuse intent +is clear. + +#### How to Reference Shared Code + +1. Write `{{ rowId }}` in the transformation script at the point where the snippet should + be substituted. +2. When creating or updating the transformation, pass `shared_code_id` (the parent config + ID) and `shared_code_row_ids` (list of row IDs referenced in the scripts). +3. `rowId` is case-sensitive and must match the row ID used when the snippet was created. +4. All row IDs referenced in scripts must appear in `shared_code_row_ids`; unused entries + in the list cause a validation error. + +#### Naming Convention + +The parent config ID follows `shared-codes.` by convention, +but always use the actual ID returned by `get_shared_codes` for an existing library rather +than assuming the conventional name. + +### 2. New Tool: `get_shared_codes` + +A dedicated read-only tool for discovering the project's shared code libraries. + +**Signature** + +```python +async def get_shared_codes( + ctx: Context, + transformation_component_ids: Sequence[str] = tuple(), +) -> GetSharedCodesOutput +``` + +**Parameters** + +- `transformation_component_ids` — optional filter. When empty, returns all shared code + configs. Accepted values are the transformation component IDs known to the server + (`keboola.snowflake-transformation`, `keboola.google-bigquery-transformation`, + `keboola.python-transformation-v2`, `keboola.r-transformation-v2`). + +**Behavior** + +1. Call `storage_client.configuration_list("keboola.shared-code")` to list all parent + configs. +2. For each config, read `configuration.componentId` to know the transformation type. +3. Apply the `transformation_component_ids` filter if provided. +4. For each matching config call `storage_client.configuration_detail(...)` to get rows + and their `code_content`. +5. Return a structured list of configs with their rows. + +**New output models** (add to `model.py`): + +```python +class SharedCodeRow(BaseModel): + row_id: str # the Mustache key (e.g. "dumpfiles") + name: str + code: str # code_content array joined with "\n" + +class SharedCodeConfig(BaseModel): + config_id: str # e.g. "shared-codes.snowflake-transformation" + transformation_component_id: str # e.g. "keboola.snowflake-transformation" + rows: list[SharedCodeRow] + +class GetSharedCodesOutput(BaseModel): + shared_codes: list[SharedCodeConfig] +``` + +**Annotations**: `readOnlyHint: true` + +### 3. Shared Code CRUD — Reuse Existing Generic Tools (with extensions) [amended] + +The existing `create_config`, `add_config_row`, and `update_config_row` tools cover +shared-code CRUD without introducing new write tools, but they needed targeted +extensions to handle the platform's flat-body wire format and explicit IDs: + +| Operation | Tool | Key parameters | +|---|---|---| +| Create parent library | `create_config` | `component_id="keboola.shared-code"`, **`configuration_id="shared-codes."`** (required for shared-code), `parameters={"componentId":""}` | +| Add snippet row | `add_config_row` | `component_id="keboola.shared-code"`, `row_id=""`, `parameters={"code_content":[""]}` | +| Update snippet | `update_config_row` | `parameter_updates=[{"op":"set","path":"code_content","value":[""]}]` | +| Disable snippet | `update_config_row` | `is_disabled=True` (no `delete_config_row` tool exists) | + +**Required extensions:** + +1. **`add_config_row.row_id`** (9af709fc) — optional string parameter. When non-empty, + forwarded to SAPI as `rowId`. Required so the LLM can set Mustache keys at row creation. + `storage.py:configuration_row_create` extended to forward the field. + +2. **`create_config.configuration_id`** (67b893c7) — optional string parameter, REQUIRED + for `keboola.shared-code` parent libraries (caller passes the conventional + `shared-codes.` value). Forwarded to SAPI as `configurationId`. + `storage.py:configuration_create` extended to forward the field. + +3. **Flat-body special case** (67b893c7) — `create_config` and `add_config_row` + normally wrap the caller's `parameters` dict under a `"parameters"` key in the + configuration body. For `component_id="keboola.shared-code"` they now write the + caller's dict at the **configuration root** instead, so `componentId` / + `code_content` land where the platform's resolver looks for them. + +4. **`get_shared_codes` reads from root with parameters fallback** (67b893c7) — to + keep legacy configs (created by the wrapped wire format) discoverable, the tool + reads `componentId` / `code_content` from the configuration root first and falls + back to `parameters.` if absent. + +5. **`configuration_detail.include`** (30faec45) — optional comma-separated CSV + forwarded as `?include=...`. `get_shared_codes` and the bulk `get_configs` path + pass `include=['rows']` so rows are populated. + +6. **`ConfigToolOutput.configuration_row_id`** (91c112d9) — `add_config_row` surfaces + the row ID assigned by SAPI and logs a warning when the assigned ID does not match + the requested `row_id` (would indicate SAPI rejected or transformed the value). + +### 4. Linking Shared Code to Transformation Configurations [amended] + +`shared_code_id` and `shared_code_row_ids` live at the **configuration root**, not under +`parameters`. None of the existing tools could write these fields. In addition to writing +the linkage, every create/update path now also **auto-emits marker code blocks** (see §6). + +#### 4a. `create_sql_transformation` (`tools.py:382`) + +Add two optional parameters: + +```python +shared_code_id: str = "" +shared_code_row_ids: Sequence[str] = tuple() +``` + +When non-empty: (1) set them directly on `configuration_payload` alongside `parameters` +and `storage`; (2) auto-emit a `Shared Code (-)` marker code block per row in +the first parameters block via `apply_shared_code_markers` (91c112d9). + +#### 4b. `update_sql_transformation` (`tools.py:537`) + +Add two new operation types to the `TfParamUpdate` discriminated union: + +- **`set_shared_code`** — sets `shared_code_id` and `shared_code_row_ids` on the + configuration root; replaces any existing values; **re-syncs marker blocks** to match + the new linkage (adds missing markers, removes orphaned ones) +- **`remove_shared_code`** — removes both fields from the configuration root AND deletes + any `Shared Code (...)` marker blocks from `parameters.blocks` + +Marker synchronization is also performed after every other `parameter_updates` +application via `sync_shared_code_markers_in_dict`, so even unrelated block edits +maintain the marker invariant. + +#### 4c. `create_config` (`tools.py:1005`) + +For Python and R transformations (and any other component) created via the +generic tool, add: + +```python +shared_code_id: str = "" +shared_code_row_ids: Sequence[str] = tuple() +``` + +Include them in `configuration_payload` when non-empty. When the target `component_id` +is a transformation backend that supports shared code, also auto-emit marker code blocks +via `sync_shared_code_markers_in_dict` (aa8fc636) — same UI-canonical behavior as the +SQL path, so Python/R don't need a separate authoring pattern. + +#### 4d. `update_config` (`tools.py:1291`) + +Add the same two optional parameters. Write them to the configuration root (not to +`parameters`) during the update; also auto-emit marker blocks for transformation +components (aa8fc636). This handles adding, changing, or clearing shared code linkage on +existing Python/R transformation configurations. + +### 5. `TransformationConfiguration` Model (`model.py`) + +Add optional fields so existing configurations with shared code linkage round-trip +correctly through `update_sql_transformation`'s deserialize/re-serialize cycle: + +```python +shared_code_id: Optional[str] = None +shared_code_row_ids: list[str] = Field(default_factory=list) +``` + +Without this change, `update_sql_transformation` would silently drop `shared_code_id` and +`shared_code_row_ids` from any transformation that already has them set, breaking the +linkage on every update. + +### 6. Marker Handling and Linkage Validation [added post-implementation] + +The platform's runtime expands `{{ rowId }}` **only** when it is the entire content of a +script array element. Three behaviors implement this contract: + +#### 6a. Auto-emit canonical marker blocks (91c112d9, aa8fc636) + +Whenever a create/update tool sees `shared_code_id` + non-empty `shared_code_row_ids` for +a transformation component, it appends one code block per row: + +```json +{ "name": "Shared Code (-)", + "script": ["{{ row_id }}"] } +``` + +This is the same shape the Keboola UI produces, so configurations created via the MCP are +indistinguishable from UI-authored ones. The LLM never has to author these markers — it +just supplies the linkage fields. + +Implementations: +- `apply_shared_code_markers(tf_cfg, sid, row_ids)` — Pydantic-model path (create) +- `sync_shared_code_markers_in_dict(updated_configuration)` — raw-dict path (update, + also used by `create_config` / `update_config` for Python/R) +- `build_shared_code_marker_codes(sid, row_ids)`, `shared_code_marker_code_name(sid, rid)`, + `is_shared_code_marker(name, script)` — shared helpers + +#### 6b. Skip auto-emit when the user already references the row (31cee8b5) + +If the user already authored a NON-marker code block whose script is exactly +`["{{ rowId }}"]` (a pure-placeholder script element), the tool **skips** emitting an +extra `Shared Code (...)` marker for that row — emitting one would execute the snippet +twice at run time. + +Inline placeholders (e.g. `["SELECT 1; {{ rowId }}"]`) are deliberately ignored by this +skip rule because the platform does not substitute those — the marker is still required +for the placeholder to resolve at all. + +`user_substitution_eligible_row_ids(blocks)` returns the set of row IDs that should be +skipped, and is consulted from both `apply_shared_code_markers` and +`sync_shared_code_markers_in_dict`. + +#### 6c. Hard-reject placeholder without linkage (aa8fc636) + +`validate_shared_code_linkage(parameters, shared_code_id, shared_code_row_ids)` scans +`parameters.blocks[*].codes[*].script` for `{{ rowId }}` placeholders and raises +`ValueError` when: +- a placeholder is referenced but `shared_code_id` is empty, or +- a referenced row is not in `shared_code_row_ids`. + +Wired into `create_sql_transformation`, `update_sql_transformation_internal`, +`create_config`, and `update_config` for transformation components. This prevents the +silent-failure mode where a placeholder is left dangling and the snippet is never +substituted at run time. + +#### 6d. Known caveat — marker ordering for dependent code + +The auto-emit appends markers at the **end** of the first parameters block. If the +user's code DEPENDS on the side-effect of the shared snippet (e.g. shared code does +`SET (region) = ('EU')` and user code reads `$region`, or shared code creates a temp +table that user code selects from), the marker must run **before** the dependent code. + +The tool does not reorder automatically — the caller must move the marker via +`update_sql_transformation` with `remove_code` + `add_code(position="start")`, or author +a pure-placeholder code block at the start themselves (which §6b then de-dups). + +Tracked as a follow-up: either default the auto-emit position to `start`, or accept an +`auto_emit_position` parameter, or document the workaround prominently in the system +prompt. + +## Workflows + +### Discovery (before writing transformation code) + +``` +get_shared_codes(transformation_component_ids=["keboola.snowflake-transformation"]) +→ returns existing snippets with their row_ids and code +→ use {{ rowId }} in transformation scripts where appropriate +``` + +### Creating a New Shared Code Snippet [amended] + +``` +1. Check if a parent config already exists via get_shared_codes +2. If not: + create_config( + component_id="keboola.shared-code", + configuration_id="shared-codes.snowflake-transformation", # REQUIRED — conventional ID + name="Shared Codes for Snowflake Transformations", + parameters={"componentId": "keboola.snowflake-transformation"} + # platform stores `componentId` at the config root; the tool unwraps for you + ) + → captures returned config_id (matches the conventional ID you passed in) + +3. add_config_row( + component_id="keboola.shared-code", + configuration_id="shared-codes.snowflake-transformation", + name="Dump files helper", + row_id="dumpfiles", + parameters={"code_content": ["SELECT ..."]} + # the row body is also flat — `code_content` lands at the row config root + ) + → snippet is now available as {{ dumpfiles }} +``` + +### Referencing Shared Code in a New Transformation [amended] + +``` +create_sql_transformation( + name="My Transformation", + sql_code_blocks=[ + Code(name="User code", script="SELECT * FROM dumped_table") + ], + shared_code_id="shared-codes.snowflake-transformation", + shared_code_row_ids=["dumpfiles"] +) +# The tool auto-emits the marker block: +# { name: "Shared Code (shared-codes.snowflake-transformation-dumpfiles)", +# script: ["{{ dumpfiles }}"] } +# appended after the user-authored code. Reorder if the user code depends on +# side-effects of the shared snippet (see §6d). +``` + +### Adding Shared Code Reference to an Existing Transformation + +``` +update_sql_transformation( + component_id="keboola.snowflake-transformation", + configuration_id="", + parameter_updates=[ + SetSharedCode( + shared_code_id="shared-codes.snowflake-transformation", + shared_code_row_ids=["dumpfiles"] + ) + ] +) +``` + +## Critical Files + +| File | Change | +|---|---| +| `feature_spec/shared_code_support/RFC.md` | This document | +| `src/keboola_mcp_server/resources/prompts/project_system_prompt.md` | Add "Shared Code" section after "Transformations" | +| `src/keboola_mcp_server/tools/components/model.py` | Add `shared_code_id`/`shared_code_row_ids` to `TransformationConfiguration`; add `SharedCodeRow`, `SharedCodeConfig`, `GetSharedCodesOutput` models | +| `src/keboola_mcp_server/tools/components/tools.py` | New `get_shared_codes`; extend `create_sql_transformation`, `update_sql_transformation` (new `TfParamUpdate` variants), `create_config`, `update_config`, `add_config_row` | +| `src/keboola_mcp_server/clients/storage.py` | Add `rowId` parameter to `configuration_row_create` if absent | +| `TOOLS.md` | Auto-regenerated via `tox -e check-tools-docs` | + +## Testing [amended] + +Original RFC coverage retained; added during implementation: + +- **Wire format**: `test_create_config_flat_body_for_shared_code`, + `test_add_config_row_flat_body_for_shared_code`, + `test_get_shared_codes_reads_root_with_parameters_fallback` (67b893c7). +- **`include=rows`**: mock-signature updates pinning that `configuration_detail` is + called with `include=['rows']` from `get_shared_codes` and the bulk `get_configs` + detail path (30faec45). +- **Marker emission**: `test_create_sql_transformation_emits_shared_code_marker_blocks` + (parametrized for snowflake / bigquery), + `test_update_sql_transformation_set_then_remove_shared_code_syncs_markers`, + `test_create_config_emits_markers_for_python_transformation` (91c112d9, aa8fc636). +- **Linkage validation**: + `test_create_sql_transformation_rejects_placeholder_without_linkage`, + `test_create_sql_transformation_rejects_placeholder_missing_from_row_ids` (aa8fc636). +- **Marker de-dup**: 151 lines covering pure-placeholder detection, marker + de-duplication on create/update, inline-placeholder ignore (31cee8b5). +- **Row ID surfacing**: `test_add_config_row_surfaces_assigned_row_id` (91c112d9). +- **Integration tests**: 7 live-stack tests with a `shared_code_parent_factory` + fixture for cleanup — covered in PR #499. +- Run `tox` (pytest + black + flake8 + check-tools-docs) before pushing. + +## Implementation Deltas (post-RFC) + +| Commit | Delta | +|---|---| +| 9af709fc | Initial implementation matching the original RFC. | +| 67b893c7 | Wire format: shared-code body is FLAT (`componentId` / `code_content` at config root, not under `parameters`). `create_config` accepts `configuration_id` (REQUIRED for shared-code parent libraries — auto-UUIDs not recognised by UI/runtime). `get_shared_codes` reads from root with parameters fallback. | +| 30faec45 | `configuration_detail.include` parameter added; `get_shared_codes` requests `include=['rows']` (Storage API omits rows by default). | +| 91c112d9 | Runtime substitution operates on a script ARRAY ELEMENT, not text inside a string. Tools auto-emit UI-canonical `Shared Code (-)` marker code blocks per linked row. `get_configs` bulk path passes `include=['rows']`. `add_config_row` surfaces the SAPI-assigned `configuration_row_id`. | +| aa8fc636 | Hard-reject `{{ rowId }}` placeholders without matching root linkage (`validate_shared_code_linkage`). Marker auto-emission extended to `create_config` / `update_config` for Python/R/DuckDB transformation components, mirroring the SQL path. System prompt rewritten and shortened from 103 → 32 lines. | +| 31cee8b5 | Skip auto-emit when the user already has a NON-marker code block with `["{{ rowId }}"]` as its pure script element — emitting another would execute the snippet twice. Inline placeholders inside other SQL strings are ignored. | + +## Out of Scope + +- Automatic detection of duplicated code across transformations — the LLM decides based on + user intent, no heuristics +- Variable support (`variables_id`, `variables_values_id`) — separate feature +- Shared code for non-transformation components +- Auto-emit marker position (currently appended after user code; caller must reorder if + user code depends on side-effects of the snippet) — see §6d diff --git a/integtests/test_mcp_server.py b/integtests/test_mcp_server.py index 9ea7ff3c8..1aabcc702 100644 --- a/integtests/test_mcp_server.py +++ b/integtests/test_mcp_server.py @@ -185,6 +185,7 @@ async def _assert_basic_setup(client: Client): 'get_flows', 'get_jobs', 'get_project_info', + 'get_shared_codes', 'get_tables', 'modify_flow', 'modify_python_js_data_app', diff --git a/integtests/tools/components/test_tools.py b/integtests/tools/components/test_tools.py index d73d4cf69..fae3303c0 100644 --- a/integtests/tools/components/test_tools.py +++ b/integtests/tools/components/test_tools.py @@ -29,17 +29,27 @@ GetComponentsOutput, GetConfigsDetailOutput, GetConfigsListOutput, + GetSharedCodesOutput, SimplifiedTfBlocks, TfAddScript, TfParamUpdate, + TfRemoveSharedCode, TfRenameBlock, TfRenameCode, TfSetCode, + TfSetSharedCode, TfStrReplace, TransformationConfiguration, ) from keboola_mcp_server.tools.components.sql_utils import split_sql_statements +from keboola_mcp_server.tools.components.tools import ( + get_shared_codes, + update_config, + update_sql_transformation, +) from keboola_mcp_server.tools.components.utils import ( + PYTHON_TRANSFORMATION_ID, + SHARED_CODE_COMPONENT_ID, clean_bucket_name, expand_component_types, get_sql_transformation_id_from_sql_dialect, @@ -1050,3 +1060,389 @@ async def test_get_config_examples_with_invalid_component(mcp_context: Context): result = await get_config_examples(ctx=mcp_context, component_id='completely-non-existent-component-12345') assert result == '' + + +# ============================================================================ +# SHARED CODE TESTS +# ============================================================================ + + +@pytest_asyncio.fixture +async def shared_code_parent_factory(mcp_context: Context) -> AsyncGenerator[Any, None]: + """ + Factory fixture: creates `keboola.shared-code` parent configs on demand and cleans them + up at teardown. Yields a callable `make(target_component_id, name=...) -> ConfigToolOutput`. + """ + client = KeboolaClient.from_state(mcp_context.session.state) + created: list[tuple[str, str]] = [] # (component_id, configuration_id) + + async def make(target_component_id: str, name: str = '') -> ConfigToolOutput: + # Shared-code parents must be created with the conventional + # `shared-codes.` ID — create_config now rejects an + # empty configuration_id since an auto-assigned UUID is invisible to the UI/runtime. + conventional_id = f'shared-codes.{target_component_id.split(".", 1)[-1]}' + config = await create_config( + ctx=mcp_context, + name=name or f'Shared Codes for {target_component_id}', + description='Created by integtest — safe to delete', + component_id=SHARED_CODE_COMPONENT_ID, + parameters={'componentId': target_component_id}, + configuration_id=conventional_id, + ) + created.append((SHARED_CODE_COMPONENT_ID, config.configuration_id)) + return config + + try: + yield make + finally: + for component_id, configuration_id in created: + try: + await client.storage_client.configuration_delete( + component_id=component_id, + configuration_id=configuration_id, + skip_trash=True, + ) + except Exception: # noqa: BLE001 + LOG.exception('Failed to clean up shared-code config %s/%s', component_id, configuration_id) + + +@pytest.mark.asyncio +async def test_get_shared_codes_empty_or_filtered(mcp_context: Context): + """`get_shared_codes` on a project with no `keboola.shared-code` configs returns an empty list.""" + # No fixtures created — project may already have shared codes from other tests/seed data, so + # we just assert the output shape rather than emptiness. Filtering to an invalid combo gives []. + result = await get_shared_codes(ctx=mcp_context) + assert isinstance(result, GetSharedCodesOutput) + + # Filtering by a known-empty component (no shared code library created for it in this project) + # must not error and must return shared_codes consistent with the requested filter. + filtered = await get_shared_codes(ctx=mcp_context, transformation_component_ids=[PYTHON_TRANSFORMATION_ID]) + for cfg in filtered.shared_codes: + assert cfg.transformation_component_id == PYTHON_TRANSFORMATION_ID + + +@pytest.mark.asyncio +async def test_get_shared_codes_rejects_unknown_filter(mcp_context: Context): + """An unknown component ID in the filter must raise without hitting the API.""" + with pytest.raises(ValueError, match='Unknown transformation component IDs'): + await get_shared_codes(ctx=mcp_context, transformation_component_ids=['nonsense.fake']) + + +@pytest.mark.asyncio +async def test_add_config_row_forwards_row_id_to_shared_code_parent( + mcp_context: Context, + shared_code_parent_factory: Any, +): + """ + Verifies the storage-layer rowId forwarding fix: when `add_config_row` is called with a + `row_id`, the created row's identifier in SAPI is exactly that value (the Mustache key). + """ + sql_dialect = await WorkspaceManager.from_state(mcp_context.session.state).get_sql_dialect() + target_component_id = get_sql_transformation_id_from_sql_dialect(sql_dialect) + + parent = await shared_code_parent_factory(target_component_id) + client = KeboolaClient.from_state(mcp_context.session.state) + + row_id = 'integ_dumpfiles' + await add_config_row( + ctx=mcp_context, + name='Dump files helper', + description='Reusable snippet for integration tests', + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id=parent.configuration_id, + parameters={'code_content': ['SELECT 1 AS integ_marker']}, + row_id=row_id, + ) + + detail = await client.storage_client.configuration_detail( + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id=parent.configuration_id, + include=['rows'], + ) + rows = cast(list, detail.get('rows') or []) + assert any( + row.get('id') == row_id for row in rows + ), f'Expected row with id={row_id!r} in shared-code config; got rows: {[r.get("id") for r in rows]}' + + +@pytest.mark.asyncio +async def test_get_shared_codes_returns_created_library( + mcp_context: Context, + shared_code_parent_factory: Any, +): + """End-to-end: create a shared-code library with one row and confirm `get_shared_codes` lists it.""" + sql_dialect = await WorkspaceManager.from_state(mcp_context.session.state).get_sql_dialect() + target_component_id = get_sql_transformation_id_from_sql_dialect(sql_dialect) + + parent = await shared_code_parent_factory(target_component_id, name='Integ shared library') + row_id = 'integ_now' + await add_config_row( + ctx=mcp_context, + name='now()', + description='returns current timestamp', + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id=parent.configuration_id, + parameters={'code_content': ['SELECT current_timestamp']}, + row_id=row_id, + ) + + result = await get_shared_codes(ctx=mcp_context, transformation_component_ids=[target_component_id]) + libraries = [cfg for cfg in result.shared_codes if cfg.config_id == parent.configuration_id] + assert len(libraries) == 1, 'Created library not surfaced by get_shared_codes' + library = libraries[0] + assert library.transformation_component_id == target_component_id + assert any(row.row_id == row_id for row in library.rows) + + +@pytest.mark.asyncio +async def test_create_sql_transformation_with_shared_code_linkage( + mcp_context: Context, + shared_code_parent_factory: Any, +): + """`create_sql_transformation` writes shared_code_id + shared_code_row_ids at the config root.""" + client = KeboolaClient.from_state(mcp_context.session.state) + sql_dialect = await WorkspaceManager.from_state(mcp_context.session.state).get_sql_dialect() + expected_component_id = get_sql_transformation_id_from_sql_dialect(sql_dialect) + + parent = await shared_code_parent_factory(expected_component_id) + row_id = 'integ_marker' + await add_config_row( + ctx=mcp_context, + name='Marker snippet', + description='reusable SELECT', + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id=parent.configuration_id, + parameters={'code_content': ['SELECT 1 AS integ_marker']}, + row_id=row_id, + ) + + tf = await create_sql_transformation( + ctx=mcp_context, + name='Integ TF with shared code', + description='references {{ integ_marker }}', + sql_code_blocks=[ + SimplifiedTfBlocks.Block.Code(name='Reused', script='SELECT * FROM ({{ integ_marker }}) AS x'), + ], + created_table_names=[], + shared_code_id=parent.configuration_id, + shared_code_row_ids=[row_id], + ) + + try: + detail = await client.storage_client.configuration_detail( + component_id=tf.component_id, configuration_id=tf.configuration_id + ) + config_root = cast(dict, detail.get('configuration') or {}) + assert config_root.get('shared_code_id') == parent.configuration_id + assert config_root.get('shared_code_row_ids') == [row_id] + finally: + await client.storage_client.configuration_delete( + component_id=tf.component_id, + configuration_id=tf.configuration_id, + skip_trash=True, + ) + + +@pytest.mark.asyncio +async def test_update_sql_transformation_set_and_remove_shared_code( + mcp_context: Context, + shared_code_parent_factory: Any, +): + """TfSetSharedCode then TfRemoveSharedCode round-trip the linkage at the config root.""" + client = KeboolaClient.from_state(mcp_context.session.state) + sql_dialect = await WorkspaceManager.from_state(mcp_context.session.state).get_sql_dialect() + expected_component_id = get_sql_transformation_id_from_sql_dialect(sql_dialect) + + parent = await shared_code_parent_factory(expected_component_id) + for row_id in ('integ_a', 'integ_b'): + await add_config_row( + ctx=mcp_context, + name=row_id, + description='int', + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id=parent.configuration_id, + parameters={'code_content': ['SELECT 1']}, + row_id=row_id, + ) + + tf = await create_sql_transformation( + ctx=mcp_context, + name='Integ TF for set/remove', + description='will receive a shared code link via update', + sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='Body', script='SELECT 1')], + created_table_names=[], + ) + + try: + # SET + await update_sql_transformation( + mcp_context, + change_description='link shared code', + configuration_id=tf.configuration_id, + parameter_updates=[ + TfSetSharedCode( + op='set_shared_code', + shared_code_id=parent.configuration_id, + shared_code_row_ids=['integ_a', 'integ_b'], + ), + ], + ) + detail = await client.storage_client.configuration_detail( + component_id=tf.component_id, configuration_id=tf.configuration_id + ) + root = cast(dict, detail.get('configuration') or {}) + assert root.get('shared_code_id') == parent.configuration_id + assert root.get('shared_code_row_ids') == ['integ_a', 'integ_b'] + + # REMOVE + await update_sql_transformation( + mcp_context, + change_description='unlink shared code', + configuration_id=tf.configuration_id, + parameter_updates=[TfRemoveSharedCode(op='remove_shared_code')], + ) + detail = await client.storage_client.configuration_detail( + component_id=tf.component_id, configuration_id=tf.configuration_id + ) + root = cast(dict, detail.get('configuration') or {}) + assert 'shared_code_id' not in root + assert 'shared_code_row_ids' not in root + finally: + await client.storage_client.configuration_delete( + component_id=tf.component_id, + configuration_id=tf.configuration_id, + skip_trash=True, + ) + + +@pytest.mark.asyncio +async def test_update_sql_transformation_preserves_shared_code_on_parameter_only_update( + mcp_context: Context, + shared_code_parent_factory: Any, +): + """Regression guard: a parameter-only update must not silently drop existing shared-code fields.""" + client = KeboolaClient.from_state(mcp_context.session.state) + sql_dialect = await WorkspaceManager.from_state(mcp_context.session.state).get_sql_dialect() + expected_component_id = get_sql_transformation_id_from_sql_dialect(sql_dialect) + + parent = await shared_code_parent_factory(expected_component_id) + row_id = 'integ_keep' + await add_config_row( + ctx=mcp_context, + name=row_id, + description='keep me', + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id=parent.configuration_id, + parameters={'code_content': ['SELECT 1']}, + row_id=row_id, + ) + + tf = await create_sql_transformation( + ctx=mcp_context, + name='Integ TF preserve test', + description='linkage must survive parameter-only update', + sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='B', script='SELECT 1')], + created_table_names=[], + shared_code_id=parent.configuration_id, + shared_code_row_ids=[row_id], + ) + + try: + await update_sql_transformation( + mcp_context, + change_description='rename block only', + configuration_id=tf.configuration_id, + parameter_updates=[TfRenameBlock(op='rename_block', block_id='b0', block_name='Renamed')], + ) + + detail = await client.storage_client.configuration_detail( + component_id=tf.component_id, configuration_id=tf.configuration_id + ) + root = cast(dict, detail.get('configuration') or {}) + assert ( + root.get('shared_code_id') == parent.configuration_id + ), 'shared_code_id was silently dropped by parameter-only update' + assert root.get('shared_code_row_ids') == [row_id] + finally: + await client.storage_client.configuration_delete( + component_id=tf.component_id, + configuration_id=tf.configuration_id, + skip_trash=True, + ) + + +@pytest.mark.asyncio +async def test_update_config_set_clear_preserve_shared_code( + mcp_context: Context, + shared_code_parent_factory: Any, + configs: list[ConfigDef], +): + """`update_config`: non-empty sets linkage, empty clears it, None preserves existing linkage.""" + client = KeboolaClient.from_state(mcp_context.session.state) + # Use any config from the project (component-agnostic — shared-code fields are meta-fields). + target = configs[0] + component_id = target.component_id + configuration_id = target.configuration_id + + sql_dialect = await WorkspaceManager.from_state(mcp_context.session.state).get_sql_dialect() + parent = await shared_code_parent_factory(get_sql_transformation_id_from_sql_dialect(sql_dialect)) + + # Snapshot current root keys so we can restore at teardown. + snapshot = await client.storage_client.configuration_detail( + component_id=component_id, configuration_id=configuration_id + ) + original_root = cast(dict, snapshot.get('configuration') or {}) + + try: + # SET + await update_config( + ctx=mcp_context, + change_description='set shared code linkage', + component_id=component_id, + configuration_id=configuration_id, + shared_code_id=parent.configuration_id, + shared_code_row_ids=['integ_x'], + ) + detail = await client.storage_client.configuration_detail( + component_id=component_id, configuration_id=configuration_id + ) + root = cast(dict, detail.get('configuration') or {}) + assert root.get('shared_code_id') == parent.configuration_id + assert root.get('shared_code_row_ids') == ['integ_x'] + + # PRESERVE (None means don't touch) + await update_config( + ctx=mcp_context, + change_description='unrelated update', + component_id=component_id, + configuration_id=configuration_id, + description='still has shared code', + ) + detail = await client.storage_client.configuration_detail( + component_id=component_id, configuration_id=configuration_id + ) + root = cast(dict, detail.get('configuration') or {}) + assert root.get('shared_code_id') == parent.configuration_id, 'linkage was dropped when shared_code_id=None' + + # CLEAR (empty string) + await update_config( + ctx=mcp_context, + change_description='clear shared code linkage', + component_id=component_id, + configuration_id=configuration_id, + shared_code_id='', + ) + detail = await client.storage_client.configuration_detail( + component_id=component_id, configuration_id=configuration_id + ) + root = cast(dict, detail.get('configuration') or {}) + assert 'shared_code_id' not in root + assert 'shared_code_row_ids' not in root + finally: + # Restore the original configuration so we don't pollute other tests. + await client.storage_client.configuration_update( + component_id=component_id, + configuration_id=configuration_id, + configuration=original_root, + change_description='integtest cleanup: restore original config', + ) diff --git a/pyproject.toml b/pyproject.toml index 820c70fb5..feb9d2dac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "keboola-mcp-server" -version = "1.73.3" +version = "1.74.0" description = "MCP server for interacting with Keboola Connection" readme = "README.md" requires-python = ">=3.10" diff --git a/src/keboola_mcp_server/clients/storage.py b/src/keboola_mcp_server/clients/storage.py index 4733a6882..b9caf4d6d 100644 --- a/src/keboola_mcp_server/clients/storage.py +++ b/src/keboola_mcp_server/clients/storage.py @@ -561,6 +561,7 @@ async def configuration_create( name: str, description: str, configuration: dict[str, Any], + configuration_id: Optional[str] = None, ) -> JsonDict: """ Creates a new configuration for a component. @@ -569,16 +570,22 @@ async def configuration_create( :param name: The name of the configuration. :param description: The description of the configuration. :param configuration: The configuration definition as a dictionary. + :param configuration_id: Optional explicit configuration ID (forwarded to SAPI as + `configurationId`). Required for `keboola.shared-code` parent libraries, which + must use the conventional `shared-codes.` ID for + the UI and runtime expansion to find them. Leave `None` to let SAPI auto-assign. :return: The SAPI call response - created configuration or raise an error. """ endpoint = f'branch/{self._branch_id}/components/{component_id}/configs' - payload = { + payload: dict[str, Any] = { 'name': name, 'description': description, 'configuration': await self._encrypt_secrets(component_id, configuration), } + if configuration_id: + payload['configurationId'] = configuration_id return cast(JsonDict, await self.post(endpoint=endpoint, data=payload)) async def configuration_delete(self, component_id: str, configuration_id: str, skip_trash: bool = False) -> None: @@ -596,12 +603,20 @@ async def configuration_delete(self, component_id: str, configuration_id: str, s if skip_trash: await self.delete(endpoint=endpoint) - async def configuration_detail(self, component_id: str, configuration_id: str) -> JsonDict: + async def configuration_detail( + self, + component_id: str, + configuration_id: str, + include: Optional[Sequence[str]] = None, + ) -> JsonDict: """ Retrieves information about a given configuration. :param component_id: The id of the component. :param configuration_id: The id of the configuration. + :param include: Optional list of SAPI `include` values (e.g. `["rows"]`) to request + additional fields the API only returns on demand. Without `include=rows` the + response omits the configuration's row data. :return: The parsed json from the HTTP response. :raises ValueError: If the component_id or configuration_id is invalid. """ @@ -610,8 +625,9 @@ async def configuration_detail(self, component_id: str, configuration_id: str) - if not isinstance(configuration_id, str) or configuration_id == '': raise ValueError(f"Invalid configuration_id '{configuration_id}'.") endpoint = f'branch/{self._branch_id}/components/{component_id}/configs/{configuration_id}' + params = {'include': ','.join(include)} if include else None - return cast(JsonDict, await self.get(endpoint=endpoint)) + return cast(JsonDict, await self.get(endpoint=endpoint, params=params)) async def configuration_list(self, component_id: str) -> list[JsonDict]: """ @@ -739,6 +755,7 @@ async def configuration_row_create( name: str, description: str, configuration: dict[str, Any], + row_id: Optional[str] = None, ) -> JsonDict: """ Creates a new row configuration for a component configuration. @@ -748,13 +765,18 @@ async def configuration_row_create( :param name: The name of the row configuration. :param description: The description of the row configuration. :param configuration: The configuration data to create row configuration. + :param row_id: Optional explicit row ID (forwarded to SAPI as `rowId`). When provided, + it becomes the row's identifier — used e.g. as the Mustache placeholder key for + `keboola.shared-code` rows. When omitted, SAPI generates a numeric ID. :return: The SAPI call response - created row configuration or raise an error. """ - payload = { + payload: dict[str, Any] = { 'name': name, 'description': description, 'configuration': await self._encrypt_secrets(component_id, configuration), } + if row_id: + payload['rowId'] = row_id return cast( JsonDict, diff --git a/src/keboola_mcp_server/resources/prompts/project_system_prompt.md b/src/keboola_mcp_server/resources/prompts/project_system_prompt.md index b4403e44e..ec59bec8e 100644 --- a/src/keboola_mcp_server/resources/prompts/project_system_prompt.md +++ b/src/keboola_mcp_server/resources/prompts/project_system_prompt.md @@ -66,6 +66,52 @@ See the [Development Branches](#development-branches) section for more details. test changes in other tables, a branch-specific FQN may be used temporarily, but it must be switched back to the production path before merging. +### Shared Code + +**Shared code** is Keboola's reusable-snippet primitive for transformations. Snippets live under the +`keboola.shared-code` component (one parent library per transformation backend, rows are individual snippets); +transformations link to them via `shared_code_id` + `shared_code_row_ids` at the configuration root. + +**Before writing transformation code:** call `get_shared_codes(transformation_component_ids=[…])` and reuse an +existing snippet whenever the same logic would appear in two or more transformations. + +**Wire format (canonical IDs and required fields):** + +| Operation | Tool call | +|---|---| +| Create parent library | `create_config(component_id="keboola.shared-code", configuration_id="shared-codes.", parameters={"componentId":""})` — `configuration_id` MUST be the conventional `shared-codes.`. Auto-generated UUIDs are not recognised by the runtime. | +| Add a row | `add_config_row(component_id="keboola.shared-code", row_id="", parameters={"code_content":[""]})` — `row_id` is the case-sensitive Mustache key. | +| Edit a snippet | `update_config_row(parameter_updates=[{"op":"set","path":"code_content","value":[""]}])`. Disable with `is_disabled=True` (no delete-row tool). | +| Reference from a SQL transformation | `create_sql_transformation` / `update_sql_transformation` with `shared_code_id` + `shared_code_row_ids`. The tool auto-emits a `Shared Code (-)` marker per row — skipped when a user-authored code block already has `["{{ rowId }}"]` as its own pure script element. | +| Reference from Python / R | `create_config` / `update_config` with the same `shared_code_id` + `shared_code_row_ids`. Same auto-emit + skip-if-already-referenced behavior as the SQL path. | +| Change linkage on an existing SQL transformation | `update_sql_transformation(parameter_updates=[{"op":"set_shared_code","shared_code_id":"…","shared_code_row_ids":["…"]}])`, or `{"op":"remove_shared_code"}` to unlink. | + +**Tool-enforced rules** — the tool rejects the call otherwise: + +- **A pure `{{ rowId }}` placeholder requires linkage** — when a script array element is *exactly* + `["{{ rowId }}"]`, `shared_code_id` must be set and that row must appear in `shared_code_row_ids`, + or the call is rejected. (Inline `{{ rowId }}` occurrences *inside* a larger string are NOT rejected — + see the runtime note below.) +- **`shared_code_row_ids` requires `shared_code_id`** — passing row IDs without a library ID is rejected. +- **Row IDs are case-sensitive.** + +**Runtime requirements** — not checked by the tool, but the snippet fails at run time if violated: + +- **Row content must be a complete executable statement** (e.g. `CREATE OR REPLACE TABLE …`, + `from datetime import …`). The runtime substitutes a pure `{{ rowId }}` script array element with the + row's `code_content` array and runs the result as its own query. Fragments (column-list, `WHERE` + clause, sub-expression) will fail at run time. +- **Only a pure `["{{ rowId }}"]` array element is substituted.** A `{{ rowId }}` embedded inline inside + a larger SQL string is left as literal text (it shares Mustache syntax with configuration variables), + so keep placeholders on their own script element. +- **Marker ordering** — the tool appends one auto-emitted `Shared Code (-)` marker per linked + row *after* the user-authored code in the first parameters block, and on every sync it strips ALL + recognised marker blocks and re-appends the canonical set there. So you cannot reorder an auto-emitted + marker with `remove_code`/`add_code` — it will be moved back. If your user code depends on a shared + snippet running first (e.g. it sets a variable or creates a temp table your code reads), author your own + **non-marker** code block whose script is the pure placeholder `["{{ rowId }}"]` at the desired position; + the tool recognises it as already-referenced and skips emitting a duplicate marker for that row. + ### Development Branches When working in development branches the storage objects (tables, buckets) created or edited in the branch will have different FQNs than in production. diff --git a/src/keboola_mcp_server/tools/components/model.py b/src/keboola_mcp_server/tools/components/model.py index 1e840fefc..391ca5710 100644 --- a/src/keboola_mcp_server/tools/components/model.py +++ b/src/keboola_mcp_server/tools/components/model.py @@ -657,6 +657,21 @@ class Table(BaseModel): parameters: Parameters = Field(description='The parameters for the transformation') storage: Storage = Field(description='The storage configuration for the transformation') + shared_code_id: Optional[str] = Field( + default=None, + description=( + 'The configuration ID of the parent `keboola.shared-code` configuration this transformation ' + 'references (e.g. `shared-codes.snowflake-transformation`). `None` when no shared code is used.' + ), + ) + shared_code_row_ids: Optional[list[str]] = Field( + default=None, + description=( + 'The list of shared code row IDs (Mustache placeholder keys) referenced by this transformation. ' + 'Each entry must correspond to a row in the parent shared-code configuration and to a ' + '`{{ rowId }}` placeholder in one of the script blocks. `None` when no shared code is used.' + ), + ) # Type alias for TransformationConfiguration.Parameters for convenience @@ -804,6 +819,28 @@ def validate_code_id_requires_block_id(self) -> 'TfStrReplace': return self +class TfSetSharedCode(BaseModel, frozen=True): + """ + Link the transformation to shared code snippets at the configuration root. + + Sets both `shared_code_id` (the parent `keboola.shared-code` configuration ID) and + `shared_code_row_ids` (the list of Mustache placeholder keys referenced from the script). + Replaces any existing shared-code linkage on the transformation. + """ + + op: Literal['set_shared_code'] + shared_code_id: str = Field(description='The parent `keboola.shared-code` configuration ID') + shared_code_row_ids: list[str] = Field( + description='The list of shared code row IDs (Mustache keys) the script references' + ) + + +class TfRemoveSharedCode(BaseModel, frozen=True): + """Remove the shared-code linkage from the transformation (clears both root fields).""" + + op: Literal['remove_shared_code'] + + # Discriminated union of all transformation parameter update operations TfParamUpdate = Annotated[ Union[ @@ -816,11 +853,47 @@ def validate_code_id_requires_block_id(self) -> 'TfStrReplace': TfSetCode, TfAddScript, TfStrReplace, + TfSetSharedCode, + TfRemoveSharedCode, ], Field(discriminator='op'), ] +# ============================================================================ +# SHARED CODE MODELS +# ============================================================================ + + +class SharedCodeRow(BaseModel): + """One reusable code snippet stored as a row of a `keboola.shared-code` configuration.""" + + row_id: str = Field(description='The row ID — also the Mustache placeholder key (e.g. `dumpfiles`)') + name: str = Field(description='Human-readable label for the snippet') + code: str = Field(description="The code body (joined from the row's `code_content` array)") + + +class SharedCodeConfig(BaseModel): + """A `keboola.shared-code` parent configuration grouping reusable snippets for one transformation backend.""" + + config_id: str = Field(description='The parent configuration ID (used as `shared_code_id` in transformations)') + name: str = Field(description='Human-readable label for the shared-code configuration') + transformation_component_id: str = Field( + description=( + 'The transformation component ID this shared code is intended for, e.g. ' + '`keboola.snowflake-transformation`, `keboola.google-bigquery-transformation`, ' + '`keboola.python-transformation-v2`, `keboola.r-transformation-v2`' + ) + ) + rows: list[SharedCodeRow] = Field(description='Available shared code snippets in this configuration') + + +class GetSharedCodesOutput(BaseModel): + """Output of the `get_shared_codes` tool.""" + + shared_codes: list[SharedCodeConfig] = Field(description='Shared code configurations and their rows') + + # ============================================================================ # TOOL OUTPUT MODELS # ============================================================================ @@ -831,6 +904,14 @@ class ConfigToolOutput(BaseModel): component_id: str = Field(description='The ID of the component.') configuration_id: str = Field(description='The ID of the configuration.') + configuration_row_id: Optional[str] = Field( + default=None, + description=( + 'For row-level operations (e.g. `add_config_row`), the actual row ID assigned by SAPI. ' + 'For `keboola.shared-code` rows this is the Mustache placeholder key. Use it when ' + 'addressing the row in subsequent calls such as `update_config_row`.' + ), + ) description: str = Field(description='The description of the configuration.') version: int = Field(description='The version number of the configuration.') timestamp: datetime = Field(description='The timestamp of the operation.') diff --git a/src/keboola_mcp_server/tools/components/tools.py b/src/keboola_mcp_server/tools/components/tools.py index 24a51b5bf..7b33f4ad4 100644 --- a/src/keboola_mcp_server/tools/components/tools.py +++ b/src/keboola_mcp_server/tools/components/tools.py @@ -60,20 +60,28 @@ GetConfigsDetailOutput, GetConfigsListOutput, GetConfigsOutput, + GetSharedCodesOutput, + SharedCodeConfig, + SharedCodeRow, SimplifiedTfBlocks, TfParamUpdate, + TfRemoveSharedCode, + TfSetSharedCode, TransformationConfiguration, VariableDefinition, ) from keboola_mcp_server.tools.components.utils import ( BIGQUERY_TRANSFORMATION_ID, FOLDER_SUPPORTING_COMPONENT_IDS, + SHARED_CODE_COMPONENT_ID, + SHARED_CODE_TRANSFORMATION_IDS, SNOWFLAKE_TRANSFORMATION_ID, VARIABLES_COMPONENT_ID, _apply_vars_to_parent_cfg, add_ids, apply_configuration_variables, apply_folder_metadata, + apply_shared_code_markers, build_folder_hint, check_suitable, clear_configuration_folder_metadata, @@ -89,8 +97,11 @@ set_cfg_update_metadata, set_configuration_folder_metadata, set_nested_value, + sync_shared_code_markers_in_dict, update_params, update_transformation_parameters, + validate_shared_code_linkage, + validate_shared_code_params, ) from keboola_mcp_server.tools.constants import CONFIG_DIFF_PREVIEW_TAG from keboola_mcp_server.tools.validation import ( @@ -137,6 +148,13 @@ def add_component_tools(mcp: KeboolaMcpServer) -> None: annotations=ToolAnnotations(readOnlyHint=True), ) ) + mcp.add_tool( + FunctionTool.from_function( + get_shared_codes, + tags={COMPONENT_TOOLS_TAG}, + annotations=ToolAnnotations(readOnlyHint=True), + ) + ) # Configuration management tools mcp.add_tool( @@ -283,7 +301,7 @@ async def fetch_config_detail(spec: FullConfigId) -> Configuration: raw_configuration = cast( JsonDict, await client.storage_client.configuration_detail( - component_id=component_id, configuration_id=configuration_id + component_id=component_id, configuration_id=configuration_id, include=['rows'] ), ) @@ -378,6 +396,114 @@ async def fetch_component_with_links(component_id: str) -> Component: return GetComponentsOutput(components=components, links=[links_manager.get_used_components_link()]) +@tool_errors() +async def get_shared_codes( + ctx: Context, + transformation_component_ids: Annotated[ + Sequence[str], + Field( + description=( + 'Optional filter limiting results to shared-code libraries that belong to the given ' + 'transformation components. Accepted values are `keboola.snowflake-transformation`, ' + '`keboola.google-bigquery-transformation`, `keboola.python-transformation-v2`, and ' + '`keboola.r-transformation-v2`. When empty, every shared-code library in the project ' + 'is returned.' + ), + ), + ] = tuple(), +) -> GetSharedCodesOutput: + """ + Discovers Keboola shared-code libraries and their reusable snippets in the current project. + + Shared code is a Keboola primitive for storing reusable SQL/Python/R snippets that transformations + reference via Mustache placeholders (`{{ rowId }}`). Each transformation backend can have a parent + `keboola.shared-code` configuration whose rows are the individual snippets. The `rowId` of each row + is the case-sensitive Mustache key used in transformation scripts; the `code_content` of the row is + the snippet body expanded at runtime. + + WHEN TO USE: + - Before writing or editing transformation code, check whether the project already maintains a + reusable snippet you can reference via `{{ rowId }}` instead of duplicating logic inline. + - When the user asks about existing shared code libraries or wants to inventory reusable snippets. + + RETURNS: + - A list of `SharedCodeConfig` entries, each with `config_id` (use as `shared_code_id` in + transformations), `transformation_component_id` (which backend the library belongs to), + and the `rows` available — each row\'s `row_id` is the Mustache placeholder key. + """ + client = KeboolaClient.from_state(ctx.session.state) + + filter_set = {component_id for component_id in transformation_component_ids if component_id} + if filter_set and not filter_set.issubset(SHARED_CODE_TRANSFORMATION_IDS): + unknown = sorted(filter_set - SHARED_CODE_TRANSFORMATION_IDS) + raise ValueError( + f'Unknown transformation component IDs in filter: {unknown}. ' + f'Accepted values: {sorted(SHARED_CODE_TRANSFORMATION_IDS)}.' + ) + + parent_configs = await client.storage_client.configuration_list(SHARED_CODE_COMPONENT_ID) + + async def build_shared_code_config(parent: JsonDict) -> Optional[SharedCodeConfig]: + # Shared-code parents and rows use a FLAT configuration body — `componentId` and + # `code_content` live at the configuration root, not under `parameters`. Historical + # configs created via the generic create_config wrapper may still have them nested, + # so read root first and fall back to `parameters.` for compatibility. + parent_configuration = cast(dict[str, Any], parent.get('configuration') or {}) + parent_parameters = cast(dict[str, Any], parent_configuration.get('parameters') or {}) + transformation_component_id = cast( + str, parent_configuration.get('componentId') or parent_parameters.get('componentId') or '' + ) + if not transformation_component_id: + # A shared-code parent with no `componentId` (neither at root nor under `parameters`) is + # malformed — its backend transformation type is ambiguous. Skip it rather than emit a + # config with an empty transformation_component_id that breaks filtering/consumption. + LOG.warning( + 'Skipping shared-code config %r: no `componentId` at the configuration root or under `parameters`.', + parent.get('id'), + ) + return None + if filter_set and transformation_component_id not in filter_set: + return None + + detail = await client.storage_client.configuration_detail( + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id=str(parent.get('id')), + include=['rows'], + ) + rows: list[SharedCodeRow] = [] + for raw_row in cast(list[dict[str, Any]], detail.get('rows') or []): + # Disabled snippets are the documented soft-delete path (`update_config_row(is_disabled=True)`); + # skip them so callers never link a transformation to a disabled shared-code row. + if raw_row.get('isDisabled'): + continue + row_config = cast(dict[str, Any], raw_row.get('configuration') or {}) + row_parameters = cast(dict[str, Any], row_config.get('parameters') or {}) + code_content = row_config.get('code_content') + if code_content is None: + code_content = row_parameters.get('code_content') or [] + if isinstance(code_content, list): + code = '\n'.join(str(item) for item in code_content) + else: + code = str(code_content) + rows.append( + SharedCodeRow( + row_id=str(raw_row.get('id') or ''), + name=str(raw_row.get('name') or ''), + code=code, + ) + ) + return SharedCodeConfig( + config_id=str(parent.get('id') or ''), + name=str(parent.get('name') or ''), + transformation_component_id=transformation_component_id, + rows=rows, + ) + + results = await process_concurrently(parent_configs, build_shared_code_config) + configs = [item for item in unwrap_results(results, 'Failed to fetch shared code library') if item is not None] + return GetSharedCodesOutput(shared_codes=configs) + + # ============================================================================ # CONFIGURATION MANAGEMENT TOOLS # ============================================================================ @@ -423,6 +549,28 @@ async def create_sql_transformation( str, Field(description=folder_field_description('transformation', 'transformations')), ] = '', + shared_code_id: Annotated[ + str, + Field( + description=( + 'Optional. The configuration ID of the parent `keboola.shared-code` library this transformation ' + 'should reference (e.g. `shared-codes.snowflake-transformation`). When provided together with ' + '`shared_code_row_ids`, every `{{ rowId }}` Mustache placeholder used in the SQL script is ' + "expanded at runtime to the matching row's code. Discover available libraries via " + '`get_shared_codes`. Leave empty when not using shared code.' + ), + ), + ] = '', + shared_code_row_ids: Annotated[ + Sequence[str], + Field( + description=( + 'Optional. The list of shared code row IDs (Mustache placeholder keys) referenced from the SQL ' + 'script. Each entry must (a) exist as a row in the parent shared-code configuration and (b) appear ' + 'as `{{ rowId }}` in at least one script block. Row IDs are case-sensitive.' + ), + ), + ] = tuple(), variables: Annotated[ Optional[list[VariableDefinition]], Field( @@ -457,6 +605,11 @@ async def create_sql_transformation( - If there are 20 or more SQL transformations in the project, consider organizing them with a folder: existing folder names are surfaced in the response's change_summary — use one of them or create a new one. + SHARED CODE LINKAGE: + - To reuse snippets from the project's `keboola.shared-code` library, embed `{{ rowId }}` placeholders in the + script AND pass `shared_code_id` + `shared_code_row_ids`. Both must be set together; the placeholders alone + have no effect. Discover existing libraries with `get_shared_codes` before creating new ones. + USAGE: - Use when you want to create a new SQL transformation. @@ -480,6 +633,27 @@ async def create_sql_transformation( transformation_configuration_payload = await create_transformation_configuration( codes=sql_code_blocks, transformation_name=name, output_tables=created_table_names, sql_dialect=sql_dialect ) + validate_shared_code_params(shared_code_id, shared_code_row_ids) + if shared_code_id: + transformation_configuration_payload.shared_code_id = shared_code_id + transformation_configuration_payload.shared_code_row_ids = list(shared_code_row_ids) + # Match the Keboola UI's emit: each row_id gets its own `Shared Code (...)` code block + # whose script is just `{{rowId}}`. The runtime expansion is driven by these marker + # blocks, not by `{{rowId}}` text occurrences inside the user's other queries. + apply_shared_code_markers( + transformation_configuration_payload, + shared_code_id=shared_code_id, + shared_code_row_ids=shared_code_row_ids, + ) + + # Enforce: any `{{rowId}}` placeholder in the script array must have a matching entry in + # `shared_code_row_ids` and a non-empty `shared_code_id`. Otherwise the runtime cannot + # resolve the placeholder and the snippet is silently skipped. + validate_shared_code_linkage( + parameters=transformation_configuration_payload.model_dump(exclude_none=True).get('parameters', {}), + shared_code_id=shared_code_id or '', + shared_code_row_ids=list(shared_code_row_ids), + ) client = KeboolaClient.from_state(ctx.session.state) links_manager = await ProjectLinksManager.from_client(client) @@ -490,7 +664,7 @@ async def create_sql_transformation( component_id=component_id, name=name, description=description, - configuration=transformation_configuration_payload.model_dump(by_alias=True), + configuration=transformation_configuration_payload.model_dump(by_alias=True, exclude_none=True), ) configuration_id = str(new_raw_transformation_configuration['id']) @@ -1003,26 +1177,79 @@ async def update_sql_transformation_internal( updated_configuration = copy.deepcopy(updated_configuration) msg: str = '' + shared_code_messages: list[str] = [] if parameter_updates: - current_param_dict = updated_configuration.get('parameters', {}) - current_raw_parameters = TransformationConfiguration.Parameters.model_validate(current_param_dict) - simplified_parameters = await current_raw_parameters.to_simplified_parameters() - - updated_params, msg = update_transformation_parameters( - parameters=simplified_parameters, - updates=parameter_updates, - sql_dialect=sql_dialect, - ) - updated_raw_parameters = await updated_params.to_raw_parameters() + # Root-level operations (shared code) live alongside `parameters`/`storage`, not within them. + # Partition the updates so we can dispatch root ops against `updated_configuration` directly + # and leave the block/code ops to the existing parameters pipeline. + root_updates: list[TfParamUpdate] = [] + block_updates: list[TfParamUpdate] = [] + for update in parameter_updates: + if isinstance(update, (TfSetSharedCode, TfRemoveSharedCode)): + root_updates.append(update) + else: + block_updates.append(update) + + for update in root_updates: + if isinstance(update, TfSetSharedCode): + if not update.shared_code_id: + raise ValueError( + 'The `set_shared_code` operation requires a non-empty `shared_code_id`. ' + 'Use the `remove_shared_code` operation to clear an existing linkage instead of ' + 'passing an empty ID.' + ) + updated_configuration['shared_code_id'] = update.shared_code_id + updated_configuration['shared_code_row_ids'] = list(update.shared_code_row_ids) + shared_code_messages.append( + f'Linked shared code {update.shared_code_id!r} ' + f'(rows: {", ".join(update.shared_code_row_ids) or ""}).' + ) + else: # TfRemoveSharedCode + removed_id = updated_configuration.pop('shared_code_id', None) + updated_configuration.pop('shared_code_row_ids', None) + shared_code_messages.append( + f'Removed shared code linkage (was {removed_id!r}).' + if removed_id + else 'Removed shared code linkage (none was set).' + ) - parameters_cfg = validate_root_parameters_configuration( - component=transformation, - parameters=updated_raw_parameters.model_dump(exclude_none=True), - initial_message='Applying the "parameter_updates" resulted in an invalid configuration.', - configuration_id=configuration_id, - ) - updated_configuration['parameters'] = parameters_cfg + if block_updates: + current_param_dict = updated_configuration.get('parameters', {}) + current_raw_parameters = TransformationConfiguration.Parameters.model_validate(current_param_dict) + simplified_parameters = await current_raw_parameters.to_simplified_parameters() + + updated_params, msg = update_transformation_parameters( + parameters=simplified_parameters, + updates=block_updates, + sql_dialect=sql_dialect, + ) + updated_raw_parameters = await updated_params.to_raw_parameters() + + parameters_cfg = validate_root_parameters_configuration( + component=transformation, + parameters=updated_raw_parameters.model_dump(exclude_none=True), + initial_message='Applying the "parameter_updates" resulted in an invalid configuration.', + configuration_id=configuration_id, + ) + updated_configuration['parameters'] = parameters_cfg + + if shared_code_messages: + msg = ' '.join(filter(None, [msg, *shared_code_messages])) + + # After any TfSet/TfRemoveSharedCode op (and after any block_updates), make sure the + # parameters.blocks reflect the current shared-code linkage with the UI-canonical + # `Shared Code (...)` marker blocks. Without these markers the platform's runtime + # expansion never substitutes `{{rowId}}`. + sync_shared_code_markers_in_dict(updated_configuration) + + # Reject configurations where a `{{rowId}}` placeholder is used without a matching + # entry in `shared_code_row_ids` (or with `shared_code_id` empty). + validate_shared_code_linkage( + parameters=updated_configuration.get('parameters', {}), + shared_code_id=str(updated_configuration.get('shared_code_id') or ''), + shared_code_row_ids=list(updated_configuration.get('shared_code_row_ids') or []), + ) if storage is not None: storage_cfg = validate_root_storage_configuration( @@ -1101,6 +1328,42 @@ async def create_config( list[dict[str, Any]], Field(description='The list of processors that will run after the configured component runs.'), ] = None, + shared_code_id: Annotated[ + str, + Field( + description=( + 'Optional. The configuration ID of the parent `keboola.shared-code` library this configuration ' + 'references at the root level. Useful when creating Python (`keboola.python-transformation-v2`), ' + 'R (`keboola.r-transformation-v2`) transformation configurations that need to reuse ' + 'shared snippets. Must be paired with `shared_code_row_ids` and matching `{{ rowId }}` placeholders ' + "in the component's script. Leave empty when not using shared code." + ), + ), + ] = '', + shared_code_row_ids: Annotated[ + Sequence[str], + Field( + description=( + 'Optional. The list of shared code row IDs (Mustache placeholder keys) referenced from the ' + 'configuration. Each entry must exist as a row in the parent shared-code configuration and ' + "appear as `{{ rowId }}` in the component's script. Row IDs are case-sensitive." + ), + ), + ] = tuple(), + configuration_id: Annotated[ + str, + Field( + description=( + 'Optional explicit configuration ID. When non-empty, forwarded to SAPI as `configurationId`. ' + 'REQUIRED for `keboola.shared-code` parent libraries — pass the conventional ' + '`shared-codes.` value (e.g. ' + '`shared-codes.snowflake-transformation`, `shared-codes.google-bigquery-transformation`, ' + '`shared-codes.python-transformation-v2`, `shared-codes.r-transformation-v2`). The UI and ' + 'runtime expansion look up shared-code libraries by this exact ID. Leave empty to let SAPI ' + 'auto-assign for any other component.' + ), + ), + ] = '', variables: Annotated[ Optional[list[VariableDefinition]], Field( @@ -1127,6 +1390,16 @@ async def create_config( USAGE: - Use when you want to create a new root configuration for a specific component. + SHARED CODE: + - For `keboola.shared-code` parent libraries: pass `component_id="keboola.shared-code"`, + `parameters={"componentId": ""}`, AND `configuration_id="shared-codes."` + (e.g. `shared-codes.snowflake-transformation`). The platform stores `componentId` AT THE + CONFIGURATION ROOT for shared-code (not nested under `parameters`); this tool unwraps the + provided parameters dict accordingly. The conventional `configuration_id` is required — + auto-generated IDs are not recognised by the runtime expansion. + - For Python/R transformations that should reuse shared snippets, set `shared_code_id` and + `shared_code_row_ids` and embed `{{ rowId }}` Mustache placeholders in the component's script. + WHEN NOT TO USE: - `keboola.orchestrator` / `keboola.flow` → use flows tools - `keboola.data-apps` → use data applications tools @@ -1158,7 +1431,38 @@ async def create_config( initial_message='The "parameters" field is not valid.', ) - configuration_payload = {'storage': storage_cfg, 'parameters': parameters} + # `keboola.shared-code` parent libraries use a flat configuration body — the platform's UI + # and runtime expansion read `componentId` at the configuration root, not under `parameters`. + # For every other component the generic wrapper applies. + if component_id == SHARED_CODE_COMPONENT_ID: + # The UI and runtime expansion only recognise shared-code libraries by the conventional + # `shared-codes.` ID — a SAPI-auto-assigned UUID (or a typo) + # would create an orphaned library that never resolves. Require the caller to pass the + # exact conventional ID derived from the target `componentId`. + target_component_id = str((parameters or {}).get('componentId') or '') + if not target_component_id: + raise ValueError( + 'Creating a `keboola.shared-code` library requires `parameters={"componentId": ' + '""}` — the target transformation the library belongs to. ' + 'Without it the library has no backend association and cannot be resolved.' + ) + expected_configuration_id = f'shared-codes.{target_component_id.split(".", 1)[-1]}' + if not configuration_id: + raise ValueError( + 'Creating a `keboola.shared-code` library requires an explicit `configuration_id` using the ' + 'conventional `shared-codes.` value (e.g. ' + '`shared-codes.snowflake-transformation`). Auto-assigned IDs are not recognised by the runtime ' + 'expansion or the UI.' + ) + if configuration_id != expected_configuration_id: + raise ValueError( + f'`configuration_id={configuration_id!r}` does not match the conventional shared-code library ID ' + f'for `componentId={target_component_id!r}`. Use {expected_configuration_id!r} — the UI and runtime ' + f'expansion look libraries up by this exact ID.' + ) + configuration_payload: dict[str, Any] = dict(parameters or {}) + else: + configuration_payload = {'storage': storage_cfg, 'parameters': parameters} if processors_before: processors_before = await validate_processors_configuration( @@ -1176,6 +1480,23 @@ async def create_config( ) set_nested_value(configuration_payload, 'processors.after', processors_after) + validate_shared_code_params(shared_code_id, shared_code_row_ids) + if shared_code_id: + configuration_payload['shared_code_id'] = shared_code_id + configuration_payload['shared_code_row_ids'] = list(shared_code_row_ids) + + # Symmetry with `create_sql_transformation`: when the target is a transformation backend + # that supports shared code (Python/R/SQL via the generic create_config path), + # auto-emit the UI-canonical `Shared Code (...)` marker code blocks and validate the + # placeholder linkage so the runtime expansion can resolve every `{{rowId}}` reference. + if component_id in SHARED_CODE_TRANSFORMATION_IDS: + sync_shared_code_markers_in_dict(configuration_payload) + validate_shared_code_linkage( + parameters=configuration_payload.get('parameters', {}), + shared_code_id=shared_code_id, + shared_code_row_ids=list(shared_code_row_ids), + ) + new_raw_configuration = cast( dict[str, Any], await client.storage_client.configuration_create( @@ -1183,6 +1504,7 @@ async def create_config( name=name, description=description, configuration=configuration_payload, + configuration_id=configuration_id or None, ), ) @@ -1258,6 +1580,17 @@ async def add_config_row( list[dict[str, Any]], Field(description='The list of processors that will run after the configured component row runs.'), ] = None, + row_id: Annotated[ + str, + Field( + description=( + 'Optional explicit row ID. When provided, becomes the row identifier in SAPI ' + '(forwarded as `rowId`). For `keboola.shared-code` rows this is the Mustache placeholder ' + 'key used in transformation scripts (e.g. `dumpfiles` → referenced as `{{ dumpfiles }}`). ' + 'Row IDs are case-sensitive. Leave empty to let SAPI auto-assign a numeric ID.' + ), + ), + ] = '', ) -> ConfigToolOutput: """ Creates a component configuration row in the specified configuration_id, using the specified name, @@ -1273,6 +1606,10 @@ async def add_config_row( USAGE: - Use when you want to create a new row configuration for a specific component configuration. + SHARED CODE ROWS: + - For `keboola.shared-code` rows, set `row_id` to the Mustache placeholder key (e.g. `dumpfiles`) + and put the snippet body in `parameters` as `{"code_content": [""]}`. + WHEN NOT TO USE: - `keboola.orchestrator` / `keboola.flow` → use flows tools - `keboola.data-apps` → use data applications tools @@ -1309,7 +1646,12 @@ async def add_config_row( configuration_id=configuration_id, ) - configuration_payload = {'storage': storage_cfg, 'parameters': parameters} + # `keboola.shared-code` rows use a flat configuration body — the platform reads + # `code_content` at the row configuration root, not under `parameters`. + if component_id == SHARED_CODE_COMPONENT_ID: + configuration_payload: dict[str, Any] = dict(parameters or {}) + else: + configuration_payload = {'storage': storage_cfg, 'parameters': parameters} if processors_before: processors_before = await validate_processors_configuration( @@ -1335,11 +1677,24 @@ async def add_config_row( name=name, description=description, configuration=configuration_payload, + row_id=row_id or None, ), ) + assigned_row_id = str(new_raw_configuration.get('id') or '') + if row_id and assigned_row_id and assigned_row_id != row_id: + LOG.warning( + 'add_config_row requested row_id=%r but SAPI assigned %r for component=%s config=%s. ' + 'Use the assigned id when addressing this row.', + row_id, + assigned_row_id, + component_id, + configuration_id, + ) + LOG.info( - f'Created new configuration for component "{component_id}" with configuration id ' f'"{configuration_id}".' + f'Created new configuration row "{assigned_row_id}" for component "{component_id}" ' + f'in configuration "{configuration_id}".' ) await set_cfg_update_metadata( @@ -1358,6 +1713,7 @@ async def add_config_row( return ConfigToolOutput( component_id=component_id, configuration_id=configuration_id, + configuration_row_id=assigned_row_id or None, description=description, version=new_raw_configuration['version'], timestamp=datetime.now(timezone.utc), @@ -1449,6 +1805,27 @@ async def update_config( Optional[str], Field(description=folder_field_description('configuration', 'configurations')), ] = None, + shared_code_id: Annotated[ + Optional[str], + Field( + description=( + 'Optional. Updates the shared-code linkage on the configuration root. ' + 'Non-empty string: sets `shared_code_id` (parent `keboola.shared-code` config ID) and replaces ' + '`shared_code_row_ids` with the value below. Empty string `""`: clears the linkage (removes ' + 'both root fields). `None` (default): leaves the existing linkage untouched. ' + 'Use for Python/R transformations; SQL transformations use update_sql_transformation.' + ), + ), + ] = None, + shared_code_row_ids: Annotated[ + Sequence[str], + Field( + description=( + 'Optional. The list of shared code row IDs (Mustache placeholder keys). Only applied when ' + '`shared_code_id` is non-empty; ignored otherwise. Row IDs are case-sensitive.' + ), + ), + ] = tuple(), variables: Annotated[ Optional[list[VariableDefinition]], Field( @@ -1472,6 +1849,7 @@ async def update_config( - Modifying configuration parameters (credentials, settings, API keys, etc.) - Updating storage mappings (input/output tables or files) - Changing configuration name or description + - Adding/removing shared-code linkage on Python/R transformations (via `shared_code_id`) - Any combination of the above WHEN NOT TO USE: @@ -1515,6 +1893,8 @@ async def update_config( storage=storage, processors_before=processors_before, processors_after=processors_after, + shared_code_id=shared_code_id, + shared_code_row_ids=shared_code_row_ids, ) vars_config_id_to_delete: str | None = None @@ -1586,6 +1966,8 @@ async def update_config_internal( storage: dict[str, Any] | None = None, processors_before: list[dict[str, Any]] | None = None, processors_after: list[dict[str, Any]] | None = None, + shared_code_id: Optional[str] = None, + shared_code_row_ids: Optional[Sequence[str]] = None, ) -> tuple[JsonDict, JsonDict]: check_suitable('update_config', component_id) @@ -1635,6 +2017,27 @@ async def update_config_internal( ) configuration_payload['parameters'] = parameters_cfg + # Shared code linkage lives at the configuration root alongside `parameters`/`storage`. + # Non-empty `shared_code_id` sets the linkage; empty string explicitly clears it. + # `None` (the default) means "leave existing linkage untouched". + if shared_code_id is not None: + if shared_code_id: + configuration_payload['shared_code_id'] = shared_code_id + configuration_payload['shared_code_row_ids'] = list(shared_code_row_ids or ()) + else: + configuration_payload.pop('shared_code_id', None) + configuration_payload.pop('shared_code_row_ids', None) + + # Symmetry with `update_sql_transformation`: when the target supports shared code, + # (re)sync the marker code blocks and enforce the placeholder linkage rule. + if component_id in SHARED_CODE_TRANSFORMATION_IDS: + sync_shared_code_markers_in_dict(configuration_payload) + validate_shared_code_linkage( + parameters=configuration_payload.get('parameters', {}), + shared_code_id=str(configuration_payload.get('shared_code_id') or ''), + shared_code_row_ids=list(configuration_payload.get('shared_code_row_ids') or []), + ) + return current_config, configuration_payload @@ -1882,17 +2285,43 @@ async def update_config_row_internal( set_nested_value(configuration_payload, 'processors.after', processors_after) if parameter_updates: - current_params = configuration_payload.get('parameters', {}) - updated_params = update_params(current_params, parameter_updates) + if component_id == SHARED_CODE_COMPONENT_ID: + # Shared-code rows use a flat body — `code_content` lives at the row root, not under + # `parameters`, and `get_shared_codes` reads the root first. Apply updates against the + # root (mirroring `add_config_row`) so an edit to e.g. `code_content` is actually visible. + # `storage`/`processors` are structural siblings, not shared-code parameters: hold them + # aside so they are neither validated against the row parameter schema nor dropped by the + # validator (which returns only the parameter body), then re-attach them afterwards. + updated_root = update_params(configuration_payload, parameter_updates) + siblings = {key: updated_root[key] for key in ('storage', 'processors') if key in updated_root} + flat_body = {key: value for key, value in updated_root.items() if key not in siblings} + # Legacy shared-code rows may still nest the snippet under `parameters` (wrapper-created + # configs). validate_row_parameters_configuration unwraps a `parameters` key and returns + # only its contents, which would drop the flat-root fields we just updated. Flatten any + # such wrapper into the root first (root wins), so the validated body stays flat. + legacy_params = flat_body.pop('parameters', None) + if isinstance(legacy_params, dict): + flat_body = {**legacy_params, **flat_body} + validated_flat = validate_row_parameters_configuration( + component=component, + parameters=flat_body, + initial_message='Applying the "parameter_updates" resulted in an invalid row configuration.', + configuration_id=configuration_id, + configuration_row_id=configuration_row_id, + ) + configuration_payload = {**validated_flat, **siblings} + else: + current_params = configuration_payload.get('parameters', {}) + updated_params = update_params(current_params, parameter_updates) - parameters_cfg = validate_row_parameters_configuration( - component=component, - parameters=updated_params, - initial_message='Applying the "parameter_updates" resulted in an invalid row configuration.', - configuration_id=configuration_id, - configuration_row_id=configuration_row_id, - ) - configuration_payload['parameters'] = parameters_cfg + parameters_cfg = validate_row_parameters_configuration( + component=component, + parameters=updated_params, + initial_message='Applying the "parameter_updates" resulted in an invalid row configuration.', + configuration_id=configuration_id, + configuration_row_id=configuration_row_id, + ) + configuration_payload['parameters'] = parameters_cfg return current_row, configuration_payload diff --git a/src/keboola_mcp_server/tools/components/utils.py b/src/keboola_mcp_server/tools/components/utils.py index fdeea5909..4de8bed30 100644 --- a/src/keboola_mcp_server/tools/components/utils.py +++ b/src/keboola_mcp_server/tools/components/utils.py @@ -66,8 +66,19 @@ BIGQUERY_TRANSFORMATION_ID = 'keboola.google-bigquery-transformation' PYTHON_TRANSFORMATION_ID = 'keboola.python-transformation-v2' R_TRANSFORMATION_ID = 'keboola.r-transformation-v2' +SHARED_CODE_COMPONENT_ID = 'keboola.shared-code' VARIABLES_COMPONENT_ID = 'keboola.variables' +# Transformation component IDs that may have a shared-code library associated with them. +SHARED_CODE_TRANSFORMATION_IDS: frozenset[str] = frozenset( + { + SNOWFLAKE_TRANSFORMATION_ID, + BIGQUERY_TRANSFORMATION_ID, + PYTHON_TRANSFORMATION_ID, + R_TRANSFORMATION_ID, + } +) + # Component IDs for which update_config actively manages folder metadata (set/clear/hint). # For all other components the folder parameter is accepted but silently skipped to avoid # unnecessary API calls on components where folder organisation is not expected. @@ -370,6 +381,247 @@ async def create_transformation_configuration( return TransformationConfiguration(parameters=raw_parameters, storage=storage) +# Matches a script element whose ENTIRE content is a single `{{ rowId }}` placeholder +# (allowing surrounding whitespace, including trailing `\n` injected by the SQL joiner). +# Only such elements are substituted by the platform's runtime expansion — so a code block +# whose script is "pure" placeholder is already equivalent to an auto-emitted marker. +PURE_SHARED_CODE_PLACEHOLDER_RE = re.compile(r'\A\s*\{\{\s*([A-Za-z0-9_-]+)\s*\}\}\s*\Z') + + +def extract_shared_code_row_ids_from_blocks(blocks: Sequence[Any]) -> set[str]: + """ + Walks `parameters.blocks[*].codes[*].script` and returns the set of shared-code row IDs + referenced by `{{ rowId }}` placeholders. Tolerates `script` being a string or a list. + + Only placeholders that are the SOLE content of a script element (e.g. `["{{ dumpfiles }}"]`) + are treated as shared-code references — that is the only form the platform substitutes for + shared code. Inline occurrences such as `["token={{ api_token }}"]` are ignored: Keboola + configuration variables reuse the same Mustache syntax, so treating every `{{...}}` as a + shared-code row would wrongly block transformations that legitimately use variables. + """ + referenced: set[str] = set() + for block in blocks or (): + codes = block.get('codes', []) if isinstance(block, dict) else getattr(block, 'codes', []) + for code in codes or (): + script = code.get('script') if isinstance(code, dict) else getattr(code, 'script', None) + items = script if isinstance(script, list) else [script] + for item in items: + if not isinstance(item, str): + continue + m = PURE_SHARED_CODE_PLACEHOLDER_RE.match(item) + if m: + referenced.add(m.group(1)) + return referenced + + +def user_substitution_eligible_row_ids(blocks: Sequence[Any]) -> set[str]: + """ + Returns row IDs whose `{{ rowId }}` placeholder already appears as the sole content of a + script element in a NON-marker user code block — i.e. positions where the platform's + runtime expansion will substitute the snippet natively. The marker emitter consults this + so it does not append a duplicate `Shared Code (...)` block (which would execute the same + snippet twice at run time). + + Inline occurrences such as `["SELECT 1; {{ rowId }}"]` are deliberately ignored: the + platform does not substitute those, so the marker is still required. + """ + referenced: set[str] = set() + for block in blocks or (): + codes = block.get('codes', []) if isinstance(block, dict) else getattr(block, 'codes', []) + for code in codes or (): + name = code.get('name', '') if isinstance(code, dict) else getattr(code, 'name', '') + script = code.get('script') if isinstance(code, dict) else getattr(code, 'script', None) + if is_shared_code_marker(name, script): + continue + items = script if isinstance(script, list) else [script] + for item in items: + if not isinstance(item, str): + continue + m = PURE_SHARED_CODE_PLACEHOLDER_RE.match(item) + if m: + referenced.add(m.group(1)) + return referenced + + +def validate_shared_code_params(shared_code_id: str | None, shared_code_row_ids: Sequence[str]) -> None: + """ + Enforces the documented contract that `shared_code_id` and `shared_code_row_ids` are set + together: passing row IDs without a `shared_code_id` is a no-op that silently drops them, + so reject it explicitly instead of quietly ignoring the input. + """ + if shared_code_row_ids and not shared_code_id: + raise ValueError( + f'`shared_code_row_ids={list(shared_code_row_ids)}` was provided without a `shared_code_id`. ' + f'Set `shared_code_id` to the parent `keboola.shared-code` library ID, or omit the row IDs.' + ) + + +def validate_shared_code_linkage( + parameters: Mapping[str, Any] | None, + shared_code_id: str, + shared_code_row_ids: Sequence[str], +) -> None: + """ + Enforces the platform's shared-code linkage rule on a transformation configuration: + if a `{{ rowId }}` placeholder is the sole content of any `parameters.blocks[*].codes[*].script` + element, then `shared_code_id` must be non-empty AND every referenced `rowId` must be present + in `shared_code_row_ids`. Inline placeholders (e.g. `["SELECT 1, {{ rowId }};"]`) are NOT + treated as shared-code references — they share the same Mustache syntax as configuration + variables and are not natively substituted; the SQL transformation tools emit marker code + blocks for those via `shared_code_id` + `shared_code_row_ids` separately. + + Raises `ValueError` with a precise message when the rule is violated. + """ + blocks = (parameters or {}).get('blocks', []) + referenced = extract_shared_code_row_ids_from_blocks(blocks) + if not referenced: + return + declared = set(shared_code_row_ids or ()) + if not shared_code_id: + raise ValueError( + f'Transformation script references shared-code placeholders {sorted(referenced)} ' + f'but `shared_code_id` is not set at the configuration root. Pass `shared_code_id` ' + f'and `shared_code_row_ids` so the runtime can resolve the placeholders.' + ) + missing = referenced - declared + if missing: + raise ValueError( + f'Transformation script references shared-code rows {sorted(missing)} that are not ' + f'in `shared_code_row_ids={sorted(declared)}`. Add the missing row IDs or remove the ' + f'placeholders.' + ) + + +def shared_code_marker_code_name(shared_code_id: str, row_id: str) -> str: + """ + Returns the canonical name the Keboola UI uses for a shared-code marker code block: + `Shared Code ({shared_code_id}-{row_id})`. + """ + return f'Shared Code ({shared_code_id}-{row_id})' + + +def is_shared_code_marker(code_name: str, code_script: Any) -> bool: + """ + Detects whether a code block is a Keboola shared-code marker: name starts with + `Shared Code (` and the script is just a single Mustache placeholder. + """ + if not isinstance(code_name, str) or not code_name.startswith('Shared Code ('): + return False + script_text: str + if isinstance(code_script, list): + if len(code_script) != 1: + return False + script_text = str(code_script[0]) + else: + script_text = str(code_script or '') + # Require the script to be EXACTLY one Mustache placeholder. A loose startswith('{{')/ + # endswith('}}') check would misclassify a multi-placeholder element like "{{ a }}\n{{ b }}" + # as a marker and strip user-authored code during marker sync. + return PURE_SHARED_CODE_PLACEHOLDER_RE.match(script_text) is not None + + +def build_shared_code_marker_codes( + shared_code_id: str, + shared_code_row_ids: Sequence[str], +) -> list[TransformationConfiguration.Parameters.Block.Code]: + """ + Builds the raw shared-code marker code blocks the Keboola UI emits for each referenced row. + The runtime expansion processes these placeholders by replacing each `{{ rowId }}` with the + row's `code_content` and running the result as a standalone query. + + Returns one Code per row_id, with `name=shared_code_marker_code_name(...)` and + `script=["{{rowId}}"]`. + """ + return [ + TransformationConfiguration.Parameters.Block.Code( + name=shared_code_marker_code_name(shared_code_id, row_id), + script=[f'{{{{{row_id}}}}}'], + ) + for row_id in shared_code_row_ids + ] + + +def sync_shared_code_markers_in_dict(updated_configuration: dict[str, Any]) -> None: + """ + Dict-form sibling of `apply_shared_code_markers`. Operates on a raw configuration dict + (as round-tripped by `update_sql_transformation_internal`) and rewrites the first block's + code list so it carries one shared-code marker per current `shared_code_row_ids` entry — + or none when the linkage has been cleared. + + The marker layout (`Shared Code (-)` + `script=["{{rid}}"]`) is what the Keboola + UI emits and what the platform runtime expansion drives off of; user-authored code blocks + are preserved unchanged. + """ + shared_code_id = str(updated_configuration.get('shared_code_id') or '') + row_ids = list(updated_configuration.get('shared_code_row_ids') or []) + + parameters = updated_configuration.setdefault('parameters', {}) + blocks = parameters.setdefault('blocks', []) + + if not blocks: + if not row_ids: + return + blocks.append({'name': 'Blocks', 'codes': []}) + + # Strip existing shared-code markers from EVERY block, not just the first — markers may + # have been moved into another block during manual reordering; leaving stale copies would + # execute the same snippet twice at run time. The canonical set is re-inserted below into + # the first block only. + for block in blocks: + block['codes'] = [ + c for c in block.get('codes', []) if not is_shared_code_marker(c.get('name', ''), c.get('script')) + ] + + target_block = blocks[0] + marker_codes: list[dict[str, Any]] = [] + if shared_code_id and row_ids: + # Skip rows the user already references as a pure-placeholder script element + # somewhere in the config — those positions are substituted natively, so adding + # a marker would execute the snippet twice at run time. + already_substituted = user_substitution_eligible_row_ids(blocks) + marker_codes = [ + { + 'name': shared_code_marker_code_name(shared_code_id, rid), + 'script': [f'{{{{{rid}}}}}'], + } + for rid in row_ids + if rid not in already_substituted + ] + target_block['codes'] = target_block['codes'] + marker_codes + + +def apply_shared_code_markers( + transformation_config: TransformationConfiguration, + shared_code_id: str, + shared_code_row_ids: Sequence[str], +) -> None: + """ + Mutates the given transformation config so that its FIRST block contains exactly one + shared-code marker per row_id (and no stale markers from a previous linkage). + + The Keboola UI mirrors this layout: every shared-code reference appears as a sibling + code block whose script is just `{{rowId}}`. The runtime expansion uses the marker + blocks (not text-substituted occurrences in the user's surrounding SQL) so omitting + them causes the shared snippet to never run. + + If `shared_code_row_ids` is empty, all existing markers are removed. + """ + blocks = transformation_config.parameters.blocks + if not blocks: + if not shared_code_row_ids: + return + blocks.append(TransformationConfiguration.Parameters.Block(name='Blocks', codes=[])) + + target_block = blocks[0] + user_codes = [code for code in target_block.codes if not is_shared_code_marker(code.name, code.script)] + # Skip rows the user already references as a pure-placeholder script element somewhere + # in the config — those positions are substituted natively, so adding a marker would + # execute the snippet twice at run time. + already_substituted = user_substitution_eligible_row_ids(blocks) + pending_row_ids = [rid for rid in shared_code_row_ids if rid not in already_substituted] + target_block.codes = user_codes + build_shared_code_marker_codes(shared_code_id, pending_row_ids) + + async def set_cfg_creation_metadata(client: KeboolaClient, component_id: str, configuration_id: str) -> None: """ Sets the configuration metadata to indicate it was created by MCP. diff --git a/tests/test_server.py b/tests/test_server.py index 31c25a1df..328bd81d8 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -63,6 +63,7 @@ async def test_list_tools(self): 'get_project_info', 'get_semantic_context', 'get_semantic_schema', + 'get_shared_codes', 'get_tables', 'modify_flow', 'modify_python_js_data_app', diff --git a/tests/tools/components/test_tools.py b/tests/tools/components/test_tools.py index ba01e47e9..8e2da78ce 100644 --- a/tests/tools/components/test_tools.py +++ b/tests/tools/components/test_tools.py @@ -39,10 +39,13 @@ GetComponentsOutput, GetConfigsDetailOutput, GetConfigsListOutput, + GetSharedCodesOutput, SimplifiedTfBlocks, TfParamUpdate, + TfRemoveSharedCode, TfRenameBlock, TfSetCode, + TfSetSharedCode, TfStrReplace, VariableDefinition, ) @@ -53,6 +56,7 @@ get_components, get_config_examples, get_configs, + get_shared_codes, run_sync_action, update_config, update_config_row, @@ -61,6 +65,7 @@ from keboola_mcp_server.tools.components.utils import ( BIGQUERY_TRANSFORMATION_ID, FOLDER_SUPPORTING_COMPONENT_IDS, + SHARED_CODE_COMPONENT_ID, SNOWFLAKE_TRANSFORMATION_ID, VARIABLES_COMPONENT_ID, clean_bucket_name, @@ -290,7 +295,7 @@ async def test_get_configs_detail( # Verify the calls were made with the correct arguments keboola_client.storage_client.configuration_detail.assert_called_once_with( - component_id=mock_component['id'], configuration_id=mock_configuration['id'] + component_id=mock_component['id'], configuration_id=mock_configuration['id'], include=['rows'] ) @@ -316,7 +321,7 @@ async def test_get_configs_detail_multiple( keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) # Return different configs based on the configuration_id - async def mock_config_detail(component_id: str, configuration_id: str): + async def mock_config_detail(component_id: str, configuration_id: str, include=None): if configuration_id == mock_configuration['id']: return {**mock_configuration, 'component': mock_component, 'configurationMetadata': mock_metadata} else: @@ -438,7 +443,7 @@ async def test_get_configs_detail_ignores_other_params( # Verify configuration_detail was called for the specified config keboola_client.storage_client.configuration_detail.assert_called_once_with( - component_id=mock_component['id'], configuration_id=mock_configuration['id'] + component_id=mock_component['id'], configuration_id=mock_configuration['id'], include=['rows'] ) @@ -1348,6 +1353,7 @@ async def test_create_config( name=name, description=description, configuration={'storage': storage, 'parameters': parameters}, + configuration_id=None, ) @@ -1404,6 +1410,7 @@ async def test_add_config_row( name=name, description=description, configuration={'storage': storage, 'parameters': parameters}, + row_id=None, ) @@ -2667,3 +2674,1373 @@ async def detail_side_effect(*args: Any, **kwargs: Any) -> dict[str, Any]: assert vars_delete_calls assert not vars_update_calls assert 'variables_id' not in main_cfg + + +# ============================================================================ +# SHARED CODE TESTS +# ============================================================================ + + +@pytest.mark.parametrize( + ('row_id', 'expected_forwarded'), + [ + pytest.param('dumpfiles', 'dumpfiles', id='explicit_row_id_forwarded'), + pytest.param('', None, id='empty_row_id_omitted'), + ], +) +@pytest.mark.asyncio +async def test_add_config_row_forwards_row_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + row_id: str, + expected_forwarded: str | None, +): + """`add_config_row` must forward an explicit row_id as the SAPI `rowId` (Mustache key for shared code).""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + component_id = mock_component['id'] + configuration_id = 'parent-config' + + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) + keboola_client.storage_client.configuration_row_create = mocker.AsyncMock( + return_value={'id': expected_forwarded or 'auto-id', 'version': 1} + ) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await add_config_row( + ctx=context, + name='shared snippet', + description='reusable SELECT', + component_id=component_id, + configuration_id=configuration_id, + parameters={'code_content': ['SELECT 1']}, + row_id=row_id, + ) + + keboola_client.storage_client.configuration_row_create.assert_called_once_with( + component_id=component_id, + config_id=configuration_id, + name='shared snippet', + description='reusable SELECT', + configuration={'storage': {}, 'parameters': {'code_content': ['SELECT 1']}}, + row_id=expected_forwarded, + ) + + +@pytest.mark.parametrize( + ('sql_dialect', 'expected_component_id'), + [ + ('Snowflake', SNOWFLAKE_TRANSFORMATION_ID), + ('BigQuery', BIGQUERY_TRANSFORMATION_ID), + ], +) +@pytest.mark.asyncio +async def test_create_sql_transformation_with_shared_code( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], + sql_dialect: str, + expected_component_id: str, +): + """When shared_code_id/row_ids are provided they must appear at the configuration root.""" + context = mcp_context_components_configs + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value=sql_dialect) + + keboola_client = KeboolaClient.from_state(context.session.state) + component = {**mock_component, 'id': expected_component_id} + configuration = {**mock_configuration, 'id': 'tf-with-sc'} + + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=configuration) + + shared_code_id = f'shared-codes.{expected_component_id.split(".")[-1]}' + shared_code_row_ids = ['dumpfiles', 'cleanup'] + + await create_sql_transformation( + ctx=context, + name='tf_with_sc', + description='uses shared code', + sql_code_blocks=[ + SimplifiedTfBlocks.Block.Code(name='Reused', script='{{ dumpfiles }}\n{{ cleanup }}'), + ], + shared_code_id=shared_code_id, + shared_code_row_ids=shared_code_row_ids, + ) + + call = keboola_client.storage_client.configuration_create.call_args + assert call is not None + payload = call.kwargs['configuration'] + assert payload['shared_code_id'] == shared_code_id + assert payload['shared_code_row_ids'] == shared_code_row_ids + + +@pytest.mark.asyncio +async def test_create_sql_transformation_without_shared_code_omits_fields( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """Default-empty shared_code_id must not introduce null/empty linkage fields into the payload.""" + context = mcp_context_components_configs + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + keboola_client = KeboolaClient.from_state(context.session.state) + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + + await create_sql_transformation( + ctx=context, + name='no_sc', + description='no shared code', + sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='c', script='SELECT 1')], + ) + + payload = keboola_client.storage_client.configuration_create.call_args.kwargs['configuration'] + assert 'shared_code_id' not in payload + assert 'shared_code_row_ids' not in payload + + +@pytest.mark.parametrize( + ('parameter_updates', 'existing_root_extras', 'expected_root'), + [ + pytest.param( + [TfSetSharedCode(op='set_shared_code', shared_code_id='lib-1', shared_code_row_ids=['a', 'b'])], + {}, + {'shared_code_id': 'lib-1', 'shared_code_row_ids': ['a', 'b']}, + id='set_shared_code_adds_root_fields', + ), + pytest.param( + [TfSetSharedCode(op='set_shared_code', shared_code_id='lib-2', shared_code_row_ids=['x'])], + {'shared_code_id': 'lib-1', 'shared_code_row_ids': ['a', 'b']}, + {'shared_code_id': 'lib-2', 'shared_code_row_ids': ['x']}, + id='set_shared_code_replaces_existing', + ), + pytest.param( + [TfRemoveSharedCode(op='remove_shared_code')], + {'shared_code_id': 'lib-1', 'shared_code_row_ids': ['a']}, + {}, + id='remove_shared_code_clears_root_fields', + ), + pytest.param( + [TfRemoveSharedCode(op='remove_shared_code')], + {}, + {}, + id='remove_shared_code_is_noop_when_no_linkage', + ), + ], +) +@pytest.mark.asyncio +async def test_update_sql_transformation_shared_code_ops( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + parameter_updates: list[TfParamUpdate], + existing_root_extras: dict[str, Any], + expected_root: dict[str, Any], +): + """TfSetSharedCode / TfRemoveSharedCode patch the configuration root, not parameters.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + existing_configuration = { + 'id': 'tf-1', + 'name': 'tf', + 'description': 'd', + 'configuration': { + 'parameters': { + 'blocks': [{'name': 'B', 'codes': [{'name': 'C', 'script': ['SELECT 1;']}]}], + }, + 'storage': {'input': {'tables': []}, 'output': {'tables': []}}, + **existing_root_extras, + }, + 'version': 1, + } + updated_configuration = {**existing_configuration, 'version': 2} + + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing_configuration) + keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=updated_configuration) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await update_sql_transformation( + context, + change_description='shared code change', + configuration_id='tf-1', + parameter_updates=parameter_updates, + ) + + sent_config = keboola_client.storage_client.configuration_update.call_args.kwargs['configuration'] + for key in ('shared_code_id', 'shared_code_row_ids'): + assert sent_config.get(key) == expected_root.get(key) + assert 'parameters' in sent_config + assert sent_config['parameters']['blocks'] + + +@pytest.mark.asyncio +async def test_update_sql_transformation_preserves_unrelated_root_fields( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """Root-level shared_code fields must survive a parameter-only update (no silent drop).""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + existing_configuration = { + 'id': 'tf-2', + 'name': 'tf', + 'description': 'd', + 'configuration': { + 'parameters': { + 'blocks': [{'name': 'B', 'codes': [{'name': 'C', 'script': ['SELECT 1;']}]}], + }, + 'storage': {'input': {'tables': []}, 'output': {'tables': []}}, + 'shared_code_id': 'lib-keep', + 'shared_code_row_ids': ['keep'], + }, + 'version': 3, + } + + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing_configuration) + keboola_client.storage_client.configuration_update = mocker.AsyncMock( + return_value={**existing_configuration, 'version': 4} + ) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await update_sql_transformation( + context, + change_description='rename only', + configuration_id='tf-2', + parameter_updates=[ + TfRenameBlock(op='rename_block', block_id='b0', block_name='Renamed'), + ], + ) + + sent_config = keboola_client.storage_client.configuration_update.call_args.kwargs['configuration'] + assert sent_config['shared_code_id'] == 'lib-keep' + assert sent_config['shared_code_row_ids'] == ['keep'] + + +@pytest.mark.asyncio +async def test_create_config_with_shared_code( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """create_config writes shared_code_id and shared_code_row_ids at the config root for Python/R/DuckDB.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + component_id = mock_component['id'] + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await create_config( + ctx=context, + name='py-tf', + description='python tf with shared code', + component_id=component_id, + parameters={'script': ['print("hi")']}, + shared_code_id='shared-codes.python-transformation-v2', + shared_code_row_ids=['imports'], + ) + + payload = keboola_client.storage_client.configuration_create.call_args.kwargs['configuration'] + assert payload['shared_code_id'] == 'shared-codes.python-transformation-v2' + assert payload['shared_code_row_ids'] == ['imports'] + + +@pytest.mark.parametrize( + ('shared_code_id', 'shared_code_row_ids', 'existing_extras', 'expected_root'), + [ + pytest.param( + 'lib-1', + ['a'], + {}, + {'shared_code_id': 'lib-1', 'shared_code_row_ids': ['a']}, + id='set_linkage', + ), + pytest.param( + '', + (), + {'shared_code_id': 'lib-1', 'shared_code_row_ids': ['a']}, + {}, + id='clear_linkage_with_empty_string', + ), + pytest.param( + None, + (), + {'shared_code_id': 'lib-1', 'shared_code_row_ids': ['a']}, + {'shared_code_id': 'lib-1', 'shared_code_row_ids': ['a']}, + id='none_preserves_existing_linkage', + ), + ], +) +@pytest.mark.asyncio +async def test_update_config_shared_code_linkage( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + shared_code_id: str | None, + shared_code_row_ids: tuple[str, ...], + existing_extras: dict[str, Any], + expected_root: dict[str, Any], +): + """update_config: non-empty sets linkage, empty clears it, None leaves existing untouched.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + component_id = mock_component['id'] + + existing = { + 'id': 'cfg-1', + 'name': 'cfg', + 'description': 'd', + 'configuration': { + 'parameters': {'k': 'v'}, + 'storage': {}, + **existing_extras, + }, + 'version': 1, + } + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) + keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing) + keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={**existing, 'version': 2}) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await update_config( + ctx=context, + change_description='tweak shared code', + component_id=component_id, + configuration_id='cfg-1', + shared_code_id=shared_code_id, + shared_code_row_ids=shared_code_row_ids, + ) + + payload = keboola_client.storage_client.configuration_update.call_args.kwargs['configuration'] + for key in ('shared_code_id', 'shared_code_row_ids'): + assert payload.get(key) == expected_root.get(key) + + +@pytest.mark.parametrize( + ('filter_ids', 'expected_config_ids'), + [ + pytest.param( + (), + {'shared-codes.snowflake-transformation', 'shared-codes.python-transformation-v2'}, + id='no_filter_returns_all', + ), + pytest.param( + (SNOWFLAKE_TRANSFORMATION_ID,), {'shared-codes.snowflake-transformation'}, id='filter_by_snowflake' + ), + pytest.param((BIGQUERY_TRANSFORMATION_ID,), set(), id='filter_with_no_matches_returns_empty'), + ], +) +@pytest.mark.asyncio +async def test_get_shared_codes( + mocker: MockerFixture, + mcp_context_components_configs: Context, + filter_ids: tuple[str, ...], + expected_config_ids: set[str], +): + """get_shared_codes lists keboola.shared-code configs and applies the component-id filter.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + list_response = [ + { + 'id': 'shared-codes.snowflake-transformation', + 'name': 'Snowflake snippets', + 'configuration': {'componentId': SNOWFLAKE_TRANSFORMATION_ID}, + }, + { + 'id': 'shared-codes.python-transformation-v2', + 'name': 'Python snippets', + 'configuration': {'componentId': 'keboola.python-transformation-v2'}, + }, + ] + detail_by_id = { + 'shared-codes.snowflake-transformation': { + 'rows': [ + { + 'id': 'dumpfiles', + 'name': 'Dump files', + 'configuration': {'code_content': ['SELECT * FROM info', 'WHERE 1=1']}, + }, + ], + }, + 'shared-codes.python-transformation-v2': { + 'rows': [ + { + 'id': 'imports', + 'name': 'imports', + 'configuration': {'code_content': ['import pandas as pd']}, + }, + ], + }, + } + + async def fake_detail(component_id: str, configuration_id: str, include=None) -> dict[str, Any]: + assert component_id == SHARED_CODE_COMPONENT_ID + assert include == ['rows'] + return detail_by_id[configuration_id] + + keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=list_response) + keboola_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=fake_detail) + + result = await get_shared_codes(ctx=context, transformation_component_ids=filter_ids) + + assert isinstance(result, GetSharedCodesOutput) + assert {cfg.config_id for cfg in result.shared_codes} == expected_config_ids + for cfg in result.shared_codes: + if cfg.config_id == 'shared-codes.snowflake-transformation': + assert cfg.transformation_component_id == SNOWFLAKE_TRANSFORMATION_ID + assert [row.row_id for row in cfg.rows] == ['dumpfiles'] + assert cfg.rows[0].code == 'SELECT * FROM info\nWHERE 1=1' + + +@pytest.mark.parametrize( + ('sql_dialect', 'expected_component_id'), + [ + ('Snowflake', SNOWFLAKE_TRANSFORMATION_ID), + ('BigQuery', BIGQUERY_TRANSFORMATION_ID), + ], +) +@pytest.mark.asyncio +async def test_create_sql_transformation_emits_shared_code_marker_blocks( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], + sql_dialect: str, + expected_component_id: str, +): + """ + Linking shared code via `shared_code_id` + `shared_code_row_ids` must inject a + `Shared Code (...)` marker code block per row id (mirroring the Keboola UI). Without + these markers the platform's runtime expansion never substitutes `{{rowId}}` and the + snippet is silently skipped at run time. + """ + context = mcp_context_components_configs + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value=sql_dialect) + + keboola_client = KeboolaClient.from_state(context.session.state) + component = {**mock_component, 'id': expected_component_id} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + + shared_code_id = f'shared-codes.{expected_component_id.split(".")[-1]}' + row_ids = ['audit_columns', 'cleanup'] + + await create_sql_transformation( + ctx=context, + name='tf_with_markers', + description='emits markers automatically', + sql_code_blocks=[ + SimplifiedTfBlocks.Block.Code(name='Main work', script='SELECT 1 AS k'), + ], + shared_code_id=shared_code_id, + shared_code_row_ids=row_ids, + ) + + sent = keboola_client.storage_client.configuration_create.call_args.kwargs['configuration'] + codes = sent['parameters']['blocks'][0]['codes'] + assert codes[0]['name'] == 'Main work', 'user-authored code must be preserved as the first code' + + marker_codes = codes[1:] + assert [c['name'] for c in marker_codes] == [ + f'Shared Code ({shared_code_id}-audit_columns)', + f'Shared Code ({shared_code_id}-cleanup)', + ] + assert [c['script'] for c in marker_codes] == [['{{audit_columns}}'], ['{{cleanup}}']] + assert sent['shared_code_id'] == shared_code_id + assert sent['shared_code_row_ids'] == row_ids + + +@pytest.mark.asyncio +async def test_update_sql_transformation_set_then_remove_shared_code_syncs_markers( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """ + `TfSetSharedCode` must add UI-canonical marker blocks (and replace any prior ones); + `TfRemoveSharedCode` must drop them while leaving user codes intact. + """ + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + existing_configuration = { + 'id': 'tf-markers', + 'name': 'tf', + 'description': 'd', + 'configuration': { + 'parameters': { + 'blocks': [ + { + 'name': 'B', + 'codes': [ + {'name': 'User code', 'script': ['SELECT 1;']}, + ], + } + ], + }, + 'storage': {'input': {'tables': []}, 'output': {'tables': []}}, + }, + 'version': 1, + } + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing_configuration) + keboola_client.storage_client.configuration_update = mocker.AsyncMock( + return_value={**existing_configuration, 'version': 2} + ) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + # SET + await update_sql_transformation( + context, + change_description='link shared code', + configuration_id='tf-markers', + parameter_updates=[ + TfSetSharedCode( + op='set_shared_code', + shared_code_id='shared-codes.snowflake-transformation', + shared_code_row_ids=['audit_columns'], + ), + ], + ) + sent_after_set = keboola_client.storage_client.configuration_update.call_args.kwargs['configuration'] + codes_after_set = sent_after_set['parameters']['blocks'][0]['codes'] + assert codes_after_set[0]['name'] == 'User code' + assert codes_after_set[-1]['name'] == 'Shared Code (shared-codes.snowflake-transformation-audit_columns)' + assert codes_after_set[-1]['script'] == ['{{audit_columns}}'] + + # Now simulate the storage backend returning the post-SET state, then REMOVE. + keboola_client.storage_client.configuration_detail = mocker.AsyncMock( + return_value={ + **existing_configuration, + 'configuration': { + **existing_configuration['configuration'], + 'parameters': {'blocks': [{'name': 'B', 'codes': codes_after_set}]}, + 'shared_code_id': 'shared-codes.snowflake-transformation', + 'shared_code_row_ids': ['audit_columns'], + }, + } + ) + await update_sql_transformation( + context, + change_description='unlink shared code', + configuration_id='tf-markers', + parameter_updates=[TfRemoveSharedCode(op='remove_shared_code')], + ) + sent_after_remove = keboola_client.storage_client.configuration_update.call_args.kwargs['configuration'] + codes_after_remove = sent_after_remove['parameters']['blocks'][0]['codes'] + assert [c['name'] for c in codes_after_remove] == [ + 'User code' + ], 'marker code must be dropped on remove_shared_code; user code preserved' + assert 'shared_code_id' not in sent_after_remove + assert 'shared_code_row_ids' not in sent_after_remove + + +@pytest.mark.asyncio +async def test_update_sql_transformation_set_shared_code_rejects_empty_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """`set_shared_code` with an empty `shared_code_id` must be rejected — callers must use + `remove_shared_code` to clear linkage, not write an inconsistent empty-id root.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + existing_configuration = { + 'id': 'tf-empty', + 'name': 'tf', + 'description': 'd', + 'configuration': {'parameters': {'blocks': []}, 'storage': {'input': {'tables': []}}}, + 'version': 1, + } + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing_configuration) + keboola_client.storage_client.configuration_update = mocker.AsyncMock() + + with pytest.raises(ValueError, match='requires a non-empty `shared_code_id`'): + await update_sql_transformation( + context, + change_description='bad set', + configuration_id='tf-empty', + parameter_updates=[TfSetSharedCode(op='set_shared_code', shared_code_id='', shared_code_row_ids=['x'])], + ) + keboola_client.storage_client.configuration_update.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_sql_transformation_rejects_placeholder_without_linkage( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """`{{ rowId }}` in a script without `shared_code_id`/`shared_code_row_ids` must raise.""" + context = mcp_context_components_configs + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + keboola_client = KeboolaClient.from_state(context.session.state) + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + + with pytest.raises(ValueError, match='`shared_code_id` is not set'): + await create_sql_transformation( + ctx=context, + name='broken', + description='bare placeholder, no linkage', + sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='Bad', script='{{ dumpfiles }}')], + ) + keboola_client.storage_client.configuration_create.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_sql_transformation_rejects_placeholder_missing_from_row_ids( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """A `{{ rowId }}` not listed in `shared_code_row_ids` must raise.""" + context = mcp_context_components_configs + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + keboola_client = KeboolaClient.from_state(context.session.state) + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + + with pytest.raises(ValueError, match='not in `shared_code_row_ids='): + await create_sql_transformation( + ctx=context, + name='broken_linkage', + description='references unlisted row', + sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='Bad', script='{{ unlisted }}')], + shared_code_id='shared-codes.snowflake-transformation', + shared_code_row_ids=['dumpfiles'], + ) + keboola_client.storage_client.configuration_create.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_sql_transformation_rejects_row_ids_without_shared_code_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """Passing `shared_code_row_ids` without a `shared_code_id` is a silent no-op — reject it.""" + context = mcp_context_components_configs + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + keboola_client = KeboolaClient.from_state(context.session.state) + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + + with pytest.raises(ValueError, match='without a `shared_code_id`'): + await create_sql_transformation( + ctx=context, + name='orphan_row_ids', + description='row ids but no library id', + sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='Q', script='SELECT 1')], + shared_code_row_ids=['dumpfiles'], + ) + keboola_client.storage_client.configuration_create.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_sql_transformation_allows_inline_config_variable_without_linkage( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """An inline `{{ variable }}` (Keboola config variable, not the sole content of a script + element) must NOT be mistaken for a shared-code reference — it shares Mustache syntax but is + not natively substituted, so creation must succeed without `shared_code_id`.""" + context = mcp_context_components_configs + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + keboola_client = KeboolaClient.from_state(context.session.state) + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + result = await create_sql_transformation( + ctx=context, + name='tf_with_variable', + description='uses a config variable inline', + sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='Query', script="SELECT '{{ api_token }}' AS t")], + ) + + assert result.success is True + keboola_client.storage_client.configuration_create.assert_called_once() + + +@pytest.mark.asyncio +async def test_create_config_emits_markers_for_python_transformation( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """`create_config` for Python/R must auto-emit `Shared Code (...)` marker blocks (SQL symmetry).""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + python_component = {**mock_component, 'id': 'keboola.python-transformation-v2'} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=python_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=python_component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await create_config( + ctx=context, + name='py_tf', + description='Python tf with shared code', + component_id='keboola.python-transformation-v2', + parameters={ + 'blocks': [{'name': 'Main', 'codes': [{'name': 'Init', 'script': ['print("ok")']}]}], + 'packages': [], + }, + shared_code_id='shared-codes.python-transformation-v2', + shared_code_row_ids=['imports'], + ) + + sent = keboola_client.storage_client.configuration_create.call_args.kwargs['configuration'] + codes = sent['parameters']['blocks'][0]['codes'] + assert codes[0]['name'] == 'Init', 'user code preserved' + assert codes[-1]['name'] == 'Shared Code (shared-codes.python-transformation-v2-imports)' + assert codes[-1]['script'] == ['{{imports}}'] + assert sent['shared_code_id'] == 'shared-codes.python-transformation-v2' + assert sent['shared_code_row_ids'] == ['imports'] + + +@pytest.mark.parametrize( + ('sql_dialect', 'expected_component_id'), + [ + ('Snowflake', SNOWFLAKE_TRANSFORMATION_ID), + ('BigQuery', BIGQUERY_TRANSFORMATION_ID), + ], +) +@pytest.mark.asyncio +async def test_create_sql_transformation_skips_marker_when_user_code_is_pure_placeholder( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], + sql_dialect: str, + expected_component_id: str, +): + """ + When a user-authored code block's script IS exactly `{{ rowId }}`, the platform's runtime + will substitute the snippet there. Emitting an additional `Shared Code (...)` marker for + the same row would execute the snippet twice, so the marker must be suppressed. + """ + context = mcp_context_components_configs + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value=sql_dialect) + + keboola_client = KeboolaClient.from_state(context.session.state) + component = {**mock_component, 'id': expected_component_id} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + + shared_code_id = f'shared-codes.{expected_component_id.split(".")[-1]}' + + await create_sql_transformation( + ctx=context, + name='tf_no_dup', + description='user already references shared row', + sql_code_blocks=[ + SimplifiedTfBlocks.Block.Code(name='Audit (shared)', script='{{ audit_columns }}'), + SimplifiedTfBlocks.Block.Code(name='Summary', script='SELECT 1 AS k'), + ], + shared_code_id=shared_code_id, + shared_code_row_ids=['audit_columns'], + ) + + sent = keboola_client.storage_client.configuration_create.call_args.kwargs['configuration'] + codes = sent['parameters']['blocks'][0]['codes'] + assert [c['name'] for c in codes] == [ + 'Audit (shared)', + 'Summary', + ], 'user code preserved verbatim; no auto-emitted marker for a row already referenced as a pure placeholder' + assert sent['shared_code_id'] == shared_code_id + assert sent['shared_code_row_ids'] == ['audit_columns'] + + +@pytest.mark.asyncio +async def test_create_sql_transformation_emits_marker_for_inline_placeholder( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """ + A `{{ rowId }}` embedded INSIDE a single SQL statement (not on its own line, e.g. quoted + inside a string literal) is NOT promoted to its own script element by the SQL splitter + and therefore NOT substituted by the platform. The marker must still be emitted so the + snippet actually runs at the configuration root. + """ + context = mcp_context_components_configs + workspace_manager = WorkspaceManager.from_state(context.session.state) + workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') + + keboola_client = KeboolaClient.from_state(context.session.state) + component = {**mock_component, 'id': SNOWFLAKE_TRANSFORMATION_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + + shared_code_id = 'shared-codes.snowflake-transformation' + + await create_sql_transformation( + ctx=context, + name='tf_inline', + description='inline placeholder still needs marker', + sql_code_blocks=[ + SimplifiedTfBlocks.Block.Code(name='Inline use', script="SELECT '{{ audit_columns }}' AS lit"), + ], + shared_code_id=shared_code_id, + shared_code_row_ids=['audit_columns'], + ) + + sent = keboola_client.storage_client.configuration_create.call_args.kwargs['configuration'] + codes = sent['parameters']['blocks'][0]['codes'] + assert codes[0]['name'] == 'Inline use' + assert codes[-1]['name'] == f'Shared Code ({shared_code_id}-audit_columns)', ( + 'inline placeholder (not the sole content of any script element) is not natively ' + 'substituted, so the marker must still be appended' + ) + + +@pytest.mark.asyncio +async def test_create_config_python_skips_marker_when_user_code_is_pure_placeholder( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """Python transformation: skip the auto-emit when a user code already has `["{{ rid }}"]`.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + python_component = {**mock_component, 'id': 'keboola.python-transformation-v2'} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=python_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=python_component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await create_config( + ctx=context, + name='py_tf_no_dup', + description='Python tf where user already references shared row', + component_id='keboola.python-transformation-v2', + parameters={ + 'blocks': [ + { + 'name': 'Audit', + 'codes': [{'name': 'shared imports', 'script': ['{{ audit_imports }}']}], + }, + { + 'name': 'Summary', + 'codes': [{'name': 'print', 'script': ['print("ok")']}], + }, + ], + 'packages': [], + }, + shared_code_id='shared-codes.python-transformation-v2', + shared_code_row_ids=['audit_imports'], + ) + + sent = keboola_client.storage_client.configuration_create.call_args.kwargs['configuration'] + first_block_codes = sent['parameters']['blocks'][0]['codes'] + assert [c['name'] for c in first_block_codes] == [ + 'shared imports' + ], 'no auto-emitted marker should appear next to a user code whose script is a pure placeholder' + # No marker should leak into the second block either. + second_block_codes = sent['parameters']['blocks'][1]['codes'] + assert all('Shared Code (' not in c['name'] for c in second_block_codes) + + +@pytest.mark.asyncio +async def test_add_config_row_surfaces_assigned_row_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """`add_config_row` exposes the actual row ID SAPI assigned, so callers can update it later.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + shared_code_component = {**mock_component, 'id': SHARED_CODE_COMPONENT_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.configuration_row_create = mocker.AsyncMock( + return_value={'id': 'audit_columns', 'version': 1} + ) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + result = await add_config_row( + ctx=context, + name='Audit', + description='snippet', + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id='shared-codes.snowflake-transformation', + parameters={'code_content': ['SELECT 1']}, + row_id='audit_columns', + ) + assert result.configuration_row_id == 'audit_columns' + + +@pytest.mark.asyncio +async def test_create_config_for_shared_code_parent_uses_flat_body_and_conventional_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], + mock_configuration: dict[str, Any], +): + """ + Shared-code parent libraries must be created with `componentId` at the configuration root + (not nested under `parameters`) and with the conventional `shared-codes.` ID, + not an auto-assigned UUID — otherwise the UI and runtime expansion cannot find the library. + """ + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + shared_code_component = {**mock_component, 'id': SHARED_CODE_COMPONENT_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await create_config( + ctx=context, + name='Shared codes for Snowflake', + description='reusable snippets', + component_id=SHARED_CODE_COMPONENT_ID, + parameters={'componentId': SNOWFLAKE_TRANSFORMATION_ID}, + configuration_id='shared-codes.snowflake-transformation', + ) + + call = keboola_client.storage_client.configuration_create.call_args + assert call.kwargs['configuration_id'] == 'shared-codes.snowflake-transformation' + assert call.kwargs['configuration'] == { + 'componentId': SNOWFLAKE_TRANSFORMATION_ID + }, 'Shared-code parent body must be flat (componentId at root), not wrapped under "parameters"' + + +@pytest.mark.asyncio +async def test_add_config_row_for_shared_code_uses_flat_body( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """Shared-code rows must persist `code_content` at the row configuration root, not under `parameters`.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + shared_code_component = {**mock_component, 'id': SHARED_CODE_COMPONENT_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.configuration_row_create = mocker.AsyncMock( + return_value={'id': 'dumpfiles', 'version': 1} + ) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await add_config_row( + ctx=context, + name='Dump files helper', + description='reusable SELECT', + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id='shared-codes.snowflake-transformation', + parameters={'code_content': ['SELECT 1']}, + row_id='dumpfiles', + ) + + call = keboola_client.storage_client.configuration_row_create.call_args + assert call.kwargs['row_id'] == 'dumpfiles' + assert call.kwargs['configuration'] == { + 'code_content': ['SELECT 1'] + }, 'Shared-code row body must be flat (code_content at root), not wrapped under "parameters"' + + +@pytest.mark.asyncio +async def test_get_shared_codes_reads_root_and_parameters_paths( + mocker: MockerFixture, + mcp_context_components_configs: Context, +): + """ + Reads `componentId` / `code_content` from the configuration root first (current platform + wire format) and falls back to `parameters.` for legacy/wrapper-created configs. + """ + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + list_response = [ + # Canonical: componentId at root. + { + 'id': 'shared-codes.snowflake-transformation', + 'name': 'Snowflake snippets', + 'configuration': {'componentId': SNOWFLAKE_TRANSFORMATION_ID}, + }, + # Legacy: componentId nested under parameters (matches old wrapper output). + { + 'id': 'legacy-shared-codes-python', + 'name': 'Legacy Python snippets', + 'configuration': {'parameters': {'componentId': 'keboola.python-transformation-v2'}}, + }, + ] + detail_by_id = { + 'shared-codes.snowflake-transformation': { + 'rows': [ + { + 'id': 'dumpfiles', + 'name': 'Dump files', + 'configuration': {'code_content': ['SELECT 1']}, # canonical: root + } + ], + }, + 'legacy-shared-codes-python': { + 'rows': [ + { + 'id': 'imports', + 'name': 'imports', + 'configuration': {'parameters': {'code_content': ['import pandas as pd']}}, # legacy + } + ], + }, + } + keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=list_response) + keboola_client.storage_client.configuration_detail = mocker.AsyncMock( + side_effect=lambda component_id, configuration_id, include=None: detail_by_id[configuration_id] + ) + + result = await get_shared_codes(ctx=context) + by_id = {cfg.config_id: cfg for cfg in result.shared_codes} + + assert by_id['shared-codes.snowflake-transformation'].transformation_component_id == SNOWFLAKE_TRANSFORMATION_ID + assert by_id['shared-codes.snowflake-transformation'].rows[0].code == 'SELECT 1' + + assert by_id['legacy-shared-codes-python'].transformation_component_id == 'keboola.python-transformation-v2' + assert by_id['legacy-shared-codes-python'].rows[0].code == 'import pandas as pd' + + +@pytest.mark.asyncio +async def test_get_shared_codes_rejects_unknown_component_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, +): + """An unknown component ID in the filter must raise before hitting the API.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + keboola_client.storage_client.configuration_list = mocker.AsyncMock() + + with pytest.raises(ValueError, match='Unknown transformation component IDs'): + await get_shared_codes(ctx=context, transformation_component_ids=['nonsense.component']) + + keboola_client.storage_client.configuration_list.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_config_shared_code_parent_requires_conventional_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """Creating a `keboola.shared-code` parent without an explicit `configuration_id` must be rejected + before hitting SAPI — an auto-assigned UUID is invisible to the UI and runtime expansion.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + shared_code_component = {**mock_component, 'id': SHARED_CODE_COMPONENT_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock() + + with pytest.raises(ValueError, match='requires an explicit `configuration_id`'): + await create_config( + ctx=context, + name='Shared codes for Snowflake', + description='reusable snippets', + component_id=SHARED_CODE_COMPONENT_ID, + parameters={'componentId': SNOWFLAKE_TRANSFORMATION_ID}, + ) + + keboola_client.storage_client.configuration_create.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_config_shared_code_parent_rejects_nonconventional_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """A `configuration_id` that doesn't match `shared-codes.` must be rejected — + the UI/runtime look libraries up by the exact conventional ID, so a typo would orphan the config.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + shared_code_component = {**mock_component, 'id': SHARED_CODE_COMPONENT_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock() + + with pytest.raises(ValueError, match='does not match the conventional shared-code library ID'): + await create_config( + ctx=context, + name='Shared codes for Snowflake', + description='reusable snippets', + component_id=SHARED_CODE_COMPONENT_ID, + parameters={'componentId': SNOWFLAKE_TRANSFORMATION_ID}, + configuration_id='shared-codes.snowflake-typo', + ) + + keboola_client.storage_client.configuration_create.assert_not_called() + + +@pytest.mark.asyncio +async def test_create_config_shared_code_parent_requires_component_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """A shared-code parent created without `parameters.componentId` has no backend association and + must be rejected before hitting SAPI.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + shared_code_component = {**mock_component, 'id': SHARED_CODE_COMPONENT_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.configuration_create = mocker.AsyncMock() + + with pytest.raises(ValueError, match='requires `parameters=.*componentId'): + await create_config( + ctx=context, + name='Shared codes', + description='reusable snippets', + component_id=SHARED_CODE_COMPONENT_ID, + parameters={}, + configuration_id='shared-codes.snowflake-transformation', + ) + + keboola_client.storage_client.configuration_create.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_shared_codes_skips_disabled_rows( + mocker: MockerFixture, + mcp_context_components_configs: Context, +): + """Disabled shared-code rows (the documented soft-delete path) must not be surfaced as reusable snippets.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + keboola_client.storage_client.configuration_list = mocker.AsyncMock( + return_value=[ + { + 'id': 'shared-codes.snowflake-transformation', + 'name': 'Snowflake snippets', + 'configuration': {'componentId': SNOWFLAKE_TRANSFORMATION_ID}, + } + ] + ) + detail = { + 'rows': [ + {'id': 'active', 'name': 'Active', 'configuration': {'code_content': ['SELECT 1']}}, + {'id': 'disabled', 'name': 'Disabled', 'isDisabled': True, 'configuration': {'code_content': ['SELECT 2']}}, + ], + } + keboola_client.storage_client.configuration_detail = mocker.AsyncMock( + side_effect=lambda component_id, configuration_id, include=None: detail + ) + + result = await get_shared_codes(ctx=context) + row_ids = [row.row_id for cfg in result.shared_codes for row in cfg.rows] + assert row_ids == ['active'], f'disabled rows must be filtered out; got {row_ids}' + + +@pytest.mark.asyncio +async def test_get_shared_codes_skips_config_without_component_id( + mocker: MockerFixture, + mcp_context_components_configs: Context, +): + """A shared-code parent with no `componentId` is malformed and must be skipped, not surfaced + with an empty transformation_component_id.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + keboola_client.storage_client.configuration_list = mocker.AsyncMock( + return_value=[ + # Malformed: no componentId anywhere. + {'id': 'orphan', 'name': 'No component', 'configuration': {}}, + # Valid one alongside it. + { + 'id': 'shared-codes.snowflake-transformation', + 'name': 'Snowflake snippets', + 'configuration': {'componentId': SNOWFLAKE_TRANSFORMATION_ID}, + }, + ] + ) + keboola_client.storage_client.configuration_detail = mocker.AsyncMock( + side_effect=lambda component_id, configuration_id, include=None: {'rows': []} + ) + + result = await get_shared_codes(ctx=context) + config_ids = [cfg.config_id for cfg in result.shared_codes] + assert config_ids == [ + 'shared-codes.snowflake-transformation' + ], f'malformed parent must be skipped; got {config_ids}' + assert all(cfg.transformation_component_id for cfg in result.shared_codes), 'no empty transformation IDs allowed' + + +@pytest.mark.asyncio +async def test_update_config_row_shared_code_applies_updates_to_flat_root( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """Editing a shared-code row must write `code_content` at the row root (where `get_shared_codes` reads it), + not under `parameters` — otherwise the edit is silently invisible.""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + shared_code_component = {**mock_component, 'id': SHARED_CODE_COMPONENT_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.configuration_row_detail = mocker.AsyncMock( + return_value={ + 'id': 'dumpfiles', + # A structural `storage` sibling that must survive a flat-root parameter update. + 'configuration': {'code_content': ['SELECT 1'], 'storage': {'input': {'tables': []}}}, + } + ) + keboola_client.storage_client.configuration_row_update = mocker.AsyncMock(return_value={'version': 2}) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await update_config_row( + ctx=context, + change_description='update snippet', + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id='shared-codes.snowflake-transformation', + configuration_row_id='dumpfiles', + parameter_updates=[ConfigParamSet(op='set', path='code_content', value=['SELECT 2'])], + ) + + call = keboola_client.storage_client.configuration_row_update.call_args + sent = call.kwargs['configuration'] + assert sent.get('code_content') == ['SELECT 2'], 'shared-code edit must land at the flat root' + assert 'code_content' not in sent.get('parameters', {}), 'shared-code edit must not be nested under parameters' + assert sent.get('storage') == {'input': {'tables': []}}, 'structural storage sibling must be preserved' + + +@pytest.mark.asyncio +async def test_update_config_row_shared_code_flattens_legacy_parameters_wrapper( + mocker: MockerFixture, + mcp_context_components_configs: Context, + mock_component: dict[str, Any], +): + """A legacy shared-code row that nests the snippet under `parameters` must be flattened: a flat-root + `code_content` update must stick (the validator unwraps `parameters` and would otherwise drop it).""" + context = mcp_context_components_configs + keboola_client = KeboolaClient.from_state(context.session.state) + + shared_code_component = {**mock_component, 'id': SHARED_CODE_COMPONENT_ID} + keboola_client.ai_service_client = mocker.MagicMock() + keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=shared_code_component) + keboola_client.storage_client.configuration_row_detail = mocker.AsyncMock( + # Legacy wrapper shape: code_content nested under `parameters`. + return_value={'id': 'dumpfiles', 'configuration': {'parameters': {'code_content': ['SELECT 1']}}} + ) + keboola_client.storage_client.configuration_row_update = mocker.AsyncMock(return_value={'version': 2}) + keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() + + await update_config_row( + ctx=context, + change_description='update legacy snippet', + component_id=SHARED_CODE_COMPONENT_ID, + configuration_id='shared-codes.snowflake-transformation', + configuration_row_id='dumpfiles', + parameter_updates=[ConfigParamSet(op='set', path='code_content', value=['SELECT 2'])], + ) + + sent = keboola_client.storage_client.configuration_row_update.call_args.kwargs['configuration'] + assert sent.get('code_content') == ['SELECT 2'], 'flat-root update must stick even for a legacy wrapper row' + assert 'parameters' not in sent, 'legacy `parameters` wrapper must be flattened away' diff --git a/tests/tools/components/test_utils.py b/tests/tools/components/test_utils.py index 1e1aee55f..b1efcb7e8 100644 --- a/tests/tools/components/test_utils.py +++ b/tests/tools/components/test_utils.py @@ -32,9 +32,11 @@ create_transformation_configuration, expand_component_types, get_config_folders, + is_shared_code_marker, set_configuration_folder_metadata, set_nested_value, structure_summary, + sync_shared_code_markers_in_dict, update_params, update_transformation_parameters, ) @@ -1569,3 +1571,60 @@ async def test_clear_configuration_folder_metadata( configuration_id='cfg-1', metadata_id=metadata_id, ) + + +@pytest.mark.parametrize( + ('name', 'script', 'expected'), + [ + ('Shared Code (sid-rid)', ['{{rid}}'], True), + ('Shared Code (sid-rid)', '{{ rid }}', True), + # Multiple placeholders in one element must NOT be treated as a marker. + ('Shared Code (sid-rid)', ['{{ a }}\n{{ b }}'], False), + # Placeholder plus surrounding SQL must NOT be a marker. + ('Shared Code (sid-rid)', ["SELECT '{{ rid }}'"], False), + # Right script shape but wrong name prefix. + ('User code', ['{{rid}}'], False), + # More than one script element. + ('Shared Code (sid-rid)', ['{{a}}', '{{b}}'], False), + ], + ids=['list-pure', 'str-pure', 'two-placeholders', 'inline', 'wrong-name', 'two-elements'], +) +def test_is_shared_code_marker(name: str, script: Any, expected: bool) -> None: + """Only a name-prefixed block whose single script element is exactly one Mustache placeholder + is a marker — otherwise marker sync would strip legitimate user code.""" + assert is_shared_code_marker(name, script) is expected + + +def test_sync_shared_code_markers_strips_markers_from_all_blocks() -> None: + """Stale `Shared Code (...)` markers in any block (e.g. moved during reordering) must be removed + before the canonical set is re-inserted into the first block — otherwise the snippet runs twice.""" + config = { + 'shared_code_id': 'shared-codes.snowflake-transformation', + 'shared_code_row_ids': ['dumpfiles'], + 'parameters': { + 'blocks': [ + {'name': 'First', 'codes': [{'name': 'User query', 'script': ['SELECT 1']}]}, + { + 'name': 'Second', + 'codes': [ + { + 'name': 'Shared Code (shared-codes.snowflake-transformation-dumpfiles)', + 'script': ['{{dumpfiles}}'], + }, + {'name': 'Other user code', 'script': ['SELECT 2']}, + ], + }, + ], + }, + } + + sync_shared_code_markers_in_dict(config) + + blocks = config['parameters']['blocks'] + # The stale marker in the second block must be gone; its user code stays. + second_names = [c['name'] for c in blocks[1]['codes']] + assert second_names == ['Other user code'], f'stale marker not stripped from non-first block: {second_names}' + # Exactly one canonical marker exists, in the first block. + all_markers = [c['name'] for block in blocks for c in block['codes'] if c['name'].startswith('Shared Code (')] + assert all_markers == ['Shared Code (shared-codes.snowflake-transformation-dumpfiles)'] + assert blocks[0]['codes'][-1]['name'] == 'Shared Code (shared-codes.snowflake-transformation-dumpfiles)' diff --git a/uv.lock b/uv.lock index 84ecf500f..bd41ce1e9 100644 --- a/uv.lock +++ b/uv.lock @@ -22,7 +22,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "caio" }, + { name = "caio", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/e2/d7cb819de8df6b5c1968a2756c3cb4122d4fa2b8fc768b53b7c9e5edb646/aiofile-3.9.0.tar.gz", hash = "sha256:e5ad718bb148b265b6df1b3752c4d1d83024b93da9bd599df74b9d9ffcf7919b", size = 17943, upload-time = "2024-10-08T10:39:35.846Z" } wheels = [ @@ -41,7 +41,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13'", ] dependencies = [ - { name = "caio" }, + { name = "caio", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/41/2fea7e193e061ce54eacc3b7bc0e6a99e4fcff43c78cf0a76dd781ed8334/aiofile-3.11.1.tar.gz", hash = "sha256:1f91912c6643d2a4e49ca4ae3514f0bf3867ce948a36d99a6411b8f4755f4cf9", size = 19342, upload-time = "2026-05-16T08:18:33.538Z" } wheels = [ @@ -1141,7 +1141,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -1332,7 +1332,7 @@ wheels = [ [[package]] name = "keboola-mcp-server" -version = "1.73.3" +version = "1.74.0" source = { editable = "." } dependencies = [ { name = "cryptography" }, @@ -2509,8 +2509,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "jeepney", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [